Merge branch 'working' into chatbot

# Conflicts:
#	.wolf/memory.md

.wolf/anatomy.md

@@ -1,12 +1,37 @@
 # anatomy.md
 
-> Auto-maintained by OpenWolf. Last scanned: 2026-07-23T08:56:01.886Z
-> Files: 788 tracked | Anatomy hits: 0 | Misses: 0
+> Auto-maintained by OpenWolf. Last scanned: 2026-07-31T09:01:40.540Z
+> Files: 840 tracked | Anatomy hits: 0 | Misses: 0
 
 ## ../../../../../../tmp/claude-1000/
 
 - `capture_single_job.sh` — Deploys a 3-node cluster from a given repo root's freshly-built binaries, (~544 tok)
 
+## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/c8633ffb-8da0-40a7-9811-48f1830c37b4/scratchpad/
+
+- `cross-spec-gap-analysis.md` — Cross-Spec Gap Analysis: admin-config-tool × config-upgrade-snapshot × clusterconfig-schema-split (2 (~2057 tok)
+
+## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/d3f40b2e-8c73-4820-8ad3-1400be8761ac/scratchpad/
+
+- `update_bug1471.py` (~439 tok)
+- `update_bug1474.py` (~1227 tok)
+
+## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/d6d1f70d-af0b-465b-9cd6-e719e38692ce/scratchpad/
+
+- `configpush_baddriver.c` — include <stdio.h> (~847 tok)
+- `impostor.py` — serve (~207 tok)
+- `manual_ha_deploy.sh` (~381 tok)
+- `verify_decode_reply.py` — Verify the actual decode_reply() implementation (mirrors the planned (~783 tok)
+- `verify_envelope.py` — Throwaway script to verify the Request/RequestReply envelope wrap+unwrap (~937 tok)
+
+## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/d6d1f70d-af0b-465b-9cd6-e719e38692ce/scratchpad/ha-configpush-replydecode/
+
+- `reject_reply_decode_driver.py` — Throwaway driver for raft-adapter-conformance task 7.3 remediation. (~1375 tok)
+
+## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/d6d1f70d-af0b-465b-9cd6-e719e38692ce/scratchpad/ha-vote-postconnect/
+
+- `impostor_peer.py` — serve, main (~584 tok)
+
 ## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-raft-tla/00c9b3f1-df3c-4ce0-a997-5c6a316be4ec/scratchpad/
 
 - `hammer.sh` — Tight repro loop for bug-1436: hammer verify_no_loss-style REPL sessions (~570 tok)
@@ -54,17 +79,20 @@
 
 ## ../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/
 
-- `MEMORY.md` — Memory Index (~2126 tok)
+- `MEMORY.md` — Memory Index (~2345 tok)
 - `project_bug1306_hook_rc_clobber.md` (~693 tok)
 - `project_bug1417_vote_index_mapping.md` — Bug-1417: Real-vote reply index mapping mismatch (~784 tok)
-- `project_bug1418_prework_decision.md` (~724 tok)
+- `project_bug1418_prework_decision.md` (~628 tok)
 - `project_bug1418_term_desync_apply_divergent.md` — Bug-1418: Term desync + apply-divergence promotion block → chaotic liveness stall (~743 tok)
 - `project_bug1419_wal_dup_submit_false_divergent.md` — Declares occurrence (~955 tok)
 - `project_bug1462_config_replication_fixed.md` — Declares client (~620 tok)
 - `project_bug847_wal_batch_payload_loss.md` (~1213 tok)
 - `project_raft_election_resonance.md` (~556 tok)
 - `project_resource_distribution_mapping_scoping.md` (~571 tok)
+- `project_scheduler_core_design_invariant.md` (~406 tok)
+- `reference_nix_capnpc_pin_fix.md` (~485 tok)
 - `reference_nonoverlay_fs_soak.md` (~371 tok)
+- `reference_xce_build_shared_dir_clobber.md` (~415 tok)
 
 ## ../../../../../claude/tmp/gdbwrap/
 
@@ -284,8 +312,8 @@
 
 ## .claude/skills/xce-test-ha/
 
-- `run-ha-soak.sh` — run-ha-soak.sh — driver for /xce-test-ha: deploy a 3-node XCE HA cluster, then (~30050 tok)
-- `SKILL.md` — XCE HA Failover Soak Test (~6970 tok)
+- `run-ha-soak.sh` — run-ha-soak.sh — driver for /xce-test-ha: deploy a 3-node XCE HA cluster, then (~41761 tok)
+- `SKILL.md` — XCE HA Failover Soak Test (~9766 tok)
 - `soak-verdict-selftest.sh` — soak-verdict-selftest.sh — deterministic unit test for the verdict parser in (~2637 tok)
 - `soak-verdict.sh` — soak-verdict.sh — pure (side-effect-free) verdict helpers for run-ha-soak.sh. (~2781 tok)
 
@@ -345,8 +373,8 @@
 - `design.md` — Technical Design: admin-config-tool (~13515 tok)
 - `requirements.md` — Requirements Document (~5992 tok)
 - `research.md` — Research & Design Decisions: admin-config-tool (~6829 tok)
-- `spec.json` (~332 tok)
-- `tasks.md` — Implementation Plan (~7241 tok)
+- `spec.json` (~573 tok)
+- `tasks.md` — Implementation Plan (~9380 tok)
 
 ## .kiro/specs/clusterconfig-schema-split/
 
@@ -367,9 +395,11 @@
 ## .kiro/specs/config-upgrade-snapshot/
 
 - `brief.md` — Brief: config-upgrade-snapshot (~1108 tok)
-- `requirements.md` — Requirements Document (~3658 tok)
-- `research.md` — Gap Analysis — config-upgrade-snapshot (2026-06-24) (~5335 tok)
-- `spec.json` (~269 tok)
+- `design.md` — Design Document: config-upgrade-snapshot (~13143 tok)
+- `requirements.md` — Requirements Document (~7093 tok)
+- `research.md` — Gap Analysis — config-upgrade-snapshot (2026-06-24) (~9323 tok)
+- `spec.json` (~136 tok)
+- `tasks.md` — Implementation Plan (~12616 tok)
 
 ## .kiro/specs/controller-status/
 
@@ -531,7 +561,7 @@
 - `design.md` — Design Document (~11209 tok)
 - `requirements.md` — Requirements Document (~4240 tok)
 - `research.md` — Research & Design Decisions (~5375 tok)
-- `spec.json` (~666 tok)
+- `spec.json` (~785 tok)
 - `tasks.md` — Implementation Plan (~14377 tok)
 - `TLA-gap-analysis.md` — Gap Analysis: XCE HA implementation vs `raft/raft.tla` (~2070 tok)
 
@@ -562,6 +592,14 @@
 - `spec.json` (~787 tok)
 - `tasks.md` — Implementation Plan (~4086 tok)
 
+## .kiro/specs/raft-adapter-conformance/
+
+- `design.md` — Design Document: raft-adapter-conformance (~9311 tok)
+- `requirements.md` — Requirements Document (~4577 tok)
+- `research.md` — Gap Analysis: raft-adapter-conformance (~8747 tok)
+- `spec.json` (~230 tok)
+- `tasks.md` — Implementation Plan (~4416 tok)
+
 ## .kiro/specs/raft-apply-decoupling/
 
 - `brief.md` — Brief: raft-apply-decoupling (~2239 tok)
@@ -639,6 +677,11 @@
 - `spec.json` — Declares is (~226 tok)
 - `tasks.md` — Implementation Tasks: resval-id-decoupling (~2424 tok)
 
+## .kiro/specs/scheduler-allocation/
+
+- `requirements.md` — Requirements Document (~9034 tok)
+- `spec.json` (~210 tok)
+
 ## .kiro/specs/scheduler-controller-msg-gap/
 
 - `design.md` — Design: scheduler-controller-msg-gap (~5006 tok)
@@ -647,11 +690,23 @@
 - `spec.json` (~230 tok)
 - `tasks.md` — Tasks: scheduler-controller-msg-gap (~4650 tok)
 
+## .kiro/specs/scheduler-core/
+
+- `brief.md` — Brief: scheduler-core (~1988 tok)
+- `requirements.md` — Requirements Document (~24180 tok)
+- `spec.json` (~202 tok)
+
 ## .kiro/specs/scheduler-framework/
 
-- `brief.md` — Brief: scheduler-framework (~5679 tok)
-- `requirements.md` — Requirements Document (~6317 tok)
-- `spec.json` (~138 tok)
+- `brief.md` — Brief: scheduler-framework (~6115 tok)
+- `requirements.md` — Requirements Document (~17160 tok)
+- `spec.json` (~211 tok)
+
+## .kiro/specs/scheduler-preemption/
+
+- `brief.md` — Brief: scheduler-preemption (~2910 tok)
+- `requirements.md` — Requirements Document (~2285 tok)
+- `spec.json` (~213 tok)
 
 ## .kiro/specs/worker-join-refactor/
 
@@ -739,6 +794,7 @@
 
 - `jul-06-jul-11.md` — Jul 6 → Jul 11 Progress Report (~2311 tok)
 - `jul-13-jul-18.md` — Jul 13 → Jul 18 Progress Report (~3919 tok)
+- `jul19-jul26-2026.md` — Work Summary: Jul 19 – Jul 26, 2026 (~3482 tok)
 - `jun29-jul04-2026.md` — Work Summary: June 30 – July 4, 2026 (~857 tok)
 
 ## scripts/
@@ -1035,63 +1091,71 @@
 ## src/controller/
 
 - `apiserver.c` — include <errno.h> (~5962 tok)
-- `CLAUDE.md` — XCE Controller (~3491 tok)
-- `cluster.c` — include "config.h" (~19620 tok)
+- `CLAUDE.md` — XCE Controller (~4527 tok)
+- `cluster.c` — include "config.h" (~20931 tok)
 - `commit.c` — Declares is (~12825 tok)
-- `config_repl.c` — define _GNU_SOURCE (~32202 tok)
-- `config_repl.h` — pragma once (~6155 tok)
+- `config_repl.c` — define _GNU_SOURCE (~33995 tok)
+- `config_repl.h` — pragma once (~6467 tok)
 - `ha_convergence.c` — Declares char (~7830 tok)
-- `ha.c` — define _GNU_SOURCE (~111827 tok)
-- `ha.h` — ifndef HA_H (~29766 tok)
+- `ha.c` — define _GNU_SOURCE (~124295 tok)
+- `ha.h` — ifndef HA_H (~34070 tok)
 - `job.c` — include <errno.h> (~24947 tok)
 - `join.c` — include "config.h" (~2795 tok)
-- `main.c` — include <argp.h> (~4682 tok)
-- `meson.build` (~2391 tok)
-- `myapp.c` — define _GNU_SOURCE (~5215 tok)
-- `myconf.c` — define _GNU_SOURCE (~1088 tok)
-- `persist.c` — define _GNU_SOURCE (~72815 tok)
+- `main.c` — include <argp.h> (~4767 tok)
+- `meson.build` (~2535 tok)
+- `myapp.c` — define _GNU_SOURCE (~5868 tok)
+- `myconf.c` — define _GNU_SOURCE (~1101 tok)
+- `persist.c` — define _GNU_SOURCE (~85714 tok)
+- `repl.c` — include <assert.h> (~8624 tok)
 - `sched.c` — include <errno.h> (~5719 tok)
+- `worker.c` — include <errno.h>; worker_from_capnp hook_detail-aware existing-worker apply (bug-1472 fix) (~7228 tok)
 
 ## src/controller/include/
 
 - `cluster.h` — if !defined(_CLUSTER_H_) (~3203 tok)
 - `commit.h` — pragma once (~3635 tok)
+- `worker.h` — if !defined(_WORKER_H_) (~199 tok)
 
 ## src/controller/testsuite/
 
-- `controller_test.c` — include <argp.h> (~2071 tok)
+- `configcompare.c` — define _GNU_SOURCE (~2122 tok)
+- `configcompare.h` — pragma once (~1107 tok)
+- `controller_test.c` — include <argp.h> (~2156 tok)
 - `snapdiff.c` — define _GNU_SOURCE (~833 tok)
 - `snapdiff.h` — pragma once (~569 tok)
-- `snapverify.c` — define _GNU_SOURCE (~2505 tok)
-- `snapverify.h` — pragma once (~645 tok)
-- `test_apply_committed_entry.c` — Declares directly (~7622 tok)
+- `snapverify.c` — define _GNU_SOURCE (~4034 tok)
+- `snapverify.h` — pragma once (~1250 tok)
+- `test_apply_committed_entry.c` — Declares directly; +3 WAL_WORKER_CTRL/UPDATE tests (bug-1472 fix: applies-to-existing, replace-not-or, JOIN-duplicate regression guard) (~12391 tok)
 - `test_apply_correctness_verification.c` — Declares driving (~16906 tok)
 - `test_apply_pump_integration.c` — Declares hook (~16913 tok)
 - `test_apply_pump.c` (~19569 tok)
 - `test_apply_reapply_idempotency.c` — Declares xce_job_t (~4026 tok)
 - `test_apply_seam.c` (~2936 tok)
 - `test_apply_seam.c` — raft-apply-decoupling task 2.1: calls apply_job_start/finish/fail/update_transition() directly on a bare jobmgr+job_t (no cluster_t), bypassing on_job_start/finish/fail, to prove the extracted guard cores work outside the receipt path (~2900 tok)
-- `test_commit_barrier_reply.c` — Declares uint8_t (~41011 tok)
-- `test_commit_barrier_rollout.c` — Declares explicitly (~26488 tok)
+- `test_bug857_regression.c` — Declares void (~6090 tok)
+- `test_commit_barrier_reply.c` — Declares uint8_t (~43215 tok)
+- `test_commit_barrier_rollout.c` — Declares explicitly (~27677 tok)
 - `test_commit_durability.c` — define _GNU_SOURCE (~32879 tok)
 - `test_commit_resolve.c` (~14018 tok)
-- `test_config_repl.c` — define _GNU_SOURCE (~45079 tok)
-- `test_convergence_verification.c` — define _GNU_SOURCE; pre-existing ha_walack_on_ack/ha_catchup_continuation_fn fixture, NO slowlane_init — the surface that broke when catchup offload defaulted ON (see bug-1451) (~18715 tok)
+- `test_config_repl.c` — define _GNU_SOURCE (~87209 tok)
+- `test_configcompare.c` — define _GNU_SOURCE (~2076 tok)
+- `test_convergence_verification.c` — define _GNU_SOURCE (~18893 tok)
 - `test_demotion_convergence.c` — define _GNU_SOURCE; dc_setup +slowlane_init, +test_demotion_offload_task_returns_promptly / test_demotion_inline_task_blocks_RED (2026-07-18, bug-1450 RED/GREEN) (~18600 tok)
 - `test_e2e_capnp.c` — include <stdint.h> (~3729 tok)
 - `test_e2e_checkpoint.c` — define _GNU_SOURCE (~3544 tok)
 - `test_e2e_gc.c` — define _GNU_SOURCE (~1678 tok)
 - `test_e2e_hooks.c` — define _GNU_SOURCE (~2184 tok)
 - `test_egress_filter.c` — Declares xce_job_t (~5072 tok)
-- `test_ha_core_safety.c` — Declares char (~36309 tok)
+- `test_ha_core_safety.c` — Declares char; HA/raft-adapter core-safety suite; includes task 5.1's test_5_1_wal_catchup_reject_sends_nack_reply (WAL catch-up reject -> ha_send_wal_ack_reply_ex NACK regression-lock) (~41000 tok)
 - `test_ha_elect_ctx_lifecycle.c` — Declares char (~2900 tok)
 - `test_ha_election.c` — Declares char (~16377 tok)
 - `test_ha_liveness.c` — Declares directly (~24748 tok)
 - `test_ha_partition.c` (~5459 tok)
 - `test_ha_rdhup_integration.c` — Declares char (~7341 tok)
+- `test_ha_role.c` — include <signal.h> (~11466 tok)
 - `test_ha_vep_teardown.c` (~6218 tok)
 - `test_ha_vote_gate.c` — define _GNU_SOURCE (~14165 tok)
-- `test_ha_vote_send.c` — define _GNU_SOURCE (~6961 tok)
+- `test_ha_vote_send.c` — define _GNU_SOURCE (~9950 tok)
 - `test_job_array_apiserver_push.c` — Declares uint8_t (~3366 tok)
 - `test_job_array_apply.c` — Declares from (~10687 tok)
 - `test_job_array_fence.c` (~6110 tok)
@@ -1100,17 +1164,26 @@
 - `test_persist_checkpoint.c` — define _GNU_SOURCE (~2415 tok)
 - `test_persist_hooks.c` — define _GNU_SOURCE (~4305 tok)
 - `test_persist_recovery.c` — define _GNU_SOURCE (~3778 tok)
-- `test_persist_snapshot_entities.c` — unsigned: queue (~14897 tok)
+- `test_persist_snapshot_entities.c` — unsigned: queue (~25632 tok)
 - `test_persist_truncate.c` — define _GNU_SOURCE (~5311 tok)
-- `test_repl_unit.c` — include <assert.h> (~6230 tok)
+- `test_repl_unit.c` — include <assert.h>; +2 bug-1474 slot-allocation guard tests (~9760 tok)
 - `test_replay_apply_seam.c` — Declares through (~3707 tok)
 - `test_snapdiff.c` — define _GNU_SOURCE (~1033 tok)
-- `test_snapverify.c` — define _GNU_SOURCE (~2008 tok)
+- `test_snapverify.c` — define _GNU_SOURCE (~4696 tok)
+- `test_term_safety_adapter.c` — define _GNU_SOURCE (~28939 tok)
+- `test_upgrade_snapshot_apply.c` — Declares char (~12444 tok)
+- `test_upgrade_snapshot_restore.c` — Declares char (~3581 tok)
 - `test_walcompare.c` — define _GNU_SOURCE (~6845 tok)
-- `test.c` — include <argp.h> (~30294 tok)
+- `test_waldump.c` — define _GNU_SOURCE (~4292 tok)
+- `test.c` — include <argp.h> (~31907 tok)
 - `walcompare.c` — define _GNU_SOURCE (~5176 tok)
 - `walcompare.h` — pragma once (~1474 tok)
-- `waldump.c` — define _GNU_SOURCE (~5516 tok)
+- `waldump.c` — define _GNU_SOURCE (~6488 tok)
+- `waldump.h` — pragma once (~1150 tok)
+
+## src/docs/
+
+- `deployment.md` — XCE Deployment Guide (~4942 tok)
 
 ## src/frontend/apiserver/backend/
 
@@ -1123,6 +1196,10 @@
 - `wire_dispatch_support.go` — go:build cgo (~2254 tok)
 - `wire_dispatch_test.go` — go:build cgo (~1337 tok)
 
+## src/frontend/apiserver/config/
+
+- `auth.yaml` — Authentication provider: config_file | pam | ldap | oauth2 (~187 tok)
+
 ## src/frontend/xcecli/cmd/auth/
 
 - `logon.go` — NewLogonCmd (~1388 tok)
@@ -1186,16 +1263,19 @@
 
 ## src/include/
 
-- `client.h` — if !defined(_CLIENT_H_) (~1707 tok)
+- `client.h` — if !defined(_CLIENT_H_) (~1899 tok)
 - `clusterconfig.h` — if !defined(_CLUSTERCONFIG_H_) (~248 tok)
 - `config.h` — if !defined(_CONFIG_H_) (~1257 tok)
 - `delay.h` — if !defined(_DELAY_H_) (~414 tok)
 - `kvmgr.h` — if !defined(__KVMGR_H__) (~631 tok)
-- `myconf.h` — if !defined(_MYCONF_H_) (~874 tok)
+- `myconf.h` — if !defined(_MYCONF_H_) (~1055 tok)
 - `netio.h` — if !defined(_NETIO_H_) (~2608 tok)
-- `persist.h` — pragma once (~24030 tok)
-- `typemap.h` — if !defined(_TYPEMAP_H_) (~5207 tok)
+- `persist.h` — pragma once (~24145 tok)
+- `repl.h` — pragma once (~2331 tok)
+- `request.h` — if !defined(_REQUEST_H_) (~2738 tok)
+- `typemap.h` — if !defined(_TYPEMAP_H_) (~5497 tok)
 - `wal_types.h` — pragma once (~2076 tok)
+- `xceapi.h` — if !defined(_XCEAPI_H_) (~650 tok)
 - `zarray.h` — if !defined(_ZARRAY_H_) (~1286 tok)
 
 ## src/include/xce/
@@ -1205,15 +1285,16 @@
 ## src/lib/
 
 - `client_internal.h` — if !defined(_CLIENT_INTERNAL_H_) (~4475 tok)
-- `client.c` — include <errno.h> (~6767 tok)
+- `client.c` — include <errno.h> (~7163 tok)
 - `cluster.c` — include <errno.h> (~1526 tok)
 - `delay.c` — include <errno.h> (~1520 tok)
 - `jobj.c` — include <errno.h> (~6090 tok)
 - `kvmgr.c` — include <errno.h> (~3053 tok)
 - `meson.build` (~1703 tok)
 - `netio.c` — include <arpa/inet.h> (~9456 tok)
-- `request.c` — include <errno.h> (~17130 tok)
+- `request.c` — include <errno.h> (~18036 tok)
 - `vtaskmgr.c` — include <errno.h> (~3822 tok)
+- `xceapi.c` — include <errno.h> (~4190 tok)
 - `zarray.c` — include <assert.h> (~5924 tok)
 
 ## src/lib/raft/
@@ -1238,7 +1319,7 @@
 
 ## src/nixSupport/
 
-- `xce.nix` (~573 tok)
+- `xce.nix` (~727 tok)
 
 ## src/scheduler/include/
 
@@ -1259,7 +1340,7 @@
 
 ## src/schema/
 
-- `controller.capnp` — Declares OPCode (~9442 tok)
+- `controller.capnp` — Declares OPCode (~10085 tok)
 
 ## src/tools/
 
@@ -1285,7 +1366,7 @@
 - `__init__.py` — xceadmin: CLI for building and deploying XCE cluster configuration. (~22 tok)
 - `build.py` — ``config build`` orchestration for xceadmin (design.md, ``xceadmin.build``; (~1578 tok)
 - `cli.py` — xceadmin command-line entry point (design.md, ``xceadmin.cli``; (~1127 tok)
-- `discovery.py` — Generic, opcode-agnostic redirect-following for xceadmin (design.md, (~1021 tok)
+- `discovery.py` — Generic, opcode-agnostic redirect-following for xceadmin (design.md, (~1775 tok)
 - `push.py` — ``config push`` orchestration for xceadmin (design.md, ``xceadmin.push``; (~1818 tok)
 
 ## src/xceadmin/src/xcecommon/
@@ -1299,13 +1380,17 @@
 
 ## src/xceadmin/tests/
 
-- `conftest.py` — Shared pytest fixtures for xceadmin's test suite. (~412 tok)
-- `test_cli.py` — Tests for the xceadmin CLI (design.md, ``xceadmin.cli``; task 1.1's (~2609 tok)
+- `_live_e2e_common.py` — Shared helpers for xceadmin's live E2E test suite (task 6.2's (~1206 tok)
+- `conftest.py` — Shared pytest fixtures for xceadmin's test suite. (~699 tok)
+- `test_cli.py` — Tests for the xceadmin CLI (design.md, ``xceadmin.cli``; task 1.1's (~2616 tok)
 - `test_config_build.py` — Golden-file-style build tests for ``xceadmin.build`` (design.md, (~3661 tok)
-- `test_config_push_mock.py` — Mock-controller tests for ``xceadmin.push`` (design.md, ``xceadmin.push``; (~3564 tok)
-- `test_discovery.py` — Unit tests for ``xceadmin.discovery`` (design.md, ``xceadmin.discovery``; (~1259 tok)
+- `test_config_push_mock.py` — Mock-controller tests for ``xceadmin.push`` (design.md, ``xceadmin.push``; (~3873 tok)
+- `test_discovery.py` — Unit tests for ``xceadmin.discovery`` (design.md, ``xceadmin.discovery``; (~2177 tok)
+- `test_e2e_live_ha_push.py` — Live 3-node HA E2E coverage for `config push` -- success path + the standby-redirect-gap regression lock; round-2 fixture adds _wait_for_standbys + settle + port recheck (still flaky -- bug-1485, primary crashes post-promotion) (~7600 tok)
+- `test_e2e_live_push.py` — Live single-node E2E coverage for `config push` (task 6.2; Requirements (~2779 tok)
 - `test_errors.py` — Tests for the xcecommon.errors exception hierarchy (task 1.2). (~2221 tok)
 - `test_mapping.py` — Unit tests for ``xcecommon.mapping`` (design.md, ``xcecommon.mapping``; (~6323 tok)
+- `test_repro_bug1485.py` — THROWAWAY repro for bug-1485 -- NOT part of the real test suite. (~1910 tok)
 - `test_schema_bundling.py` — Cross-cutting bundling tests for ``xcecommon.schema`` and (~1526 tok)
 - `test_schema.py` — Tests for xcecommon.schema (task 2.2). (~3150 tok)
 - `test_validation.py` — Tests for xcecommon.validation (task 2.1). (~5022 tok)

.wolf/cerebrum.md

@@ -10,8 +10,13 @@
 - Run soak tests with specific round counts and deploy directories for precise regression tracking
 - Prefer parallel agent runs for independent analysis work; sequential for dependent tasks
 - Keep test logs for analysis, don't skip intermediate output even if a test hangs
+- NEVER push code to Rongsong's branch (`origin/working`, rshen/xce.git) unless the user explicitly allows it. We only sync Rongsong's code INTO our own branch (local working / myxce fork). When "push" is mentioned without a target repo, default to myxce (user's fork) and confirm first; `git push origin working` requires explicit permission (2026-08-01).
 
 ## Key Learnings
+- (2026-07-28, bug-1479 ROOT-CAUSED not fixed) A permanent quorum-loss family member unrelated to admission control: when repl.c's `repl_broadcast` (live single-entry WAL tail-follow) gets an `ae_reject_logmismatch` NACK back from a standby (a genuinely NORMAL Raft outcome after rapid leader-election churn leaves a conflicting uncommitted tail entry at the same LSN with a different term — confirmed byte-precise via `xce_test waldump`: ctrl-2's own lsn=13 was its own deposed term=3 noop, while the canonical/surviving log has a DIFFERENT term=4 noop at the same lsn), NOTHING in ha.c's `repl_ack_decode_and_dispatch` NACK branch (`!success`) does anything but check for a higher observed term and then literally `WARN("...success=0 — ignoring")` — no `repl_on_ack` call (the ONLY site that decrements `pending_ack_count`), no nextIndex backoff-and-retry, no catch-up trigger. Consequence: `pending_ack_count` climbs 1-per-broadcast-attempt with zero decrements until `repl_broadcast`'s own `MAX_PENDING_ACKS=16` defensive eviction fires and marks the standby `in_catch_up=1` — which correctly excludes it from `repl_active_standby_count`/quorum, but NOTHING then drives an actual catch-up stream/snapshot-install for it (the only other `repl_complete_catchup` call site, config_repl.c's ACK-driven `catchup_lane_fn`, can never engage a standby that never sends a genuine ACK in the first place). Net effect: a standby with ONE conflicting log entry from election churn (routine, expected) becomes PERMANENTLY excluded from quorum with no self-healing path — in a 3-node cluster this is one unlucky election sequence away from total write-quorum loss. **General lesson**: when a symptom looks like "resource exhaustion" (admission cap, connection pool, whatever), always verify the actual counter's INCREMENT and DECREMENT sites are symmetric across EVERY reply outcome (success AND failure/reject), not just the happy path — a counter that only decrements on success but increments unconditionally will silently drift to its ceiling under ANY sustained failure condition, which looks identical to a genuine leak from the outside (repeated timeouts / repeated rejects) but has a completely different fix (handle the reject/failure path, not "find the missing release call"). Confirmed the ORIGINAL admission-slot-leak hypothesis for this bug was WRONG by checking `commit_resolve()`/`wait_release_locked()` — those DO release correctly on every timeout, 1:1 (17 timeouts, 17 releases) — before chasing the real mechanism.
+- (2026-07-28, waldump REPL usage) `xce_test` (built at `src/xce.build/controller/xce_test` or `src/build.dbg/controller/xce_test`) has no `waldump` CLI flag — `--help` only shows generic argp options. It's a REPL: pipe commands via stdin, e.g. `printf 'waldump <data_dir_or_wal_seg>\nquit\n' | ASAN_OPTIONS=detect_leaks=0 timeout 60 <path-to-xce_test>` (mirrors the `xtest()` shell helper in both `clean_failover_preservation.sh` and `run-ha-soak.sh`). Output columns are `LSN | TERM | TYPE | PLEN | PCRC | INFO` — the PLEN column is payload length, NOT a second type code (a WAL_CONFIG_MARKER shows as `UNKNOWN | 40 |` because waldump's type-name table has no case for type 50 yet — 40 there is the marker's 40-byte payload length, not its wire type number). This is a fast, byte-precise way to independently confirm/refute term or index divergence between two members' WALs without instrumenting production code.
+- (2026-07-27/28, bug-1475 FIXED) A term/version index that grows its backing array geometrically and zero-fills the newly-grown region to mean "absent" has a subtle, TIME-DEPENDENT bug: zero-filling with the same sentinel value used for a legitimate "recorded as literal 0" entry makes a deliberately-never-recorded slot (here: persist.c's per-LSN term_index, where a WAL_CHECKPOINT's own LSN is BY DESIGN never passed to record_entry_term) silently and RETROACTIVELY flip from "absent" to "present, value 0" the instant ANY later index is recorded and the array-growth watermark (`term_index_count`) crosses it — even though nothing ever explicitly wrote to that slot. Two readers that check the SAME never-recorded slot at DIFFERENT times (e.g. a sender that reads it AFTER appending later entries, vs. a receiver that reads it BEFORE appending anything past it) get DIFFERENT answers for what should be a stable, position-only fact — this produced a permanent AppendEntries logOk rejection (bug-1475: sender computed prev_log_term=1 (WAL_LEGACY_TERM), receiver computed prev_log_term=0 (absent), for the identical checkpoint LSN). **General fix pattern**: reserve a DIFFERENT sentinel (here `TERM_INDEX_UNSET = UINT64_MAX`, filled via `memset(...,0xFF,...)` on array growth) distinct from every legitimate stored value, so a never-explicitly-written slot reads "absent" PERMANENTLY, independent of later growth — never let "grown but unwritten" collide with a real, meaningful value (here, literal term 0 / WAL_LEGACY_TERM) in the same sentinel. **Do-not-repeat**: when hand-tracing (not instrumenting) a bug involving an in-memory index that has a documented "grows on demand, gaps read as absent" contract, don't trust that the gap actually STAYS absent across the FULL lifetime of the process/test — verify with real instrumentation whether "absent" is a stable property of the position or merely a snapshot of "not yet grown past." This was found only after two prior, plausible-looking production fixes (config_repl.c's tracker-advance + ha.c's JIT log_len resync, both already landed) still left the regression test RED — real fprintf instrumentation at the streamer, the JIT resync, and raft.c's logOk check (all added and fully removed afterward) pinned the exact moment prev_log_term flipped from 0 to 1. See `.wolf/buglog.json` bug-1475 for the full three-part fix and `test_bug1475_catchup_survives_midstream_checkpoint` (test_config_repl.c) for the regression proof.
+
 - (2026-07-11, controller_unit_test isolated-group + leak-detection runs) `controller_unit_test` supports `--run STR` (repeatable, arg key `-t`) to select one or more TEST()-registered groups by exact function name (matches `argp`-parsed `-t`/`--run`, dispatch via `check_test()`/`TEST()` macro in `controller_test.c`) — use it to isolate a single suite instead of parsing the full-suite log. Full-suite direct runs interleave two PASS-print conventions: older suites only print via Unity's own per-assertion output (no `<file>:<line>:<name>:PASS` banner), while newer TDD suites (apply/HA-prevote family) add a custom `printf("%s: PASS\n", ...)` + banner — so grepping for `../controller/testsuite/<file>.c` banners under-counts groups that don't use the newer convention; don't infer suite pass/fail or crash-truncation from banner presence alone, always check the process exit code. Confirmed via meson (`Ok:1/Fail:0`, one lumped "controller test") AND a clean direct exit(0) with zero grep hits for AddressSanitizer/heap-buffer-overflow/UAF/double-free/FAIL across a 15.7k-line full-suite log — the log's abrupt-looking tail (last group's own internal UNITY_BEGIN/END subtotal printed twice) is normal, not truncation; verify with `echo $?` instead. The historical "test_ha_prevote 18/18" figure from an earlier session was the COMBINED total of `test_ha_prevote` (12 tests, confirmed via `grep -c RUN_TEST` in the .c file) + `test_ha_failover_prevote` (6 tests) — the two are separate TEST() groups in `controller_test.c`, not one suite of 18; report them separately (12/12 + 6/6) to avoid a false "regression" reading.
 - (2026-07-08, HA soak test sequential vs chaotic modes) Sequential mode soak validates clean failover lifecycle: 10 rounds, 36 jobs, zero loss, all 3 nodes led primary — deterministic validation of promotion + catch-up + reapply paths working as designed. Chaotic mode deliberately overlaps kills across election/catch-up windows to surface edge cases: rounds 1-4 pass verification, round 5 hits primary-discovery lag (bug-854 root-caused earlier, excluded from verdict), rounds 6-7 stall from election convergence lag + duplicate-node-guard (bug-1418, pre-existing guard). The election stall in chaotic round 6 is likely compound: (a) tight overlapping kill cadence outpaces election+restart timing (heartbeat-timeout bounds ~500ms but nix shell spinup adds latency), (b) a node restarted but not fully converged rejoins before the prior kill round's primary finishes its term tenure. Sequential mode PASS verdict is solid baseline. Chaotic mode needs either slower kill spacing OR a real-fs host deploy (current sandbox has overlayfs fdatasync EIO bug-924) to distinguish fault-tolerance (still holds) from environment-induced timing collapse.
 - (2026-07-04, raft-apply-decoupling task 6.1 round 2 — job-state merge-staleness fix) Round 1's `apply_committed_entry` wired the live pump but left a real corruption: `record_job_fail_fact`/`record_job_update_fact` (job.c) precomputed a FINAL absolute state at RECEIPT time (a scratch-copy merge for FAIL, an inline backstop+PDONE merge for postdone) and baked it into `wal_jobstate_t.state`; under deferred apply, a second fact for the same job (e.g. FINISH) could apply BETWEEN receipt and this fact's own apply, making the baked snapshot stale — a POST_EXEC-stage FAIL received while a job is still `RUN` (FINISH not yet applied) bakes `RUN|EXIT|PERR`, and when FINISH applies first (`DONE`) then FAIL's blind absolute overwrite clobbers it back to `RUN|EXIT|PERR` — DONE silently lost. FIX PATTERN: `wal_jobstate_t._pad` (previously always-zero, unused) is repurposed as a discriminator — `_pad==0` still means "legacy absolute state, safe" (job_ctrl's mark_job_fail/mark_job_update paths, which mutate+capture state synchronously under one lock, no staleness window); a nonzero `_pad` (`WAL_JOB_FAIL_STAGE_FLAG|stage` for FAIL, `WAL_JOB_UPDATE_POSTDONE_MARK` for postdone) marks "recompute at apply time against ACTUAL current state" — `apply_committed_entry` (persist.c) dispatches on this bit per WAL type, calling the REAL `apply_job_fail_transition`/new `apply_job_postdone_transition` (job.c, mirrors the existing guard-core family) instead of `apply_job_update_transition`'s blind overwrite. New low-level `persist_submit_job_fact_ex(p, type, jobid, state, pad)` added as the primitive; `persist_submit_job_fact` becomes a pad=0 thin wrapper — ZERO existing call sites (including ~20 in test_commit_barrier_reply.c) needed touching. GENERAL PATTERN for "wire format has 2 producers, one safe one not": don't change the safe producer's shape — add a discriminator field/flag so the consumer can tell them apart, and make "0"/"absent" mean exactly the old/safe behavior so every existing writer keeps working unchanged.
@@ -106,9 +111,11 @@
 - (2026-07-10) A chaotic-HA-soak "COMMITTED JOB LOSS" verdict with `committed_divergence=0` and all jobs present at final settlement is a VISIBILITY-LATENCY gap, not a durability violation: the job was legitimately committed by the old primary before it died, but the new leader's post-election `safe_committed_lsn` (computed only from THIS election's granted-voter sample, which can be conservative when a second voter is mid-restart) is lower until fresh quorum reconfirmation — correct Raft Figure-8 safety, not a bug by itself. Per this project's test philosophy (skill xce-test-ha: "a committed job an elected primary cannot list IS a real loss regardless of eventual reappearance") this is still a hard soak FAIL and must be fixed — but ONLY by shrinking the reconfirmation window (e.g. sending the first post-promotion heartbeat immediately), never by unsafely exposing unconfirmed old-term entries early just to make the check pass.
 - (2026-07-10, bug-1433, controller-invariant-audit) `peer_ctrl_classify` (ha.c ~5935) hardcodes `VTASK_LIVENESS` for EVERY inbound peer-control packet (vote replies, PreVote replies, WAL/config ACKs) — a task class documented as "never admission-gated," i.e. never expected to block. `raft_cb_become_leader` (the raft library's `.on_become_leader` callback) is invoked synchronously from THIS dispatch path whenever a real multi-node winning vote reply arrives — NOT from the separate `VTASK_LIVE_BLOCK`-dispatched election-*start* path (`ha_dispatch_election_start`, which only covers RDHUP-failover/single-node self-grant). Any blocking call added inside `raft_cb_become_leader` (or anything it calls) is reachable on a worker class the design guarantees never blocks — same bug class as bug-1198 (closed by moving `persist_checkpoint`'s snapshot-send fdatasync off VTASK_LIVENESS entirely). Before adding ANY blocking call to the promotion path, trace peer_ctrl_classify's actual dispatch, don't trust an existing nearby comment claiming "already on VTASK_LIVE_BLOCK" — one was WRONG (see bug-1433 in buglog.json for the full trace). `persist_apply_pump_run_sync` still has this residual (bounded `pread`, lower risk than `fdatasync`, not fixed this session — see buglog.json bug-1433 for why it can't simply be removed).
 - (2026-07-10, bug-1434, controller-invariant-audit) `persist_promotion_advance_committed` and `ha_standby_advance_committed_and_apply` both did an unlocked `load; if(target<=cur) return; store` on `p->committed_lsn` — safe ONLY if truly single-writer, but `raft_cb_become_leader` calls the former BEFORE setting `role = CTRL_PRIMARY` (~170 lines later), so a still-in-flight WAL catch-up entry on another thread can call the latter concurrently during that window — classic lost-update race that can regress `committed_lsn` (monotonicity violation). General lesson: when two independent call sites do "monotonic-max store to a shared atomic, no lock" and you can't prove they're strictly serialized by role/state gating (gating checked AT CALL TIME can still race with a WINDOW where the gate value itself is about to change), use a CAS retry loop (`persist_committed_lsn_advance_atomic`), not a bare load-then-store — it's safe under ANY interleaving with zero lock-ordering risk.
+- (2026-07-28, bug-1477) `cluster_init_phase2()` (cluster.c) is the SOLE place that runs `commit_engine_init()` -- which is itself the ONLY place that registers `persist_t`'s `on_durable_advance` wake hook (persist.c:796-799 says so explicitly: "starts unregistered ... the commit engine registers it"). Without that hook, `persist_apply_pump_run`/`apply_committed_entry` NEVER runs on that node, for ANY committed LSN, for its whole process lifetime -- a node can still fully participate in WAL replication/ACK (that path is independent, in ha.c) while never materializing ANY committed state. `cluster_init_phase2` short-circuits with `-EINVAL` (cluster.c:1349-1358) whenever `c->myconf->cluster_config == NULL` (config not loaded yet) -- normal during cold boot -- and its THREE raft-driven callers (`raft_cb_become_follower` ha.c:2000-2012, `raft_cb_become_leader` ha.c:1772-1783, `ha_elect_evaluate_and_signal`'s two branches ha.c:5609-5629/5699-5710) all set `ha->phase2_done=1` UNCONDITIONALLY after calling it, with NO check of its return value -- unlike `myapp.c`'s own `skip_election` fast-path caller (~line 415-425, bug-849 lineage) which correctly gates the latch on success. Worse: a low-priority node (highest `my_priority`, longest startup stagger) that finds "leader already known" on its very first check takes `goto skip_election` and NEVER runs `ha_start_election_async` at all -- so it never reaches ANY of the three raft-callback retry opportunities, ever, for its whole life, if it never later contests its own election. If its one-shot `myapp.c` attempt races ahead of config load, `cluster_init_phase2`/`commit_engine_init` never runs again, PERIOD -- a real, permanent HA-bootstrap gap, predates config-upgrade-snapshot by ~1 month (myapp.c skip_election = commit 20e59d0b 2026-06-21; ha.c raft-callback structure = commit 8f0d2d55 task 6.2), NOT introduced by it. It stayed invisible until config-upgrade-snapshot's task 5 (`persist_checkpoint_marker_triggered`) created a NEW cross-member invariant (every member independently writes a local checkpoint at a synchronized marker-boundary LSN) that a node with a never-wired apply pump silently violates -- surfacing as a PERMANENT (not transient/racy) `ae_reject_logmismatch` catch-up wedge that superficially resembles bug-1475 but is a categorically different, unrecoverable failure (the missing entry is genuinely, permanently absent -- not merely stale bookkeeping) since a WAL_CHECKPOINT entry is local-only/non-replicated by design and can never arrive over the wire to fill the gap. Diagnostic tell: grep the suspect node's log for "commit engine wired" -- if it NEVER appears (vs. multiple times on healthy peers, once per boot/restart/election), the commit engine/apply pump never wired on that node at all.
 
 ## Do-Not-Repeat
 
+- (2026-07-29, bug-1482) When a code-review finding says "X is never freed, add a free() call" citing a `_for_test` wrapper's own doc comment as the ownership contract, VERIFY the doc comment against the actual callee before complying literally — don't just trust it. `myreq_handler_for_test`'s comment claimed "the caller still owns/frees `pkt` itself", but `myreq_handler` unconditionally calls `xce_decode(pkt, ...)` as its first step, and `xce_decode()` (src/lib/request.c:274-275) itself unconditionally does `free_packet(pkt); FREEUP(pkt);` before returning — pkt is fully consumed/freed internally, no leak exists. Blindly adding the suggested `free_packet(pkt); free(pkt);` after the call produced an immediate ASAN heap-use-after-free/double-free (independent of `detect_leaks`, since UAF detection isn't gated by leak detection) — caught only by actually rebuilding and running the authoritative test command, not by code inspection alone. Fix was to correct the misleading doc comment instead of adding the free. **Lesson: for any wrapper that decodes/consumes a buffer (xce_decode-style), trace the actual free call sites transitively before trusting an ownership comment — a stale/wrong doc comment is itself a bug, and "add the missing free" is not always the right fix for a reported leak.**
 - (2026-07-19, bug-1452) `commit_engine_fini(&ce)` MUST run AFTER `persist_stop(p)`, never before, whenever the `persist_t` was `persist_start`'d (i.e. has a live `fsync_worker` thread). `commit_engine_fini`'s own header comment says this explicitly ("Callers are expected to have stopped the flusher (persist_stop) first") but `test_persist_truncate.c`'s three continuation tests called it backwards (`commit_engine_fini` then `teardown_persist`→`persist_stop`). `persist_set_durable_cb(p, NULL, NULL)` inside `commit_engine_fini` is an unsynchronized plain-pointer unhook — while the flusher thread is still alive it can already be mid-flight in `commit_on_durable_advance`/`commit_advance_locked` (no lock held, per persist.c:398-399's documented lock-ordering) and race the teardown: the guard `ce->p == NULL` check passes on a still-valid read, then a SEPARATE later re-read of `ce->p` inside `commit_advance_locked` (commit.c:208) observes the value `commit_engine_fini` just nulled — SEGV at exactly `offsetof(persist_t, committed_lsn)` (0x38). This was misdiagnosed for a day as caused by the g_drive_async_demotion/catchup_offload flags (bug-1450/1451) because flipping them changed thread-scheduling timing enough to trigger the always-latent race more reliably — the SAME crash signature was independently seen on 2026-07-07, before those flags existed, proving the flags were never causal. Fix: call `persist_stop(p)` immediately before `commit_engine_fini(&ce)`; the teardown helper's own internal `persist_stop` becomes a safe no-op second call. **Same backwards ordering also exists in `test_demotion_convergence.c` (line 233 `commit_engine_fini` before line 235 `persist_stop`) against a real `persist_start`'d fixture — not yet fixed, flag for a follow-up sweep of every `commit_engine_fini` call site in the controller test suite.**
 - (2026-07-03, raft-apply-decoupling task 1.5) `ninja -C build`'s INCREMENTAL rebuild in this worktree's shared `build/` dir can silently produce a stale/wrong `controller_unit_test` binary that reports a build success but gives WRONG test results for the actual HEAD source — distinct from bug-924 (which is the `meson test` driver hanging on overlayfs fdatasync). Symptom that tipped this off: `test_config_repl`'s `test_commit_advance_retry_clears_push_before_commit_fence` was independently measured as 70/70-passing by one task's reviewer and 10/10-failing by the next task's implementer, on what both believed was the same commit. Root-caused by isolating each measurement in its own throwaway `git worktree add <tmp-path> <commit>` + fresh `meson setup --buildtype=debug -Db_sanitize=address` + `ninja -C <tmp-build>` — proved the SAME commit passes cleanly with a fresh build and fails deterministically only in the main worktree's incrementally-rebuilt `build/`. Fix was `rm -rf build && meson setup build ... && ninja -C build` in the main worktree. **Lesson: whenever a test result looks surprising or contradicts a very recent measurement on the same commit, do NOT immediately conclude a source regression — first rule out a stale incremental build by either a full `rm -rf build` rebuild or an isolated `git worktree add` + fresh setup.** Never mutate the current worktree in place to "check an old commit" (e.g. `git checkout <sha> -- .`) — it silently reverts working-tree files against current HEAD; use `git worktree add` instead.
 - (2026-06-24, raft-sim-conformance task 1.2) A CONFORMANCE cluster (`raft_sim_conformance_init`) wires `on_send_vote_request`/`on_send_append_entries` to the unified `c->bag[]` (sim_conf_send_*), NOT to the legacy `c->pending[]`/`c->ae_pending[]`. So `raft_sim_deliver_pending` / `raft_sim_deliver_append_entries` (which drain pending[]/ae_pending[]) DO NOTHING on a conformance cluster — a 3-node conformance election will leave node 0 stuck in CANDIDATE (assert "Expected 2 Was 1" = RAFT_LEADER vs RAFT_CANDIDATE). Bag delivery is task 2.1. To force a leader in a task-1.2 test WITHOUT bag delivery, use a 1-node conformance cluster: `raft_start_election` short-circuits node 0 straight to RAFT_LEADER (no peers to canvass). The conformance `applied[]` decode reads the committed entry's `node->log[k-1].bytes` (the legacy `raft_sim_entry_t` byte WAL, reused) as a little-endian uint64 (memcpy), NOT the `raft_sim_log_entry_t {term,value}` struct (that type is only used by the bag message `entry_value` field).
@@ -515,3 +522,87 @@ Fixed two `xce-audit-invariants controller` thread-role violations (`ha_demotion
 ## Do-Not-Repeat (2026-07-21, admin-config-tool task 1.3, bug-1459) — a nix-provided `python3`'s site-packages lives under the read-only `/nix/store`, so `pip install`/`pip install -e .` against it directly fails with "externally-managed-environment" (PEP 668), even from inside `nix develop`. Fix pattern: `python3 -m venv --system-site-packages "$VENV_DIR"` (inherits the nix-provided heavy binary deps like `pycapnp` for free, so pip never re-downloads/rebuilds them) then run `pip`/`python` via `"$VENV_DIR/bin/..."`, not bare. This is the general pattern for any future nix + `pip install` component in this repo, not xceadmin-specific.
 
 ## Do-Not-Repeat (2026-07-21, admin-config-tool task 1.3, bug-1460) — a Python package's non-`.py` data files (here: `xcecommon/schema_files/*.capnp` and `schema_files/json/*.schema.json`) are silently DROPPED from a wheel built by `pip install .` (non-editable) unless declared via `[tool.setuptools.package-data]` in `pyproject.toml`. An editable install (`pip install -e .`) never catches this — it resolves straight into the checked-out source tree, bypassing wheel packaging entirely. Any bundled-non-code-asset pattern in this repo's Python components needs an explicit `package-data` entry, and the ONLY way to verify it works is to actually run the non-editable/`release` install profile and inspect `site-packages`, not just the editable one.
+
+## Do-Not-Repeat (2026-07-23, bug-1464) — every single-component `/xce-build <comp>` invocation does `rm -rf` on the SHARED `xce.build` dir before compiling; all components (controller, xceadmin, etc.) default to the same build dir per profile. Running `/xce-build controller` then `/xce-build xceadmin` separately wipes whichever was built first — only the most recent single-component build survives. When a downstream task (e.g. an HA soak's `deploy-ha.sh`) needs binaries from MULTIPLE components present simultaneously, use `/xce-build all` (one `setup_build()` + one unfiltered `meson compile`), not sequential per-component calls.
+
+## Key Learning (2026-07-23, bug-1418) — a `blocked_on` entry in a spec.json (or a project memory note) claiming a bug is "NOT FIXED" can itself be stale; the 2026-07-21 blocked_on note on config-upgrade-snapshot was ~2 weeks out of date against `.wolf/buglog.json`'s own 2026-07-09 resolution entry for bug-1418. Before trusting/acting on any blocked_on or "still open" claim, cross-check buglog.json's own entries for that bug id first — re-verified fresh here with a 10-round `--chaotic` soak (PASS, zero divergence/loss), then cleared config-upgrade-snapshot's blocked_on to [].
+
+## Key Learning (2026-07-23, config-upgrade-snapshot task 1) — this codebase's `controller.capnp` convention: EVERY struct gets `$C.mapname(...)`, even a brand-new wire-only "frame" concept with no corresponding "live" runtime C struct. For that case, define a small hand-written `typedef struct { ... } capnp_X_t;` in `src/include/typemap.h` (matching field-for-field, `Text`→`char*`, `UIntN`→`uintN_t`) and `$C.mapname` the schema struct onto it — exactly the pattern already used by `SnapshotHeader`/`capnp_snap_hdr_t`, `PersistConfig`/`capnp_persist_cfg_t`, `ReplEnvelope`/`capnp_repl_envelope_t` (all in typemap.h's "Persistence Cap'n Proto mapped types" section). This gives you generated `encode_X`/`decode_X`/`free_X` conversion functions for free (via the file-level `$C.codecgen` annotation) instead of hand-writing capnp field accessors. Do NOT emit a bare `struct X { name @0: Text; ... }` with no mapname even if a design doc's sketch shows it that way literally — cross-check real schema precedent first. Used this for `HostAdminState`/`QueueAdminState` → `capnp_host_admin_state_t`/`capnp_queue_admin_state_t`.
+
+## Do-Not-Repeat (2026-07-23, bug-918 recurrence 3) — when `nix develop`/`/xce-build` fails with a fixed-output-derivation hash mismatch on `capnpc-1.0.1.drv`'s `source.drv`, do NOT assume `nix flake lock --update-input xbuild` (the documented fix from the 2026-06-22 occurrence) will help without checking first — `nix flake metadata` may already show `xbuild` pinned to the LATEST available `shen390s/my-nix-channel@master` commit, in which case re-locking is a verified no-op (confirmed: ran it, `git diff -- src/flake.lock` was empty). Also confirmed via a direct `nix develop` in the canonical non-worktree repo (`/home/rshen/projects/xce/xce/src`) that it fails IDENTICALLY — this is a live, machine-wide upstream break (c-capnproto's mutable `main.tar.gz` drifted again), not a worktree-isolation artifact, and is NOT fixable from inside this repo at all. **Workaround that DOES work for schema-only verification**: `find /nix/store -maxdepth 1 -iname '*capnp*'` to check for already-REALIZED (not just `.drv`) `capnpc-1.0.1`/`capnproto-1.2.0` store paths from a build that completed before the drift — if present, invoke their `bin/capnp`/`bin/capnpc-c` DIRECTLY (`capnp compile -o <full-path-to-capnpc-c>:<outdir> -I <schema_dir> --src-prefix <schema_dir> controller.capnp`), bypassing `nix develop` entirely. This proves a schema change compiles and inspect the generated `.c`/`.h` for correct struct/function generation, but does NOT unblock a full project build — `controller_unit_test` also needs `librtti` (private SSH git flake input), which had NO `.drv` or realized store path on this machine at all (never built here), so full build+link+test remains infeasible until upstream fixes the pin or a machine with a pre-cached librtti toolchain is used. When this recurs, check bug-918's full `fix` field in buglog.json first — it now documents this exact triage sequence.
+
+## Key Learning (2026-07-24, bug-918 RESOLVED, recurrence 4) — the actual durable fix for the c-capnproto fixed-output-derivation hash mismatch (recurrences 1-3 above): a local `overrideAttrs` in `src/nixSupport/xce.nix`'s `capnpc` derivation pinning `src = pkgs.fetchFromGitHub { owner = "shen390s"; repo = "c-capnproto"; rev = "<immutable commit sha>"; sha256 = "<matching hash>"; }` — this makes THIS repo's build independent of upstream xbuild's package.nix, which pins the MUTABLE `main` ref and breaks every time that branch moves. `librtti` was NOT actually a blocker on this machine (recurrence 3's concern that it had "never been built/cached" turned out to be stale/machine-state-dependent) — a full `meson setup --reconfigure && ninja controller/controller_unit_test` built clean and the full suite ran (978 Tests 0 Failures). **Do-not-repeat**: for ANY future fixed-output-derivation hash mismatch traced to an upstream flake pinning a mutable branch/ref (not a tag/commit), the fix is a local `overrideAttrs` pinning `src` to an explicit immutable commit in the CONSUMING repo, not re-locking flake.lock or waiting on upstream — that path is a dead end once the upstream input is already at its latest commit (recurrence 3 already proved this).
+
+## Do-Not-Repeat (2026-07-24, config-upgrade-snapshot task 1, bug-1465) — bumping a shared on-disk format-version constant (here `persist.c`'s `SNAPSHOT_FORMAT_VERSION`) breaks ANY test file that hardcodes its own independent copy of that value to structurally validate bytes it reads directly (bypassing the real read-path on purpose, to test the write side in isolation) — even in a DIFFERENT, already-shipped spec's test file that the current task has no reason to otherwise touch. `test_config_repl.c`'s `fake_fmt` sites were caught and fixed in the same session the bump was made; `test_persist_snapshot_entities.c`'s independent `SNAPENT_TEST_FORMAT_VERSION` constant was NOT (different file, no shared symbol/include tying them together) and only surfaced as 3 failures once the full suite finally ran (blocked for a session by the bug-918 nix issue above). **General rule**: after bumping any shared format/version constant, `grep -rn` the whole test tree for OTHER hardcoded copies of the old literal value (not just call sites that reference the primary constant by name) before declaring the bump done — a stale hardcoded copy in an unrelated spec's test file is a silent regression that only a full-suite run catches.
+
+## Key Learning (2026-07-24, config-upgrade-snapshot task 2.3) — `persist.c` deliberately has ZERO function-call dependency on `config_repl.c` (not even the trivial `#include "config_repl.h"`). Two precedents establish this as an intentional domain boundary, not an oversight: `scan_latest_committed_config_ref` (persist.c) inline-decodes WAL_CONFIG_MARKER payloads instead of calling any config_repl.c decode helper ("decoded inline here so persist stays free of a config_repl layering dependency"); task 2.3's new `read_active_config_bytes` (persist.c, embeds active config bytes into every checkpoint) fully inline-reimplements `conf_read_current_filename`+`conf_read_raw`'s path-resolution-and-read logic (~140 lines) rather than calling those existing, already-public (`config_repl.h`) functions, even though `ha.c` DOES call `conf_read_raw` cross-module via a bare `extern` declaration at 2 sites (config push to standbys) — design.md's Architecture boundary map explains the asymmetry: `ha.c` is the intentional CONSUMER of "config replication (config_repl.c) owns config-bytes sourcing," so a ha.c→config_repl.c call is expected in-boundary integration, while `persist.c` owns materialization/on-disk-format and is kept deliberately free of that domain's dependency. **When touching persist.c again** (e.g. task 5's marker-apply trigger, task 6's Activation Fence — which design.md explicitly says must "verify against LIVE on-disk config bytes each attempt" mirroring `config_activate_committed`'s re-read-every-time pattern, `config_repl.c:275-302`) — check whether the new code lands in persist.c (keep it config_repl.c-dependency-free, duplicate the small read-with-validation routine if needed) or ha.c (fine to call config_repl.c directly, precedent already established). Do not "clean up" persist.c's duplication by refactoring it to call config_repl.c — that would be reversing a deliberate, design.md-documented boundary decision, not a simplification.
+
+## Key Learning (2026-07-24, config-upgrade-snapshot task 5) — a lock-reentrancy or stale-value hazard in a NEW `apply_committed_entry()` case can NEVER be caught by a test that calls `apply_committed_entry()` directly and in isolation. `persist_apply_pump_run()` holds `apply_pump_lock` across its ENTIRE per-LSN dispatch loop (including the call into `apply_committed_entry`); a naive `persist_checkpoint()` call from inside a new case would self-deadlock on that same non-recursive mutex, but only if the test actually drives the call through `persist_apply_pump_run()`/`persist_apply_pump_kick()` — a bare direct call never holds the lock, so it can't catch the deadlock. Same for a stale-`last_applied_lsn` hazard: `persist_apply_pump_run`/`replay_wal` both advance `last_applied_lsn` STRICTLY AFTER `apply_committed_entry` returns, never before — so a test needs a leading unrelated WAL entry before the entry under test to make `last_applied_lsn` genuinely differ from the tested entry's own lsn at dispatch time, or a "reads `last_applied_lsn` instead of the `lsn` parameter" bug is invisible. **General rule for any future `apply_committed_entry` case whose body triggers a SEPARATE persist_t operation that also touches `apply_pump_lock` or LSN-position state**: the test file MUST drive it through the real `persist_apply_pump_run`/`persist_apply_pump_kick` production call site (disk-backed persist_t + cluster_t fixture, mirrors `test_replay_apply_seam.c`/`test_apply_pump.c`, NOT `test_apply_committed_entry.c`'s bare in-memory fixture) with at least one leading entry, not just a bare direct call.
+
+## Do-Not-Repeat (2026-07-24, config-upgrade-snapshot task 5) — in a Unity test file that does real `persist_start()`/`persist_init()` (spins up a live `fsync_worker` background thread holding a `persist_t*` pointer into the test function's own stack frame), NEVER put the fixture's teardown call at the END of the test function after a run of `TEST_ASSERT_*` calls. A failed `TEST_ASSERT_*` does a `longjmp` straight back to `RunTest`, skipping every line after it — including the teardown call — which leaves the background thread alive holding a dangling pointer into a stack frame the harness is about to reuse for the NEXT test in the same binary. This manifests as an ASAN `stack-use-after-return` crash inside `flusher_fsync_forced_fail`/`fsync_worker`, attributed to a LATER, unrelated test's frame, making it look like that later test is broken when the REAL bug is an earlier test's missing unconditional teardown. This exact pathology is already documented in `test_apply_pump.c`'s own file header ("longjmp-past-teardown pathology from task 3.1's round-2 lesson") — re-discovered the hard way while drafting `test_upgrade_snapshot_apply.c`'s RED-phase run. **Required pattern**: gather every assertion-relevant observable into a plain local variable FIRST (no `TEST_ASSERT` between "the SUT ran" and "teardown runs"), call teardown UNCONDITIONALLY, THEN run every `TEST_ASSERT` against the captured locals.
+
+## Key Learning (2026-07-24, config-upgrade-snapshot task 5, bug-1466) — `persist_write_checkpoint_marker()` (called by ANY checkpoint, periodic OR the new task-5 marker-triggered one) appends its local-only, never-replicated `WAL_CHECKPOINT` entry by incrementing `persist_t.current_lsn` DIRECTLY, bypassing `ha_primary_log_append()`/`raft_log_append()` entirely. This silently violates `ha_primary_log_append`'s own documented co-tracking invariant (`ha.c:4057`, "p->current_lsn == s->log_len + 1 always holds") — the raft LIBRARY's `s->log_len` (a separate in-memory counter, `lib/raft/raft.c:574/585`) does NOT advance for a checkpoint marker, so it under-counts the true physical WAL tip from that point on. Consequence: `config_append_marker`'s/`ha_primary_log_append`'s RETURNED lsn (sourced from the library's `idx = s->log_len + 1`) becomes UNRELIABLE as an exact physical-position identifier for any entry appended after an earlier checkpoint has interleaved — this is a PRE-EXISTING gap (the periodic WAL-threshold checkpoint trigger has always had it) that task 5's marker-triggered checkpoint turns from "rare, needs >=100 WAL records to fire" into "the common case for any second config marker committed after an earlier one." **Verified this does NOT corrupt on-disk data or commit/durability correctness**: `commit_advance_locked`'s `majority_candidate_locked` (commit.c) is driven entirely by `persist_t`'s own `durable_lsn`/`current_lsn`, never by `s->log_len` (which the code's own comment calls "advisory only" for commit-ceiling purposes, ha-term-safety commit-ceiling-optA) — and `persist_wal_async` always self-assigns from `current_lsn`, ignoring the library's `index` parameter for a primary's own append (`ha_cb_persist_entry_stub`), so there is no on-disk LSN collision either. The blast radius is confined to any caller trusting the RETURNED lsn for precise identity/ordering once a checkpoint has occurred earlier in the session. **Do NOT attempt to fix the deeper `s->log_len` resync inside a `persist.c`-scoped task** — it requires touching `ha.c`/`lib/raft/raft.c` (owned by `ha-term-safety`/`raft-log-replication`), which is a bigger, deliberate architectural decision (e.g. adding a `raft_advance_log_len`-style API), not a same-task patch. Test-side fix that IS correct and in-scope: read the entry's TRUE physical lsn directly off `persist_t.current_lsn - 1` right after appending, instead of trusting the raft-library-derived return value — exactly how production code that needs precise positioning (`persist.c`'s apply seam, which receives `lsn` from the WAL entry's own on-disk header) already effectively does it. Full finding in `.wolf/buglog.json` bug-1466.
+
+## Key Learning (2026-07-25, config-upgrade-snapshot task 8.6, bug-1469) — task 8.6's live `--config-upgrade` soak EMPIRICALLY REPRODUCED bug-857 on its first real run, via a THIRD distinct mechanism neither the original (disproven lost_and_found) hypothesis nor task 8.5's unit-level fix (`schedule_checkpoint` WAL-threshold-vs-marker-boundary race) covered: `do_upgrade_config_with_bytes` (config_repl.c, the handler for the node that ORIGINATES/commits an upgrade) saves new config bytes to disk and bumps `config_generation`, but never calls `config_activate_committed()`/`load_primary_config()` to re-parse those bytes INTO `c->myconf` — so the originating primary's own in-memory queue table (and whatever the checkpoint writer walks to emit the queue-entity list) stays stale even though its Activation Fence (`upgrade_materialization_verify`, ha.c) reports "VERIFIED" (that function only hash-checks on-disk bytes + a checkpoint's own cfgref stamp, never touches `c->myconf`). `config_activate_committed` is only invoked from `handle_repl_config_push` (a STANDBY's push-receipt path) or a member's own boot/recovery — confirmed via ctrl-1's log: ZERO `config_activate_committed` calls in its original (primary) process instance, TWO in its restarted instance. If the originating primary is killed before some LATER independent event triggers that call (exactly what the chaotic-kill-interleaved-with-upgrade scenario does by design), its last checkpoint permanently freezes a self-contradictory state: correct new config bytes, stale old queue-entity set. **Do-not-repeat**: when investigating ANY "config activation" or "materialization" claim in this codebase, do not trust an Activation-Fence/"VERIFIED" log line as proof that `c->myconf`'s in-memory parsed structures were refreshed — trace whether `config_activate_committed`/`load_primary_config` was ACTUALLY called on that specific code path (grep the live log for the function name itself), since the fence's hash-check and a checkpoint's cfgref stamp can both be satisfied by disk-bytes-only activity. Full finding in `.wolf/buglog.json` bug-1469; NOT fixed (out of task 8.6's scope — flagged for tasks 1-7's owner). Task 8.6's own harness (dispatch wiring, `run_config_upgrade_round`, SKILL.md docs) is otherwise correct and not implicated — confirmed the on-disk-checkpoint-comparison methodology is the design-intended one and waiting longer would not have self-corrected (no further WAL activity to trigger a follow-up checkpoint).
+
+## Key Learning (2026-07-25, config-upgrade-snapshot task 8.6) — a live HA soak against a NEW client RPC can surface bugs the RPC's own author never had a way to trigger; two distinct, real materialization bugs found this way, only one safely fixable in-task
+Building task 8.6's `--config-upgrade` live soak required first discovering `OP_upgradeConfig` had NO working client anywhere (`xce_upgradeconfig_req_raw`/`xce_upgradeconfig`/`xcecli_upgradeconfig` built fresh in request.c/xceapi.c/client.c) AND that `cluster.c`'s dispatch gate for it was inverted (rejected primary, accepted standby — backwards; commit `fd93997f`, 2026-05-31, flipped it from a since-superseded primary-push design; already independently diagnosed as a blocking risk by the sibling `admin-config-tool` spec's design.md and explicitly left for whoever needed live `OP_upgradeConfig` first — that was this task). Fixing the dispatch gate (bug-1468) was a small, obviously-correct routing fix, in scope. Once the RPC worked, the FIRST real live run surfaced bug-1469: `apply_committed_entry`'s `WAL_CONFIG_MARKER` case called `config_activate_committed()` (which refreshes `c->myconf`) AFTER the marker-triggered checkpoint write, not before — so the checkpoint captured stale config. Reordering fixed HALF of it (verified: full 1025-test suite clean, checkpoint-content correctness improved). But re-running the SAME live soak after that fix showed the IDENTICAL divergence signature on a different node — root cause turned out DEEPER: `config_activate_committed`→`load_primary_config` only re-parses bytes into `myconf->cluster_config`, it NEVER touches `c->qmgr` (the runtime queue table `collect_queue_frames`/the checkpoint writer actually walks). The ONLY code that ever populates `c->qmgr` from config is `build_queues()` (queue.c), called EXACTLY ONCE per process lifetime at boot (`xce_config_init` hook), and it's unconditional/blind-create — unsafe to just re-call live (would attempt to recreate already-existing queues). This means NO live (non-restart) mechanism exists anywhere in this codebase to add a config-upgrade's NEW queue to an already-running member's runtime state — directly contradicting this whole spec's stated goal ("no controller restart required for a configuration upgrade"). This is genuinely deep, tasks-1-7 territory (needs a new create-if-missing sync function mirroring `restore_host_state`'s established pattern, PLUS an undecided queue-REMOVAL semantics design question) — NOT something to improvise under a test-infra task's time budget. **Resolution used**: changed the soak's own upgrade-target config diff from "add a queue" to "update an existing queue's description" — a legitimate, still fully meaningful config-identity test (Requirements 4.2/4.3/6.1/6.4 don't require adding an entity) that sidesteps the unrelated, unfixed mechanism entirely, rather than either (a) faking a pass by weakening the assertion, or (b) rushing an unreviewed fix into live queue-provisioning code. A third, much smaller (1-byte) checkpoint non-determinism remained even after that (bug-1471, host-frame encoding, root cause not yet isolated) — logged for follow-up rather than guessed at. **General lesson**: when a new regression-test/soak harness for feature X finds a bug in X's own materialization logic, first ask whether the SAME class of gap could apply more broadly than the one field your test happened to touch (here: ANY new queue OR host from ANY live upgrade, not just this soak's specific queue) before writing a fix — and if the honest answer requires new production logic + an undecided design question, stop and document rather than patch under the wrong task's time pressure.
+
+## Do-Not-Repeat (2026-07-25, config-upgrade-snapshot task 8.6) — a long-running/compacted session can leave substantial prior work on disk with no visible conversational memory of it; always `git status`/`git diff --stat` early when picking up a "continue"
+Mid-task, discovered via `git status` that `persist.c`/`test_config_repl.c` were already modified with a detailed, real bug fix (bug-1469) I had no memory of writing — from an earlier, compacted portion of the same long-running session (confirmed via claude-mem observation hooks showing timestamps hours earlier than the visible transcript, and via the fix's own content matching exactly what I'd have derived independently). Nearly redid Part A-D from scratch before noticing Part E (the whole soak script + SKILL.md docs) was ALREADY fully written on disk too. Lesson: when resuming any task (especially on a bare "continue"), run `git status --short` + `git diff --stat` against the expected file set FIRST, before writing anything — don't trust a feeling of "starting fresh" just because the visible conversation transcript looks short. The claude-mem observation timestamps (visible in PreToolUse:Read hook context) are a reliable secondary signal for how much wall-clock work actually preceded the current visible slice.
+
+## Key Learning (2026-07-25, config-upgrade-snapshot task 8.6, bug-1471/bug-1472) — a "1-byte checkpoint divergence near a padding-sized gap" is not proof of padding; decode the actual field before guessing
+bug-1471's two live hypotheses (uninitialized capnp padding vs. genuine content discrepancy) were distinguished definitively by (1) writing a throwaway byte-precise capnp decoder that reuses the SAME wire shape the real `write_HostAdminState`/`encode_HostAdminState` generated code produces (1 data word holding `status` via `capn_write32` at offset 0 + 4 bytes of trailing padding, 1 pointer word for `name` Text) rather than eyeballing `xxd` output, and (2) directly disassembling the pinned c-capnproto runtime (`/nix/store/*-capnpc-1.0.1/lib/libCapnC_Runtime.so`, since no `.c` source is shipped in the nix store — only the built `.so` — `objdump -d` on `capn_init_malloc`'s `create_local` callback) to confirm its segment allocator calls `calloc(1, size)`, not `malloc`, so it IS zero-initialized. Both lines of evidence agreed: the differing byte (file offset 42879) was byte index 2 of the 4-byte `status` field itself (0x00500000 = `XCE_WORKER_CLOSED|XCE_WORKER_CLOSED_BY_ADMIN` on ctrl-1 vs 0x0 on ctrl-2/ctrl-3), NOT the always-zero padding word right after it — a genuine content bug, not noise. Root cause: `worker_from_capnp` (worker.c:827-856, introduced 2026-05-08, ~2.5 months before this spec) is called for EVERY `WAL_WORKER_CTRL` apply regardless of JOIN/UPDATE/DEAD detail, and its idempotency guard returns `-EEXIST` (silently discarding the decoded fields, including `status`) whenever the named worker already exists — correct for JOIN replay, WRONG for UPDATE/DEAD (where "already exists" is the NORMAL case). This means ANY `hostctrl -c`/`hostctrl -o` against an already-joined host never propagates its admin-status change to standbys, unconditionally, cluster-wide — a real, general, pre-existing bug entirely outside config-upgrade-snapshot's boundary (filed as bug-1472, NOT patched here — scope discipline). **Do-not-repeat**: when a buglog entry frames two hypotheses as "padding vs. content" for a capnp field, always decode the field itself using the codebase's own generated wire-layout constants (`X_word_count`/`X_pointer_count`) before assuming either — "looks like it's near a text/entity boundary" is not evidence of which one it is.
+
+## Key Learning (2026-07-25, config-upgrade-snapshot task 8.9, bug-1474) — a live soak that FAILS is not automatically a test-script bug; forensically distinguish "my assumption was wrong" from "production genuinely did the wrong thing" via direct log evidence, byte-for-byte, before touching either
+Building `run_config_upgrade_wal_trim_round` (`--config-upgrade-wal-trim`) to prove bug-1147's WAL-trim retention guard holds under the marker-triggered path required a "baseline capture" step (record the primary's currently-active WAL segment as a genuinely-caught-up standby's floor, right before killing it) — a reasonable design, but its correctness is NOT self-evident from "the round ran without erroring." When the round's own final assertion failed (floor segment trimmed away, only the active segment survived) on BOTH of 2 independent live runs, the instinct to weaken/second-guess the test's own logic was strong — instead, cross-referenced `ctrl-2.log`'s (the primary's) own `commit: ack standby_idx=N` lines, `on_standby_connected`/`repl registered late-joiner standby` lines, and the exact wall-clock kill timestamp (`ctrl-1.log`'s own last line) to reconstruct, byte-for-byte, WHAT the replicator's slot state actually was at trim time. This revealed a REAL, 2-of-2-reproducible production defect: the primary's heartbeat task's "reconnect to any peer missing from standby_endpoints" sweep spuriously treats an ALREADY-connected peer (one connected only via a promoted-vote-channel, not yet a heartbeat-task connection) as missing, and the resulting duplicate "late-joiner" reconnect's `repl_add_standby` call picks the LOWEST-INDEXED free replicator slot with no regard for whether that slot still holds an UNRELATED, unexpired bug-1147 disconnect guard — silently clobbering it via `memset`. Filed as bug-1474 (NOT fixed — out of task 8.9's scope, mirrors the bug-1471/1472 precedent). **Do-not-repeat**: when a live-soak assertion fails, do NOT assume the test's own timing/logic is at fault just because the mechanism is complex — grep the ACTUAL controller logs for the specific slot/connection/ack events around the failure window first; a "the guard should have protected X but didn't" failure is exactly the shape a genuine upstream defect takes, and 2 independent runs showing the IDENTICAL mechanism is strong evidence it's real, not a timing fluke.
+
+## Key Learning (2026-07-26) — bug-1474 fixed: repl_add_standby's first-empty-slot scan must never clobber an unexpired bug-1147 disconnect guard
+Prior entry (2026-07-25) noted `repl_remove_standby` correctly sets the `last_disconnected_lsn`/`last_disconnect_time` guard and `repl_expire_disconnect_slots` correctly clears it on expiry — but nothing protected the guard from PREMATURE clobbering by `repl_add_standby`'s slot-allocation scan, which picked the first `ep==NULL` slot by lowest index with zero awareness of whether that slot still held an unexpired guard for a different peer. This let an unrelated peer's `repl_add_standby` call (triggered in the wild by a separate ha.c duplicate-reconnect defect, but the vulnerability exists regardless of the trigger) unconditionally `memset` away a guard `repl_trim_lsn` needed, letting `trim_wal_segments` delete WAL segments a genuinely disconnected-but-reconnecting standby still needed. **Fix pattern**: two-pass slot allocation — Pass 1 prefers a genuinely virgin slot (`ep==NULL && last_disconnected_lsn==0`); Pass 2 (only if no virgin slot exists) falls back to the guard-holding slot with the OLDEST `last_disconnect_time` (closest to its own natural `repl_expire_disconnect_slots` expiry, so reusing it discards the least remaining protection) rather than refusing the connect (`-ENOSPC` would be worse for cluster availability than an early guard loss in an already-degraded multi-disconnect scenario). **Do-Not-Repeat**: when a struct slot has BOTH a "live" state (`ep != NULL`) and a "recently-freed-but-still-meaningful" state (`ep==NULL && guard_field != 0`), a first-empty-slot allocator that only checks the live/dead boundary will silently destroy the recently-freed state's data — always give the allocator a way to distinguish "truly free" from "free but still guarding something," and pick a deliberate, explained tie-break policy (here: oldest-guard-first) for the exhausted-fallback case rather than an accidental index-order fallthrough. TDD: 2 new tests in `test_repl_unit.c` (`test_repl_add_standby_prefers_virgin_slot_over_guard`, `test_repl_add_standby_exhausted_guards_prefers_oldest_disconnect`), RED confirmed against the pre-fix scan, GREEN at 1030/1030 (0 failures, no regressions).
+
+## Key Learning (2026-07-25, bug-1472 fix) — the `Worker` capnp struct's `resvals` field maps to `resvals_to_sched`/`nresvals_to_sched`, NOT to `xce_worker_t.host.resvals` (the live per-host resource-value `zarray_t` that scheduling actually reads and that `set_host_slots()`/`update_resvals()` mutate). Confirmed by reading `src/schema/controller.capnp`'s `struct Worker` (`resvals @5: List(NameValue) $C.mapname("resvals_to_sched") $C.maplistcount("nresvals_to_sched")` — no field maps to `host.resvals` at all) and the generated `decode_Worker`/`encode_Worker` in `controller.capnp.c`. Consequence: the WAL_WORKER_CTRL wire format (`persist_worker_hook_fun` → `capnp_encode_worker(&w->host, ...)`, same single encode call site for JOIN/UPDATE/DEAD) can NEVER replicate a `mark_worker_unavail()`-style resource-slot zeroing to another member, no matter how the apply side is fixed — this is a genuine schema gap, not an apply-logic bug. Any future work on `HOOK_DETAIL_WORKER_DEAD` replication (see bug-1472-remainder in buglog.json) needs a schema change (a new wire-encodable resource-value-slots field) or a non-replication-based strategy, not just an apply-side fix mirroring bug-1472's UPDATE fix. **Also confirmed**: `encode_Worker`/`decode_Worker` DO faithfully round-trip `status`, `dead_since`, and `attrs` (`decode_HostAttrs_ptr` allocates and copies) for all three hook_detail values, since the encode call always captures the worker's CURRENT (post-mutation) `w->host` state uniformly — so a DEAD-case fix is NOT blocked by field-fidelity for those three, only by the missing `resvals` wire mapping.
+
+## Key Learning (2026-07-25, bug-1472 fix) — `worker_from_capnp`'s pre-existing fresh-restore path (`restore_worker(c, host)` on a MISS) leaks the decoded `xce_worker_t`'s `name`, `attrs` (+ its `hosttype`/`hostmodel` strings), and `resvals_to_sched` list on every call — NOT introduced by this fix, already present before it, out of scope to fix here, but worth knowing before touching this code again. Root cause: `copy_worker` (`src/lib/wobj.c:51-81`, called by `makeWorker`/`makeWorker_`/`override_worker`) has THREE different ownership behaviors for the SAME `xce_worker_t*` argument depending on the field: `name` and `attrs->hosttype`/`attrs->hostmodel` are re-created via `makeString()` (a COPY — caller's originals are untouched and must be freed by the caller); `dispatch_window_config` and `thresholds` are transferred via `MOVE_PTR` (caller must NOT free these — ownership passes to the new worker); `resvals_to_sched`/`nresvals_to_sched` are DROPPED entirely (`dst->host.nresvals_to_sched=0; dst->host.resvals_to_sched=NULL` — neither copied nor moved, silently discarded). None of `worker_from_capnp`'s existing callers (`restore_worker`, and this fix's own -EEXIST/JOIN branch) call `free_Worker(&host)` after `makeWorker`/`copy_worker` returns, so `name`/`attrs`/`resvals_to_sched` leak every time. This has never been caught because this repo's controller test config runs with `ASAN_OPTIONS=detect_leaks=0` (required separately for an unrelated reason — see bug-918/controller_unit_test hang note). **Do-not-repeat**: if you ever decode an `xce_worker_t` via `decode_Worker` and DON'T hand it to `makeWorker`/`copy_worker`/`restore_worker` (e.g. bug-1472's own new UPDATE branch, which only reads `host.status`), you MUST call `free_Worker(&host)` yourself — copy_worker's mixed copy/move/drop semantics mean there is no single safe default, and the existing call sites' silence is leaked technical debt, not a pattern to copy.
+
+## Key Learning (2026-07-25) — bug-1472 FIXED: `worker_from_capnp` now decodes `hook_detail` and applies UPDATE in place to an existing worker
+`worker_from_capnp` (`src/controller/worker.c`) previously returned `-EEXIST` unconditionally whenever the decoded worker's name was already registered, silently discarding the decoded fields regardless of whether the WAL entry was a JOIN, UPDATE, or DEAD. Fixed by reading the WAL_WORKER_CTRL payload's leading `hook_detail` discriminator (already decoded by the caller at `persist.c:3728-3729` but never passed down or re-read by `worker_from_capnp` itself) and branching: `HOOK_DETAIL_WORKER_JOIN` keeps the original -EEXIST no-op (regression-guarded); `HOOK_DETAIL_WORKER_UPDATE` now REPLACES (bitwise, not OR — see worker_ctrl's assignment-style `&= ~(...)`/`|= (...)` semantics, an OR-based fix would make HOST_OPEN unable to ever clear the closed bits again) `XCE_WORKER_ADMIN_MASK` bits on the existing worker under its own write lock, then returns 0 so `persist.c`'s existing generic "0 ⇒ fire the hook_detail-keyed notification" fallthrough (persist.c:3737-3785, unmodified) fires the correct `HOOK_DETAIL_WORKER_UPDATE` hook; `HOOK_DETAIL_WORKER_DEAD` is deliberately left as -EEXIST/no-op (see the two Key Learning entries above for why — filed as bug-1472-remainder, not silently ignored). `persist.c` needed NO changes — confirmed its WAL_WORKER_CTRL apply case already treats any non-EEXIST, non-negative `worker_from_capnp` return generically. TDD: RED confirmed by temporarily reverting just `worker.c` (via `git stash push -- <2 files>` / `git apply` round-trip, keeping the new tests) and re-running — exactly the 2 new UPDATE tests failed, the JOIN-duplicate regression-guard test already passed (proving it exercises the untouched path). GREEN: full suite 1030→1033 tests (this worktree's immediate pre-fix baseline, not a stale prior-session number), 0 failures, net +3 = exactly the 3 new tests (`test_ace_worker_ctrl_update_applies_to_existing_worker`, `test_ace_worker_ctrl_update_replaces_not_ors_admin_bits`, `test_ace_worker_ctrl_join_duplicate_existing_is_noop`), zero regressions.
+
+## Key Learning (2026-07-28, config-upgrade-snapshot task 8.9, bug-1478 FIXED) — a function's own doc comment can describe a NARROWER behavior than the code actually implements; the gap between the two is where a real bug hides
+`ha.c`'s `repl_ack_decode_and_dispatch` computed the primary's replication/convergence target as `target_lsn = max(committed_lsn, last_written)` (`last_written = persist.current_lsn - 1`, the RAW physical WAL tip), with a comment saying this exists "so standbys can catch up even when committed_lsn is stuck at 0" — but the code applied the `max()` UNCONDITIONALLY, not only in that `committed_lsn==0` case. This was harmless until config-upgrade-snapshot task 5's marker-triggered checkpoint (a local-only, never-replicated `WAL_CHECKPOINT` entry that `persist_write_checkpoint_marker` appends by bumping `current_lsn` directly, bypassing `raft_log_append`/the commit engine — the same architectural gap already flagged, unfixed at its root, as bug-1466) started interleaving a real, mid-operation local-only entry: `last_written` then permanently sat one LSN past the REAL, majority-gated `committed_lsn` (confirmed via `commit_advance_locked`'s own log never advancing past 21, while `ha_dispatch_repl_action_for_standby`'s log reported "committed_lsn=22" forever) — and since a checkpoint's own consumed LSN slot is BY DESIGN never streamed/applied by any standby, `HA_REPL_CONVERGED` became permanently unreachable, forever re-streaming an empty range every heartbeat. **Ground-truthed via `xce_test waldump <wal_dir>`** (an existing REPL command in the `xce_test` binary, `src/controller/testsuite/waldump.c` — prints `LSN | TERM | TYPE | PLEN | PCRC | INFO` for every entry in a WAL directory, plus a gap/tip summary) directly on the preserved live-soak deploy dir, which is what turned "I think lsn=22 is the checkpoint" into "confirmed: lsn=21 is WAL_CONFIG_MARKER (type UNKNOWN/40B), lsn=22 is WAL_CHECKPOINT (0B)" — use this tool FIRST whenever a WAL-position/LSN-identity question comes up in a live soak investigation, instead of inferring entry types from log lines alone. **Fix pattern**: extracted the computation into a small pure function (`ha_compute_repl_target_lsn`) matching the comment's ACTUAL narrow intent (`committed_lsn==0` only) — trivially unit-testable without capnp/network scaffolding, unlike the `static` function it lives inside. **Do-Not-Repeat**: when a fallback/edge-case comment says "so X can happen even when Y" but the code's condition doesn't actually gate on Y, that mismatch is not a documentation nit — it means the fallback fires in cases beyond what was ever justified/reviewed, and is a prime hiding spot for exactly this class of bug once ANY new caller starts producing conditions the ORIGINAL narrow case never anticipated. Full finding + fix in `.wolf/buglog.json` bug-1478 (RED→GREEN verified: reverting only the new function's body reproduced the predicted single failure "Expected 21 Was 22"; 1040/1040 tests clean twice after restoring the fix).
+
+## Do-Not-Repeat (2026-07-28, bug-1476) — a buglog "FIXED... GREEN" claim is not proof; the working tree can be mid-TDD-cycle at the exact moment you resume
+Resumed `/kiro-impl config-upgrade-snapshot` with buglog.json's bug-1476 entry claiming "FIXED (2026-07-28)... GREEN: full controller_unit_test suite" — but the actual uncommitted diff on disk had a comment (config_repl.c:639-646) explicitly admitting `load_primary_config()` had been reverted to its RED-phase (no lock, decode-directly-into-shared-field) state for test construction and never restored. Running the full suite fresh immediately surfaced the real state: `test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free` FAILED. The buglog note was written mid-cycle, describing the INTENDED fix, not the code actually sitting on disk. Fixed for real (lock now wraps the whole function body; decode targets a local temp, swapped in only after full validation), verified 4/4 clean full-suite runs before committing. **Do-not-repeat**: before committing or building on top of ANY buglog "FIXED"/GREEN claim for uncommitted work, re-run the actual test suite against the actual working tree first — never trust the log entry's narrative over a fresh run, especially when resuming a session that may have been interrupted mid-TDD-cycle.
+
+## Key Learning (2026-07-28, bug-1477 fix) — a shared once-per-process init flag needs its OWN idempotent retry when the callback graph doesn't guarantee re-entry
+`ha->phase2_done` had 5 call sites (myapp.c's skip_election path + 4 in ha.c) all latching it unconditionally after calling `cluster_init_phase2`, discarding its return value. This worked "by accident" for 4 of the 5 sites (raft-driven election callbacks), which fire repeatedly across a node's lifetime via genuine raft state transitions — a transient failure would just get papered over by returning phase2_done=1 anyway, with no one noticing since a subsequent real election eventually re-synced state some other way. But the 5th site (myapp.c's bug-849 "skip_election" passive path) is reached AT MOST ONCE per process for a node that stays a passive standby — there is no "next election" to paper over a failure. **Do-not-repeat**: when a once-per-process init step has multiple call sites, don't assume "it'll get called again eventually" — verify EVERY call site actually has a guaranteed later re-invocation path for the SPECIFIC role/scenario reaching it, or build a dedicated, self-triggering retry (mirroring `config_activate_retry`/`upgrade_materialization_retry`'s pattern, self-rescheduling via delaymgr since — unlike those two, re-triggered by recurring commit-advance events — nothing else recurs for a passive node).
+
+## Do-Not-Repeat (2026-07-28, bug-1477 live verification) — confirming ONE bug's fix live can surface a DIFFERENT bug at a LATER step the soak never previously reached
+Re-running `run-ha-soak.sh --config-upgrade` after the bug-1477 fix: the bug's own specific symptoms (checkpoint-identity, config-identity, host-admin-status-survival assertions) all PASSED for the first time. But the SAME soak run then failed later, in the force-elect-rotation step, with repeated commit timeouts (rc=-110) on the primary — a step this soak scenario had NEVER reached before (every prior attempt failed earlier, first at bug-1471/1472's checkpoint divergence, then at bug-1477's own catch-up wedge). **Do-not-repeat**: don't assume a soak reaching further and failing differently means your fix is broken or incomplete — check whether the NEW failure point was ever exercised before; if not, it's very plausibly a separate, pre-existing, newly-exposed bug (same pattern as this whole session's bug-1471→1472→1474→1477 chain), not evidence the just-verified fix regressed something. Investigate the new failure on its own terms (what's different about THIS code path) rather than assuming causality from the just-changed code.
+
+## Key Learning (2026-07-29, bug-1479 FIXED — 3-layer fix, each layer only verifiable live) — a correct adapter-side fix can be unit-green and still be dead code in production; only a live re-run proves the wire is actually connected
+bug-1479 (permanent quorum loss when a standby log-conflicts during chaotic election churn) needed THREE compounding fixes, and each of the first two looked complete after a clean unit-test run yet still failed the EXACT SAME live scenario (`run-ha-soak.sh --config-upgrade`'s `force_elect_node` loop) for a NEW reason each time:
+1. **`ha_raft_on_repl_nack`** (primary-side): routes a WAL-ACK-reply NACK into the raft library's nextIndex backoff + immediate `repl_start_catchup`. Unit-correct, 1048/1048 green — but live soak v1 still failed identically (0/3 acked), because nothing ever produced the `success=0` reply this code was waiting for.
+2. **Standby-side NACK sending** (`ha_apply_term_entry_via_library`'s logOk-reject path now calls `ha_send_wal_ack_reply_ex(..., success=0)` instead of silently dropping the reject — `ha_send_wal_ack_reply` had hardcoded `success=1` unconditionally, the ONLY WalAckReply sender in the whole controller). Unit-correct, verified via a new capture-hook test — but live soak v2 confirmed the NACK NOW reaches the primary (log-grepped: `WAL NACK from standby ... success=0`) and STILL failed identically, because `ha_stream_wal_to_standby`'s catch-up range was hard-capped at `committed_lsn`, which can never advance without this exact standby's own ack — a self-referential deadlock invisible from EITHER side's code in isolation.
+3. **`ha_compute_repl_stream_ceiling_lsn`**: widens the STREAM_WAL ceiling (and the convergence check gating it) to the leader's own log tail, matching raft.tla's real AppendEntries semantics (leaderCommit is advisory, never a send-cap) — while re-using bug-1478's exact guard (never widen onto a trailing local-only checkpoint's unstreamable LSN) so fixing gap 3 didn't reopen gap-1478. Live soak v3: PASS, all 3 force-elect rounds, 1053/1053 unit tests.
+**Do-not-repeat**: when an adapter/wire-protocol fix passes its own unit tests, that only proves the function is INTERNALLY correct — it does NOT prove anything upstream/downstream actually calls it with real data. For any fix whose job is "handle X when X arrives," always ALSO verify (via a live/integration run, log-grepped, not inferred) that X actually arrives in production BEFORE declaring victory — a unit-green "correct but unreachable" fix is functionally identical to no fix at all, and the only way to catch that is to re-run the EXACT scenario that surfaced the original bug, all the way to its real end state, every time a layer is added.
+
+## Key Learning (2026-07-29, raft-adapter-conformance task 5.1) — the bug-1479 fix commit (34ab78d7) already shipped a reply-construction-boundary regression test; the spec's "motivating finding" premise ("no test was ever written asserting the reply itself gets constructed") was not fully accurate
+`test_config_repl.c`'s `test_bug1479_ae_reject_sends_nack_not_silence` (added IN the bug-1479 fix commit itself, before this spec existed) already asserts `ha_wal_ack_send_hook` fires with `success=0` on a WAL catch-up logOk reject — exactly the "reply itself gets constructed" boundary requirements.md's motivating finding claimed was untested. Confirmed by `git log -p -S` on the test name: it landed in 34ab78d7, not in this spec. Task 5.1's explicit `_Boundary:_` (`test_ha_core_safety.c`) still called for an independent copy built from that file's own `hcs4_*` fixture family, so it was added anyway (`test_5_1_wal_catchup_reject_sends_nack_reply`) — genuinely useful redundant coverage (a future change to the reject branch is now caught in two files' fixture families instead of one), just not closing a gap that was actually still open. **Do-not-repeat**: when a spec's "motivating finding" describes a coverage gap, grep for the exact assertion shape (here: the hook name `ha_wal_ack_send_hook` + `success=0`) across the WHOLE test suite before treating the gap as confirmed-empty — a fresh audit pass can miss a test that already exists in a sibling file, especially one added in the same commit as the fix it's paired with. Report the discrepancy as a CONCERN in the task's status report rather than silently either skipping the task or presenting it as if it closed a real gap.
+
+## Do-Not-Repeat (2026-07-30, admin-config-tool task 6.3) — a bare `re.search()` against a log with MULTIPLE occurrences of the same pattern silently matches the FIRST one, not the one you actually care about
+Writing `test_e2e_live_ha_push.py`'s standby-ACK evidence check as `_STANDBY_SAVED_RE.search(standby.log_file.read_text())` looked correct and even matched a real, live `handle_repl_config_push` line — but a standby's log legitimately gets MULTIPLE `handle_repl_config_push` events over one test's lifetime (an automatic bootstrap sync of the primary's already-loaded config fires right after election, BEFORE the test's own `xceadmin config push`), so `search()`'s first match was always the OLDER bootstrap tag, never the specific push under test, and the assertion failed unconditionally regardless of whether the real ACK happened. Fixed with `re.finditer(...)` checked against ALL occurrences. **Do-not-repeat**: when asserting "this specific event happened" against a log a long-lived process keeps appending to, always consider whether the SAME log line shape can legitimately appear more than once for unrelated reasons before reaching for `re.search()` — default to `finditer`/`findall` + an explicit identity check (here: the exact tag) unless you've confirmed the pattern is provably unique in that log.
+
+## Key Learning (2026-07-30, admin-config-tool task 6.3) — design.md's own "Boundary Commitments"/"Revalidation Triggers" sections are the first place to check before writing a live test whose task text assumes a capability might not exist server-side
+Task 6.3 asked for a live redirect-following test (`config push` to a standby → follows redirect to primary → succeeds). Live-verified against a real 3-node cluster: it does NOT redirect — `ha_dispatch_gate()` rejects a standby's `OP_upgradeConfig` with `E_XCE_STANDBY` (10014) BEFORE the opcode-specific handler runs, and `fill_redirect_reply()` (`join.c`) is only ever wired for `OPCode_join`, never `OPCode_upgradeConfig`. This was NOT a surprise requiring fresh investigation — `design.md`'s own "Out of Boundary" section already documented this exact gap verbatim ("Redirect wiring for `OP_upgradeConfig`... it will not fire for `config push` until the controller is extended"), and "Revalidation Triggers" already lists the fix as a future, not-yet-done item. **Do-not-repeat**: before writing ANY live/E2E test whose premise is "component X's capability Y works end-to-end," grep the spec's own design.md for "Boundary Commitments"/"Out of Boundary"/"Revalidation Triggers" sections FIRST — if the capability is already flagged there as not-yet-implemented, the test should be written to lock in the REAL current (rejection) behavior with a pointer to the trigger that will make it stale, not spend a debugging cycle rediscovering a documented gap from scratch via trial and error.
+
+## Do-Not-Repeat (2026-07-30, admin-config-tool task 6.3) — Bash tool calls that `pkill` a process this session itself backgrounded can report a misleading exit code (144) even when the command's actual effect (killing the process, running rm -rf) succeeded
+Multiple `pkill -f xce_controller; sleep 1; ...` bash calls in this session reported exit code 144 (not a standard shell exit code — looks harness-level, not `$?`-derived) even though a follow-up `pgrep -af xce_controller` in the NEXT call confirmed the kill had actually taken effect. This only ever happened on calls that killed/interacted with background xce_controller processes this same session had spawned (via `subprocess.Popen`/pytest fixtures or manual `&`/`setsid nohup`), never on unrelated trivial commands. **Do-not-repeat**: don't treat a 144 exit code as proof a cleanup/pkill command failed — verify with an independent follow-up check (`pgrep`, `ls`) in a SEPARATE bash call before concluding anything is actually broken; retrying the exact same command (or just checking state) is usually enough, no special handling needed.
+
+## Do-Not-Repeat (2026-07-31, bug-1464 extended) — `xce-build.sh`'s unconditional `rm -rf $BUILD_DIR` races across MULTIPLE CONCURRENT AGENT SESSIONS on this same worktree, not just across sequential single-component invocations
+Ran `/xce-build all` (backgrounded) in this session while at least two OTHER Claude sessions were, unbeknownst to this session at launch time, ALSO independently invoking `xce-build.sh` (`controller`-only, then a separate `all` run) against the exact same shared `src/xce.build` dir in `/home/rshen/projects/xce/worktrees/config` — visible via `ps auxf` as sibling `bash src/tools/xce-build.sh ...` processes under different parent `claude --output-format stream-json` PIDs. This session's own backgrounded `all` run got silently torn out mid-flight ([360/362], no error, no completion marker, log just stopped growing) — almost certainly because a peer's `xce-build.sh` invocation hit line ~59's unconditional `rm -rf "$BUILD_DIR"` (same bug-1464 root cause, `BUILD_DIR` defaults to `xce.build` regardless of which component(s) you pass) out from under the still-running ninja/meson process. **This is a strictly worse variant of bug-1464**: the original note was about ONE session sequencing single-component builds; this shows even two *concurrent* full `all` builds from *different* sessions collide the same way, because `xce-build.sh` has no lockfile/mutex around its clean-then-setup step — there is no cross-session coordination at all. **Do-not-repeat**: before trusting a backgrounded `xce-build.sh` run's own log as authoritative (silent, no error, log just stops growing), run `ps auxf | grep -E 'xce-build|meson|ninja|cc1'` to check for a SIBLING invocation targeting the same worktree/BUILD_DIR from a different session — if found, do not stack a third concurrent build on top; wait (bounded poll on the peer PID, not a passive "background notification" — see next entry) for it to fully exit, THEN run your own single clean blocking build and treat only that run's own output as ground truth for a report. If this recurs, the real fix belongs in `xce-build.sh` itself (e.g. `flock` around the `rm -rf`+`meson setup` step), not in caller-side workarounds.
+
+## Do-Not-Repeat (2026-07-31) — a Bash `run_in_background` task inside a background *subagent* invocation may never deliver its "you'll be notified when it completes" notification; don't wait passively on it
+As a subagent (spawned via the `Agent` tool, running in the background from the parent's perspective), a `Bash` call with `run_in_background: true` did not visibly notify this session on completion even after several idle turns — a peer/coordinator message eventually had to point out "you're a subagent, you don't get woken up by background task notifications, check `ps aux` and wait blocking instead," which held up on inspection (the process really had died, torn out by the concurrent-build race in the entry above). **Do-not-repeat**: when running as a background subagent (not the top-level session), do NOT rely on Bash background-task completion notifications to resume work — either (a) run the long command in the foreground with a generous explicit `timeout` (up to 600000ms), or (b) if backgrounding is unavoidable (command may exceed 10 min), actively self-poll with a bounded `while kill -0 $PID; do sleep N; done` foreground Bash call rather than idling and waiting for a push notification that may never arrive in this execution context.
+
+## Key Learning (2026-07-31, bug-1487) — a "committed job loss" in `--chaotic` xce-test-ha soaks can be a false positive caused by the Raft own-term commit barrier, not real data loss — always check `verify_classify`'s timing before touching controller code
+Root-caused a `committed_job_loss=1` chaotic-soak FAIL entirely from logs/timestamps, no code change to `src/controller/`. Mechanism: a newly-elected primary cannot fold a PRIOR-term entry into its own `committed_lsn` watermark just because it holds it durably (Raft Figure-8 safety; `ha_arm_own_term_gate` in `ha.c` — every promotion appends an own-term `WAL_NOOP` and only advances `committed_lsn` past the inherited prefix once THAT noop itself gets a fresh majority ack under the new term). `expect-job` reads `c->jobmgr`, populated only by the apply pump once entries cross `committed_lsn` — so a genuinely majority-committed job can be transiently invisible on the brand-new primary until that fresh ack lands. In steady state this closes in <200ms, but `--chaotic` mode's own "restart the previous round's downed node with no convergence wait" design means the peer whose ack is needed can itself be mid-catch-up, stretching the window to several seconds (measured live: 6.232s). `run-ha-soak.sh`'s chaotic-mode `verify_classify()` ran its `expect-job` check immediately on primary detection with zero settle/retry (unlike `verify_no_loss`, which already had a bug-1422 3-attempt/2s retry, and unlike `wait_for_full_convergence`, which chaotic mode deliberately skips between kills). Fix was a bounded retry in `verify_classify` alone (5×2s), never touching `src/controller/`. **Do-not-repeat**: before treating any xce-test-ha `committed_job_loss` count as a real controller bug, first check (a) whether the SAME job shows up as present in every LATER verify/final-sweep in the same run (a real loss never comes back; this class of false-positive always does), and (b) whether `verify_classify`/whichever check flagged it has an equivalent settle/retry to its sibling `verify_no_loss` — chaotic and sequential modes have historically drifted out of sync on this (bug-1422 fixed sequential only; bug-1487 caught chaotic's matching gap 3 weeks later).

.wolf/hooks/_session.json

@@ -1,428 +1,386 @@
 {
-  "session_id": "session-2026-07-23-2018",
-  "started": "2026-07-23T03:18:13.693Z",
+  "session_id": "session-2026-07-30-0409",
+  "started": "2026-07-30T11:09:15.945Z",
   "files_read": {
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/include/typemap.h": {
+    "/home/rshen/projects/xce/worktrees/config/.kiro/specs/admin-config-tool/tasks.md": {
       "count": 1,
-      "tokens": 5207,
-      "first_read": "2026-07-23T03:38:15.897Z"
+      "tokens": 9072,
+      "first_read": "2026-07-30T12:19:14.341Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/sched.c": {
-      "count": 2,
-      "tokens": 5719,
-      "first_read": "2026-07-23T03:38:44.680Z"
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_push.py": {
+      "count": 7,
+      "tokens": 2779,
+      "first_read": "2026-07-30T12:19:14.613Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/limits.yaml": {
+    "/home/rshen/projects/xce/worktrees/config/src/tools/deploy-ha.sh": {
       "count": 3,
-      "tokens": 475,
-      "first_read": "2026-07-23T03:44:36.683Z"
+      "tokens": 1054,
+      "first_read": "2026-07-30T12:19:29.564Z"
     },
-    "/home/claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md": {
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/src/xcecommon/discovery.py": {
+      "count": 1,
+      "tokens": 0,
+      "first_read": "2026-07-30T12:19:29.992Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/src/xceadmin/discovery.py": {
+      "count": 1,
+      "tokens": 1775,
+      "first_read": "2026-07-30T13:02:08.621Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/src/xceadmin/push.py": {
+      "count": 1,
+      "tokens": 1818,
+      "first_read": "2026-07-30T13:02:15.180Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/conftest.py": {
+      "count": 2,
+      "tokens": 699,
+      "first_read": "2026-07-30T13:06:35.262Z"
+    },
+    "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/c8633ffb-8da0-40a7-9811-48f1830c37b4/tasks/b732ki2y3.output": {
+      "count": 1,
+      "tokens": 0,
+      "first_read": "2026-07-30T13:21:55.900Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/include/xce/errnox.h": {
       "count": 2,
       "tokens": 0,
-      "first_read": "2026-07-23T04:17:53.113Z"
+      "first_read": "2026-07-30T13:30:09.495Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/common/request/request.go": {
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py": {
+      "count": 8,
+      "tokens": 7600,
+      "first_read": "2026-07-30T13:31:16.766Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/controller/ha.c": {
+      "count": 2,
+      "tokens": 124295,
+      "first_read": "2026-07-30T13:42:42.468Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/controller/cluster.c": {
+      "count": 2,
+      "tokens": 20931,
+      "first_read": "2026-07-30T13:43:09.865Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/_live_e2e_common.py": {
+      "count": 3,
+      "tokens": 1206,
+      "first_read": "2026-07-30T13:43:53.361Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/tools/test-heartbeat-tolerance.sh": {
+      "count": 1,
+      "tokens": 2367,
+      "first_read": "2026-07-30T13:56:04.113Z"
+    },
+    "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/c8633ffb-8da0-40a7-9811-48f1830c37b4/tasks/bnpittld0.output": {
+      "count": 1,
+      "tokens": 0,
+      "first_read": "2026-07-30T14:06:34.840Z"
+    },
+    "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/c8633ffb-8da0-40a7-9811-48f1830c37b4/tasks/byx7ipj7f.output": {
+      "count": 6,
+      "tokens": 0,
+      "first_read": "2026-07-30T14:17:02.982Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/.claude/skills/kiro-impl/templates/debugger-prompt.md": {
+      "count": 1,
+      "tokens": 812,
+      "first_read": "2026-07-30T14:33:14.242Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/xceadmin/pyproject.toml": {
+      "count": 1,
+      "tokens": 362,
+      "first_read": "2026-07-30T14:35:12.474Z"
+    },
+    "/home/claude/.claude/projects/-home-rshen-projects-xce-worktrees-config/d6d1f70d-af0b-465b-9cd6-e719e38692ce/tool-results/b9vaeqw0k.txt": {
+      "count": 1,
+      "tokens": 0,
+      "first_read": "2026-07-31T03:34:12.233Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/lib/client.c": {
+      "count": 1,
+      "tokens": 7163,
+      "first_read": "2026-07-31T03:50:38.584Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/lib/client_internal.h": {
+      "count": 1,
+      "tokens": 4475,
+      "first_read": "2026-07-31T03:50:49.994Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/lib/client_failover.c": {
+      "count": 1,
+      "tokens": 0,
+      "first_read": "2026-07-31T04:02:26.918Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/.claude/skills/xce-test-ha/run-ha-soak.sh": {
       "count": 5,
-      "tokens": 2738,
-      "first_read": "2026-07-23T06:41:22.155Z"
+      "tokens": 41010,
+      "first_read": "2026-07-31T04:50:52.430Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/xce-build.sh": {
+    "/home/claude/tmp/xce-ha-chaos30-run1.log": {
       "count": 1,
-      "tokens": 1703,
-      "first_read": "2026-07-23T06:44:39.588Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/cmd/jobs/peek.go": {
-      "count": 3,
       "tokens": 0,
-      "first_read": "2026-07-23T06:48:49.031Z"
+      "first_read": "2026-07-31T04:51:29.049Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/steering/security.md": {
+    "/home/rshen/projects/xce/worktrees/config/src/lib/utils.c": {
+      "count": 1,
+      "tokens": 0,
+      "first_read": "2026-07-31T04:57:05.222Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/controller/job.c": {
+      "count": 1,
+      "tokens": 24947,
+      "first_read": "2026-07-31T04:57:56.735Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/docs/deployment.md": {
       "count": 2,
-      "tokens": 1142,
-      "first_read": "2026-07-23T07:06:09.487Z"
+      "tokens": 4816,
+      "first_read": "2026-07-31T08:39:59.108Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/apiserver/middleware/startserver.go": {
+    "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c": {
+      "count": 1,
+      "tokens": 5516,
+      "first_read": "2026-07-31T08:54:13.432Z"
+    },
+    "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/test_waldump.c": {
       "count": 1,
       "tokens": 0,
-      "first_read": "2026-07-23T07:15:02.581Z"
+      "first_read": "2026-07-31T08:57:57.611Z"
     },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/hosts.yaml": {
-      "count": 3,
-      "tokens": 248,
-      "first_read": "2026-07-23T07:34:39.878Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/users.yaml": {
-      "count": 2,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:34:40.669Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/queues.yaml": {
-      "count": 2,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:34:41.074Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/resources.yaml": {
-      "count": 3,
-      "tokens": 1360,
-      "first_read": "2026-07-23T07:34:41.904Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/validation.py": {
+    "/home/rshen/projects/xce/worktrees/config/src/controller/persist.c": {
       "count": 1,
-      "tokens": 3771,
-      "first_read": "2026-07-23T07:35:49.930Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/schema_files/json/queues.schema.json": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:36:36.426Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/schema_files/json/users.schema.json": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:36:36.868Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/include/xce/types.h": {
-      "count": 5,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:38:59.047Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/schema_files/controller.capnp": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:39:10.240Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/mapping.py": {
-      "count": 4,
-      "tokens": 6156,
-      "first_read": "2026-07-23T07:39:20.024Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xceadmin/build.py": {
-      "count": 3,
-      "tokens": 1578,
-      "first_read": "2026-07-23T07:39:42.825Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/schema_files/json/resources.schema.json": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:40:32.392Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/worker/src/resource_sync.c": {
-      "count": 2,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:47:51.591Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/schema/resources.schema.json": {
-      "count": 2,
-      "tokens": 0,
-      "first_read": "2026-07-23T07:50:16.987Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/tests/test_config_build.py": {
-      "count": 3,
-      "tokens": 3661,
-      "first_read": "2026-07-23T08:01:18.465Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/tests/test_mapping.py": {
-      "count": 1,
-      "tokens": 6233,
-      "first_read": "2026-07-23T08:05:43.506Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/flake.nix": {
-      "count": 1,
-      "tokens": 571,
-      "first_read": "2026-07-23T08:10:45.772Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.claude/skills/kiro-spec-quick/templates/specs/init.json": {
-      "count": 1,
-      "tokens": 125,
-      "first_read": "2026-07-23T08:24:55.733Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.claude/skills/kiro-spec-quick/templates/specs/requirements-init.md": {
-      "count": 1,
-      "tokens": 40,
-      "first_read": "2026-07-23T08:24:56.256Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.claude/skills/kiro-spec-requirements/rules/ears-format.md": {
-      "count": 1,
-      "tokens": 739,
-      "first_read": "2026-07-23T08:27:19.147Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.claude/skills/kiro-spec-requirements/rules/requirements-review-gate.md": {
-      "count": 1,
-      "tokens": 924,
-      "first_read": "2026-07-23T08:27:20.034Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.claude/skills/kiro-spec-requirements/templates/specs/requirements.md": {
-      "count": 1,
-      "tokens": 325,
-      "first_read": "2026-07-23T08:27:20.408Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/schema/controller.capnp": {
-      "count": 2,
-      "tokens": 9442,
-      "first_read": "2026-07-23T08:28:40.209Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/scheduler/src/resource.c": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T08:31:33.974Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/scheduler/src/scheduler.c": {
-      "count": 2,
-      "tokens": 10738,
-      "first_read": "2026-07-23T08:32:19.648Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/worker.c": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T08:34:40.757Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/xcelimit.c": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T08:37:56.101Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/resource-distribution-mapping/requirements.md": {
-      "count": 1,
-      "tokens": 378,
-      "first_read": "2026-07-23T08:42:17.402Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/scheduler/src/schedobj.c": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T08:45:35.515Z"
-    },
-    "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/resreq.c": {
-      "count": 1,
-      "tokens": 0,
-      "first_read": "2026-07-23T08:46:42.409Z"
+      "tokens": 85714,
+      "first_read": "2026-07-31T09:25:16.220Z"
     }
   },
   "files_written": [
     {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/cluster.c",
-      "action": "edit",
-      "tokens": 182,
-      "at": "2026-07-23T03:27:12.632Z"
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/_live_e2e_common.py",
+      "action": "create",
+      "tokens": 1206,
+      "at": "2026-07-30T13:08:37.963Z"
     },
     {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/sched.c",
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_push.py",
+      "action": "edit",
+      "tokens": 150,
+      "at": "2026-07-30T13:08:53.196Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_push.py",
+      "action": "edit",
+      "tokens": 154,
+      "at": "2026-07-30T13:09:23.515Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "create",
+      "tokens": 4314,
+      "at": "2026-07-30T13:13:28.961Z"
+    },
+    {
+      "file": "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/d6d1f70d-af0b-465b-9cd6-e719e38692ce/scratchpad/manual_ha_deploy.sh",
+      "action": "create",
+      "tokens": 408,
+      "at": "2026-07-30T13:20:31.874Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 1089,
+      "at": "2026-07-30T13:31:47.698Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 288,
+      "at": "2026-07-30T13:32:02.450Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 1084,
+      "at": "2026-07-30T13:32:32.689Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 865,
+      "at": "2026-07-30T13:59:11.573Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 131,
+      "at": "2026-07-30T13:59:23.445Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 424,
+      "at": "2026-07-30T13:59:37.846Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 255,
+      "at": "2026-07-30T13:59:50.809Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 110,
+      "at": "2026-07-30T14:10:20.969Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 29,
+      "at": "2026-07-30T14:10:30.683Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 91,
+      "at": "2026-07-30T14:27:48.483Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_e2e_live_ha_push.py",
+      "action": "edit",
+      "tokens": 26,
+      "at": "2026-07-30T14:27:57.885Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/xceadmin/tests/test_repro_bug1485.py",
+      "action": "create",
+      "tokens": 1910,
+      "at": "2026-07-30T14:38:48.005Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/.kiro/specs/admin-config-tool/tasks.md",
+      "action": "edit",
+      "tokens": 404,
+      "at": "2026-07-30T14:57:07.885Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/.claude/skills/xce-test-ha/run-ha-soak.sh",
+      "action": "edit",
+      "tokens": 136,
+      "at": "2026-07-31T04:10:31.220Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/.claude/skills/xce-test-ha/run-ha-soak.sh",
+      "action": "edit",
+      "tokens": 1266,
+      "at": "2026-07-31T05:03:59.690Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/.claude/skills/xce-test-ha/run-ha-soak.sh",
+      "action": "edit",
+      "tokens": 68,
+      "at": "2026-07-31T05:06:39.208Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/docs/deployment.md",
+      "action": "edit",
+      "tokens": 353,
+      "at": "2026-07-31T08:40:15.401Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/docs/deployment.md",
+      "action": "edit",
+      "tokens": 209,
+      "at": "2026-07-31T08:48:31.523Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.h",
+      "action": "edit",
+      "tokens": 224,
+      "at": "2026-07-31T08:58:40.147Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
+      "action": "edit",
+      "tokens": 53,
+      "at": "2026-07-31T08:58:51.647Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
+      "action": "edit",
+      "tokens": 717,
+      "at": "2026-07-31T08:59:15.171Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
+      "action": "edit",
+      "tokens": 109,
+      "at": "2026-07-31T08:59:29.637Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
+      "action": "edit",
+      "tokens": 198,
+      "at": "2026-07-31T08:59:42.696Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
+      "action": "edit",
+      "tokens": 248,
+      "at": "2026-07-31T08:59:55.620Z"
+    },
+    {
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
       "action": "edit",
       "tokens": 45,
-      "at": "2026-07-23T03:38:37.461Z"
+      "at": "2026-07-31T09:00:12.945Z"
     },
     {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/sched.c",
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
       "action": "edit",
-      "tokens": 54,
-      "at": "2026-07-23T03:38:55.639Z"
+      "tokens": 52,
+      "at": "2026-07-31T09:00:25.189Z"
     },
     {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/sched.c",
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/waldump.c",
       "action": "edit",
-      "tokens": 44,
-      "at": "2026-07-23T03:39:03.310Z"
+      "tokens": 102,
+      "at": "2026-07-31T09:00:37.310Z"
     },
     {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/limits.yaml",
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/test_waldump.c",
       "action": "edit",
-      "tokens": 260,
-      "at": "2026-07-23T03:44:44.946Z"
+      "tokens": 41,
+      "at": "2026-07-31T09:00:48.309Z"
     },
     {
-      "file": "/home/claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1462_config_replication_fixed.md",
-      "action": "create",
-      "tokens": 676,
-      "at": "2026-07-23T04:17:43.805Z"
-    },
-    {
-      "file": "/home/claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md",
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/test_waldump.c",
       "action": "edit",
-      "tokens": 178,
-      "at": "2026-07-23T04:18:08.721Z"
+      "tokens": 1422,
+      "at": "2026-07-31T09:01:22.297Z"
     },
     {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/common/request/request.go",
-      "action": "edit",
-      "tokens": 83,
-      "at": "2026-07-23T06:51:51.324Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/cmd/jobs/peek.go",
-      "action": "edit",
-      "tokens": 15,
-      "at": "2026-07-23T06:52:02.766Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/cmd/jobs/peek.go",
-      "action": "edit",
-      "tokens": 21,
-      "at": "2026-07-23T06:53:19.253Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/common/request/request.go",
-      "action": "edit",
-      "tokens": 55,
-      "at": "2026-07-23T06:55:09.384Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/frontend/xcecli/common/request/request.go",
-      "action": "edit",
-      "tokens": 114,
-      "at": "2026-07-23T06:55:21.016Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/steering/security.md",
-      "action": "edit",
-      "tokens": 472,
-      "at": "2026-07-23T07:06:58.377Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/steering/security.md",
-      "action": "edit",
-      "tokens": 352,
-      "at": "2026-07-23T07:22:00.047Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/mapping.py",
-      "action": "edit",
-      "tokens": 316,
-      "at": "2026-07-23T07:53:19.608Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/mapping.py",
-      "action": "edit",
-      "tokens": 264,
-      "at": "2026-07-23T07:55:23.984Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xcecommon/mapping.py",
-      "action": "edit",
-      "tokens": 148,
-      "at": "2026-07-23T07:55:37.484Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/src/xceadmin/build.py",
+      "file": "/home/rshen/projects/xce/worktrees/config/src/controller/testsuite/test_waldump.c",
       "action": "edit",
       "tokens": 80,
-      "at": "2026-07-23T07:56:24.677Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/resources.yaml",
-      "action": "edit",
-      "tokens": 239,
-      "at": "2026-07-23T07:57:00.436Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/hosts.yaml",
-      "action": "edit",
-      "tokens": 22,
-      "at": "2026-07-23T07:57:33.198Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/hosts.yaml",
-      "action": "edit",
-      "tokens": 112,
-      "at": "2026-07-23T07:57:44.825Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/users.yaml",
-      "action": "edit",
-      "tokens": 46,
-      "at": "2026-07-23T07:59:36.138Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/queues.yaml",
-      "action": "edit",
-      "tokens": 27,
-      "at": "2026-07-23T08:00:27.195Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/configs/queues.yaml",
-      "action": "edit",
-      "tokens": 23,
-      "at": "2026-07-23T08:00:36.996Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/tests/test_mapping.py",
-      "action": "edit",
-      "tokens": 218,
-      "at": "2026-07-23T08:06:54.073Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/tests/test_config_build.py",
-      "action": "edit",
-      "tokens": 611,
-      "at": "2026-07-23T08:07:12.095Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/xceadmin/tests/test_config_build.py",
-      "action": "edit",
-      "tokens": 155,
-      "at": "2026-07-23T08:07:26.235Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/resource-distribution-mapping/spec.json",
-      "action": "create",
-      "tokens": 132,
-      "at": "2026-07-23T08:25:58.835Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/resource-distribution-mapping/requirements.md",
-      "action": "create",
-      "tokens": 432,
-      "at": "2026-07-23T08:26:13.948Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/resource-distribution-mapping/research.md",
-      "action": "create",
-      "tokens": 1769,
-      "at": "2026-07-23T08:51:54.075Z"
-    },
-    {
-      "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/resource-distribution-mapping/requirements.md",
-      "action": "edit",
-      "tokens": 170,
-      "at": "2026-07-23T08:52:06.680Z"
-    },
-    {
-      "file": "/home/claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_resource_distribution_mapping_scoping.md",
-      "action": "create",
-      "tokens": 620,
-      "at": "2026-07-23T08:54:48.635Z"
-    },
-    {
-      "file": "/home/claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md",
-      "action": "edit",
-      "tokens": 152,
-      "at": "2026-07-23T08:56:01.903Z"
+      "at": "2026-07-31T09:01:40.552Z"
     }
   ],
   "edit_counts": {
-    "src/controller/cluster.c": 1,
-    "src/controller/sched.c": 3,
-    "src/configs/limits.yaml": 1,
-    "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1462_config_replication_fixed.md": 1,
-    "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md": 2,
-    "src/frontend/xcecli/common/request/request.go": 3,
-    "src/frontend/xcecli/cmd/jobs/peek.go": 2,
-    ".kiro/steering/security.md": 2,
-    "src/xceadmin/src/xcecommon/mapping.py": 3,
-    "src/xceadmin/src/xceadmin/build.py": 1,
-    "src/configs/resources.yaml": 1,
-    "src/configs/hosts.yaml": 2,
-    "src/configs/users.yaml": 1,
-    "src/configs/queues.yaml": 2,
-    "src/xceadmin/tests/test_mapping.py": 1,
-    "src/xceadmin/tests/test_config_build.py": 2,
-    ".kiro/specs/resource-distribution-mapping/spec.json": 1,
-    ".kiro/specs/resource-distribution-mapping/requirements.md": 2,
-    ".kiro/specs/resource-distribution-mapping/research.md": 1,
-    "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_resource_distribution_mapping_scoping.md": 1
+    "src/xceadmin/tests/_live_e2e_common.py": 1,
+    "src/xceadmin/tests/test_e2e_live_push.py": 2,
+    "src/xceadmin/tests/test_e2e_live_ha_push.py": 12,
+    "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-config/d6d1f70d-af0b-465b-9cd6-e719e38692ce/scratchpad/manual_ha_deploy.sh": 1,
+    "src/xceadmin/tests/test_repro_bug1485.py": 1,
+    ".kiro/specs/admin-config-tool/tasks.md": 1,
+    ".claude/skills/xce-test-ha/run-ha-soak.sh": 3,
+    "src/docs/deployment.md": 2,
+    "src/controller/testsuite/waldump.h": 1,
+    "src/controller/testsuite/waldump.c": 8,
+    "src/controller/testsuite/test_waldump.c": 3
   },
-  "anatomy_hits": 21,
-  "anatomy_misses": 18,
-  "repeated_reads_warned": 32,
+  "anatomy_hits": 20,
+  "anatomy_misses": 10,
+  "repeated_reads_warned": 31,
   "cerebrum_warnings": 0,
-  "stop_count": 20
+  "stop_count": 74
 }