From b639fae162f71e4c9880283f874e0b91f1656e13 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Wed, 2 Sep 2026 08:39:52 +0000 Subject: [PATCH] feat(scheduler-sim-repl-verbosity): task 1.2 - implement job identifier range compression Implements repl_fmt_jobid_range() in repl.c (per design.md's "no new production .c file" commitment), grouping job-array identifiers by base and compressing contiguous index runs into [lo-hi] ranges. All 5 of task 1.1's RED-phase tests now pass. Went through one review round: an initial GREEN implementation placed the function in a new file (repl_jobid_range.c) to sidestep a linker issue in the lean test target from task 1.1. Review found and verified a cleaner fix: link the test against the full scheduler_srcs object list (mirroring the existing sched_scheduler_connect_test precedent), which resolves the same symbols without a new file. Remediated accordingly; bug-1762 in buglog.json documents the investigation. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01YAWMhfRJFnmmnG8h97yUDd --- .../scheduler-sim-repl-verbosity/tasks.md | 2 +- .wolf/buglog.json | 879 +++++++++--------- src/scheduler/src/repl.c | 263 ++++++ src/scheduler/src/repl_internal.h | 35 + src/scheduler/testsuite/meson.build | 43 +- 5 files changed, 788 insertions(+), 434 deletions(-) diff --git a/.kiro/specs/scheduler-sim-repl-verbosity/tasks.md b/.kiro/specs/scheduler-sim-repl-verbosity/tasks.md index e7df9ce9..5d7dd762 100644 --- a/.kiro/specs/scheduler-sim-repl-verbosity/tasks.md +++ b/.kiro/specs/scheduler-sim-repl-verbosity/tasks.md @@ -6,7 +6,7 @@ - Observable: the new test cases execute and fail against the current codebase, which has no compression behavior yet - _Requirements: 4.1, 4.2, 4.3, 4.4, 4.5_ -- [ ] 1.2 Implement job identifier range compression so the new tests pass +- [x] 1.2 Implement job identifier range compression so the new tests pass - Group identifiers by their shared array submission and render contiguous index runs as a single compressed range, comma-joining non-contiguous runs - Never drop or fabricate an identifier, and fail safely (without truncating or crashing) when the output buffer is too small - Observable: all tests from 1.1 pass; the helper is ready for the event-echo and round-summary work to adopt diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 2bfebc09..8ac8245d 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -23,7 +23,7 @@ "error_message": "Wrong reference: g_tmpdir should be g_instdir", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "Used \"g_tmpdir\" instead of \"g_instdir\"", - "fix": "Changed g_tmpdir → g_instdir", + "fix": "Changed g_tmpdir \u2192 g_instdir", "tags": [ "wrong-reference", "c" @@ -53,7 +53,7 @@ "timestamp": "2026-05-21T08:30:20.719Z", "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer", "file": "src/controller/ha.c", - "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) — double-free at lines 3594-3595 and 3600-3601", + "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) \u2014 double-free at lines 3594-3595 and 3600-3601", "fix": "Set ep->data = NULL before endpoint_fini so FREEUP becomes a no-op; then free(sctx) explicitly in the error paths", "tags": [ "md", @@ -70,7 +70,7 @@ { "id": "bug-007", "timestamp": "2026-05-21T12:53:13.789Z", - "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 — reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", + "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 \u2014 reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", "file": "src/controller/ha.c", "root_cause": "ha_arm_heartbeat_timeout_vtask freed the old liveness_poll_ctx_t immediately after setting cancelled=1, but the delaymgr task was already in flight reading ctx->cancelled=0 and then continued to read ctx->ha after free", "fix": "ha_arm_heartbeat_timeout_vtask no longer frees old ctx; liveness_poll_fn now frees ctx when it fires with cancelled=1 (guarded by #ifndef XCE_TESTING to avoid bad-free on stack-allocated test ctxs)", @@ -317,7 +317,7 @@ "timestamp": "2026-05-26T13:51:32.274Z", "error_message": "Missing error handling in unknown", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: - [ ] `test_heartbeat_carries_committed_lsn`: heartbeat payl; `committed_lsn`; standby updates `ha.last_known_committed_ls", "tags": [ "auto-detected", @@ -333,7 +333,7 @@ "timestamp": "2026-05-26T13:52:49.044Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: // Catch-up gate (Clarification 6): behind nodes must not in; if (c->ha.in_catch_up) {", "tags": [ "auto-detected", @@ -349,7 +349,7 @@ "timestamp": "2026-05-27T03:02:34.912Z", "error_message": "Missing error handling in unknown", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: name, key, applied_lsn, and config_tag immediately after TCP; - [ ] `test_ctrl_hello_registers_standby_in_repl`: handle_ct", "tags": [ "auto-detected", @@ -430,7 +430,7 @@ "error_message": "Wrong reference: vcpus should be nres", "file": "src/include/clusterconfig.h", "root_cause": "Used \"vcpus\" instead of \"nres\"", - "fix": "Changed vcpus → nres", + "fix": "Changed vcpus \u2192 nres", "tags": [ "auto-detected", "wrong-reference", @@ -558,7 +558,7 @@ "error_message": "Wrong reference: capn_str_null should be str", "file": "src/controller/config_repl.c", "root_cause": "Used \"capn_str_null\" instead of \"str\"", - "fix": "Changed capn_str_null → str", + "fix": "Changed capn_str_null \u2192 str", "tags": [ "wrong-reference", "c" @@ -605,7 +605,7 @@ "error_message": "Wrong reference: in_catch_up should be acked_config_lsn", "file": "src/controller/repl.c", "root_cause": "Used \"in_catch_up\" instead of \"acked_config_lsn\"", - "fix": "Changed in_catch_up → acked_config_lsn", + "fix": "Changed in_catch_up \u2192 acked_config_lsn", "tags": [ "wrong-reference", "c" @@ -636,7 +636,7 @@ "error_message": "Wrong reference: ha_state_t should be _Atomic", "file": "src/controller/ha.h", "root_cause": "Used \"ha_state_t\" instead of \"_Atomic\"", - "fix": "Changed ha_state_t → _Atomic", + "fix": "Changed ha_state_t \u2192 _Atomic", "tags": [ "wrong-reference", "h" @@ -651,7 +651,7 @@ "error_message": "Wrong reference: socket should be epoll", "file": "src/controller/ha.c", "root_cause": "Used \"socket\" instead of \"epoll\"", - "fix": "Changed socket → epoll", + "fix": "Changed socket \u2192 epoll", "tags": [ "auto-detected", "wrong-reference", @@ -699,7 +699,7 @@ "error_message": "Wrong reference: XCE_ERR_STANDBY should be E_XCE_STANDBY", "file": "src/lib/request.c", "root_cause": "Used \"XCE_ERR_STANDBY\" instead of \"E_XCE_STANDBY\"", - "fix": "Changed XCE_ERR_STANDBY → E_XCE_STANDBY", + "fix": "Changed XCE_ERR_STANDBY \u2192 E_XCE_STANDBY", "tags": [ "wrong-reference", "c" @@ -714,7 +714,7 @@ "error_message": "Wrong reference: XCE_ERR_NOT_READY should be E_XCE_NOT_READY", "file": "src/lib/testsuite/test_libxce_retry.c", "root_cause": "Used \"XCE_ERR_NOT_READY\" instead of \"E_XCE_NOT_READY\"", - "fix": "Changed XCE_ERR_NOT_READY → E_XCE_NOT_READY", + "fix": "Changed XCE_ERR_NOT_READY \u2192 E_XCE_NOT_READY", "tags": [ "wrong-reference", "c" @@ -729,7 +729,7 @@ "error_message": "Wrong reference: data should be ctrl_hello_reply_t", "file": "src/include/typemap.h", "root_cause": "Used \"data\" instead of \"ctrl_hello_reply_t\"", - "fix": "Changed data → ctrl_hello_reply_t", + "fix": "Changed data \u2192 ctrl_hello_reply_t", "tags": [ "wrong-reference", "h" @@ -743,7 +743,7 @@ "timestamp": "2026-05-29T03:54:23.205Z", "error_message": "Missing error handling in struct", "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: wal_catchup_entry_t* walcatchupentrymsg;; snapshot_transfer_t* snapshottransfermsg; | Also: /* Forward declarations for types defined later in this file; typedef struct snapshot_transfer_t snapshot_transfer_fw", "tags": [ "auto-detected", @@ -808,7 +808,7 @@ "error_message": "Wrong reference: nres should be vcpus", "file": "src/include/clusterconfig.h", "root_cause": "Used \"nres\" instead of \"vcpus\"", - "fix": "Changed nres → vcpus", + "fix": "Changed nres \u2192 vcpus", "tags": [ "auto-detected", "wrong-reference", @@ -824,7 +824,7 @@ "error_message": "Wrong reference: data should be request_vote_reply_t", "file": "src/include/typemap.h", "root_cause": "Used \"data\" instead of \"request_vote_reply_t\"", - "fix": "Changed data → request_vote_reply_t", + "fix": "Changed data \u2192 request_vote_reply_t", "tags": [ "wrong-reference", "h" @@ -839,7 +839,7 @@ "error_message": "Wrong reference: cluster_init_phase2 should be cluster_register_listener", "file": "src/controller/include/cluster.h", "root_cause": "Used \"cluster_init_phase2\" instead of \"cluster_register_listener\"", - "fix": "Changed cluster_init_phase2 → cluster_register_listener", + "fix": "Changed cluster_init_phase2 \u2192 cluster_register_listener", "tags": [ "wrong-reference", "h" @@ -870,7 +870,7 @@ "error_message": "Wrong reference: CTRL_PERF_NHASH should be c", "file": "src/controller/cluster.c", "root_cause": "Used \"CTRL_PERF_NHASH\" instead of \"c\"", - "fix": "Changed CTRL_PERF_NHASH → c", + "fix": "Changed CTRL_PERF_NHASH \u2192 c", "tags": [ "wrong-reference", "c" @@ -885,7 +885,7 @@ "error_message": "Wrong reference: true should be false", "file": ".claude/skills/xce-build/SKILL.md", "root_cause": "Used \"true\" instead of \"false\"", - "fix": "Changed true → false", + "fix": "Changed true \u2192 false", "tags": [ "auto-detected", "wrong-reference", @@ -901,7 +901,7 @@ "error_message": "Wrong reference: 120 should be 300", "file": "src/controller/meson.build", "root_cause": "Used \"120\" instead of \"300\"", - "fix": "Changed 120 → 300", + "fix": "Changed 120 \u2192 300", "tags": [ "auto-detected", "wrong-reference", @@ -932,8 +932,8 @@ "timestamp": "2026-05-30T06:38:09.325Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* After election determines role: */; int is_first_boot = !c->running; /* running=1 only after cl | Also: #### `ha_dispatch_gate` — CTRL_PRIMARY_COMMITTING case; ```c | Also: ### 3-node startup — ctrl-a wins (with COMMITTING phase); participant PeerC as ctrl-c endpoint | Also: - **`ha_start_election_async()` is fully non-blocking**: Arm; - *Initial startup* (`ha->phase2_done == 0`): calls `cluster", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* After election determines role: */; int is_first_boot = !c->running; /* running=1 only after cl | Also: #### `ha_dispatch_gate` \u2014 CTRL_PRIMARY_COMMITTING case; ```c | Also: ### 3-node startup \u2014 ctrl-a wins (with COMMITTING phase); participant PeerC as ctrl-c endpoint | Also: - **`ha_start_election_async()` is fully non-blocking**: Arm; - *Initial startup* (`ha->phase2_done == 0`): calls `cluster", "tags": [ "auto-detected", "error-handling", @@ -1076,7 +1076,7 @@ "timestamp": "2026-05-30T11:43:44.829Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Push config to standbys with lower config_tag (Req 7.1) *; #ifdef XCE_TESTING", "tags": [ "auto-detected", @@ -1253,7 +1253,7 @@ "error_message": "Wrong reference: third should be second", "file": "src/controller/testsuite/test_ha_repl_ack.c", "root_cause": "Used \"third\" instead of \"second\"", - "fix": "Changed third → second", + "fix": "Changed third \u2192 second", "tags": [ "auto-detected", "wrong-reference", @@ -1332,7 +1332,7 @@ "timestamp": "2026-05-31T04:11:45.534Z", "error_message": "Missing error handling in struct", "file": ".kiro/specs/raft-library/design.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: Caller->>RaftSM: raft_on_vote_request(state, from, term); alt term < current_term", "tags": [ "auto-detected", @@ -1410,7 +1410,7 @@ { "id": "bug-206", "timestamp": "2026-05-31T08:40:00.000Z", - "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 — ha->term must be atomically updated to peer term when peer term > ours", + "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 \u2014 ha->term must be atomically updated to peer term when peer term > ours", "file": "src/controller/ha.c", "root_cause": "vote_reply_msg_handler delegated term fencing entirely to raft_cb_become_follower (via raft_on_vote_reply). In XCE_TESTING builds, vote_reply_dispatch calls xce_vote_reply_hook and returns early, bypassing raft_on_vote_reply and thus never updating ha->term.", "fix": "Added explicit CAS loop in vote_reply_msg_handler before vote_reply_dispatch to advance ha->term when vr_term > ha->term. This applies on both the production path and the XCE_TESTING hook path.", @@ -1432,7 +1432,7 @@ "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_even_peers_returns_einval", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_even_peers_returns_einval\"", - "fix": "Changed test_raft_init_zero_peers_returns_einval → test_raft_init_even_peers_returns_einval", + "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_even_peers_returns_einval", "tags": [ "wrong-reference", "c" @@ -1444,7 +1444,7 @@ { "id": "bug-208", "timestamp": "2026-05-31", - "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite — build exits with error code", + "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite \u2014 build exits with error code", "file": "src/scheduler/testsuite/test.c", "root_cause": "sched_meet_host_selection() lazily allocates w->host_eval_ctx via build_host_eval_ctx(). Four test functions (test_meet_host_selection_always_true_passes, test_meet_host_selection_always_false_fails, test_meet_host_selection_numeric_match, test_meet_host_selection_string_match) called zarray_fini() without first calling sched_worker_release_host_ctx() to free the lazily-allocated host_eval_ctx.", "fix": "Added sched_worker_release_host_ctx(&w) calls before zarray_fini() in all four affected test functions.", @@ -1481,7 +1481,7 @@ "error_message": "Wrong reference: 2000 should be 5000", "file": "src/controller/ha.h", "root_cause": "Used \"2000\" instead of \"5000\"", - "fix": "Changed 2000 → 5000", + "fix": "Changed 2000 \u2192 5000", "tags": [ "auto-detected", "wrong-reference", @@ -1529,7 +1529,7 @@ "error_message": "Wrong reference: Request_body_nodata should be Request_body_nomsg", "file": "src/controller/ha.c", "root_cause": "Used \"Request_body_nodata\" instead of \"Request_body_nomsg\"", - "fix": "Changed Request_body_nodata → Request_body_nomsg", + "fix": "Changed Request_body_nodata \u2192 Request_body_nomsg", "tags": [ "wrong-reference", "c" @@ -1544,7 +1544,7 @@ "error_message": "Wrong reference: vote_reply_msg_handler should be vote_reply_msg_handler_for_test", "file": "src/controller/ha.c", "root_cause": "Used \"vote_reply_msg_handler\" instead of \"vote_reply_msg_handler_for_test\"", - "fix": "Changed vote_reply_msg_handler → vote_reply_msg_handler_for_test", + "fix": "Changed vote_reply_msg_handler \u2192 vote_reply_msg_handler_for_test", "tags": [ "wrong-reference", "c" @@ -1575,7 +1575,7 @@ "error_message": "Wrong operator: && should be !=", "file": "src/controller/ha.c", "root_cause": "Used \"&&\" instead of \"!=\"", - "fix": "Changed operator && → !=", + "fix": "Changed operator && \u2192 !=", "tags": [ "auto-detected", "operator-fix", @@ -1687,7 +1687,7 @@ "error_message": "Wrong reference: test_ha_connect should be test_ha_ctrlhello", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_ha_connect\" instead of \"test_ha_ctrlhello\"", - "fix": "Changed test_ha_connect → test_ha_ctrlhello", + "fix": "Changed test_ha_connect \u2192 test_ha_ctrlhello", "tags": [ "wrong-reference", "c" @@ -1702,7 +1702,7 @@ "error_message": "Wrong reference: vcpu should be vcpus", "file": "src/tools/deploy-ha.sh", "root_cause": "Used \"vcpu\" instead of \"vcpus\"", - "fix": "Changed vcpu → vcpus", + "fix": "Changed vcpu \u2192 vcpus", "tags": [ "auto-detected", "wrong-reference", @@ -1766,7 +1766,7 @@ "error_message": "Wrong reference: startup_validation should be testsuite_test_ha_liveness", "file": "src/xce.build/build.ninja", "root_cause": "Used \"startup_validation\" instead of \"testsuite_test_ha_liveness\"", - "fix": "Changed startup_validation → testsuite_test_ha_liveness", + "fix": "Changed startup_validation \u2192 testsuite_test_ha_liveness", "tags": [ "wrong-reference", "ninja" @@ -1845,7 +1845,7 @@ "error_message": "Wrong reference: test_am_i_primary should be test_ha_cluster", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_am_i_primary\" instead of \"test_ha_cluster\"", - "fix": "Changed test_am_i_primary → test_ha_cluster", + "fix": "Changed test_am_i_primary \u2192 test_ha_cluster", "tags": [ "wrong-reference", "c" @@ -1875,8 +1875,8 @@ "timestamp": "2026-06-04T14:33:03.464Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ── Task 2.1: OPCode_walCatchupEntry handler ─────────────; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2500\u2500 Task 2.1: OPCode_walCatchupEntry handler \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);", "tags": [ "auto-detected", "error-handling", @@ -1891,7 +1891,7 @@ "timestamp": "2026-06-04T14:41:35.073Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: case OPCode_walCatchupEntry:; /* Task 2.1: standby applies a single WAL catch-up entry fro", "tags": [ "auto-detected", @@ -1907,7 +1907,7 @@ "timestamp": "2026-06-04T14:46:44.652Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: #include \"persist.h\" /* persist_t, apply_wal_entry */; #include \"typemap.h\" /* wal_catchup_entry_t */", "tags": [ "auto-detected", @@ -1972,7 +1972,7 @@ "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero", "file": "src/controller/testsuite/test_ha_role.c", "root_cause": "Used \"test_evaluate_repl_action_stream_wal_from_zero\" instead of \"test_evaluate_repl_action_send_snapshot_lsn_zero\"", - "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero → test_evaluate_repl_action_send_snapshot_lsn_zero", + "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero \u2192 test_evaluate_repl_action_send_snapshot_lsn_zero", "tags": [ "wrong-reference", "c" @@ -2099,7 +2099,7 @@ "error_message": "Wrong reference: uintptr_t should be void", "file": "src/controller/testsuite/test_ha_evaluate_committing.c", "root_cause": "Used \"uintptr_t\" instead of \"void\"", - "fix": "Changed uintptr_t → void", + "fix": "Changed uintptr_t \u2192 void", "tags": [ "wrong-reference", "c" @@ -2130,7 +2130,7 @@ "error_message": "Wrong reference: 10000 should be 60000", "file": "src/controller/ha.h", "root_cause": "Used \"10000\" instead of \"60000\"", - "fix": "Changed 10000 → 60000", + "fix": "Changed 10000 \u2192 60000", "tags": [ "auto-detected", "wrong-reference", @@ -2146,7 +2146,7 @@ "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS", "file": "src/controller/testsuite/test_ha_evaluate_committing.c", "root_cause": "Used \"10000\" instead of \"HA_DEFAULT_COMMITTING_TIMEOUT_MS\"", - "fix": "Changed 10000 → HA_DEFAULT_COMMITTING_TIMEOUT_MS", + "fix": "Changed 10000 \u2192 HA_DEFAULT_COMMITTING_TIMEOUT_MS", "tags": [ "wrong-reference", "c" @@ -2160,7 +2160,7 @@ "timestamp": "2026-06-05T06:50:00Z", "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN", "file": "src/controller/ha.c", - "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always — majority convergence path could never succeed.", + "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always \u2014 majority convergence path could never succeed.", "fix": "Changed ha->n_peers = n_peers to ha->n_peers = n_other (= n_peers-1 = other-node count). Moved the assignment after n_other is computed. ha_repl_ack_received quorum formula (ha->n_peers+2)/2 still yields correct majority for both old and new value.", "tags": [ "ha", @@ -2179,7 +2179,7 @@ "error_message": "Wrong reference: DEBUG2 should be INFO", "file": "src/controller/ha.c", "root_cause": "Used \"DEBUG2\" instead of \"INFO\"", - "fix": "Changed DEBUG2 → INFO", + "fix": "Changed DEBUG2 \u2192 INFO", "tags": [ "auto-detected", "wrong-reference", @@ -2363,10 +2363,10 @@ { "id": "bug-340", "timestamp": "2026-06-10T00:23:21.918Z", - "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup → CTRL_ELECTING → raft_cb_become_follower → CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) — a wedged outbound standby endpoint.", + "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup \u2192 CTRL_ELECTING \u2192 raft_cb_become_follower \u2192 CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) \u2014 a wedged outbound standby endpoint.", "file": "src/controller/ha.c, src/controller/testsuite/test_ha_liveness.c", - "root_cause": "primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints — so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout — confirming the watchdog is correct and the sole cause is the broken channel.)", - "fix": "In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action — the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) → become_follower → STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED→GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c.", + "root_cause": "primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints \u2014 so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout \u2014 confirming the watchdog is correct and the sole cause is the broken channel.)", + "fix": "In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action \u2014 the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) \u2192 become_follower \u2192 STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED\u2192GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c.", "tags": [ "logic-fix", "c", @@ -2390,7 +2390,7 @@ "error_message": "Wrong reference: last_heartbeat should be voted_for_term", "file": "src/controller/ha.h", "root_cause": "Used \"last_heartbeat\" instead of \"voted_for_term\"", - "fix": "Changed last_heartbeat → voted_for_term", + "fix": "Changed last_heartbeat \u2192 voted_for_term", "tags": [ "wrong-reference", "h" @@ -2436,7 +2436,7 @@ "timestamp": "2026-06-10T01:39:31.683Z", "error_message": "Missing error handling in unknown", "file": "src/lib/request.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: if (req->body_which == Request_body_walcatchupentrymsg; && req->body.walcatchupentrymsg != NULL) { | Also: /* wal_entry_to_wire: fill a wire WalCatchupEntry from a C e; * fields and raw payload bytes. Sub-entry nesting is one le", "tags": [ "auto-detected", @@ -2578,7 +2578,7 @@ "timestamp": "2026-06-10T07:56:21.347Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* =========================================================; * Task 1.1: repl_broadcast send path tests", "tags": [ "auto-detected", @@ -2595,7 +2595,7 @@ "error_message": "Wrong reference: WAL_JOB should be WAL_JOB_SUBMIT", "file": "src/controller/testsuite/test_repl_unit.c", "root_cause": "Used \"WAL_JOB\" instead of \"WAL_JOB_SUBMIT\"", - "fix": "Changed WAL_JOB → WAL_JOB_SUBMIT", + "fix": "Changed WAL_JOB \u2192 WAL_JOB_SUBMIT", "tags": [ "wrong-reference", "c" @@ -2610,7 +2610,7 @@ "error_message": "Wrong reference: DEBUG3 should be TRACE", "file": "src/controller/repl.c", "root_cause": "Used \"DEBUG3\" instead of \"TRACE\"", - "fix": "Changed DEBUG3 → TRACE", + "fix": "Changed DEBUG3 \u2192 TRACE", "tags": [ "auto-detected", "wrong-reference", @@ -2626,7 +2626,7 @@ "error_message": "Wrong reference: request should be ep", "file": "src/controller/cluster.c", "root_cause": "Used \"request\" instead of \"ep\"", - "fix": "Changed request → ep", + "fix": "Changed request \u2192 ep", "tags": [ "auto-detected", "wrong-reference", @@ -2641,7 +2641,7 @@ "timestamp": "2026-06-10T08:47:48.296Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Task 3.3 (ha-wal-replication): dispatch gate standby-role; RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_ro", "tags": [ "auto-detected", @@ -2692,7 +2692,7 @@ "error_message": "Wrong reference: test_catchup_full_lifecycle_integration should be test_single_node_committed_lsn_unchanged", "file": "src/controller/testsuite/test_p2_integration.c", "root_cause": "Used \"test_catchup_full_lifecycle_integration\" instead of \"test_single_node_committed_lsn_unchanged\"", - "fix": "Changed test_catchup_full_lifecycle_integration → test_single_node_committed_lsn_unchanged", + "fix": "Changed test_catchup_full_lifecycle_integration \u2192 test_single_node_committed_lsn_unchanged", "tags": [ "wrong-reference", "c" @@ -2771,7 +2771,7 @@ "error_message": "Wrong reference: signal should be stdio", "file": "src/lib/delay.c", "root_cause": "Used \"signal\" instead of \"stdio\"", - "fix": "Changed signal → stdio", + "fix": "Changed signal \u2192 stdio", "tags": [ "auto-detected", "wrong-reference", @@ -2997,7 +2997,7 @@ "timestamp": "2026-06-11T13:30:00Z", "error_message": "test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby", "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "repl_add_standby returns the slot index (>=0) on success, not 0 — second add legitimately returns 1", + "root_cause": "repl_add_standby returns the slot index (>=0) on success, not 0 \u2014 second add legitimately returns 1", "fix": "fixture asserts TEST_ASSERT_EQUAL_INT(i, repl_add_standby(...)) per standby slot", "tags": [ "controller", @@ -3014,8 +3014,8 @@ "timestamp": "2026-06-11T13:35:00Z", "error_message": "3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013)", "file": "src/controller/testsuite/controller_test.c", - "root_cause": "each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters — main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c)", - "fix": "not fixed (out of task 1.1 boundary) — documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2)", + "root_cause": "each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters \u2014 main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c)", + "fix": "not fixed (out of task 1.1 boundary) \u2014 documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2)", "tags": [ "controller", "testsuite", @@ -3033,8 +3033,8 @@ "timestamp": "2026-06-11T15:28:35.990Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ═════════════════════════════════════════════════════════; * Task 3.1 — membership re-evaluation (commit_on_membership_ | Also: /* Task 3.1 — membership re-evaluation */; RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pend", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 3.1 \u2014 membership re-evaluation (commit_on_membership_ | Also: /* Task 3.1 \u2014 membership re-evaluation */; RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pend", "tags": [ "auto-detected", "error-handling", @@ -3049,8 +3049,8 @@ "timestamp": "2026-06-11T16:19:51.195Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 4.2 — wired gate: registry truth at ha_dispatch_gate; RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_th", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 4.2 \u2014 wired gate: registry truth at ha_dispatch_gate; RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_th", "tags": [ "auto-detected", "error-handling", @@ -3081,8 +3081,8 @@ "timestamp": "2026-06-11T22:39:24.704Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ═════════════════════════════════════════════════════════; * Task 5.1 — integration: ACK feed and the job-submit sync-a | Also: #include \"typemap.h\" /* wal_catchup_entry_t (task 5; #include \"wal_types.h\" /* WAL_JOB_SUBMIT (task 5.1 sy", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 5.1 \u2014 integration: ACK feed and the job-submit sync-a | Also: #include \"typemap.h\" /* wal_catchup_entry_t (task 5; #include \"wal_types.h\" /* WAL_JOB_SUBMIT (task 5.1 sy", "tags": [ "auto-detected", "error-handling", @@ -3194,7 +3194,7 @@ "error_message": "Wrong reference: test_single_node_committed_lsn_unchanged should be test_single_node_committed_lsn_advances", "file": "src/controller/testsuite/test_p2_integration.c", "root_cause": "Used \"test_single_node_committed_lsn_unchanged\" instead of \"test_single_node_committed_lsn_advances\"", - "fix": "Changed test_single_node_committed_lsn_unchanged → test_single_node_committed_lsn_advances", + "fix": "Changed test_single_node_committed_lsn_unchanged \u2192 test_single_node_committed_lsn_advances", "tags": [ "wrong-reference", "c" @@ -3420,7 +3420,7 @@ "timestamp": "2026-06-12T11:05:00-07:00", "error_message": "va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*", "file": "src/controller/job.c (caller) / src/controller/persist.c (consumer)", - "root_cause": "update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) — an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident)", + "root_cause": "update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) \u2014 an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident)", "fix": "FIXED (TDD RED->GREEN). Added a dedicated HOOK_DETAIL_JOB_RUSAGE_UPDATE arm to persist_job_hook_fun (persist.c) that reads the real xce_job_rusages_t* payload and looks the job up by jobid via the cluster_t* the hook already carries (c->jobmgr), instead of falling through to the default arm that misread it as a job_t*. Fix candidate (1) from the original investigation was used -- a dedicated RUSAGE arm -- not candidate (2) (firing the hook with (c, j) like other job hooks), because sched_job_rusage_update_fun/apiserver_job_rusage_update_fun (this hook's other two listeners, sched.c/apiserver.c) genuinely need the full rusages payload (pids/pgids/usages), so changing what job.c fires would have broken them. Regression test (test_commit_barrier_reply.c's test_update_op_rusage_receipt_reply_facts_retained, extended): drives the real do_update_op/update_job_rusages call path and scans the resulting WAL_JOB_RUSAGE entry, asserting state == JOB_STAT_PEND (the job's real state) -- RED under the old code (read back 0, the zeroed padding past xce_job_rusages_t's end), GREEN under the fix. Also fixed test_persist_hooks.c's test_job_hook_rusage_writes_wal_entry, which previously masked the bug by passing a job_t* stub instead of the real xce_job_rusages_t* shape -- now builds a minimal real cluster+jobmgr+job and passes the correct payload type; added a sibling test for the job-not-found graceful-skip path (returns 0, appends nothing, no crash). Full controller_unit_test suite: 1096 tests, 2 pre-existing unrelated failures (test_config_repl.c, filed as bug-1561 -- overlayfs fdatasync artifact, zero diff in that file this session), no regressions.\n\n[Original entry]\nNOT FIXED (discovered while writing task 3.3 receipt-ACK verdict pins; production out of boundary). Test test_update_op_rusage_receipt_reply_facts_retained uses a padded union so the pin stays in-bounds under ASAN without masking the bug; fix candidates: dedicated RUSAGE arm in persist_job_hook_fun reading xce_job_rusages_t*, or fire the hook with (c, j) like the other job hooks", "tags": [ "pre-existing", @@ -3603,7 +3603,7 @@ "error_message": "Wrong reference: test_ha_cluster should be test_ha_role_lifecycle", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_ha_cluster\" instead of \"test_ha_role_lifecycle\"", - "fix": "Changed test_ha_cluster → test_ha_role_lifecycle", + "fix": "Changed test_ha_cluster \u2192 test_ha_role_lifecycle", "tags": [ "wrong-reference", "c" @@ -3662,7 +3662,7 @@ }, { "error_message": "AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async <- ha_arm_own_term_gate <- ha_promote_to_primary, during test_promotion_registers_persist_hooks (test_ha_role_lifecycle.c case 3)", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect — OUT OF task 2.2 boundary)", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", "root_cause": "task-2.2's ha_promote_to_primary now appends a WAL_NOOP via persist_wal_async, which broadcasts to registered standbys. The lifecycle fixture (lifecycle_cluster_setup) registers fake `int g_lc_eps[]` endpoints via repl_add_standby and NEVER installs repl_broadcast_send_hook, so repl_broadcast calls xce_wal_batch_req_raw(s->ep) and reads ep->peer (offset 36) past the 4-byte int. IDENTICAL to the prior test_p2_integration tests 8/9 bug already in buglog.", "fix": "Test-side fix (parent must apply, mirrors the documented p2_integration fix): in test_ha_role_lifecycle.c add `extern int (*repl_broadcast_send_hook)(void*, wal_catchup_entry_t**, int);` and a no-op drop hook, install it in lifecycle_cluster_setup before promotion (reset to NULL in teardown), and cast endpoints to (void*). Production ha.c is correct per design (persist_wal_async appends+broadcasts by design).", "tags": [ @@ -3682,9 +3682,9 @@ }, { "error_message": "task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and current_lsn==noop_lsn+1, but lifecycle_cluster_setup primes current_lsn = appended_lsn + 1, so a real persist_wal_async appends the noop at appended_lsn+1 (current_lsn->appended_lsn+2), giving noop_lsn=appended_lsn+1 != asserted appended_lsn.", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect — OUT OF task 2.2 boundary)", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", "root_cause": "persist_wal_async assigns hdr.lsn = current_lsn++ (entry takes the CURRENT value). The fixture comment assumes the next append takes current_lsn-1, which is off by one. To make the noop land at `appended_lsn`, the prime must be `current_lsn = appended_lsn`, not `appended_lsn + 1`.", - "fix": "Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design §Promotion Rewrite / brief.", + "fix": "Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design \u00a7Promotion Rewrite / brief.", "tags": [ "ha-single-leader-role", "task-2.2", @@ -3785,7 +3785,7 @@ "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", "file": "src/controller/ha.c", "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", - "fix": "Changed CTRL_PRIMARY_COMMITTING → CTRL_PRIMARY", + "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", "tags": [ "wrong-reference", "c" @@ -3816,7 +3816,7 @@ "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", "file": "src/controller/testsuite/test_ha_connection_reuse.c", "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", - "fix": "Changed CTRL_PRIMARY_COMMITTING → CTRL_PRIMARY", + "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", "tags": [ "wrong-reference", "c" @@ -3863,7 +3863,7 @@ "error_message": "Wrong reference: test_nonblocking_single_node_election_enters_committing should be test_nonblocking_single_node_election_enters_primary_directly", "file": "src/controller/testsuite/test_ha_startup.c", "root_cause": "Used \"test_nonblocking_single_node_election_enters_committing\" instead of \"test_nonblocking_single_node_election_enters_primary_directly\"", - "fix": "Changed test_nonblocking_single_node_election_enters_committing → test_nonblocking_single_node_election_enters_primary_directly", + "fix": "Changed test_nonblocking_single_node_election_enters_committing \u2192 test_nonblocking_single_node_election_enters_primary_directly", "tags": [ "wrong-reference", "c" @@ -3897,7 +3897,7 @@ "error_message": "Wrong reference: uint32_t should be ha_config_t", "file": "src/controller/ha.h", "root_cause": "Used \"uint32_t\" instead of \"ha_config_t\"", - "fix": "Changed uint32_t → ha_config_t", + "fix": "Changed uint32_t \u2192 ha_config_t", "tags": [ "wrong-reference", "h" @@ -3911,8 +3911,8 @@ "timestamp": "2026-06-13T14:43:54.194Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/ha-async-io.md", - "root_cause": "Had `CTRL_PRIMARY_COMMITTING → CTRL_PRIMARY`", - "fix": "Changed to 's own-term commit rule (§5.4.2): an election winn", + "root_cause": "Had `CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY`", + "fix": "Changed to 's own-term commit rule (\u00a75.4.2): an election winn", "tags": [ "auto-detected", "wrong-value", @@ -3927,8 +3927,8 @@ "timestamp": "2026-06-13T14:54:21.781Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-deploy-ha/SKILL.md", - "root_cause": "Had `→ CTRL_PRIMARY_COMMITTING`", - "fix": "Changed to `HA FSM (raft): → CTRL_PRIMARY (term=...)`", + "root_cause": "Had `\u2192 CTRL_PRIMARY_COMMITTING`", + "fix": "Changed to `HA FSM (raft): \u2192 CTRL_PRIMARY (term=...)`", "tags": [ "auto-detected", "wrong-value", @@ -3943,8 +3943,8 @@ "timestamp": "2026-06-13T15:31:21.740Z", "error_message": "Incorrect value in code", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_consistency_invariants.md", - "root_cause": "Had \"XCE HA Raft+CP umbrella — sub-specs 1&2 of 4 IMPL", - "fix": "Changed to \"XCE HA Raft+CP umbrella — sub-specs 1,2,3 of 4 IM", + "root_cause": "Had \"XCE HA Raft+CP umbrella \u2014 sub-specs 1&2 of 4 IMPL", + "fix": "Changed to \"XCE HA Raft+CP umbrella \u2014 sub-specs 1,2,3 of 4 IM", "tags": [ "auto-detected", "wrong-value", @@ -3960,7 +3960,7 @@ "error_message": "Wrong reference: stdint should be stdbool", "file": "src/controller/include/cluster.h", "root_cause": "Used \"stdint\" instead of \"stdbool\"", - "fix": "Changed stdint → stdbool", + "fix": "Changed stdint \u2192 stdbool", "tags": [ "wrong-reference", "h" @@ -3975,7 +3975,7 @@ "error_message": "Wrong reference: xlist_destroy should be xlist_fini", "file": "src/controller/testsuite/test_egress_filter.c", "root_cause": "Used \"xlist_destroy\" instead of \"xlist_fini\"", - "fix": "Changed xlist_destroy → xlist_fini", + "fix": "Changed xlist_destroy \u2192 xlist_fini", "tags": [ "wrong-reference", "c" @@ -4102,7 +4102,7 @@ "error_message": "AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart", "file": "src/controller/testsuite/test_demotion_convergence.c", "root_cause": "dc_run_pending_task peeked tasks.head and ran entry(args) WITHOUT dequeuing. The once-guard correctly makes the 2nd ha_rejoin_reconcile schedule no new task, so head still pointed at the first (already-run) task whose body had freed its heap ctx (args). The 2nd dc_run_pending_task re-ran that task on the freed ctx. The convergence task bodies free their own args (production delaymgr killer frees only the delay_task_t node, never args; production execv never re-runs a one-shot).", - "fix": "Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) — mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct).", + "fix": "Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) \u2014 mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct).", "tags": [ "asan", "use-after-free", @@ -4138,7 +4138,7 @@ "timestamp": "2026-06-14T04:57:22.178Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /*; * ha_walack_continuation_for_test: test-only entry point for", "tags": [ "auto-detected", @@ -4170,8 +4170,8 @@ "timestamp": "2026-06-14T06:37:35.904Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: printf(\"\\n>> Regression pins: fan-out / snapshot fallback / ; \"re-inclusion (task 6.1 — PASS against current code)\\n\\n\"); | Also: /* ── Fan-out counting send-hook (task 6.1 (a)/(d), umbrella; * repl_broadcast routes each enqueue through repl_broadcast_", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: printf(\"\\n>> Regression pins: fan-out / snapshot fallback / ; \"re-inclusion (task 6.1 \u2014 PASS against current code)\\n\\n\"); | Also: /* \u2500\u2500 Fan-out counting send-hook (task 6.1 (a)/(d), umbrella; * repl_broadcast routes each enqueue through repl_broadcast_", "tags": [ "auto-detected", "error-handling", @@ -4271,7 +4271,7 @@ "error_message": "WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0", "file": "src/controller/config_repl.c", "root_cause": "ha_stream_wal_to_standby filter `if (hdr.lsn > standby_lsn)` excludes lsn 0 when standby_lsn==0 (0>0 is false). standby_lsn=0 is ambiguous: it means BOTH 'empty standby (applied nothing)' AND 'standby has applied lsn 0'. The streamer cannot distinguish, computes start_lsn=standby_lsn+1=1, and drops the genuine lsn-0 entry. Only non-cosmetic because lsn-0 entries can be REAL data (first-ever WAL entry of a fresh cluster is lsn 0 from a real write OR the promotion NOOP).", - "fix": "FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry — no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine).", + "fix": "FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry \u2014 no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine).", "tags": [ "ha", "replication", @@ -4292,7 +4292,7 @@ "error_message": "Wrong reference: test_pin_dispatch_gate_degraded_allowlist should be test_empty_standby_fresh_cluster_streams_not_snapshot", "file": "src/controller/testsuite/test_convergence_verification.c", "root_cause": "Used \"test_pin_dispatch_gate_degraded_allowlist\" instead of \"test_empty_standby_fresh_cluster_streams_not_snapshot\"", - "fix": "Changed test_pin_dispatch_gate_degraded_allowlist → test_empty_standby_fresh_cluster_streams_not_snapshot", + "fix": "Changed test_pin_dispatch_gate_degraded_allowlist \u2192 test_empty_standby_fresh_cluster_streams_not_snapshot", "tags": [ "wrong-reference", "c" @@ -4371,7 +4371,7 @@ "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t → uint32_t", + "fix": "Changed myconf_t \u2192 uint32_t", "tags": [ "wrong-reference", "h" @@ -4426,8 +4426,8 @@ "timestamp": "2026-06-15T09:16:53.679025Z", "error_message": "jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works.", "file": "src/lib/request.c", - "root_cause": "SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() — the ONLY place that sets c->running=1 — is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked.", - "fix": "Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown — the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run — moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal).", + "root_cause": "SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() \u2014 the ONLY place that sets c->running=1 \u2014 is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked.", + "fix": "Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown \u2014 the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run \u2014 moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal).", "tags": [ "joblist", "workerlist", @@ -4483,7 +4483,7 @@ "timestamp": "2026-06-16T00:10:49.295Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-prevote/research.md", - "root_cause": "Had 't understand PreVote (older binary) — does the ca", + "root_cause": "Had 't understand PreVote (older binary) \u2014 does the ca", "fix": "Changed to 's \"", "tags": [ "auto-detected", @@ -4579,8 +4579,8 @@ "timestamp": "2026-06-16T04:50:38.855Z", "error_message": "Missing error handling in struct", "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 3.1 — voter-side PreVote dispatch gate (stickiness +; * no mutation, reply term unchanged) */", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 3.1 \u2014 voter-side PreVote dispatch gate (stickiness +; * no mutation, reply term unchanged) */", "tags": [ "auto-detected", "error-handling", @@ -4596,7 +4596,7 @@ "error_message": "Wrong reference: test_commit_engine should be test_ha_prevote", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_prevote\"", - "fix": "Changed test_commit_engine → test_ha_prevote", + "fix": "Changed test_commit_engine \u2192 test_ha_prevote", "tags": [ "wrong-reference", "c" @@ -4659,7 +4659,7 @@ "error_message": "Wrong reference: test_commit_engine should be test_ha_failover_prevote", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_failover_prevote\"", - "fix": "Changed test_commit_engine → test_ha_failover_prevote", + "fix": "Changed test_commit_engine \u2192 test_ha_failover_prevote", "tags": [ "wrong-reference", "c" @@ -4721,7 +4721,7 @@ "timestamp": "2026-06-16T15:55:00Z", "error_message": "controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap", "file": "src/controller/persist.c", - "root_cause": "fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned — up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms).", + "root_cause": "fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned \u2014 up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms).", "fix": "Replaced the nanosleep with pthread_cond_timedwait on a new fsync_mutex/fsync_cond (added to persist_t, init in persist_init, destroyed in persist_fini). persist_stop now sets fsync_stop and pthread_cond_signal(fsync_cond) under fsync_mutex before join, so the worker wakes immediately. fdatasync still runs under p->lock, released before re-acquiring fsync_mutex (no lock nesting). Full controller suite now completes in ~10s (was >300s); 0 failures.", "tags": [ "controller", @@ -4764,10 +4764,10 @@ { "id": "bug-606", "timestamp": "2026-06-16T17:46:00Z", - "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win → CTRL_ELECTING → become_follower (term bumped) → ~10s → repeat, term climbing unbounded (3→9→∞), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", + "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win \u2192 CTRL_ELECTING \u2192 become_follower (term bumped) \u2192 ~10s \u2192 repeat, term climbing unbounded (3\u21929\u2192\u221e), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", "file": "src/controller/ha.c (ha_begin_failover, ha_election_stagger_ms, ha_failover_stagger_fn); src/controller/ha.h; test: src/controller/testsuite/test_ha_role.c", - "root_cause": "Dueling-candidates split-vote livelock — Raft §5.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other — PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) → split vote → election times out (fixed election_timeout_ms, raft.c:105, NO jitter) → become_follower, term+1 → repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn → ha_on_primary_ctrl_fd_rdhup → PreVote → ha_start_election_async → raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it.", - "fix": "Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh → skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won → PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first → STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive — skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed — the failover-start stagger alone resolves it; left as future defense-in-depth.", + "root_cause": "Dueling-candidates split-vote livelock \u2014 Raft \u00a75.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other \u2014 PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) \u2192 split vote \u2192 election times out (fixed election_timeout_ms, raft.c:105, NO jitter) \u2192 become_follower, term+1 \u2192 repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn \u2192 ha_on_primary_ctrl_fd_rdhup \u2192 PreVote \u2192 ha_start_election_async \u2192 raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it.", + "fix": "Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh \u2192 skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won \u2192 PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first \u2192 STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive \u2014 skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed \u2014 the failover-start stagger alone resolves it; left as future defense-in-depth.", "tags": [ "ha", "raft", @@ -4810,7 +4810,7 @@ { "id": "bug-609", "timestamp": "2026-06-16T10:52:01.800Z", - "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call — transparent failover did not engage", + "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call \u2014 transparent failover did not engage", "file": "src/lib/client_failover.c", "root_cause": "handle_disconnect leaves xce_ctx_t.state=XCE_CTX_ERR after the primary's connection drops; the per-op xceapi guard (xceapi.c:14) returns -EINVAL when state!=XCE_CTX_RUN, BEFORE the op reaches a primary. xcecli_retry_decide classified -EINVAL as TERMINAL (non-retriable), so the failover loop bailed without rotating/recovering. The ctx->state ERR->RUN recovery only ran AFTER a rotate, which a terminal classification never triggered. Surfaced by the live 3-node E2E (task 6.2); the unit tests fed status codes directly to decide() and bypassed the real guard, so they missed it.", "fix": "In xcecli_retry_decide, before classifying *rc, detect ac->ctx.state==XCE_CTX_ERR (known-dead connection) and treat it as a prompt transport-style failover cause (XCE_FO_RETRY_NOW, no backoff) regardless of *rc, so the loop rotates+recovers. On budget exhaustion surface -E_XCE_NOT_READY (cluster-not-ready), never -EINVAL. Regression: test_decide_known_dead_ctx_retries_not_terminal.", @@ -4832,10 +4832,10 @@ { "id": "bug-610", "timestamp": "2026-06-16T10:52:36.535Z", - "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate → xcecli_retry_next → xcecli_joblist during live master failover (NULL ep->iomgr deref)", + "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate \u2192 xcecli_retry_next \u2192 xcecli_joblist during live master failover (NULL ep->iomgr deref)", "file": "src/lib/client_failover.c", - "root_cause": "xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) → NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it.", - "fix": "Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini → endpoint_init → endpoint_set_peer → re-bind ep->iomgr=&ctx->iomgr → endpoint_connect → re-set owner+msg/err handlers → iomgr_monitor. xce_failover_rotate now delegates the reconnect to it.", + "root_cause": "xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) \u2192 NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it.", + "fix": "Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini \u2192 endpoint_init \u2192 endpoint_set_peer \u2192 re-bind ep->iomgr=&ctx->iomgr \u2192 endpoint_connect \u2192 re-set owner+msg/err handlers \u2192 iomgr_monitor. xce_failover_rotate now delegates the reconnect to it.", "tags": [ "async-fix", "md", @@ -4909,7 +4909,7 @@ "error_message": "Wrong reference: address should be hint", "file": ".kiro/specs/libxce-failover/tasks.md", "root_cause": "Used \"address\" instead of \"hint\"", - "fix": "Changed address → hint", + "fix": "Changed address \u2192 hint", "tags": [ "auto-detected", "wrong-reference", @@ -4925,7 +4925,7 @@ "error_message": "Wrong reference: XCE_MAXHOST_LEN should be XCE_MAXHOSTNAME_LEN", "file": ".kiro/specs/libxce-failover/design.md", "root_cause": "Used \"XCE_MAXHOST_LEN\" instead of \"XCE_MAXHOSTNAME_LEN\"", - "fix": "Changed XCE_MAXHOST_LEN → XCE_MAXHOSTNAME_LEN", + "fix": "Changed XCE_MAXHOST_LEN \u2192 XCE_MAXHOSTNAME_LEN", "tags": [ "wrong-reference", "md" @@ -5084,7 +5084,7 @@ "error_message": "Wrong reference: test_p1_integration should be test_join_redirect", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_p1_integration\" instead of \"test_join_redirect\"", - "fix": "Changed test_p1_integration → test_join_redirect", + "fix": "Changed test_p1_integration \u2192 test_join_redirect", "tags": [ "wrong-reference", "c" @@ -5099,7 +5099,7 @@ "error_message": "Wrong reference: test_recover_resets_err_state_to_run should be test_decide_jitter_staggers_many_clients", "file": "src/lib/testsuite/test_libxce_retry.c", "root_cause": "Used \"test_recover_resets_err_state_to_run\" instead of \"test_decide_jitter_staggers_many_clients\"", - "fix": "Changed test_recover_resets_err_state_to_run → test_decide_jitter_staggers_many_clients", + "fix": "Changed test_recover_resets_err_state_to_run \u2192 test_decide_jitter_staggers_many_clients", "tags": [ "wrong-reference", "c" @@ -5158,7 +5158,7 @@ "error_message": "Wrong reference: 611 should be 646", "file": "src/lib/netio.c", "root_cause": "Used \"611\" instead of \"646\"", - "fix": "Changed 611 → 646", + "fix": "Changed 611 \u2192 646", "tags": [ "auto-detected", "wrong-reference", @@ -5214,9 +5214,9 @@ { "id": "bug-650", "timestamp": "2026-06-17T18:30:00Z", - "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket — the request was never transmitted yet the client waits for a reply forever.", + "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket \u2014 the request was never transmitted yet the client waits for a reply forever.", "file": "src/lib/request.c", - "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) — but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", + "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) \u2014 but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", "fix": "All six request.c senders propagate the enq result: `int enq_rc = endpoint_enq(ep,&pkt->link); return (enq_rc < 0) ? enq_rc : 0;`. A failed send now returns <0 through xce_jobsubmit_req -> xce_jobsubmit, which returns before cbuf_get; the failover loop classifies the transport error and rotates instead of blocking on a reply that never arrives. TDD: test_request_send_propagation.c builds an OPCode_nop request (encodes OK) on a dead endpoint (EPOLL_CTL_MOD ENOENT) and asserts xce_request returns <0 (verified red with old `return 0`, green with fix). libxce 169/0; controller_unit_test links clean.", "tags": [ "libxce", @@ -5258,7 +5258,7 @@ "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", "file": "src/lib/testsuite/test_netio_arm_failure.c", "root_cause": "Used \"test_enq_arm_failure_is_loud_and_fires_errhandler\" instead of \"test_enq_arm_failure_is_loud_no_errhandler\"", - "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler → test_enq_arm_failure_is_loud_no_errhandler", + "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler \u2192 test_enq_arm_failure_is_loud_no_errhandler", "tags": [ "wrong-reference", "c" @@ -5272,7 +5272,7 @@ "timestamp": "2026-06-17T19:10:00Z", "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", "file": "src/lib/netio.c", - "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() — it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", + "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() \u2014 it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", "fix": "endpoint_enq() no longer calls handle_error(); on arm failure it just returns a RETRIABLE -ECONNRESET. The caller propagates it (request.c senders return it -> client failover classifies ECONNRESET as RETRY_NOW and rotates; controller heartbeat path simply gets the error, no lock re-entry). The real teardown stays on the io_worker thread (handle_input/handle_output -> io_handler -> handle_error), which holds no caller locks. Updated test_netio_arm_failure.c: endpoint_enq must return <0 AND must NOT invoke the err_handler (g_err_count==0). libxce 169/0; controller_unit_test links clean.", "tags": [ "libxce", @@ -5330,8 +5330,8 @@ "timestamp": "2026-06-17T22:00:00Z", "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", "file": "src/controller/ha.c", - "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` — checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", - "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) — so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", + "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` \u2014 checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", + "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) \u2014 so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", "tags": [ "controller", "ha", @@ -5353,7 +5353,7 @@ "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", "file": "src/controller/persist.c, src/controller/commit.c, src/controller/repl.c", "root_cause": "'committed' does not imply 'durable'. WAL fsync is asynchronous (persist.c:173 fsync_worker, timer wal_sync_ms); persist_append and the standby WalAck path do NOT fdatasync before ACK. commit_on_ack (commit.c:303) advances committed_lsn on majority ACK = OS page cache only, durable on zero disks. On kill+restart, killed nodes lose the not-yet-fsynced tail; persist_recover hits CRC break at zeroed region (prev_crc=00000000, offset 40) and truncates replay to ~0 entries, yet the hollow node stays eligible, wins election under degraded required=1 quorum (PreVote+freshness gate cannot fence it because the committed entries were durable on nobody), self-promotes advertising committed_lsn=21 over an empty job table, and the node holding the real data converges DOWN. Violates Leader-Completeness assumption (majority-ACK != durable-on-majority) in src/controller/CLAUDE.md.", - "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 — matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro — this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", + "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 \u2014 matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro \u2014 this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", "tags": [ "controller", "ha", @@ -5511,7 +5511,7 @@ "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t → uint32_t", + "fix": "Changed myconf_t \u2192 uint32_t", "tags": [ "wrong-reference", "h" @@ -5622,7 +5622,7 @@ "error_message": "Wrong reference: else should be RUN_TEST", "file": "src/controller/testsuite/test_commit_durability.c", "root_cause": "Used \"else\" instead of \"RUN_TEST\"", - "fix": "Changed else → RUN_TEST", + "fix": "Changed else \u2192 RUN_TEST", "tags": [ "auto-detected", "wrong-reference", @@ -5871,8 +5871,8 @@ "timestamp": "2026-06-18T07:52:20.595Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* task 3.2 — Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 3.2 \u2014 Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", "tags": [ "auto-detected", "error-handling", @@ -5887,7 +5887,7 @@ "timestamp": "2026-06-18T08:04:18.857Z", "error_message": "Missing error handling in enum", "file": "src/include/persist.h", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Bounded catch-up window (ha-commit-durability task 3.3, R; * The FINITE deadline a recovery-truncated / behind member i", "tags": [ "auto-detected", @@ -5919,7 +5919,7 @@ "timestamp": "2026-06-18T08:39:54.889Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `durable_floor ≤ applied_lsn`", + "root_cause": "Had `durable_floor \u2264 applied_lsn`", "fix": "Changed to `applied_lsn`", "tags": [ "auto-detected", @@ -5967,8 +5967,8 @@ "timestamp": "2026-06-18T09:08:09.215Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ── ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2500\u2500 ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", "tags": [ "auto-detected", "error-handling", @@ -5983,7 +5983,7 @@ "timestamp": "2026-06-18T09:09:28.348Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fr; * member is not behind anything and must be promotable (no f", "tags": [ "auto-detected", @@ -6047,7 +6047,7 @@ "timestamp": "2026-06-18T09:49:58.840Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Test-injectable send hook (defined in repl.c under XCE_TE; * repl_broadcast from dereferencing the dummy endpoints as r", "tags": [ "auto-detected", @@ -6063,7 +6063,7 @@ "timestamp": "2026-06-18T10:25:02.673Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* ha_cancel_promotion_fence_timer: cancel a pending bounded; * (idempotent; no-op when none is armed). Called when the m", "tags": [ "auto-detected", @@ -6080,7 +6080,7 @@ "error_message": "Wrong operator: != should be ==", "file": "src/controller/ha.c", "root_cause": "Used \"!=\" instead of \"==\"", - "fix": "Changed operator != → ==", + "fix": "Changed operator != \u2192 ==", "tags": [ "auto-detected", "operator-fix", @@ -6095,8 +6095,8 @@ "timestamp": "2026-06-18T10:30:18.073Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* task 4.4 — bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 4.4 \u2014 bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", "tags": [ "auto-detected", "error-handling", @@ -6141,10 +6141,10 @@ { "id": "bug-760", "timestamp": "2026-06-18T13:00:00Z", - "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 — a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", + "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 \u2014 a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", "root_cause": "ha-commit-durability task 2.2 capped the primary commit self-contribution at durable_lsn (advanced asynchronously by the demand-driven group-commit flusher) and wired a durable-advance commit-wake callback. This adjacent-spec (ha-commit-barrier-rollout) test uses a REAL persist+flusher and checks committed_lsn SYNCHRONOUSLY the instant after repl_on_ack(fact_lsn); the flusher has not yet fdatasync'd LSN1, so durable_lsn==0, self-cap==0, majority candidate==0, committed_lsn stays 0. Pre-spec the test passed (self counted the appended tip immediately). Task 2.2 primed durable_lsn in the in-memory stub fixtures (barrier_setup_vcpus) but did NOT update this real-flusher test. The implementation-note claim that this is a pre-existing /tmp-overlay fdatasync-EIO artifact is FALSE in this env: fdatasync returns rc=0 on /tmp here, and the test passed pre-spec.", - "fix": "FIXED — test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", + "fix": "FIXED \u2014 test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", "tags": [ "ha-commit-durability", "commit-barrier-rollout", @@ -6213,10 +6213,10 @@ { "id": "bug-765", "timestamp": "2026-06-18T14:30:00Z", - "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) → corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", - "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob → CREATE path)", + "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) \u2192 corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", + "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob \u2192 CREATE path)", "root_cause": "Tasks were checked [x] without the dequeue/update path and rusage storage actually being wired; gap surfaced only by cross-task re-audit (per-task review missed it).", - "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation — sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 — enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", + "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation \u2014 sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 \u2014 enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", "tags": [ "scheduler", "controller", @@ -6268,7 +6268,7 @@ "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", "root_cause": "Used \"xce_sched_op_name\" instead of \"sched_op_name\"", - "fix": "Changed xce_sched_op_name → sched_op_name", + "fix": "Changed xce_sched_op_name \u2192 sched_op_name", "tags": [ "wrong-reference", "md" @@ -6314,8 +6314,8 @@ "timestamp": "2026-06-19T13:00:00Z", "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", "file": "src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table", - "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up — so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", - "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain — no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking — the systematic trigger is gone.", + "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up \u2014 so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", + "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain \u2014 no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking \u2014 the systematic trigger is gone.", "tags": [ "ha", "convergence", @@ -6371,7 +6371,7 @@ "error_message": "Wrong reference: hdr should be payload_len", "file": "src/controller/persist.c", "root_cause": "Used \"hdr\" instead of \"payload_len\"", - "fix": "Changed hdr → payload_len", + "fix": "Changed hdr \u2192 payload_len", "tags": [ "wrong-reference", "c" @@ -6386,7 +6386,7 @@ "error_message": "Wrong reference: NULL should be type", "file": "src/controller/testsuite/test_ha_role_lifecycle.c", "root_cause": "Used \"NULL\" instead of \"type\"", - "fix": "Changed NULL → type", + "fix": "Changed NULL \u2192 type", "tags": [ "auto-detected", "wrong-reference", @@ -6402,7 +6402,7 @@ "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", "file": "src/controller/testsuite/test.c (cmd_jobsub)", "root_cause": "On the 2nd+ submit (and during failover windows) the jobsub reply/jobid is not cleanly populated, leaving an uninitialized jobid struct; cmd_jobsub still prints 'jobsubmit ok' with the garbage id instead of verifying the controller reply status and failing. Forces E2E scripts (test-commit-durability.sh) to strict-regex-filter the 4-field id to avoid chasing phantom jobs.", - "fix": "PENDING — jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient — a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", + "fix": "PENDING \u2014 jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient \u2014 a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", "tags": [ "xce_test", "jobsub", @@ -6454,9 +6454,9 @@ "id": "bug-795", "timestamp": "2026-06-19", "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", - "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} — UPSTREAM ha-commit-durability Phase 6.2", - "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) — must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) — broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact — reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause.", - "fix": "FIXED AND VERIFIED LIVE -- ha-commit-durability was reopened specifically for this bug (Phase 6, tasks 6.1-6.4). Task 6.1 attempted to reproduce the cold-restart-all loss against a REAL fsync-able filesystem (ext4, not this sandbox's overlayfs) and it did not reproduce -- the original loss was an overlayfs fdatasync artifact, not a genuine product defect (tasks 6.2/6.3 root-cause/fix were correctly marked N/A as a direct consequence). Task 6.4 re-validated on real fs and closed bug-795. spec.json's validate_impl (decision: GO, 2026-07-18) independently reconfirms this with a later, fresh live re-validation: a 4-round rolling SIGKILL+restart soak plus a full cold-restart-all cycle, 8/8 PASS, 0 job loss -- all 20 committed jobs survived both rolling failover and cold-restart-all. Distinct root cause from bug-828 (a premature-truncate bug during a live election under rolling single-node failover, fixed separately in ha-term-safety commit b77d412) -- not the same defect, despite early investigation notes grouping them together as 'bug-828/795'. One adjacent, still-open, low-priority item surfaced during the same validation: an extra/duplicate committed job appeared once during the original 2026-07-02 validation run (over-counting, the opposite symptom, not yet root-caused, does not block the GO) -- filed separately, not part of this entry.\n\n[Original entry]\nNOT FIXED HERE — out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", + "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} \u2014 UPSTREAM ha-commit-durability Phase 6.2", + "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) \u2014 broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact \u2014 reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause.", + "fix": "FIXED AND VERIFIED LIVE -- ha-commit-durability was reopened specifically for this bug (Phase 6, tasks 6.1-6.4). Task 6.1 attempted to reproduce the cold-restart-all loss against a REAL fsync-able filesystem (ext4, not this sandbox's overlayfs) and it did not reproduce -- the original loss was an overlayfs fdatasync artifact, not a genuine product defect (tasks 6.2/6.3 root-cause/fix were correctly marked N/A as a direct consequence). Task 6.4 re-validated on real fs and closed bug-795. spec.json's validate_impl (decision: GO, 2026-07-18) independently reconfirms this with a later, fresh live re-validation: a 4-round rolling SIGKILL+restart soak plus a full cold-restart-all cycle, 8/8 PASS, 0 job loss -- all 20 committed jobs survived both rolling failover and cold-restart-all. Distinct root cause from bug-828 (a premature-truncate bug during a live election under rolling single-node failover, fixed separately in ha-term-safety commit b77d412) -- not the same defect, despite early investigation notes grouping them together as 'bug-828/795'. One adjacent, still-open, low-priority item surfaced during the same validation: an extra/duplicate committed job appeared once during the original 2026-07-02 validation run (over-counting, the opposite symptom, not yet root-caused, does not block the GO) -- filed separately, not part of this entry.\n\n[Original entry]\nNOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", "tags": [ "ha-commit-durability", "commit-durability", @@ -6500,8 +6500,8 @@ "timestamp": "2026-06-20T01:05:25.091Z", "error_message": "Incorrect value in code", "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"$label: DATA LOSS — a committed job vanished (bug", - "fix": "Changed to \"$label: DATA LOSS — a committed job vanished from", + "root_cause": "Had \"$label: DATA LOSS \u2014 a committed job vanished (bug", + "fix": "Changed to \"$label: DATA LOSS \u2014 a committed job vanished from", "tags": [ "auto-detected", "wrong-value", @@ -6581,7 +6581,7 @@ "error_message": "Wrong reference: Data should be Text", "file": "src/schema/controller.capnp", "root_cause": "Used \"Data\" instead of \"Text\"", - "fix": "Changed Data → Text", + "fix": "Changed Data \u2192 Text", "tags": [ "auto-detected", "wrong-reference", @@ -6597,7 +6597,7 @@ "error_message": "Wrong reference: xce_data_t should be char", "file": "src/include/typemap.h", "root_cause": "Used \"xce_data_t\" instead of \"char\"", - "fix": "Changed xce_data_t → char", + "fix": "Changed xce_data_t \u2192 char", "tags": [ "wrong-reference", "h" @@ -6628,7 +6628,7 @@ "error_message": "Wrong operator: >= should be ==", "file": "src/controller/testsuite/test.c", "root_cause": "Used \">=\" instead of \"==\"", - "fix": "Changed operator >= → ==", + "fix": "Changed operator >= \u2192 ==", "tags": [ "auto-detected", "operator-fix", @@ -6642,7 +6642,7 @@ "id": "bug-817", "timestamp": "2026-06-19", "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", - "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) — UPSTREAM, not controller-status", + "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) \u2014 UPSTREAM, not controller-status", "root_cause": "libxce client failover does not cleanly advance past a dead FIRST master to the live nodes; surfaces a typed EINVAL instead of rotating. Pre-existing (controller-status consumes failover unchanged). Surfaced by controller-status task 5.1 E2E (authoritative wait-primary timed out after a primary kill while a dead node led the masters list, even though a new primary was serving).", "fix": "FIXED -- bug-817 and bug-854 are the same defect (client-side positional primary-discovery: a client fails to reach the live primary when the dead controller happens to be first in the configured masters list). libxce-failover's own spec.json validation note states this was 'found and fixed in prior follow-up commits, already present in the validated build.' Independently corroborated by component-failover-adoption/research.md (written 2026-08 this same project week): 'xcecli_start (client.c:125-188) already rotates through ALL configured masters on the very first connect (bug-854 fix, comment at line 136)' -- cited there as current, standing production behavior. Three later specs (ha-term-safety, raft-ha-core-safety, raft-adapter-conformance) all isolate/route-around bug-854 in their own soak verdicts rather than re-fixing it, consistent with it already being fixed elsewhere.\n\n[Original entry]\nNOT fixed in controller-status (out of boundary). Route to libxce-failover. Confirmed affects plain joblist identically. Workaround: ensure the masters list does not lead with a known-dead node, or restart killed nodes (as the durability soak does).", "tags": [ @@ -6769,7 +6769,7 @@ "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", "file": "src/controller/ha.c (vote_reply_dispatch), src/controller/persist.c (persist_wal_write_replicated)", "root_cause": "vote_reply_decode_and_dispatch decoded the granting voter's advertised peerLastLogLsn (vr_lsn at ha.c ~4254) but vote_reply_dispatch DISCARDED it ('(void)peer_lsn; raft_on_vote_reply(...)'). raft_on_vote_reply only bumps s->votes_granted; it never records the peer LSN. The only writer of ctx->votes[idx].vote_granted/peer_lsn (handle_request_vote_reply) was, in production, called ONLY from the connect/error failure paths with (granted=0, lsn=0); the real-grant caller was gated behind a test-only hook (xce_vote_reply_hook, never set in production). So ctx->votes[*] stayed calloc'd zero -> ha_compute_safe_committed_lsn saw n_granted=1 (self) -> safe_committed_lsn = self_lsn. A standby promoting with its own durable_floor=1 then persist_truncate_wal_above(1) destroyed committed LSN 2..5.", - "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) — it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", + "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) \u2014 it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", "tags": [ "async-fix", "md", @@ -6979,8 +6979,8 @@ "timestamp": "2026-06-20T14:30:00.000Z", "error_message": "Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy.", "file": "src/lib/hook.c (xce_add_hook idempotency); root path src/controller/{persist.c persist_start, ha.c ha_register_persist_hooks}", - "root_cause": "NOT array expansion — job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) — proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary.", - "fix": "FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners — persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) — with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) — re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) — confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup.", + "root_cause": "NOT array expansion \u2014 job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) \u2014 proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary.", + "fix": "FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners \u2014 persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) \u2014 with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) \u2014 re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) \u2014 confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup.", "tags": [ "double-register", "duplicate", @@ -7004,9 +7004,9 @@ "id": "bug-847", "timestamp": "2026-06-20T14:37:08.863Z", "error_message": "Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism.", - "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) — capnp multi-segment Data pointers", - "root_cause": "Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary — on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine — ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up.", - "fix": "FIXED — CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON — clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) — the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed — whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) — buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed.", + "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) \u2014 capnp multi-segment Data pointers", + "root_cause": "Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary \u2014 on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine \u2014 ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up.", + "fix": "FIXED \u2014 CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON \u2014 clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) \u2014 the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed \u2014 whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) \u2014 buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed.", "tags": [ "c", "reproducer-unignored", @@ -7034,7 +7034,7 @@ { "id": "bug-848", "timestamp": "2026-06-20T14:40:02.211Z", - "error_message": "On a PRESERVED-DATA restart (no wipe — user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", + "error_message": "On a PRESERVED-DATA restart (no wipe \u2014 user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", "file": "src/controller/cluster.c (cluster_init_phase2, after persist_recover) + src/controller/ha.c (ha_init applied_lsn=0; state report sr->applied_lsn)", "root_cause": "ha->applied_lsn is NEVER reconstructed from the recovered WAL on boot. ha_init zeros it (ha.c:1367); persist_recover (cluster.c:1352) replays the WAL and advances persist->current_lsn/durable_lsn, but nothing copies that recovered position into ha->applied_lsn (persist_t and ha_state_t are separate; the only setter besides ha_init is the SNAPSHOT receive path ha.c:1771 = checkpoint_lsn). So the standby state report advertises sr->applied_lsn = ha->applied_lsn = 0 (ha.c:1787) and the primary's ha_evaluate_repl_action streams [0+1 .. committed]. ASYMMETRY proving it's a bug: ELECTION advertise uses ha_candidate_advertise_lsn = persist_durable_floor (the RECOVERED position, correct), but CATCH-UP advertise uses the un-recovered applied_lsn=0. The system knows its durable tip for voting but not for catch-up. Defeats Raft incremental catch-up for every restarted node; also the upstream cause of the earlier missing-prefix cases (full re-stream from a trimmed primary -> prefix gap).", "fix": "FIXED 2026-06-20 (HA-layer). Added ha_sync_applied_lsn_from_recovery(ha,p) (ha.c/ha.h) = monotonic store of persist_durable_floor(p) into ha->applied_lsn; wired into cluster_init_phase2 right after persist_recover, plus a diagnostic INFO log of recovered durable_floor/current_lsn/applied_lsn. TDD: test_recovered_member_advertises_real_applied_lsn (test_persist_recovery.c) write->recover->sync asserts applied_lsn==durable_floor>0 and monotonic. controller_unit_test 6/6 + adjacent HA 8/8, daemon builds. CAVEAT: live ctrl-1 also showed current_lsn=1 (seg O_TRUNC'd from LSN 1) implying persist_recover may not have loaded the WAL on that restart at all; if so this HA-layer fix is necessary but not sufficient. The new INFO log reveals on the next live restart whether persist_recover is effective (durable_floor>0) or not (bug-849 candidate: recovery not running/loading on the rejoin path).", @@ -7060,10 +7060,10 @@ { "id": "bug-849", "timestamp": "2026-06-20T14:42:09.400Z", - "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only — missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", + "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only \u2014 missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", "file": "src/controller/myapp.c (standby-rejoin path ~317-320) + src/controller/cluster.c cluster_init_phase2 (holds persist_recover)", - "root_cause": "CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED — its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done — durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY — waiting for primary to connect and push config') with NO election → cluster_init_phase2 NEVER ran → NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it.", - "fix": "FIXED 2026-06-21 (design B — boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving — so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done — durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix.", + "root_cause": "CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED \u2014 its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done \u2014 durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY \u2014 waiting for primary to connect and push config') with NO election \u2192 cluster_init_phase2 NEVER ran \u2192 NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it.", + "fix": "FIXED 2026-06-21 (design B \u2014 boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving \u2014 so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done \u2014 durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix.", "tags": [ "wrong-value", "md", @@ -7091,7 +7091,7 @@ "error_message": "A rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery.", "file": "src/controller/repl.c repl_add_standby (line ~119) + repl_broadcast in_catch_up gate", "root_cause": "repl_add_standby initializes a freshly-registered standby with in_catch_up=0, so it is IMMEDIATELY eligible for repl_broadcast (which correctly skips in_catch_up standbys at repl.c:27). Mechanism (confirmed from ctrl-1.log 11:07:21): ctrl-1 reconnects -> registered in_catch_up=0 -> the new job [15..16] is live-broadcast to it (not gated) -> persist_wal_write_replicated writes them, current_lsn jumps to 17 -> THEN the catch-up stream of [1..14] arrives but persist_wal_write_replicated's idempotency guard (lsn+1 <= current_lsn) skips ALL of them (1+1<=17 ... 14+1<=17) -> permanent prefix gap [1..14]. Independent of bug-849: even with recovery (current_lsn=8), a live [15..16] arriving first jumps current_lsn to 17 and the catch-up of [8..14] is skipped (gap [8..14]). Root architectural cause: catch-up streaming and live broadcast are SEPARATE unordered paths (unlike Raft AppendEntries which is one ordered per-follower stream).", - "fix": "FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby — REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it — so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) — contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together.", + "fix": "FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby \u2014 REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it \u2014 so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) \u2014 contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together.", "tags": [ "wrong-value", "c", @@ -7120,8 +7120,8 @@ "timestamp": "2026-06-20T15:20:00.000Z", "error_message": "After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client.", "file": "src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)", - "root_cause": "Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) — or yields a bare EINVAL while RUN — failover gives up permanently. Exact stuck state needs a live repro to confirm.", - "fix": "FIXED (verified 2026-07-02, no direct code trail to when — landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note.", + "root_cause": "Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) \u2014 or yields a bare EINVAL while RUN \u2014 failover gives up permanently. Exact stuck state needs a live repro to confirm.", + "fix": "FIXED (verified 2026-07-02, no direct code trail to when \u2014 landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note.", "tags": [ "client", "stale-report", @@ -7144,7 +7144,7 @@ "error_message": "Wrong reference: test_wal_batch_entry_without_payload should be test_wal_batch_roundtrip_large_payloads", "file": "src/lib/testsuite/test_wal_wire.c", "root_cause": "Used \"test_wal_batch_entry_without_payload\" instead of \"test_wal_batch_roundtrip_large_payloads\"", - "fix": "Changed test_wal_batch_entry_without_payload → test_wal_batch_roundtrip_large_payloads", + "fix": "Changed test_wal_batch_entry_without_payload \u2192 test_wal_batch_roundtrip_large_payloads", "tags": [ "wrong-reference", "c" @@ -7191,7 +7191,7 @@ "error_message": "Wrong reference: test_repl_write_rejects_checkpoint should be test_repl_write_rejects_forward_gap", "file": "src/controller/testsuite/test_repl_apply.c", "root_cause": "Used \"test_repl_write_rejects_checkpoint\" instead of \"test_repl_write_rejects_forward_gap\"", - "fix": "Changed test_repl_write_rejects_checkpoint → test_repl_write_rejects_forward_gap", + "fix": "Changed test_repl_write_rejects_checkpoint \u2192 test_repl_write_rejects_forward_gap", "tags": [ "wrong-reference", "c" @@ -7205,7 +7205,7 @@ "timestamp": "2026-06-21T13:30:00.000Z", "error_message": "CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn.", "file": "src/controller/persist.c:588/726/1040 (hdr.term=p->term, p->term never set) + src/controller/ha.c (election term = local+1, PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term,lsn) log-match)", - "root_cause": "ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) — only CHAOTIC churn triggers it.", + "root_cause": "ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) \u2014 only CHAOTIC churn triggers it.", "fix": "FIXED AND VERIFIED LIVE -- three distinct contributing root causes were found and fixed, all owned by ha-term-safety (spec.json validate_impl decision: GO, 2026-07-17): (1) Term propagation -- repl_broadcast (persist.c:747, the live replicator) built its wire entry without setting entry.term, so live traffic wire-encoded term=0 even though the primary's own header had the real term (same root cause as bug-990, fixed together in task 3.4: entry.term=hdr->term now carried through the live send+intake path, term-0 legacy direct-apply retired). 175/175 unit tests, live-confirmed on real fs. (2) Election-Safety violation (bug-1095/bug-1096, the 'bug-851 family', found by the raft-sim-conformance oracle): raft_on_vote_reply in libxce's raft.c double-counted duplicate vote replies and never dropped stale-term replies, allowing two leaders at the same term -- fixed with a per-peer vote-reply dedup + stale-term drop, unit-tested at both the raft-sim and controller level. (3) Premature election-time truncation (bug-828, closely related): ha_enter_real_election was truncating the WAL to a stale last_known_committed_lsn before computing the safe commit point, destroying already-durable committed entries -- fixed (commit b77d412), live-verified: clean failover preserves committed jobs, walcompare committed-prefix identical across all 3 members across multiple runs. ha-term-safety's GO decision explicitly cites a fresh 6-round LIVE chaotic-multi-kill soak (bug-851's exact repro scenario) passing with zero committed divergence/job loss -- superseding an earlier in-task note (task 5.4) that had called this env-blocked before that later live run completed.\n\n[Original entry]\nNOT FIXED (HA-core + persistence => full spec workflow per CLAUDE.md, not a quick patch). Keystone lever: stamp hdr.term from live ha->term on the primary append path + carry term through the repl envelope + persist it, THEN make ha_convergence reconcile truncate any local suffix whose (term,lsn) != leader's (standard Raft log-matching). MUST pair with durably-establishing the election term before a new leader acts (no two leaders per term: persist/propagate the won term to a majority before appending the own-term noop, or bump term beyond any (term,lsn) a voter advertises). Route to ha-consistency-invariants / ha-convergence-recovery as a new sub-spec.", "tags": [ "ha", @@ -7235,8 +7235,8 @@ "timestamp": "2026-06-21T09:10:00.000Z", "error_message": "Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok.", "file": "src/lib/client.c:403 xcecli_ctrlstatus_primary + src/controller/ctrlstatus.c:104 (standby primaryName=\"\")", - "root_cause": "Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 — the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it — it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby).", - "fix": "FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source — it is only a latency optimization; the client rotation fix alone resolves the hang.", + "root_cause": "Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 \u2014 the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it \u2014 it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby).", + "fix": "FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source \u2014 it is only a latency optimization; the client rotation fix alone resolves the hang.", "tags": [ "libxce", "client", @@ -7420,7 +7420,7 @@ "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_oversize_peers_returns_einval", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_oversize_peers_returns_einval\"", - "fix": "Changed test_raft_init_zero_peers_returns_einval → test_raft_init_oversize_peers_returns_einval", + "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_oversize_peers_returns_einval", "tags": [ "wrong-reference", "c" @@ -7515,7 +7515,7 @@ "error_message": "Wrong reference: g_te_spy should be g_pe_spy", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"g_te_spy\" instead of \"g_pe_spy\"", - "fix": "Changed g_te_spy → g_pe_spy", + "fix": "Changed g_te_spy \u2192 g_pe_spy", "tags": [ "wrong-reference", "c" @@ -7577,7 +7577,7 @@ "timestamp": "2026-06-21T16:51:25.960Z", "error_message": "Missing error handling in unknown", "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: RUN_TEST(test_sim_9_1_replication_and_partition_catch_up);; RUN_TEST(test_sim_9_2_divergent_suffix_truncation);", "tags": [ "auto-detected", @@ -7594,7 +7594,7 @@ "error_message": "Wrong operator: <= should be >=", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"<=\" instead of \">=\"", - "fix": "Changed operator <= → >=", + "fix": "Changed operator <= \u2192 >=", "tags": [ "auto-detected", "operator-fix", @@ -7706,7 +7706,7 @@ "error_message": "Wrong reference: TEST_ASSERT_NOT_EQUAL_INT should be TEST_ASSERT_NOT_EQUAL", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"TEST_ASSERT_NOT_EQUAL_INT\" instead of \"TEST_ASSERT_NOT_EQUAL\"", - "fix": "Changed TEST_ASSERT_NOT_EQUAL_INT → TEST_ASSERT_NOT_EQUAL", + "fix": "Changed TEST_ASSERT_NOT_EQUAL_INT \u2192 TEST_ASSERT_NOT_EQUAL", "tags": [ "wrong-reference", "c" @@ -7737,7 +7737,7 @@ "error_message": "controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch", "file": "src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)", "root_cause": "PRE-EXISTING on checkpoint 20e59d0, NOT caused by ha-term-safety 1.1/1.2. The bug-850 contiguity guard (persist.c:705 rejects lsn>current_lsn with -EAGAIN) is committed at HEAD, but the test_wal_catchup_* tests feed forward-gap LSNs (30, 42) into a FRESH persist (current_lsn~=1) and still expect them to apply -> guard returns -EAGAIN -> applied_lsn does not advance. ha-term-safety changes touch only vote/schema/typemap/stubs; git diff --stat shows ZERO changes to persist.c/config_repl.c/test_config_repl.c. Also a separate run TIMED OUT 300s (the bug-850 DNR /tmp fsync-EIO + commit-WAIT signature; environment artifact on overlayfs).", - "fix": "FIXED (verified 2026-07-02): ran controller_unit_test directly — all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to).", + "fix": "FIXED (verified 2026-07-02): ran controller_unit_test directly \u2014 all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to).", "tags": [ "controller-test", "stale-report", @@ -7763,7 +7763,7 @@ "error_message": "nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=)", "file": "src/flake.nix", "root_cause": "flake pins c-capnproto to the MUTABLE github archive/main.tar.gz; upstream main branch advanced so the tarball content/hash drifted, breaking the fixed-output derivation. Only pre-cached nix stores still build.", - "fix": "RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) — the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression. RECURRENCE 2026-07-23 (config worktree, /xce-build controller build+report task for config-upgrade-snapshot Task 1 RED-phase capture): identical hash mismatch reproduced verbatim (same specified sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM= vs got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk= for capnpc-1.0.1.drv source.drv) on FIRST invocation and again on an identical immediate RETRY (deterministic, not transient -- unlike the unrelated ninja-race bugs, retrying does not help here). Build never reached compilation; no xce_controller, controller_unit_test, or xce_test binaries produced; controller.capnp.c/h were not regenerated. This worktree (config, at .../worktrees/config) still has the pre-fix flake.lock/pin state -- the 2026-06-22 fix (refreshing src/flake.lock) was apparently not carried into this worktree, or upstream c-capnproto main.tar.gz drifted again since. NOT fixed this run (build+report only, no source/flake edits authorized by the task). Next step: refresh src/flake.lock in this worktree (nix flake update / lock the xbuild input) the same way the 2026-06-22 fix did, then retry the controller build. RECURRENCE 3 (2026-07-23, config worktree, config-upgrade-snapshot Task 1 implementer session): identical hash mismatch reproduced again AFTER a prior subagent had already logged occurrence 2 (build+report task, no fix attempted). This session went further: (1) confirmed `nix flake lock --update-input xbuild` in src/ is a NO-OP -- xbuild is already pinned to the latest available shen390s/my-nix-channel@master commit (073f527e), so re-locking flake.lock does NOT fix this recurrence (the previous 2026-06-22 fix's mechanism -- refresh flake.lock -- no longer applies once xbuild is already current); (2) confirmed via direct `nix develop` in the CANONICAL non-worktree repo (/home/rshen/projects/xce/xce/src) that it fails IDENTICALLY (same two sha256 hashes, same capnpc-1.0.1.drv path) -- this is a live, machine-wide, upstream c-capnproto main.tar.gz drift, NOT a worktree-isolation problem, and is not fixable from within this repo at all (the broken pin lives inside the external shen390s/my-nix-channel flake, at whatever commit is currently latest); (3) WORKAROUND discovered and used successfully: this machine's /nix/store already has REALIZED (fully built, not just .drv) output paths for capnpc-1.0.1 and capnproto-1.2.0 from a build that completed before the drift -- `find /nix/store -maxdepth 1 -iname '*capnp*'` surfaces them; invoking their bin/capnp and bin/capnpc-c binaries DIRECTLY (bypassing `nix develop`'s full devShell evaluation entirely) still works and successfully compiled a controller.capnp schema change end-to-end, generating correct bindings. This unblocks schema-level capnp verification even while the full `nix develop` shell is broken, but does NOT unblock a full project build/link -- controller_unit_test additionally needs `librtti` (fetched via a private SSH git flake input), which has NEVER been built/cached on this machine at all (no .drv or realized store path exists for it), so a full build+test run remains genuinely infeasible until either upstream fixes the c-capnproto pin or a machine with a pre-cached librtti+full toolchain is used.", + "fix": "RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) \u2014 the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression. RECURRENCE 2026-07-23 (config worktree, /xce-build controller build+report task for config-upgrade-snapshot Task 1 RED-phase capture): identical hash mismatch reproduced verbatim (same specified sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM= vs got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk= for capnpc-1.0.1.drv source.drv) on FIRST invocation and again on an identical immediate RETRY (deterministic, not transient -- unlike the unrelated ninja-race bugs, retrying does not help here). Build never reached compilation; no xce_controller, controller_unit_test, or xce_test binaries produced; controller.capnp.c/h were not regenerated. This worktree (config, at .../worktrees/config) still has the pre-fix flake.lock/pin state -- the 2026-06-22 fix (refreshing src/flake.lock) was apparently not carried into this worktree, or upstream c-capnproto main.tar.gz drifted again since. NOT fixed this run (build+report only, no source/flake edits authorized by the task). Next step: refresh src/flake.lock in this worktree (nix flake update / lock the xbuild input) the same way the 2026-06-22 fix did, then retry the controller build. RECURRENCE 3 (2026-07-23, config worktree, config-upgrade-snapshot Task 1 implementer session): identical hash mismatch reproduced again AFTER a prior subagent had already logged occurrence 2 (build+report task, no fix attempted). This session went further: (1) confirmed `nix flake lock --update-input xbuild` in src/ is a NO-OP -- xbuild is already pinned to the latest available shen390s/my-nix-channel@master commit (073f527e), so re-locking flake.lock does NOT fix this recurrence (the previous 2026-06-22 fix's mechanism -- refresh flake.lock -- no longer applies once xbuild is already current); (2) confirmed via direct `nix develop` in the CANONICAL non-worktree repo (/home/rshen/projects/xce/xce/src) that it fails IDENTICALLY (same two sha256 hashes, same capnpc-1.0.1.drv path) -- this is a live, machine-wide, upstream c-capnproto main.tar.gz drift, NOT a worktree-isolation problem, and is not fixable from within this repo at all (the broken pin lives inside the external shen390s/my-nix-channel flake, at whatever commit is currently latest); (3) WORKAROUND discovered and used successfully: this machine's /nix/store already has REALIZED (fully built, not just .drv) output paths for capnpc-1.0.1 and capnproto-1.2.0 from a build that completed before the drift -- `find /nix/store -maxdepth 1 -iname '*capnp*'` surfaces them; invoking their bin/capnp and bin/capnpc-c binaries DIRECTLY (bypassing `nix develop`'s full devShell evaluation entirely) still works and successfully compiled a controller.capnp schema change end-to-end, generating correct bindings. This unblocks schema-level capnp verification even while the full `nix develop` shell is broken, but does NOT unblock a full project build/link -- controller_unit_test additionally needs `librtti` (fetched via a private SSH git flake input), which has NEVER been built/cached on this machine at all (no .drv or realized store path exists for it), so a full build+test run remains genuinely infeasible until either upstream fixes the c-capnproto pin or a machine with a pre-cached librtti+full toolchain is used.", "tags": [ "nix", "flake", @@ -7800,7 +7800,7 @@ "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_waldump\" instead of \"test_term_safety_adapter\"", - "fix": "Changed test_waldump → test_term_safety_adapter", + "fix": "Changed test_waldump \u2192 test_term_safety_adapter", "tags": [ "wrong-reference", "c" @@ -8046,7 +8046,7 @@ "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", "file": "src/controller/testsuite/test_term_safety_adapter.c", "root_cause": "Used \"ha_arm_own_term_gate\" instead of \"ha_arm_own_term_gate_for_test\"", - "fix": "Changed ha_arm_own_term_gate → ha_arm_own_term_gate_for_test", + "fix": "Changed ha_arm_own_term_gate \u2192 ha_arm_own_term_gate_for_test", "tags": [ "wrong-reference", "c" @@ -8076,7 +8076,7 @@ "timestamp": "2026-06-22T08:21:57.062Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat ; * library as a 0-entry AppendEntries so the LIBRARY performs", "tags": [ "auto-detected", @@ -8231,7 +8231,7 @@ "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "Used \"persist_roll_segment\" instead of \"persist_test_force_new_segment\"", - "fix": "Changed persist_roll_segment → persist_test_force_new_segment", + "fix": "Changed persist_roll_segment \u2192 persist_test_force_new_segment", "tags": [ "wrong-reference", "c" @@ -8243,7 +8243,7 @@ { "id": "bug-970", "timestamp": "2026-06-22T10:24:35.905518Z", - "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER — config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", + "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER \u2014 config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", "file": "src/controller/config_repl.c, src/controller/persist.c", "root_cause": "Finding1: do_upgrade_config_with_bytes still drove the term-blind committed_config_tag push/ack/advance and never called config_append_marker, so the derive primitive was dead code. Finding2: persist_checkpoint->trim_wal_segments unlinks older segments and the snapshot sentinel carried no config reference, so a committed marker in a trimmed segment vanished and config_committed_marker regressed/returned -ENOENT.", "fix": "Added config_commit_change() wired into do_upgrade_config_with_bytes step 7: appends WAL_CONFIG_MARKER as the consensus record (config_append_marker), then delegates byte delivery + the existing majority barrier to push_config_to_standbys (mechanism unchanged); committed_config_tag is now only a cache, config_committed_marker (committed-prefix derive) is authoritative. Compaction safety: persist_checkpoint scans the committed prefix for the latest marker {version_tag,sha256} (scan_latest_committed_config_ref) BEFORE trim and stamps it into the snapshot sentinel (41-byte optional tail); load_snapshot reads it back into persist.cfgref_*; config_committed_marker falls back to it when no marker survives the live WAL. Added integration test (drives config_commit_change) + checkpoint-trim test + persist_test_force_new_segment seam. cb5 upgrade pins made persist-backed so the marker append succeeds. RED confirmed (Expected 1 Was 0; Expected 0 Was -110); GREEN 0 failures direct-run.", @@ -8317,7 +8317,7 @@ "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", "file": "src/controller/config_repl.c, src/controller/ha.c, src/controller/myapp.c", "root_cause": "config_activate_committed (verify-before-activate + fence) was unit-correct but not integrated: (A) handle_repl_config_push called load_primary_config UNCONDITIONALLY before config_activate_committed, so mismatched bytes went live before the gate; (B) myapp.c boot step 7 + on_commit_advanced never called the consumer, so a recovering member loaded config blind against the committed marker; (C) ha_dispatch_gate / pre-promotion gate never read config_serve_fence, so a fenced member still served + could promote", - "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY — removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", + "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY \u2014 removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", "tags": [ "ha-term-safety", "config-replication", @@ -8358,7 +8358,7 @@ "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "Used \"ClusterConfig\" instead of \"cluster_config_t\"", - "fix": "Changed ClusterConfig → cluster_config_t", + "fix": "Changed ClusterConfig \u2192 cluster_config_t", "tags": [ "wrong-reference", "c" @@ -8543,7 +8543,7 @@ "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", "file": "src/controller/testsuite/test_term_safety_adapter.c", "root_cause": "Used \"test_repl_ack_currentterm_quorum_advances_commit\" instead of \"test_live_broadcast_receive_persists_leader_term\"", - "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit → test_live_broadcast_receive_persists_leader_term", + "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit \u2192 test_live_broadcast_receive_persists_leader_term", "tags": [ "wrong-reference", "c" @@ -8575,7 +8575,7 @@ ], "occurrences": 1, "last_seen": "2026-06-22T16:00:00.000000", - "validation": "Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) — bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline)." + "validation": "Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) \u2014 bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline)." }, { "id": "bug-1000", @@ -8598,8 +8598,8 @@ "timestamp": "2026-06-22T15:44:54.113Z", "error_message": "Missing error handling in errno", "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - _Blocked (live portion only — build+unit gate CONFIRMED GR", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - _Blocked (live portion only \u2014 build+unit gate CONFIRMED GR", "tags": [ "auto-detected", "error-handling", @@ -8646,7 +8646,7 @@ "timestamp": "2026-06-22T22:31:03.315Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);", "tags": [ "auto-detected", @@ -8662,8 +8662,8 @@ "timestamp": "2026-06-22T22:32:53.722Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 — see th", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 \u2014 see th", "tags": [ "auto-detected", "error-handling", @@ -8692,7 +8692,7 @@ { "id": "bug-1013", "timestamp": "2026-06-22T00:00:00Z", - "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes — STRICT_TERM live failover gate fails 'committed_lsn>=5'", + "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes \u2014 STRICT_TERM live failover gate fails 'committed_lsn>=5'", "file": "src/controller/commit.c", "root_cause": "ha-term-safety task 2.4 added a min(durable-majority, raft_commit_index) ceiling in majority_candidate_locked. raft_commit_index is fed only via the library ACK path (ha_raft_on_repl_ack -> raft_advance_commit -> on_commit_advanced), which advances from match_index[] driven by the asynchronous catch-up streamer and routinely LAGS the durable majority. When delivery goes via catch-up, raft_commit_index stays low and PINS committed_lsn below the safe durable watermark (a redundant, starvation-prone gate). The durable-majority barrier was already current-term-safe under single-writer + the own-term NOOP writable gate, making the ceiling redundant.", "fix": "Option A: removed the raft_commit_index fold from majority_candidate_locked (commit.c). The durable-majority barrier + own-term writable_lsn gate are now authoritative for commit-advance. commit_set_raft_commit_index / ce->raft_commit_index made INERT-ADVISORY (recorded for observability/conformance-oracle only; still fed by on_commit_advanced so the library half keeps a sink). Kept durable self-cap (persist_durable_lsn), persist-before-ACK, and assert(cand<=persist_durable_lsn). Verified live: commit advances 4->5 via durable barrier (commit.c:184); walcompare OK committed prefix byte-identical across 3 nodes up to LSN 5.", @@ -8720,8 +8720,8 @@ "timestamp": "2026-06-23T03:05:00.000Z", "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", - "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held — the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", - "fix": "No production code change required for part-2 — the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 — locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", + "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held \u2014 the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", + "fix": "No production code change required for part-2 \u2014 the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 \u2014 locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", "tags": [ "safety", "failover", @@ -8750,8 +8750,8 @@ "timestamp": "2026-06-23T03:45:00Z", "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", "file": "src/controller/ha.c", - "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp — NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", - "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning — it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step — committed jobs are physically present on disk (walcompare proves it).", + "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp \u2014 NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", + "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning \u2014 it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step \u2014 committed jobs are physically present on disk (walcompare proves it).", "tags": [ "ha", "raft", @@ -8781,7 +8781,7 @@ "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", "file": "src/controller/testsuite/clean_failover_preservation.sh", "root_cause": "Used \"$MASTERS\" instead of \"$MASTERS_LIVE\"", - "fix": "Changed $MASTERS → $MASTERS_LIVE", + "fix": "Changed $MASTERS \u2192 $MASTERS_LIVE", "tags": [ "wrong-reference", "sh" @@ -8795,8 +8795,8 @@ "timestamp": "2026-06-23", "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term), config_repl.c (ha_stream_wal_to_standby), persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK).", - "root_cause": "ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 → logOk mismatch → the new-term NOOP (the post-promotion first entry) is perpetually rejected → no WalAck → applied_lsn stuck at pre-promotion value → the heartbeat state report advertises the stale lsn → the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch → CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL.", - "fix": "FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED→GREEN proven (Expected 1 Was 0 → PASS). Live: standby converge + committed prefix identical.", + "root_cause": "ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 \u2192 logOk mismatch \u2192 the new-term NOOP (the post-promotion first entry) is perpetually rejected \u2192 no WalAck \u2192 applied_lsn stuck at pre-promotion value \u2192 the heartbeat state report advertises the stale lsn \u2192 the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch \u2192 CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL.", + "fix": "FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED\u2192GREEN proven (Expected 1 Was 0 \u2192 PASS). Live: standby converge + committed prefix identical.", "tags": [ "failover", "availability", @@ -8859,7 +8859,7 @@ "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", "file": "src/lib/netio.c", "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", - "fix": "Changed handle_ssl_handshake → endpoint_ssl_handshake", + "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", "tags": [ "wrong-reference", "c" @@ -8874,7 +8874,7 @@ "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", "file": "src/controller/cluster.c", "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", - "fix": "Changed handle_ssl_handshake → endpoint_ssl_handshake", + "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", "tags": [ "wrong-reference", "c" @@ -8889,7 +8889,7 @@ "error_message": "Wrong reference: endpoint_ssl_handshake should be handle_ssl_handshake", "file": "src/lib/netio.c", "root_cause": "Used \"endpoint_ssl_handshake\" instead of \"handle_ssl_handshake\"", - "fix": "Changed endpoint_ssl_handshake → handle_ssl_handshake", + "fix": "Changed endpoint_ssl_handshake \u2192 handle_ssl_handshake", "tags": [ "wrong-reference", "c" @@ -8903,8 +8903,8 @@ "timestamp": "2026-06-23T14:08:05.466Z", "error_message": "Missing error handling in unknown", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: # ── Steps 7–10 looped: kill primary, verify, submit, restar; #", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: # \u2500\u2500 Steps 7\u201310 looped: kill primary, verify, submit, restar; #", "tags": [ "auto-detected", "error-handling", @@ -8936,7 +8936,7 @@ "error_message": "Wrong operator: >= should be ==", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", "root_cause": "Used \">=\" instead of \"==\"", - "fix": "Changed operator >= → ==", + "fix": "Changed operator >= \u2192 ==", "tags": [ "auto-detected", "operator-fix", @@ -8967,8 +8967,8 @@ "timestamp": "2026-06-24T12:05:00.000Z", "error_message": "After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win.", "file": "src/controller/ha.c (raft_cb_become_leader / safe_committed_lsn), src/controller/cluster.c (OPCode_preVoteRequest)", - "root_cause": "ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 → majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 — divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader.", - "fix": "IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log — the new leader NEVER truncates entries it holds (Raft §5.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped — clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence.", + "root_cause": "ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 \u2192 majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 \u2014 divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader.", + "fix": "IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log \u2014 the new leader NEVER truncates entries it holds (Raft \u00a75.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped \u2014 clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence.", "tags": [ "prevote", "deadlock", @@ -9007,8 +9007,8 @@ "timestamp": "2026-06-24T00:00:00.000Z", "error_message": "Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch)", "file": "src/controller/config_repl.c, src/controller/ha.c", - "root_cause": "AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term → ae_reject_stale) correctly rejects, but it should never fire — the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever.", - "fix": "IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct — capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) — the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) → falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED — safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures.", + "root_cause": "AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term \u2192 ae_reject_stale) correctly rejects, but it should never fire \u2014 the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever.", + "fix": "IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct \u2014 capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) \u2014 the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) \u2192 falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED \u2014 safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures.", "tags": [ "catch-up", "replication", @@ -9031,7 +9031,7 @@ "timestamp": "2026-06-24T08:18:27.510Z", "error_message": "Missing error handling in unknown", "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: if (c && c->persist.wal_path[0]) {; /* bug-1067: pass the leader's CURRENT term so historical ca", "tags": [ "auto-detected", @@ -9048,7 +9048,7 @@ "error_message": "Wrong reference: standby_lsn should be leader_term", "file": "src/controller/testsuite/test_coldstart_lsn0.c", "root_cause": "Used \"standby_lsn\" instead of \"leader_term\"", - "fix": "Changed standby_lsn → leader_term", + "fix": "Changed standby_lsn \u2192 leader_term", "tags": [ "wrong-reference", "c" @@ -9063,7 +9063,7 @@ "error_message": "AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c)", "file": "src/lib/xcectx.c", "root_cause": "ctx->reply.u is an UNTAGGED union shared across opcodes (debugmsg/get/jobid/clusterconf/ctrlstatus alias the same 8 bytes). Commit 61e1391 ('fix bug-1060 bug-1067', unrelated Raft work) added speculative free-guards `if (ctx->reply.u.ctrlstatus) free_CtrlStatus_ptr(&ctx->reply.u.ctrlstatus)` (and the clusterconf twin). A non-NULL slot does NOT imply a CtrlStatus*: a failover-aborted reslist/joblist leaves ctx->reply.u.get == &getreply (a dead STACK pointer) because xce_get_with_condition/xce_resval only NULL the slot on the success path, not the xce_get_req<0 / cbuf_get<0 early-returns. The next ctrlstatus reply (worker thread) sees the foreign non-NULL slot and free_CtrlStatus walks it -> stack-use-after-return SEGV.", - "fix": "1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) — restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0.", + "fix": "1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) \u2014 restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0.", "tags": [ "asan", "use-after-return", @@ -9090,7 +9090,7 @@ "error_message": "Wrong reference: handle_reply should be xce_handle_reply", "file": "src/lib/xcectx.c", "root_cause": "Used \"handle_reply\" instead of \"xce_handle_reply\"", - "fix": "Changed handle_reply → xce_handle_reply", + "fix": "Changed handle_reply \u2192 xce_handle_reply", "tags": [ "wrong-reference", "c" @@ -9105,7 +9105,7 @@ "error_message": "Wrong reference: libxce_failover_client_hang should be project_libxce_failover_hang", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1077_reply_union_type_confusion.md", "root_cause": "Used \"libxce_failover_client_hang\" instead of \"project_libxce_failover_hang\"", - "fix": "Changed libxce_failover_client_hang → project_libxce_failover_hang", + "fix": "Changed libxce_failover_client_hang \u2192 project_libxce_failover_hang", "tags": [ "wrong-reference", "md" @@ -9136,7 +9136,7 @@ "error_message": "Wrong reference: apply should be advanced", "file": ".kiro/specs/raft-sim-conformance/design.md", "root_cause": "Used \"apply\" instead of \"advanced\"", - "fix": "Changed apply → advanced", + "fix": "Changed apply \u2192 advanced", "tags": [ "wrong-reference", "md" @@ -9231,7 +9231,7 @@ "error_message": "Wrong reference: test_sim_check_election_safety_fail_on_two_leaders should be test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"test_sim_check_election_safety_fail_on_two_leaders\" instead of \"test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix\"", - "fix": "Changed test_sim_check_election_safety_fail_on_two_leaders → test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", + "fix": "Changed test_sim_check_election_safety_fail_on_two_leaders \u2192 test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", "tags": [ "wrong-reference", "c" @@ -9338,7 +9338,7 @@ "error_message": "raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it", "file": "src/lib/raft/raft.c", "root_cause": "No DropStaleResponse: a reply with term < s->term fell through to replies_received++/votes_granted++, corrupting quorum accounting (raft.tla DropStaleResponse discards mterm < currentTerm).", - "fix": "FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace(\"vote_reply_stale_drop\", ...) at raft.c:455 — exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks).", + "fix": "FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace(\"vote_reply_stale_drop\", ...) at raft.c:455 \u2014 exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks).", "tags": [ "raft", "vote-reply", @@ -9442,7 +9442,7 @@ { "id": "bug-1112", "timestamp": "2026-06-24T15:15:03.882281Z", - "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 — lib test binary fails to link", + "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 \u2014 lib test binary fails to link", "file": "src/lib/raft/raft_sim.c", "root_cause": "Commit e445404 (task 5.3 drop/dup/reorder) added the raft.h declaration + the test_sim_5_3 test but FORGOT to add the raft_sim_scenario_drop_dup_reorder implementation to raft_sim.c. Symbol is declared+referenced but never defined, breaking the link for the whole xcelib_test binary.", "fix": "Pre-existing, outside task 5.4 boundary. Surfaced while validating task 5.4 (figure8). Needs task 5.3 implementation added to raft_sim.c. NOT fixed by 5.4.", @@ -9462,8 +9462,8 @@ "timestamp": "2026-06-24T15:16:52.683Z", "error_message": "Missing error handling in unknown", "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /*; * sim_scen_catch_up_commit: drive bounded AppendEntries-to-a | Also: * on a CONNECTED quorum, yet raft_advance_commit must NOT co; * count alone — the current-term Figure-8 rule (raft.c G4) f", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /*; * sim_scen_catch_up_commit: drive bounded AppendEntries-to-a | Also: * on a CONNECTED quorum, yet raft_advance_commit must NOT co; * count alone \u2014 the current-term Figure-8 rule (raft.c G4) f", "tags": [ "auto-detected", "error-handling", @@ -9479,7 +9479,7 @@ "error_message": "Wrong operator: != should be ==", "file": "src/lib/raft/raft_sim.c", "root_cause": "Used \"!=\" instead of \"==\"", - "fix": "Changed operator != → ==", + "fix": "Changed operator != \u2192 ==", "tags": [ "auto-detected", "operator-fix", @@ -9527,7 +9527,7 @@ "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_1_fault_switch_mechanism", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_1_fault_switch_mechanism\"", - "fix": "Changed test_sim_6_1_seed_sweep → test_sim_7_1_fault_switch_mechanism", + "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_1_fault_switch_mechanism", "tags": [ "wrong-reference", "c" @@ -9542,7 +9542,7 @@ "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_2_negative_both_directions", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_2_negative_both_directions\"", - "fix": "Changed test_sim_6_1_seed_sweep → test_sim_7_2_negative_both_directions", + "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_2_negative_both_directions", "tags": [ "wrong-reference", "c" @@ -9557,7 +9557,7 @@ "error_message": "Wrong reference: test_sim_4_2_step_liveness_and_coverage should be test_sim_8_1_export_trace_named_and_gated", "file": "src/lib/testsuite/test_raft.c", "root_cause": "Used \"test_sim_4_2_step_liveness_and_coverage\" instead of \"test_sim_8_1_export_trace_named_and_gated\"", - "fix": "Changed test_sim_4_2_step_liveness_and_coverage → test_sim_8_1_export_trace_named_and_gated", + "fix": "Changed test_sim_4_2_step_liveness_and_coverage \u2192 test_sim_8_1_export_trace_named_and_gated", "tags": [ "wrong-reference", "c" @@ -9603,7 +9603,7 @@ "timestamp": "2026-06-25T03:20:00.000Z", "error_message": "test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY", "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence — a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs.", + "root_cause": "TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence \u2014 a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs.", "fix": "Test-only, two parts. (a) Detach the durable-advance commit wake (g_dc.persist.on_durable_advance = NULL) and re-pin committed_lsn = committed_boundary right before driving the demotion task, killing the commit-engine race deterministically (helps BOTH fs paths). (b) Hybrid gate by fs type via static dc_path_is_overlayfs() (statfs f_type == OVERLAYFS_SUPER_MAGIC 0x794c7630, defaults to real-fs path on statfs error): on a real fs keep the strong cross-context fresh boot-recovery assertion (fresh->current_lsn <= committed_boundary+1); on overlayfs skip that fd-incoherent step and assert the LIVE-context truncate effect (g_dc.persist.current_lsn <= committed_boundary+1, coherent under p->lock). Both branches prove the same property. Production persist.c truncate (ftruncate+fdatasync) untouched.", "fix_validation": "Full controller suite 20/20 runs: 0 demotion_replay failures, 0 total failures. Group-only 60/60: 0 failures. Baseline (pre-fix) was 4/20. /tmp confirmed overlayfs (statfs magic 0x794c7630), so the overlayfs branch is the one exercised here.", "tags": [ @@ -9763,7 +9763,7 @@ "timestamp": "2026-06-25T23:37:38.579Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: - Commit-advance under pool load (R4.5): with the vtask pool; - Snapshot/checkpoint concurrency (R1.1/1.3/1.5): concurrent", "tags": [ "auto-detected", @@ -9809,10 +9809,10 @@ { "id": "bug-1147", "timestamp": "2026-06-26T03:46:38.904419+00:00", - "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor → lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", + "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor \u2192 lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", "file": "src/controller/persist.c:1374 (trim_wal_segments)", - "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live — write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", - "fix": "FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor — comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete).", + "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live \u2014 write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", + "fix": "FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor \u2014 comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete).", "tags": [ "durability", "permanent-gap", @@ -9889,7 +9889,7 @@ "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", "file": ".kiro/specs/raft-drive-async/design.md", "root_cause": "Used \"PeerCtrlDispatch\" instead of \"LivenessAdmission\"", - "fix": "Changed PeerCtrlDispatch → LivenessAdmission", + "fix": "Changed PeerCtrlDispatch \u2192 LivenessAdmission", "tags": [ "wrong-reference", "md" @@ -9920,7 +9920,7 @@ "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", "root_cause": "Had `VTASK_LIVENESS_RESERVED`", - "fix": "Changed to `R = clamp(n_peers − 1, 1, n_vcpus − 1)`", + "fix": "Changed to `R = clamp(n_peers \u2212 1, 1, n_vcpus \u2212 1)`", "tags": [ "auto-detected", "wrong-value", @@ -10096,7 +10096,7 @@ "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", "file": "src/lib/testsuite/test.c", "root_cause": "Used \"test_request_send_propagation\" instead of \"test_netio_bounded_connect\"", - "fix": "Changed test_request_send_propagation → test_netio_bounded_connect", + "fix": "Changed test_request_send_propagation \u2192 test_netio_bounded_connect", "tags": [ "wrong-reference", "c" @@ -10207,7 +10207,7 @@ "error_message": "Wrong operator: >= should be !=", "file": "src/controller/commit.c", "root_cause": "Used \">=\" instead of \"!=\"", - "fix": "Changed operator >= → !=", + "fix": "Changed operator >= \u2192 !=", "tags": [ "auto-detected", "operator-fix", @@ -10508,10 +10508,10 @@ { "id": "bug-1197", "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF — raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", + "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF \u2014 raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", "file": "src/controller/ha.c", "root_cause": "Moving peer-control off the sequential io_worker onto the pool co-locates teardown (frees ep->data) and inbound delivery (reads ep->data) on different threads with no unmonitor-before-free ordering. Safe today only because g_drive_async_peerctrl_dispatch_enabled defaults OFF.", - "fix": "SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer — a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON.", + "fix": "SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer \u2014 a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON.", "tags": [ "endpoint-quiescence", "fixed", @@ -10538,7 +10538,7 @@ "timestamp": "2026-06-26T00:00:00.000Z", "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", "file": "src/lib/vtaskmgr.c", - "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt — its pkt lifetime is the endpoint's, so an unconditional free would double-free.", + "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt \u2014 its pkt lifetime is the endpoint's, so an unconditional free would double-free.", "fix": "Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan.", "tags": [ "raft-drive-async", @@ -10557,10 +10557,10 @@ { "id": "bug-1198", "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline — a blocking op under the never-gated liveness reservation breaks deadlock-freedom", + "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline \u2014 a blocking op under the never-gated liveness reservation breaks deadlock-freedom", "file": "src/controller/ha.c", "root_cause": "repl_ack_decode_and_dispatch -> ha_dispatch_repl_action_for_standby can run the snapshot send inline. LIVENESS is only correct once that long op is offloaded off the pool.", - "fix": "FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment — the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed.", + "fix": "FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment \u2014 the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed.", "tags": [ "task-3.1", "review-finding", @@ -10583,7 +10583,7 @@ "timestamp": "2026-06-26T11:33:28.328Z", "error_message": "Missing error handling in task", "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Helper: check if a file exists in a directory ; static int file_exists_in(const char* dir, const char* name)", "tags": [ "auto-detected", @@ -10600,7 +10600,7 @@ "error_message": "Wrong reference: file_exists_for_trim_test should be seg_file_exists", "file": "src/controller/testsuite/test_e2e_checkpoint.c", "root_cause": "Used \"file_exists_for_trim_test\" instead of \"seg_file_exists\"", - "fix": "Changed file_exists_for_trim_test → seg_file_exists", + "fix": "Changed file_exists_for_trim_test \u2192 seg_file_exists", "tags": [ "wrong-reference", "c" @@ -10753,7 +10753,7 @@ "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", "file": "src/include/netio.h", "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", - "fix": "Changed ssl_handshake_done → tearing_down", + "fix": "Changed ssl_handshake_done \u2192 tearing_down", "tags": [ "wrong-reference", "h" @@ -10768,7 +10768,7 @@ "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", "file": "src/lib/netio.c", "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", - "fix": "Changed ssl_handshake_done → tearing_down", + "fix": "Changed ssl_handshake_done \u2192 tearing_down", "tags": [ "wrong-reference", "c" @@ -10812,9 +10812,9 @@ { "id": "bug-1224", "timestamp": "2026-06-26", - "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently — controller_unit_test never completes (timeout-killed)", + "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently \u2014 controller_unit_test never completes (timeout-killed)", "file": "src/controller/testsuite/test_ha_dispatch_integration.c", - "root_cause": "Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) → flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard.", + "root_cause": "Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) \u2192 flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard.", "fix": "Add end-of-body completion counters (g_pc_completed on both pc_probe paths incl. the park path; g_block_done at the end of block_task_entry) and poll them to the full expected count AFTER gate_open and BEFORE vtaskmgr_fini, so every pool body has fully returned (worker idle in cbuf_get) before the cancel-based fini. Verified deterministic 5/5 isolated runs + full suite EXIT=0, ASan-clean.", "tags": [ "raft-drive-async", @@ -10911,8 +10911,8 @@ "timestamp": "2026-06-27T04:59:57.805Z", "error_message": "Missing error handling in unknown", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: # ── Chaotic-mode primitives (count outcomes into the verdic; # Run `walcompare --report` across all three members and ADD | Also: # ── Chaotic mode: overlapping-kill rounds → isolated verdic; # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land insid", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: # \u2500\u2500 Chaotic-mode primitives (count outcomes into the verdic; # Run `walcompare --report` across all three members and ADD | Also: # \u2500\u2500 Chaotic mode: overlapping-kill rounds \u2192 isolated verdic; # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land insid", "tags": [ "auto-detected", "error-handling", @@ -10959,7 +10959,7 @@ "timestamp": "2026-06-27T09:28:07.944Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: - **Technical debt addressed:** the silent empty-payload shi | Also: - `.claude/skills/xce-test-ha/run-ha-soak.sh` (and `SKILL.md; - `src/controller/` log init (only if the client-probe appro | Also: | Catch-up Payload Integrity | Adapter | Stream a non-empty ; | Persist Conflict Guard | Persist | `(term,lsn,payload-crc) | Also: - The live-repro gate (task 1.3) has **already run and refut; - A green synthetic unit test over a still-red soak is not a", "tags": [ "auto-detected", @@ -10976,7 +10976,7 @@ "error_message": "Wrong operator: == should be <<", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", "root_cause": "Used \"==\" instead of \"<<\"", - "fix": "Changed operator == → <<", + "fix": "Changed operator == \u2192 <<", "tags": [ "auto-detected", "operator-fix", @@ -11039,8 +11039,8 @@ "timestamp": "2026-06-27T10:45:00.686Z", "error_message": "Missing error handling in unknown", "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* (a) non-empty incoming whose payload differs at an existi; * — a genuine payload conflict. Branch on tip vs below-tip.", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* (a) non-empty incoming whose payload differs at an existi; * \u2014 a genuine payload conflict. Branch on tip vs below-tip.", "tags": [ "auto-detected", "error-handling", @@ -11087,8 +11087,8 @@ "timestamp": "2026-06-27T11:14:27.133Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: * TASK 2.3 — cross-term catch-up streamer payload integrity ; * | Also: /* Task 2.3 cross-term catch-up streamer payload integrity:; * CORE RED — the streamer must never ship a non-empty cro", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: * TASK 2.3 \u2014 cross-term catch-up streamer payload integrity ; * | Also: /* Task 2.3 cross-term catch-up streamer payload integrity:; * CORE RED \u2014 the streamer must never ship a non-empty cro", "tags": [ "auto-detected", "error-handling", @@ -11151,7 +11151,7 @@ "timestamp": "2026-06-30T01:47:07.635Z", "error_message": "Missing await", "file": "src/controller/ha.c", - "root_cause": "Async call without await — returned Promise instead of value", + "root_cause": "Async call without await \u2014 returned Promise instead of value", "fix": "Added await to async call", "tags": [ "auto-detected", @@ -11167,7 +11167,7 @@ "timestamp": "2026-06-30T02:25:32.055Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: - _Depends: 1.5, 6.2, 5.2, 5.3_; - _Blocked: first soak run (HEAD 01419e1, Phase-2+5 fixes) i", "tags": [ "auto-detected", @@ -11184,7 +11184,7 @@ "error_message": "raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills", "file": "src/controller/ha.c (election/PreVote path)", "root_cause": "After 3 overlapping kills the only fully-up-to-date node is killed; the two survivors are both restarted-and-lagging (tips 15 and 11; leader was 19). Both hold the full committed prefix [1..11] (safe candidates) but their PreVote rounds repeatedly time out with no grant quorum (prospective terms frozen, never bumped). Distinct from the 5.2 spurious-failover flap (that is fixed and confirmed). Contributors: freshness mutual-denial between lagging survivors, in_catch_up-denies-all-votes (Req 2.5) with no live primary to catch up from, and/or peer vote connection not established between two nodes that only held a control channel to the dead primary.", - "fix": "FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED→GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync→EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus.", + "fix": "FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED\u2192GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync\u2192EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus.", "tags": [ "prevote", "ha", @@ -11214,8 +11214,8 @@ "timestamp": "2026-06-30T02:41:41.017Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 5.3 lagging-but-safe survivor electability under ove; * CORE RED — a catching-up survivor with NO live primary m", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 5.3 lagging-but-safe survivor electability under ove; * CORE RED \u2014 a catching-up survivor with NO live primary m", "tags": [ "auto-detected", "error-handling", @@ -11261,7 +11261,7 @@ "id": "bug-1281", "timestamp": "2026-06-30T03:25:00Z", "error_message": "Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall.", - "file": "src/controller (restart/recovery/election path — exact site unknown, no backtrace)", + "file": "src/controller (restart/recovery/election path \u2014 exact site unknown, no backtrace)", "root_cause": "ROOT-CAUSED via gdb SIGQUIT all-thread bt: an ABBA lock-order deadlock in src/lib async dispatch. io_worker (netio.c:672) holds ep->lock then blocks on q->lock (dataq_enq); vdispatcher dataq_deq (dataq.c:65) holds q->lock then blocks on ep->lock via the vtask_eligible->endpoint_is_task_blocked (netio.c:1001) match callback. io_worker dies -> node stops accepting/answering PreVotes -> round-4 stall. In release the same ep/q race corrupts -> SIGSEGV. See bug-1282 (the actual dispatch defect).", "fix": "The crash/hang IS bug-1282 (UPSTREAM raft-drive-async dispatch deadlock). raft-ha-core-safety 7.2 stays blocked until bug-1282 is fixed upstream, then re-run the gate. No raft-ha-core-safety code change.", "tags": [ @@ -11309,7 +11309,7 @@ "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", "root_cause": "Had \"Task 7.2 on ~/tmp\"", - "fix": "Changed to \"Task 7.2/5.4 — UNIFIED root cause\"", + "fix": "Changed to \"Task 7.2/5.4 \u2014 UNIFIED root cause\"", "tags": [ "auto-detected", "wrong-value", @@ -11324,7 +11324,7 @@ "timestamp": "2026-06-30T04:17:08.329Z", "error_message": "ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV.", "file": "src/lib/dataq.c:65 (dataq_deq holds q->lock across match cb) + src/lib/vtaskmgr.c:255 (vtask_eligible) + src/lib/netio.c:1001 (endpoint_is_task_blocked takes ep->lock) + src/lib/netio.c:672 (io_worker holds ep->lock across vtask_enq->dataq_enq)", - "root_cause": "dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock — inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes.", + "root_cause": "dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock \u2014 inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes.", "fix": "FIXED (commit 25e741d): promoted endpoint_t.busy from a 1-bit bitfield to _Atomic int; endpoint_block_tasks/endpoint_unblock_tasks/endpoint_is_task_blocked now lockless (release/acquire), removing the ep->lock acquire from the vtask-eligibility path that ran under q->lock. Verified: post-fix gdb soak has ZERO __lll_lock_wait threads + no endpoint_is_task_blocked in any lock path; controller_unit_test all green. Mirrors dispatch_inflight (bug-1197). NOTE: a SEPARATE release-only restart segfault (bug-1283) remains and still blocks 7.2.", "tags": [ "deadlock", @@ -11382,10 +11382,10 @@ { "id": "bug-1286", "timestamp": "2026-06-30T05:30:00Z", - "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart — Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", + "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart \u2014 Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", "file": "src/controller/cluster.c:251 (myreq_handler) + src/controller/ha.c:3194 (ha_dispatch_gate)", - "root_cause": "NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 — a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown).", - "fix": "FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL → &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green.", + "root_cause": "NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 \u2014 a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown).", + "fix": "FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL \u2192 &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green.", "tags": [ "segfault", "null-deref", @@ -11410,8 +11410,8 @@ "timestamp": "2026-06-30T09:15:12.861Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had 't crash) — STILL blocks the gate; likely an endpo", - "fix": "Changed to 't crash) — is now **ROOT-CAUSED** (via the new `", + "root_cause": "Had 't crash) \u2014 STILL blocks the gate; likely an endpo", + "fix": "Changed to 't crash) \u2014 is now **ROOT-CAUSED** (via the new `", "tags": [ "auto-detected", "wrong-value", @@ -11456,9 +11456,9 @@ { "id": "bug-1296", "timestamp": "2026-06-30T13:21:00.000Z", - "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP→PreVote→timeout→repeat livelock — zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait — chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request → mutual timeout → RDHUP fires again → repeat forever.", + "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP\u2192PreVote\u2192timeout\u2192repeat livelock \u2014 zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait \u2014 chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request \u2192 mutual timeout \u2192 RDHUP fires again \u2192 repeat forever.", "file": "src/controller/ha.c", - "root_cause": "TASK 5.6 dueling-candidate PreVote cross-denial — DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING — it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient.", + "root_cause": "TASK 5.6 dueling-candidate PreVote cross-denial \u2014 DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING \u2014 it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient.", "fix": "FIXED 2026-06-30 (subsumed by tasks 5.6 commit 81cb09a + 5.7 commit 74784b2). The RDHUP->PreVote->timeout liveloc was the same fixed-watchdog resonance as bug covered by 5.7: per-campaign election watchdog re-randomized to [T,2T] (Ongaro 3.4) + don't-preempt-in-flight-election (5.6). CONFIRMED by a 10-round chaotic soak: 10/10 rounds elect a new primary in a single term increment (4->13), zero term escalation, round 7+ (the old liveloc point) all clean.", "tags": [ "raft-ha-core-safety", @@ -11497,9 +11497,9 @@ { "id": "bug-1298", "timestamp": "2026-06-30T13:36:00-07:00", - "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) — endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3→4, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", + "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) \u2014 endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3\u21924, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", "file": "src/lib/netio.c / src/lib/vtaskmgr.c / src/controller/cluster.c", - "root_cause": "Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error — handle_endpoint_error → zarray_del → endpoint_killer free()s the endpoint_t — while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match → vtask_eligible → endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF.", + "root_cause": "Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error \u2014 handle_endpoint_error \u2192 zarray_del \u2192 endpoint_killer free()s the endpoint_t \u2014 while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match \u2192 vtask_eligible \u2192 endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF.", "fix": "FIXED (commit 6e05dc1): added endpoint_t reference count (refcnt _Atomic + heap_alloc flag). endpoint_get starts refcnt=1 (owner); vtask_enq takes a per-vtask ref before publishing the task to the dispatcher, run_task/taskq_item_killer release it; every heap-ep teardown site drops its owner ref via endpoint_unref (last unref fini+frees). The dispatcher can now read ep->busy on a queued vtask without the io_worker freeing it underneath. Embedded eps (heap_alloc=0) never freed by unref. Validated: 182 unit tests/0 fail; 6-round ASAN chaotic soak ZERO ASan reports, ctrl-3 survives round 4, all rounds elect.", "tags": [ "netio", @@ -11560,7 +11560,7 @@ "error_message": "6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean.", "file": "src/lib/hook.c (xce_run_hooks); job.c:603 gate is correct", "root_cause": "SHARPENED 2026-06-30 (waldump+log bisect of the 10-round soak): NOT the catch-up prefix-gap/entity-snapshot mechanism. xce_run_hooks (src/lib/hook.c:134-157) returns only the LAST-registered hook's rc (rc overwritten each loop iter; break is BEFORE_EVENT-only). The durable-majority gate works: a term-9 leader's job at LSN 30 timed out commit (peers mid-restart, repl_wait_majority_ack -> commit_submit -> -ETIMEDOUT), persist_job_hook_fun returned -ETIMEDOUT, but a later-registered JOB AFTER_EVENT hook (sched/apiserver) returned 0 and CLOBBERED it -> handle_jobsubmit saw hook_rc==0 -> replied 'jobsubmit ok' for an uncommitted entry -> term-10 leader legally overwrote LSN 30 (term-10 NOOP) -> committed-job-loss. Violates the Commit Durability invariant (reply-success => majority-durable). Evidence: all 3 WALs gapless/CRC-intact/full payloads; lost job only on ctrl-1@LSN30 term9 vs term10 NOOP on majority; ctrl-1 log has the WAIT-deadline-expired WARNs and NO 'rejecting job' line.", - "fix": "FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) — neither is job loss.", + "fix": "FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) \u2014 neither is job loss.", "tags": [ "ha-commit-durability", "pre-existing", @@ -11588,10 +11588,10 @@ { "id": "bug-1307", "timestamp": "2026-06-30T15:05:00.000Z", - "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN — silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix → Leader Completeness violation.", + "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN \u2014 silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix \u2192 Leader Completeness violation.", "file": "src/controller/ha.c", - "root_cause": "Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously → cross-denied → the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) → the early node campaigns with an incomplete log → voters grant based on stale view → winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win — with the new jitter timings this window is hit frequently.", - "fix": "CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario — 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) — that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run — no residual doubt.", + "root_cause": "Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously \u2192 cross-denied \u2192 the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) \u2192 the early node campaigns with an incomplete log \u2192 voters grant based on stale view \u2192 winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win \u2014 with the new jitter timings this window is hit frequently.", + "fix": "CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario \u2014 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) \u2014 that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run \u2014 no residual doubt.", "tags": [ "data-safety-regression", "task-5.7", @@ -11661,7 +11661,7 @@ "error_message": "Wrong reference: test_job_hook_start_writes_wal_entry should be test_job_hook_new_is_notify_only", "file": "src/controller/testsuite/test_persist_hooks.c", "root_cause": "Used \"test_job_hook_start_writes_wal_entry\" instead of \"test_job_hook_new_is_notify_only\"", - "fix": "Changed test_job_hook_start_writes_wal_entry → test_job_hook_new_is_notify_only", + "fix": "Changed test_job_hook_start_writes_wal_entry \u2192 test_job_hook_new_is_notify_only", "tags": [ "wrong-reference", "c" @@ -11692,7 +11692,7 @@ "error_message": "Wrong reference: unistd should be time", "file": "src/controller/testsuite/test_ha_rdhup_integration.c", "root_cause": "Used \"unistd\" instead of \"time\"", - "fix": "Changed unistd → time", + "fix": "Changed unistd \u2192 time", "tags": [ "auto-detected", "wrong-reference", @@ -11724,7 +11724,7 @@ "error_message": "Wrong reference: test_ha_partition should be test_persist_snapshot_entities", "file": "src/controller/testsuite/controller_test.c", "root_cause": "Used \"test_ha_partition\" instead of \"test_persist_snapshot_entities\"", - "fix": "Changed test_ha_partition → test_persist_snapshot_entities", + "fix": "Changed test_ha_partition \u2192 test_persist_snapshot_entities", "tags": [ "wrong-reference", "c" @@ -11755,7 +11755,7 @@ "error_message": "Wrong reference: pthread should be fcntl", "file": "src/controller/testsuite/test_persist_snapshot_entities.c", "root_cause": "Used \"pthread\" instead of \"fcntl\"", - "fix": "Changed pthread → fcntl", + "fix": "Changed pthread \u2192 fcntl", "tags": [ "auto-detected", "wrong-reference", @@ -11834,7 +11834,7 @@ "timestamp": "2026-07-01T13:27:42.519Z", "error_message": "Missing error handling in every", "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block | Also: /* Task 6.1 integration: leader->standby replication carries; * payload) faithfully end-to-end across BOTH live in-term re", "tags": [ "auto-detected", @@ -11851,7 +11851,7 @@ "error_message": "Wrong reference: test_pre_vote_catch_up_denies should be test_pre_vote_catch_up_grants_when_primary_unreachable", "file": "src/controller/testsuite/test_ha_vote_gate.c", "root_cause": "Used \"test_pre_vote_catch_up_denies\" instead of \"test_pre_vote_catch_up_grants_when_primary_unreachable\"", - "fix": "Changed test_pre_vote_catch_up_denies → test_pre_vote_catch_up_grants_when_primary_unreachable", + "fix": "Changed test_pre_vote_catch_up_denies \u2192 test_pre_vote_catch_up_grants_when_primary_unreachable", "tags": [ "wrong-reference", "c" @@ -11882,7 +11882,7 @@ "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t → uint32_t", + "fix": "Changed myconf_t \u2192 uint32_t", "tags": [ "wrong-reference", "h" @@ -11913,7 +11913,7 @@ "error_message": "Wrong reference: persist_config_t should be uint32_t", "file": "src/include/persist.h", "root_cause": "Used \"persist_config_t\" instead of \"uint32_t\"", - "fix": "Changed persist_config_t → uint32_t", + "fix": "Changed persist_config_t \u2192 uint32_t", "tags": [ "wrong-reference", "h" @@ -11928,7 +11928,7 @@ "error_message": "Wrong reference: WAL_SEGMENT_SIZE_BYTES should be p", "file": "src/controller/persist.c", "root_cause": "Used \"WAL_SEGMENT_SIZE_BYTES\" instead of \"p\"", - "fix": "Changed WAL_SEGMENT_SIZE_BYTES → p", + "fix": "Changed WAL_SEGMENT_SIZE_BYTES \u2192 p", "tags": [ "wrong-reference", "c" @@ -11943,7 +11943,7 @@ "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t → uint32_t", + "fix": "Changed myconf_t \u2192 uint32_t", "tags": [ "wrong-reference", "h" @@ -11957,8 +11957,8 @@ "timestamp": "2026-07-02T03:57:52.302Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"MODE=force-snapshot — forcing one full-checkpoint", - "fix": "Changed to \"MODE=force-snapshot — forcing one full-checkpoint", + "root_cause": "Had \"MODE=force-snapshot \u2014 forcing one full-checkpoint", + "fix": "Changed to \"MODE=force-snapshot \u2014 forcing one full-checkpoint", "tags": [ "auto-detected", "wrong-value", @@ -12114,7 +12114,7 @@ "error_message": "/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install (\"HA: snapshot loaded/legacy\" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip.", "file": "src/controller/config_repl.c", "root_cause": "ha_send_snapshot_to_standby (config_repl.c, ~line 2596) loops over up to 12 65536-byte chunks calling xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len) WITHOUT checking or propagating its return value. xce_snapshot_chunk_req_raw -> endpoint_enq can return -ECONNRESET (e.g. if the standby endpoint is mid-teardown or reconnecting during the empty-rejoin window) but the loop keeps incrementing offset/chunk_index regardless, and the function still logs \"transfer complete\" and returns rc=0 at the end. Confirmed via ctrl-2 log evidence: primary logs start+complete for a 735546-byte (12-chunk) snapshot within ~2ms, but ctrl-2 only ever receives a handful of leading chunks (2 in one run, 5 in another) before the stream silently stops -- no ERROR on either side.", - "fix": "FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) — the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups).", + "fix": "FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) \u2014 the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups).", "tags": [ "raft-drive-async", "snapshot-transfer", @@ -12149,10 +12149,10 @@ { "id": "bug-1389", "timestamp": "2026-07-02T09:05:00.000Z", - "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times — the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT — the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) — nondeterministic, not a fixed structural bug.", + "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times \u2014 the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT \u2014 the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) \u2014 nondeterministic, not a fixed structural bug.", "file": "src/lib/request.c (suspected: xce_snapshot_chunk_encode_raw capnp Data encode) or src/controller/ha.c (ha_handle_snapshot_chunk per-chunk append)", - "root_cause": "NOT YET PINNED to an exact line — root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 — 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' — a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing — vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race — xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation — alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug — it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities).", - "fix": "FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) — the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen \"ab\"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward — exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append — order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' — under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found — permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS.", + "root_cause": "NOT YET PINNED to an exact line \u2014 root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 \u2014 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' \u2014 a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing \u2014 vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race \u2014 xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation \u2014 alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug \u2014 it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities).", + "fix": "FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) \u2014 the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen \"ab\"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward \u2014 exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append \u2014 order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' \u2014 under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found \u2014 permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS.", "tags": [ "raft-drive-async", "snapshot-transfer", @@ -12204,10 +12204,10 @@ { "id": "bug-1396", "timestamp": "2026-07-02T14:00:00.000Z", - "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle — the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) — an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", + "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle \u2014 the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) \u2014 an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", "file": "src/tools/test-commit-durability.sh (harness) or controller job-recovery path (unconfirmed which)", - "root_cause": "NOT INVESTIGATED — deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race — ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE — a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) — rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) — almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected.", - "fix": "STILL NOT ROOT-CAUSED for the original exact symptom (successful query, count=20 vs expected=19) — 2 fresh repro attempts (2026-07-09, 3 rounds each, real ext4 fs under ~/tmp) did not reproduce an extra job; both runs showed the tracked count exactly matched after the fix below. HOWEVER, both attempts DID hit a DIFFERENT, related bug in the exact same assert_job_count() check: right after the cold-restart-all election, the aggregate expect-jobs/joblist query itself FAILED outright with 'xce_joblist: Unknown error 10014' / 'could not determine job count' (a query FAILURE, not a wrong-but-successful count) — this is the identical client-side race as bug-1422 (libxce's failover retry loop, client_failover.c, can exhaust its 20-attempt budget in a rapid post-election retry storm and surface a stale E_XCE_STANDBY). Confirmed as a false alarm both times: the very next per-id assert_no_loss check (individual expect-job calls) found every job present. Fixed assert_job_count() with the same bounded-retry pattern used for bug-1422 (3 attempts, 2s settle) — re-run after the fix: 7 passed, 0 failed, clean. This closes the QUERY-FAILURE false-negative class in this check, and makes the harness reliable enough to actually catch the ORIGINAL extra-job symptom if it recurs (previously a query failure and a genuine extra-job count were indistinguishable at a glance; now the retry-exhausted case self-heals and no longer masks a real mismatch). The original 'extra job' phenomenon itself remains open/unconfirmed — needs a longer run (original was 1-of-5 full 8-round soaks) to have a realistic chance of firing again.", + "root_cause": "NOT INVESTIGATED \u2014 deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race \u2014 ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE \u2014 a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) \u2014 rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) \u2014 almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected.", + "fix": "STILL NOT ROOT-CAUSED for the original exact symptom (successful query, count=20 vs expected=19) \u2014 2 fresh repro attempts (2026-07-09, 3 rounds each, real ext4 fs under ~/tmp) did not reproduce an extra job; both runs showed the tracked count exactly matched after the fix below. HOWEVER, both attempts DID hit a DIFFERENT, related bug in the exact same assert_job_count() check: right after the cold-restart-all election, the aggregate expect-jobs/joblist query itself FAILED outright with 'xce_joblist: Unknown error 10014' / 'could not determine job count' (a query FAILURE, not a wrong-but-successful count) \u2014 this is the identical client-side race as bug-1422 (libxce's failover retry loop, client_failover.c, can exhaust its 20-attempt budget in a rapid post-election retry storm and surface a stale E_XCE_STANDBY). Confirmed as a false alarm both times: the very next per-id assert_no_loss check (individual expect-job calls) found every job present. Fixed assert_job_count() with the same bounded-retry pattern used for bug-1422 (3 attempts, 2s settle) \u2014 re-run after the fix: 7 passed, 0 failed, clean. This closes the QUERY-FAILURE false-negative class in this check, and makes the harness reliable enough to actually catch the ORIGINAL extra-job symptom if it recurs (previously a query failure and a genuine extra-job count were indistinguishable at a glance; now the retry-exhausted case self-heals and no longer masks a real mismatch). The original 'extra job' phenomenon itself remains open/unconfirmed \u2014 needs a longer run (original was 1-of-5 full 8-round soaks) to have a realistic chance of firing again.", "tags": [ "ha-commit-durability", "low-priority", @@ -12275,10 +12275,10 @@ { "id": "bug-1400", "timestamp": "2026-07-02T14:20:00.000Z", - "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` — positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", + "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` \u2014 positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", "file": "src/controller/testsuite/test.c (cmd_jobsub, ~line 1748)", - "root_cause": "CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 — 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) — a ~128-byte leak per malformed jobsub invocation.", - "fix": "SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label — mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change.", + "root_cause": "CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 \u2014 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) \u2014 a ~128-byte leak per malformed jobsub invocation.", + "fix": "SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label \u2014 mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change.", "tags": [ "stale-report", "low-priority", @@ -12296,9 +12296,9 @@ { "id": "bug-1401", "timestamp": "2026-07-02T14:25:00.000Z", - "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) — observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", + "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) \u2014 observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", "file": "src/lib/controller.capnp.c (decode_CtrlStatus_ptr, generated codec) or its caller in request.c/xcecli", - "root_cause": "CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` — the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct — the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code.", + "root_cause": "CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` \u2014 the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct \u2014 the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code.", "fix": "src/include/client.h:119 changed to `#define xce_free_ctrlstatus(s) free_CtrlStatus_ptr(&(s))`. Added regression test test_xce_free_ctrlstatus_frees_the_struct_itself (src/lib/testsuite/test_ctrlstatus.c), which decodes via decode_CtrlStatus_ptr (the real allocation path, not the stack-struct helper the other roundtrip tests use) and frees via the xce_free_ctrlstatus macro. RED confirmed: reverting the fix reproduces the exact reported leak signature (LeakSanitizer: 120 bytes in 1 object, decode_CtrlStatus_ptr calloc). GREEN after fix: 285 lib tests / 0 failures / 0 leaks. Controller ASAN build + controller_unit_test also green post-fix (client.h is shared).", "tags": [ "asan", @@ -12334,7 +12334,7 @@ "error_message": "Wrong reference: test_xcecli_ctrlstatus_null_ctx_typed_failure should be test_xce_free_ctrlstatus_frees_the_struct_itself", "file": "src/lib/testsuite/test_ctrlstatus.c", "root_cause": "Used \"test_xcecli_ctrlstatus_null_ctx_typed_failure\" instead of \"test_xce_free_ctrlstatus_frees_the_struct_itself\"", - "fix": "Changed test_xcecli_ctrlstatus_null_ctx_typed_failure → test_xce_free_ctrlstatus_frees_the_struct_itself", + "fix": "Changed test_xcecli_ctrlstatus_null_ctx_typed_failure \u2192 test_xce_free_ctrlstatus_frees_the_struct_itself", "tags": [ "wrong-reference", "c" @@ -12585,9 +12585,9 @@ { "id": "bug-1417", "timestamp": "2026-07-07T09:30:00.000Z", - "error_message": "[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 — ctrl-2 PreVote wins but real election times out despite ctrl-3 granting RequestVote", + "error_message": "[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 \u2014 ctrl-2 PreVote wins but real election times out despite ctrl-3 granting RequestVote", "file": "src/controller/ha.c", - "root_cause": "vote_reply_dispatch passes local votes_idx (compressed, excludes self) directly to raft_on_vote_reply which expects ABSOLUTE peer index (includes self). For self_idx=1 (ctrl-2), ctrl-3's vote (absolute=2, local=1) lands in vote_replies[1]=ctrl-2's own self-slot → treated as duplicate → vote silently dropped. With ctrl-1 dead in chaotic mode, ctrl-2 only has self-vote (1 < quorum 2) → eternal timeout. Self_idx=0 and self_idx=2 are partially/fully immune due to index alignment luck. ha_repl_ack_abs_peer_idx() already exists (ha.c:3779) with correct mapping and passing tests — vote_reply_dispatch just doesn't call it.", + "root_cause": "vote_reply_dispatch passes local votes_idx (compressed, excludes self) directly to raft_on_vote_reply which expects ABSOLUTE peer index (includes self). For self_idx=1 (ctrl-2), ctrl-3's vote (absolute=2, local=1) lands in vote_replies[1]=ctrl-2's own self-slot \u2192 treated as duplicate \u2192 vote silently dropped. With ctrl-1 dead in chaotic mode, ctrl-2 only has self-vote (1 < quorum 2) \u2192 eternal timeout. Self_idx=0 and self_idx=2 are partially/fully immune due to index alignment luck. ha_repl_ack_abs_peer_idx() already exists (ha.c:3779) with correct mapping and passing tests \u2014 vote_reply_dispatch just doesn't call it.", "fix": [ "FIXED (src/controller/ha.c vote_reply_dispatch, ~line 5233): raft_on_vote_reply now receives ha_repl_ack_abs_peer_idx(ha, 1, idx) instead of the raw compressed local idx. ha_repl_ack_abs_peer_idx (ha.c:3779, already existed + unit-tested for the heartbeat-ACK path) performs the exact inverse of raft_cb_send_vote_request send-side remap: votes_idx < self_idx ? votes_idx : votes_idx + 1. ha_record_vote_reply (immediately preceding line) is UNCHANGED -- it correctly wants the compressed idx for ctx->votes[]. Confirmed pre_vote_reply_dispatch needs NO equivalent fix: raft_cb_send_pre_vote_request (ha.c:301) already stores the ABSOLUTE peer_idx in vctx->peer_idx for the PreVote path (explicit code comment there), matching raft_on_pre_vote_reply/pre_vote_replies[] absolute-index expectation -- only the real-vote send path (raft_cb_send_vote_request, ha.c:175) stores compressed. RED->GREEN: new unit test test_bug1417_middle_index_real_vote_from_peer2_not_dropped (test_ha_vote_send.c) drives a REAL 3-node election (self_idx=1) through raft_start_election then a genuine peer-2 grant via vote_reply_msg_handler_for_test (the actual decode/dispatch path, not the xce_vote_reply_hook bypass every other real-vote test in this file uses) -- reproduced Expected 2 Was 1 pre-fix, GREEN post-fix (votes_granted=2, role=RAFT_LEADER). Verified: full 79-group controller_unit_test sweep 0 new failures; 10-round --chaotic xce-test-ha soak PASS (committed_divergence=0, committed_job_loss=0, 18 jobs). Soak surfaced an UNRELATED, pre-existing --chaotic harness bug (run-ha-soak.sh run_chaotic_rounds: DOWN variable is not cleared on a skip-this-round no-primary-to-kill path, so a subsequent round calls start_node on the same already-running node without killing it first -- 6 stacked ctrl-2 processes observed piling up over rounds 5-10) -- confirmed via git diff this script was untouched by this fix, logged separately, NOT fixed here (out of bug-1417 boundary)." ], @@ -12665,7 +12665,7 @@ "timestamp": "2026-07-08T06:55:15.000Z", "error_message": "Sequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to stem from the bug-1417 fix); root-caused 2026-07-09 as the primary never discovering a follower's higher term via AppendEntries-reject responses", "file": "src/controller/ha.c", - "root_cause": "ha_raft_on_repl_ack only routes AE-reply term-adoption for success=1 ACKs; AE rejects (success=0) never reach raft_on_append_entries_reply. That function additionally drops any reply with reply_term != s->term (raft.c:832-833, stale-term drop) before it can adopt a higher term. Net effect: once a follower is ahead in term (e.g. it steps down from PreVote to CTRL_STANDBY at a higher term while an existing primary keeps campaigning/replicating at its old term), every AppendEntries the stale primary sends is rejected with ae_reject_stale, the primary never learns the higher term, and it can never regain write-quorum (needs 2/3, only self-vote counts) — a permanent livelock. First surfaced as an election regression (sequential soak: no primary elected at term>8 by round 6, chaotic stalled at round 5) suspected to be caused by the bug-1417 vote-index-mapping fix; re-reproduced 2026-07-09 as a round-4 sequential stall (primary ctrl-2 term=6, follower ctrl-3 term=7, 490 ae_reject_stale events, primary writable_lsn stuck at 17), which pinned the exact mechanism above.", + "root_cause": "ha_raft_on_repl_ack only routes AE-reply term-adoption for success=1 ACKs; AE rejects (success=0) never reach raft_on_append_entries_reply. That function additionally drops any reply with reply_term != s->term (raft.c:832-833, stale-term drop) before it can adopt a higher term. Net effect: once a follower is ahead in term (e.g. it steps down from PreVote to CTRL_STANDBY at a higher term while an existing primary keeps campaigning/replicating at its old term), every AppendEntries the stale primary sends is rejected with ae_reject_stale, the primary never learns the higher term, and it can never regain write-quorum (needs 2/3, only self-vote counts) \u2014 a permanent livelock. First surfaced as an election regression (sequential soak: no primary elected at term>8 by round 6, chaotic stalled at round 5) suspected to be caused by the bug-1417 vote-index-mapping fix; re-reproduced 2026-07-09 as a round-4 sequential stall (primary ctrl-2 term=6, follower ctrl-3 term=7, 490 ae_reject_stale events, primary writable_lsn stuck at 17), which pinned the exact mechanism above.", "fix": "Added two fallback term-discovery vectors that fire regardless of AE-reject outcome. (1) StateReport heartbeat-reply now carries the standby's current term (controller.capnp +term field @2, typemap.h state_report_t +term, ha_nop_reply_state_report populates it from ha->term) so the primary learns a higher term on every convergence cycle. (2) New library entry point raft_on_higher_term_observed(s, observed_term) mirrors UpdateTerm (strictly-greater-than only; touches term/voted_for/role; on_persist before on_become_follower for durability ordering) and is called from repl_ack_decode_and_dispatch on both the heartbeat state-report branch and the WAL-NACK branch (ha.c). Primary now steps down instead of retrying AppendEntries against a follower that rejects forever. Validated: 265 controller + 287 lib unit tests pass (zero regressions); 10-round sequential soak PASS (33 jobs, zero loss, all 3 nodes led ctrl-1/ctrl-2/ctrl-3); 10-round chaotic soak PASS (33 jobs, zero committed divergence, zero committed job loss).", "tags": [ "ha", @@ -12688,10 +12688,10 @@ { "id": "bug-1422", "timestamp": "2026-07-09T10:14:14.000Z", - "error_message": "40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed present' right after a fresh election (new primary ctrl-1, term=40) — xce_test's expect-job got E_XCE_STANDBY (10014) from a fresh client session whose own wait-primary had JUST confirmed ctrl-1 as serving 45ms earlier", + "error_message": "40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed present' right after a fresh election (new primary ctrl-1, term=40) \u2014 xce_test's expect-job got E_XCE_STANDBY (10014) from a fresh client session whose own wait-primary had JUST confirmed ctrl-1 as serving 45ms earlier", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "False-positive test-harness failure, NOT real committed-job loss: ctrl-1's own controller log confirms it stayed CTRL_PRIMARY throughout the window and successfully committed the 3 new post-failover jobs (log_append/ae_reply_success/commit_advanced at 10:14:22-23). ha_dispatch_gate's CTRL_PRIMARY branch (ha.c:3451-3531) unconditionally permits OPCode_get (joblist) even while fenced, so ctrl-1 itself could not have returned E_XCE_STANDBY for this query — the reply must have come from a standby (ctrl-2, just killed, or ctrl-3) that the client's failover retry loop landed on. Root cause traced to client_failover.c's xcecli_retry_next/xcecli_retry_decide (client_failover.c:305-319): E_XCE_STANDBY classifies as XCE_FO_RETRY_NOW (client_failover.c:59-60, prompt rotate, no backoff), and the 20-attempt retry budget (3 masters) should reliably reach the real primary — but when the WHOLE budget is exhausted while every recent classification was RETRY_NOW (as opposed to RETRY_BACKOFF), xcecli_retry_decide's budget-exhaustion branch (line 312-318) does NOT override *rc the way it does for RETRY_BACKOFF exhaustion (which maps to the distinct -E_XCE_NOT_READY, line 316-317) — it just returns false with *rc left at whatever the last attempt's raw status was (here, a stray E_XCE_STANDBY from a standby hit during the rapid post-election retry storm). verify_no_loss() (run-ha-soak.sh) then did a single one-shot check with zero retry at the driver level, unlike the job-submit loop just above it in the same file which already retries on this exact 'primary likely just promoted, regaining write-quorum' condition (run-ha-soak.sh:483-486). Only surfaced at round 36 of a 40-round run — narrow enough timing that shorter 6-10 round soaks across many prior sessions never hit it.", - "fix": "Added a bounded retry (3 total attempts, 2s settle sleep between) around the WHOLE verify_no_loss() check in run-ha-soak.sh, mirroring the existing submit-retry loop's rationale: a job still missing after a real settle window + 2 retries is a genuine loss, not this client-side retry-storm race. Did not touch client_failover.c (the battle-tested libxce-failover retry/rotation library with many other production consumers) — fixing at the test-driver level is safer and fully local. Re-ran both 40-round soaks after the fix: sequential PASS (123 jobs, zero loss, all 3 nodes led) and chaotic PASS (123 jobs, zero committed divergence, zero committed job loss).", + "root_cause": "False-positive test-harness failure, NOT real committed-job loss: ctrl-1's own controller log confirms it stayed CTRL_PRIMARY throughout the window and successfully committed the 3 new post-failover jobs (log_append/ae_reply_success/commit_advanced at 10:14:22-23). ha_dispatch_gate's CTRL_PRIMARY branch (ha.c:3451-3531) unconditionally permits OPCode_get (joblist) even while fenced, so ctrl-1 itself could not have returned E_XCE_STANDBY for this query \u2014 the reply must have come from a standby (ctrl-2, just killed, or ctrl-3) that the client's failover retry loop landed on. Root cause traced to client_failover.c's xcecli_retry_next/xcecli_retry_decide (client_failover.c:305-319): E_XCE_STANDBY classifies as XCE_FO_RETRY_NOW (client_failover.c:59-60, prompt rotate, no backoff), and the 20-attempt retry budget (3 masters) should reliably reach the real primary \u2014 but when the WHOLE budget is exhausted while every recent classification was RETRY_NOW (as opposed to RETRY_BACKOFF), xcecli_retry_decide's budget-exhaustion branch (line 312-318) does NOT override *rc the way it does for RETRY_BACKOFF exhaustion (which maps to the distinct -E_XCE_NOT_READY, line 316-317) \u2014 it just returns false with *rc left at whatever the last attempt's raw status was (here, a stray E_XCE_STANDBY from a standby hit during the rapid post-election retry storm). verify_no_loss() (run-ha-soak.sh) then did a single one-shot check with zero retry at the driver level, unlike the job-submit loop just above it in the same file which already retries on this exact 'primary likely just promoted, regaining write-quorum' condition (run-ha-soak.sh:483-486). Only surfaced at round 36 of a 40-round run \u2014 narrow enough timing that shorter 6-10 round soaks across many prior sessions never hit it.", + "fix": "Added a bounded retry (3 total attempts, 2s settle sleep between) around the WHOLE verify_no_loss() check in run-ha-soak.sh, mirroring the existing submit-retry loop's rationale: a job still missing after a real settle window + 2 retries is a genuine loss, not this client-side retry-storm race. Did not touch client_failover.c (the battle-tested libxce-failover retry/rotation library with many other production consumers) \u2014 fixing at the test-driver level is safer and fully local. Re-ran both 40-round soaks after the fix: sequential PASS (123 jobs, zero loss, all 3 nodes led) and chaotic PASS (123 jobs, zero committed divergence, zero committed job loss).", "tags": [ "xce-test-ha", "false-positive", @@ -12713,8 +12713,8 @@ "timestamp": "2026-07-09T10:30:00.000Z", "error_message": "Final resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from the 2026-07-07T18:20 analysis (term desync + apply-divergence promotion-gate liveness deadlock) are both now resolved, not by the originally-proposed gate relaxation.", "file": "src/controller/ha.c", - "root_cause": "Re-audited both factors against the CURRENT code (post raft-apply-decoupling 32/32, post bug-1421): (1) TERM DESYNC — ha_start_prevote_async (ha.c:6715) already computes its prospective term from ha->raft.term + 1, not the stale ha->term the original analysis described; ha.c:1009-1010 explicitly syncs ha->term from the raft term on every relevant transition. This half was already fixed independently of bug-1418's own tracked fix. (2) APPLY-DIVERGENCE LIVENESS DEADLOCK — the promotion gate (ha.c raft_cb_become_leader ~line 1130, and promotion_fence_timeout_fn ~line 876) unconditionally refuses promotion while persist_apply_verdict()==APPLY_DIVERGENT, by deliberate design (raft-apply-decoupling task 5.3, Req 8.4: a divergent local materialization must never be trusted to self-clear via any live/structural condition, including durable_floor>=cluster_hwm — only a fresh persist_recover() replay may clear it). The buglog's own previously-proposed fix (relax the gate to self-apply when durable_floor>=cluster_hwm) would have violated this intentional invariant and risked serving genuinely divergent state to clients. Instead, raft-apply-decoupling's tasks 5.1 (checkpoint_lsn mislabeled from WAL tip instead of the logical applied cursor, causing a false APPLY_DIVERGENT on an ordinary crash-during-checkpoint) and 7.2/bug-1416 (a boot-time TOCTOU where a returning node's persist_recover() replay-verify raced the live apply pump / live-replication apply path, producing a false APPLY_DIVERGENT on the restart/catch-up path specifically — the exact scenario bug-1418's residual occurrence hit: 'apply pump lagged behind WAL tip during kill') closed the false-positive detection paths that were producing spurious APPLY_DIVERGENT verdicts in the first place. A genuine (non-false-positive) divergence still correctly fences forever by design.", - "fix": "[Consolidated 2026-08-15: this ID previously had 4 duplicate entries in buglog.json (a pgrep-guard test-harness fix, a partial term-desync analysis, and a stale mid-investigation 'NOT FIXED' snapshot -- all superseded by this final resolution pass, which already accounted for all 3 in its own analysis). Collapsed to this single entry; the other 3 removed.]\n\nNo code change to the promotion gate itself — it is intentionally correct and must stay unconditional. The actual fix is the combination of already-shipped upstream work: bug-1421 (AE-reject term-discovery blindspot, this session) + raft-apply-decoupling tasks 5.1/5.3/7.2 (false-APPLY_DIVERGENT elimination, closed prior to this session). Validated: two 40-round soaks today (sequential 123 jobs zero-loss all-3-nodes-led; chaotic 123 jobs zero-committed-divergence zero-job-loss) — an order of magnitude more election churn than the original 10-round soak that first surfaced bug-1418 — produced zero apply-divergent promotion refusals and zero liveness deadlocks. Further corroborated by the 30-round sequential+chaotic regression run immediately after this entry.", + "root_cause": "Re-audited both factors against the CURRENT code (post raft-apply-decoupling 32/32, post bug-1421): (1) TERM DESYNC \u2014 ha_start_prevote_async (ha.c:6715) already computes its prospective term from ha->raft.term + 1, not the stale ha->term the original analysis described; ha.c:1009-1010 explicitly syncs ha->term from the raft term on every relevant transition. This half was already fixed independently of bug-1418's own tracked fix. (2) APPLY-DIVERGENCE LIVENESS DEADLOCK \u2014 the promotion gate (ha.c raft_cb_become_leader ~line 1130, and promotion_fence_timeout_fn ~line 876) unconditionally refuses promotion while persist_apply_verdict()==APPLY_DIVERGENT, by deliberate design (raft-apply-decoupling task 5.3, Req 8.4: a divergent local materialization must never be trusted to self-clear via any live/structural condition, including durable_floor>=cluster_hwm \u2014 only a fresh persist_recover() replay may clear it). The buglog's own previously-proposed fix (relax the gate to self-apply when durable_floor>=cluster_hwm) would have violated this intentional invariant and risked serving genuinely divergent state to clients. Instead, raft-apply-decoupling's tasks 5.1 (checkpoint_lsn mislabeled from WAL tip instead of the logical applied cursor, causing a false APPLY_DIVERGENT on an ordinary crash-during-checkpoint) and 7.2/bug-1416 (a boot-time TOCTOU where a returning node's persist_recover() replay-verify raced the live apply pump / live-replication apply path, producing a false APPLY_DIVERGENT on the restart/catch-up path specifically \u2014 the exact scenario bug-1418's residual occurrence hit: 'apply pump lagged behind WAL tip during kill') closed the false-positive detection paths that were producing spurious APPLY_DIVERGENT verdicts in the first place. A genuine (non-false-positive) divergence still correctly fences forever by design.", + "fix": "[Consolidated 2026-08-15: this ID previously had 4 duplicate entries in buglog.json (a pgrep-guard test-harness fix, a partial term-desync analysis, and a stale mid-investigation 'NOT FIXED' snapshot -- all superseded by this final resolution pass, which already accounted for all 3 in its own analysis). Collapsed to this single entry; the other 3 removed.]\n\nNo code change to the promotion gate itself \u2014 it is intentionally correct and must stay unconditional. The actual fix is the combination of already-shipped upstream work: bug-1421 (AE-reject term-discovery blindspot, this session) + raft-apply-decoupling tasks 5.1/5.3/7.2 (false-APPLY_DIVERGENT elimination, closed prior to this session). Validated: two 40-round soaks today (sequential 123 jobs zero-loss all-3-nodes-led; chaotic 123 jobs zero-committed-divergence zero-job-loss) \u2014 an order of magnitude more election churn than the original 10-round soak that first surfaced bug-1418 \u2014 produced zero apply-divergent promotion refusals and zero liveness deadlocks. Further corroborated by the 30-round sequential+chaotic regression run immediately after this entry.", "tags": [ "ha", "raft", @@ -12738,10 +12738,10 @@ { "id": "bug-1423", "timestamp": "2026-07-09T13:58:16.000Z", - "error_message": "30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry attempts (15s+). Root-caused as a genuine permanent election deadlock, not a client-side false positive: ctrl-3 won PreVote and entered CTRL_ELECTING (real election, term=10) with ctrl-1 dead and ctrl-2 alive, then never showed another HA state-machine event for 2+ minutes (until the harness killed the cluster on FAIL) — stuck in CTRL_ELECTING forever. Meanwhile ctrl-2 endlessly retried its own PreVote every ~15-17s, correctly denied every time by ctrl-3 on freshness grounds, but that denial never unstuck ctrl-3's already-in-flight real election.", + "error_message": "30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry attempts (15s+). Root-caused as a genuine permanent election deadlock, not a client-side false positive: ctrl-3 won PreVote and entered CTRL_ELECTING (real election, term=10) with ctrl-1 dead and ctrl-2 alive, then never showed another HA state-machine event for 2+ minutes (until the harness killed the cluster on FAIL) \u2014 stuck in CTRL_ELECTING forever. Meanwhile ctrl-2 endlessly retried its own PreVote every ~15-17s, correctly denied every time by ctrl-3 on freshness grounds, but that denial never unstuck ctrl-3's already-in-flight real election.", "file": "src/controller/ha.c", - "root_cause": "raft_cb_send_vote_request (ha.c, sends the real RequestVote RPC to each peer) has two failure branches — endpoint_connect() failing and iomgr_monitor() failing — that are exactly what happens when the RPC target is a dead/unreachable peer (here, ctrl-1, killed at the start of round 5). The endpoint_connect failure branch only called the legacy handle_request_vote_reply() when `c == NULL` (unit-test mode) — in PRODUCTION (c always non-NULL) a connect failure to a dead peer was silently dropped with NO notification to the raft library at all. The iomgr_monitor failure branch called handle_request_vote_reply() unconditionally, including in production — but per vote_reply_dispatch's own design comment, handle_request_vote_reply is the LEGACY/ctx-done-signaling path; 'raft_on_vote_reply remains the SOLE production promotion driver' (vote_reply_dispatch calls ha_record_vote_reply + raft_on_vote_reply, NOT handle_request_vote_reply). Net effect: a candidate's vote request to a dead peer never registers as a 'reply' with the production raft library's raft_state_t.replies_received counter. raft.c's Step 6 fast-path ('all peers have replied, no quorum → become follower and retry', gated on replies_received == n_peers-1) can therefore NEVER trigger whenever any configured peer is down — with quorum=2 in a 3-node cluster and one peer dead, replies_received caps at 1 (only the live peer's real reply, if any) and can never reach n_peers-1=2. Recovery then depends ENTIRELY on the raw per-campaign election_timeout (5-10s, ha_jittered_election_delay) firing via delaymgr — in the observed failure, no election_timeout_fn trace appeared in the log at all for the full 2+ minute window, leaving the candidate permanently wedged.", - "fix": "Routed both raft_cb_send_vote_request failure branches (connect failure and monitor failure) through vote_reply_dispatch(ha, ectx, votes_idx, term, vote_granted=0, 0, 0) in production (c != NULL) — the exact same call a genuine vote reply makes (ha_record_vote_reply + raft_on_vote_reply), so a dead/unreachable peer now correctly counts as 'replied, not granted' immediately on connect/monitor failure instead of only via the timeout. Test mode (c == NULL) keeps the legacy handle_request_vote_reply call unit tests depend on. Added a forward declaration of the previously-file-local-only vote_reply_dispatch near the top of ha.c so raft_cb_send_vote_request (defined much earlier in the file) can call it. Verified: full controller_unit_test suite clean (755 PASS / 0 Fail, exit 0) both before and after removing temporary diagnostic logging used during root-cause; re-ran the exact failing scenario — 30-round sequential HA soak PASS (93 jobs, zero loss, all 3 nodes led) and 30-round chaotic HA soak PASS (93 jobs, zero committed divergence, zero committed job loss) — both previously hit this stall reliably (round 5 of the sequential run, 100% reproduction before the fix).", + "root_cause": "raft_cb_send_vote_request (ha.c, sends the real RequestVote RPC to each peer) has two failure branches \u2014 endpoint_connect() failing and iomgr_monitor() failing \u2014 that are exactly what happens when the RPC target is a dead/unreachable peer (here, ctrl-1, killed at the start of round 5). The endpoint_connect failure branch only called the legacy handle_request_vote_reply() when `c == NULL` (unit-test mode) \u2014 in PRODUCTION (c always non-NULL) a connect failure to a dead peer was silently dropped with NO notification to the raft library at all. The iomgr_monitor failure branch called handle_request_vote_reply() unconditionally, including in production \u2014 but per vote_reply_dispatch's own design comment, handle_request_vote_reply is the LEGACY/ctx-done-signaling path; 'raft_on_vote_reply remains the SOLE production promotion driver' (vote_reply_dispatch calls ha_record_vote_reply + raft_on_vote_reply, NOT handle_request_vote_reply). Net effect: a candidate's vote request to a dead peer never registers as a 'reply' with the production raft library's raft_state_t.replies_received counter. raft.c's Step 6 fast-path ('all peers have replied, no quorum \u2192 become follower and retry', gated on replies_received == n_peers-1) can therefore NEVER trigger whenever any configured peer is down \u2014 with quorum=2 in a 3-node cluster and one peer dead, replies_received caps at 1 (only the live peer's real reply, if any) and can never reach n_peers-1=2. Recovery then depends ENTIRELY on the raw per-campaign election_timeout (5-10s, ha_jittered_election_delay) firing via delaymgr \u2014 in the observed failure, no election_timeout_fn trace appeared in the log at all for the full 2+ minute window, leaving the candidate permanently wedged.", + "fix": "Routed both raft_cb_send_vote_request failure branches (connect failure and monitor failure) through vote_reply_dispatch(ha, ectx, votes_idx, term, vote_granted=0, 0, 0) in production (c != NULL) \u2014 the exact same call a genuine vote reply makes (ha_record_vote_reply + raft_on_vote_reply), so a dead/unreachable peer now correctly counts as 'replied, not granted' immediately on connect/monitor failure instead of only via the timeout. Test mode (c == NULL) keeps the legacy handle_request_vote_reply call unit tests depend on. Added a forward declaration of the previously-file-local-only vote_reply_dispatch near the top of ha.c so raft_cb_send_vote_request (defined much earlier in the file) can call it. Verified: full controller_unit_test suite clean (755 PASS / 0 Fail, exit 0) both before and after removing temporary diagnostic logging used during root-cause; re-ran the exact failing scenario \u2014 30-round sequential HA soak PASS (93 jobs, zero loss, all 3 nodes led) and 30-round chaotic HA soak PASS (93 jobs, zero committed divergence, zero committed job loss) \u2014 both previously hit this stall reliably (round 5 of the sequential run, 100% reproduction before the fix).", "tags": [ "ha", "raft", @@ -12766,8 +12766,8 @@ "timestamp": "2026-07-09T17:00:00.000Z", "error_message": "PreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a dead/unreachable peer) never registered a reply with the raft library's PreVote tally, in EITHER test or production mode (worse than the real-vote path, which at least handled test mode).", "file": "src/controller/ha.c", - "root_cause": "Same missing-registration pattern as bug-1423, in raft_cb_send_pre_vote_request instead of raft_cb_send_vote_request. Found while auditing for the analogous gap after fixing bug-1423. Lower severity than bug-1423: PreVote has an unconditional round-timeout fallback (ha_prevote_round_timeout_fn, confirmed firing reliably every round across dozens of observed rounds in the bug-1423 investigation logs) that always resolves the round regardless of tally completeness, so this was never a permanent-deadlock class bug like bug-1423 was — just a latency waste: every PreVote round against a down peer burns the full round timeout instead of concluding as soon as the live peer(s) reply.", - "fix": "Routed both failure branches through pre_vote_reply_dispatch(ha, c, peer_idx, granted=0) — the same production PreVote-reply path a real reply uses (raft_on_pre_vote_reply) — mirroring bug-1423's fix exactly. Added a forward declaration near the top of ha.c (pre_vote_reply_dispatch is defined much later in the file). Verified: full controller_unit_test suite clean (0 Fail, exit 0) after the change.", + "root_cause": "Same missing-registration pattern as bug-1423, in raft_cb_send_pre_vote_request instead of raft_cb_send_vote_request. Found while auditing for the analogous gap after fixing bug-1423. Lower severity than bug-1423: PreVote has an unconditional round-timeout fallback (ha_prevote_round_timeout_fn, confirmed firing reliably every round across dozens of observed rounds in the bug-1423 investigation logs) that always resolves the round regardless of tally completeness, so this was never a permanent-deadlock class bug like bug-1423 was \u2014 just a latency waste: every PreVote round against a down peer burns the full round timeout instead of concluding as soon as the live peer(s) reply.", + "fix": "Routed both failure branches through pre_vote_reply_dispatch(ha, c, peer_idx, granted=0) \u2014 the same production PreVote-reply path a real reply uses (raft_on_pre_vote_reply) \u2014 mirroring bug-1423's fix exactly. Added a forward declaration near the top of ha.c (pre_vote_reply_dispatch is defined much later in the file). Verified: full controller_unit_test suite clean (0 Fail, exit 0) after the change.", "tags": [ "ha", "raft", @@ -12787,7 +12787,7 @@ "timestamp": "2026-07-09T21:15:00.000Z", "error_message": "Intermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path shortly after a fresh single-node election.", "file": "src/controller/persist.c or src/controller/commit.c (unrooted)", - "root_cause": "NOT ROOT-CAUSED. Flagged during raft-apply-decoupling task 7.4's throughput benchmark work. Reproduced identically on both the pre-fix and post-fix (task 6.1 apply-pump) binaries — confirmed unrelated to that fix. Resembles but is not confirmed identical to the closed bug-1416 boot-race class (persist_recover() replay-verify racing the live apply pump / live-replication apply path).", + "root_cause": "NOT ROOT-CAUSED. Flagged during raft-apply-decoupling task 7.4's throughput benchmark work. Reproduced identically on both the pre-fix and post-fix (task 6.1 apply-pump) binaries \u2014 confirmed unrelated to that fix. Resembles but is not confirmed identical to the closed bug-1416 boot-race class (persist_recover() replay-verify racing the live apply pump / live-replication apply path).", "fix": "[UPDATE 2026-08-16] Live investigation (single-node deploy via /xce-deploy-ha, controller_unit_test's own fix build): found a RELIABLE reproduction recipe the original ~2/9-sequential-attempts description never had, and narrowed the root cause to a specific mechanism, but did NOT land a fix -- reporting findings honestly per the explicit scope of this investigation pass.\n\nRELIABLE REPRO (not sequential -- CONCURRENT): deploy a fresh single-node cluster, poll the log for CTRL_PRIMARY, then fire 9 xce_test client processes (each: connect + one jobsub) SIMULTANEOUSLY (backgrounded, not sequential) immediately after election. Reproduced in 3 of 4 rounds (1-3 of 9 concurrent clients failing per round, closely matching the original ~2/9 ratio) -- 0/45 reproductions when the SAME 9 attempts were fired sequentially instead (one xce_test process at a time). Concurrency at connect time, not raw timing after election, is the actual trigger.\n\nSYMPTOM, precisely: the failing client's xcecli_jobsubmit() (client.c, wrapped in XCECLI_FAILOVER_CALL) returns rc>=0 (success-shaped, NOT a negative error) after exactly ~10.0-10.1s, with the output jobid=0 (never written) instead of the submitted JobId -- test.c's cmd_jobsub (bug-786's own guard) correctly catches this as 'no clean ack' rather than misreporting success, so no data is actually lost/corrupted -- but the client hangs ~10s before getting a clear answer, matching the bug's own description exactly.\n\nCONFIRMED CLIENT-SIDE, not server-side: cross-referenced the controller's own log (fresh --logfile per round) against the 9 concurrent attempts -- server-side WAL only ever shows 8 job-submit entries committed (lsn sequence has no gaps, no admission-cap rejection WARN, no error of any kind) when 9 concurrent clients attempted -- the failing 9th request never reaches handle_jobsubmit at all. Ruled out the obvious admission-cap suspect: commit.c's COMMIT_MODE_WAIT admission control (cap=n_vcpus-VTASK_LIVENESS_RESERVED=3 on this 4-vcpu deploy) never logged its 'WAIT rejected, admission cap exhausted' WARN in any reproduction -- 8 concurrent WAITs completing well under the cap is plausible given each takes <100ms server-side once received.\n\nTHE ~10s NUMBER, precisely sourced: xceapi.c's xce_jobsubmit() (the innermost call XCECLI_FAILOVER_CALL wraps) has a HARDCODED cbuf_deadline_10s() wait on cbuf_timedget waiting for a reply to land in the single-slot ctx->wbuf. Read cbuf_timedget itself (mylock.c) -- confirmed CORRECT, it returns -ETIMEDOUT (negative) on a genuine timeout, which xce_jobsubmit correctly propagates as a negative rc. So a bare cbuf_timedget timeout cannot by itself explain the observed rc>=0-with-jobid=0 shape; something in the retry/rotate/recover layer ABOVE it must be involved.\n\nNARROWED SUSPECT (not confirmed): client_internal.h's own documented CI1 hazard -- 'handle_disconnect may have injected a stale status into the single-slot ctx->wbuf ... xcecli_ctx_recover drains any pending entry with a non-blocking cbuf_tryget so the next attempt's cbuf_get cannot consume a prior-attempt reply' -- but this drain only runs when xce_failover_rotate() succeeds (client_failover.c:130-153, mirrored by xcecli_retry_next per its own doc comment: 'only recover when the rotate succeeded'). On this single-candidate cluster, rotate reconnects to the SAME (only) node via a real xce_ctx_reconnect() -- plausible this usually succeeds (ruling out the simplest 'rotate never succeeds on single-node, drain never runs' theory) but NOT independently verified under the exact concurrent-connect contention this repro needs; a transient reconnect failure specifically during the connection-establishment storm (9 simultaneous TLS handshakes against a listener seconds old) could leave a stale wbuf entry undrained, which a LATER retry's cbuf_timedget could then either (a) consume immediately (fast, not 10s -- doesn't match) or (b) never resolve at all if the drain path interacts badly with an in-flight send, leaving the single wbuf slot permanently empty until the full 10s deadline. Did not instrument (fprintf/gdb) the actual attempt count or wbuf state during a live failing run to confirm this mechanism specifically -- xce_test's own -vvv flag produced no additional client-side retry/rotate trace output (goes through a different, unwired logger in this CLI context), and adding real instrumentation would need a fresh lib rebuild + re-repro cycle not completed in this pass.\n\nNOT YET DONE: instrument xcecli_retry_next/xce_jobsubmit directly (attempt count, wbuf drain outcome, rotate rc, per-attempt elapsed) and re-run the now-reliable concurrent-burst repro to catch the exact mechanism in the act -- this is the recommended next step, using the repro recipe above (no gdb needed; the reproduction is now reliable enough for print-based debugging). Confirmed unrelated to raft-apply-decoupling task 6.1's apply-pump fix (reproduces identically on that spec's pre-fix and post-fix binaries per the original entry) and to this session's bug-1428/1426/1474/1472-remainder fixes (none touch xceapi.c/client.c/client_failover.c's client-side retry path).\n\n[Original entry]\nNOT FIXED. Worked around via retry for task 7.4's benchmarking purposes only. Needs a dedicated root-cause investigation with the same live-repro rigor bug-1416 got (gdb-attach via GDB_WRAP seam, or targeted debug logging around the commit-wait barrier right after a fresh single-node election).\n\n[UPDATE 2026-08-16, part 2] FIXED. Root-caused via static code trace (no gdb/instrumentation needed after all): XCECLI_FAILOVER_CALL's idempotent-replay rule (client_failover.c, xce_failover_classify -- 'a duplicate on a re-sent write means the replay guard already applied our write, treat as success') can normalize a retried resend that hits the server's EEXIST/inflight-collision rejection into overall success (rc==0) -- but that rejection reply carries NO payload (handle_jobsubmit's EEXIST/inflight-collision branches reply with RequestReply_data_nodata, job.c:1441-1467), so xce_jobsubmit() (xceapi.c) never populates its *jobid output on the winning (dup) attempt, and never did on the earlier failing attempt either (its raw status was already negative). Net: xcecli_jobsubmit() can return rc>=0 with *jobid left at the caller's initial value (0) -- matching the confirmed symptom exactly (rc>=0, jobid=0, ~10.0-10.1s: one full 10s cbuf_timedget deadline on the failing first attempt, then a fast dup-rejection reply on the resend).\n\nFIX (TDD): added jobsubmit_finalize_jobid(uint64_t* jobid, uint64_t submitted_jobid, int overall_rc) (src/lib/client_internal.h declaration, src/lib/client.c definition), called from xcecli_jobsubmit() right after XCECLI_FAILOVER_CALL returns. Sets *jobid = submit->jobid (the CLIENT-generated id already on the wire -- job ids are client-generated via idgen(); the controller commits/echoes this exact value verbatim on every genuine success reply, bug-786/job.c:1439) whenever overall_rc >= 0, otherwise leaves *jobid untouched. Deliberately placed at the jobsubmit-specific wrapper layer, not the generic classify/decide layer (which is correctly type-erased and has no concept of jobid) -- test.c's cmd_jobsub (bug-786's own gate, comparing returned jobid == locally-generated jobsubmit.jobid) was ALREADY the right verification signal and needed no changes. 3 new pure unit tests in test_libxce_retry.c (test_jobsubmit_finalize_populates_jobid_on_success/leaves_jobid_untouched_on_failure/null_jobid_safe) -- socket-free, following this codebase's established direct-state-simulation test style for I/O-adjacent logic (test_netio_on_connected.c's precedent) rather than mocking a live connection.\n\nLIVE VALIDATION: full ASAN rebuild (0 compiler errors; controller_unit_test 1101 tests, 2 pre-existing unrelated test_config_repl.c failures; libxce test suite 1/1 OK, all 3 new tests PASS). Deployed a fresh single-node cluster and ran the same concurrent-burst repro as the original investigation (fresh election -> 9 simultaneous xce_test jobsub clients) for 5 rounds / 45 total clients: 0 'no clean ack' failures, 45/45 clean acks with jobid matching the submitted id (vs. the original repro's 1-3/9 failures in 3 of 4 rounds pre-fix). Honest caveat: this is probabilistic/timing-dependent (like the original repro), so 45/45 clean is strong but not airtight proof the exact race window is unreachable -- the fix is correct by construction regardless (it closes the *jobid gap for EVERY path that can produce rc>=0, not just the one race that was reproduced), so residual risk is limited to a still-undiscovered SEPARATE mechanism producing the same symptom, not this one going unfixed.", "tags": [ "ha", @@ -12810,10 +12810,10 @@ { "id": "bug-1426", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "trim_wal_segments's retention floor has no awareness of the local apply pump's own lag — a reachable scenario since apply-lag-behind-commit is raft-apply-decoupling's entire premise.", + "error_message": "trim_wal_segments's retention floor has no awareness of the local apply pump's own lag \u2014 a reachable scenario since apply-lag-behind-commit is raft-apply-decoupling's entire premise.", "file": "src/controller/persist.c (trim_wal_segments, apply_pump_read_entry cache)", - "root_cause": "Identified during raft-apply-decoupling task 7.4's apply-pump segment-read cache fix (round-2 review finding): the new fast-path cache can (POSIX unlink-keeps-fd-alive) silently advance last_applied_lsn past already-deleted data if the pump is lagging when trim_wal_segments fires. Currently bounded by the pre-existing APPLY_DIVERGENT boot fence (deferred loud fencing on next restart, not silent corruption) — a real behavior change from the pre-cache code, not literally risk-free, but not an active defect either. Same family as an already-tracked standby-cursor variant (per task 7.4's own notes).", - "fix": "FIXED (TDD RED->GREEN). trim_wal_segments() now caps its trim floor at the local apply pump's own progress (last_applied_lsn), not just remote standby-acked positions: floor = min(repl_trim_lsn's standby/disconnect-guard floor, last_applied_lsn+1). The +1 matters -- repl_trim_lsn's floor treats equality as 'still needed, preserve' (a standby acked through LSN N might still need that segment for retry/resync), but the LOCAL apply cursor's equality case means the opposite: last_applied_lsn==N means this process is DONE with entry N and will never re-read it, so a segment starting exactly at N is safe to delete. Feeding last_applied_lsn+1 into the same shared 'seg_start_lsn < floor' comparison gets correct delete-on-equal semantics for this term without a second, differently-shaped check. This closes the real gap: apply_pump_read_entry()'s fast-path cache holds an open fd into whatever segment it's currently reading; POSIX unlink-keeps-fd-alive meant a lagging pump could keep reading via that stale fd even after trim_wal_segments deleted the segment out from under it (no immediate corruption, since the fd's data was still valid) -- but a FRESH read of the same range (a restart's persist_recover() replay, or the pump's own cache falling back to its slow path after an unrelated miss) would find the segment already gone, producing a genuine APPLY_DIVERGENT-class gap between what the live process quietly caught up to and what a fresh read of disk state can see. Regression test (test_e2e_checkpoint.c's test_trim_guard_preserves_unapplied_local_segments): no standbys registered (so the pre-fix floor alone would allow trimming everything below the active segment) but last_applied_lsn deliberately lags behind one segment -- proves the already-applied segment IS trimmed (no unnecessary retention) while the not-yet-applied one is preserved (the actual guard). Also had to advance last_applied_lsn in the 3 pre-existing trim tests in the same file (test_checkpoint_wal_trim_after_checkpoint, test_trim_guard_preserves_standby_segments, test_trim_without_standbys_proceeds_normally), which call persist_wal_async directly (bypassing the whole apply path) and so never advance last_applied_lsn on their own -- without the added stamp, this fix would have correctly-but-unexpectedly blocked ALL trimming in those fixtures, since last_applied_lsn defaults to 0. Full controller_unit_test suite: 1099 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED — deliberately deferred. task 7.4's own scope explicitly excluded redesigning trim_wal_segments's retention-floor awareness of apply-pump lag; documented via an accurate code comment instead. A future fix would give trim_wal_segments a floor input derived from the local apply cursor, not just standby-acked LSNs.", + "root_cause": "Identified during raft-apply-decoupling task 7.4's apply-pump segment-read cache fix (round-2 review finding): the new fast-path cache can (POSIX unlink-keeps-fd-alive) silently advance last_applied_lsn past already-deleted data if the pump is lagging when trim_wal_segments fires. Currently bounded by the pre-existing APPLY_DIVERGENT boot fence (deferred loud fencing on next restart, not silent corruption) \u2014 a real behavior change from the pre-cache code, not literally risk-free, but not an active defect either. Same family as an already-tracked standby-cursor variant (per task 7.4's own notes).", + "fix": "FIXED (TDD RED->GREEN). trim_wal_segments() now caps its trim floor at the local apply pump's own progress (last_applied_lsn), not just remote standby-acked positions: floor = min(repl_trim_lsn's standby/disconnect-guard floor, last_applied_lsn+1). The +1 matters -- repl_trim_lsn's floor treats equality as 'still needed, preserve' (a standby acked through LSN N might still need that segment for retry/resync), but the LOCAL apply cursor's equality case means the opposite: last_applied_lsn==N means this process is DONE with entry N and will never re-read it, so a segment starting exactly at N is safe to delete. Feeding last_applied_lsn+1 into the same shared 'seg_start_lsn < floor' comparison gets correct delete-on-equal semantics for this term without a second, differently-shaped check. This closes the real gap: apply_pump_read_entry()'s fast-path cache holds an open fd into whatever segment it's currently reading; POSIX unlink-keeps-fd-alive meant a lagging pump could keep reading via that stale fd even after trim_wal_segments deleted the segment out from under it (no immediate corruption, since the fd's data was still valid) -- but a FRESH read of the same range (a restart's persist_recover() replay, or the pump's own cache falling back to its slow path after an unrelated miss) would find the segment already gone, producing a genuine APPLY_DIVERGENT-class gap between what the live process quietly caught up to and what a fresh read of disk state can see. Regression test (test_e2e_checkpoint.c's test_trim_guard_preserves_unapplied_local_segments): no standbys registered (so the pre-fix floor alone would allow trimming everything below the active segment) but last_applied_lsn deliberately lags behind one segment -- proves the already-applied segment IS trimmed (no unnecessary retention) while the not-yet-applied one is preserved (the actual guard). Also had to advance last_applied_lsn in the 3 pre-existing trim tests in the same file (test_checkpoint_wal_trim_after_checkpoint, test_trim_guard_preserves_standby_segments, test_trim_without_standbys_proceeds_normally), which call persist_wal_async directly (bypassing the whole apply path) and so never advance last_applied_lsn on their own -- without the added stamp, this fix would have correctly-but-unexpectedly blocked ALL trimming in those fixtures, since last_applied_lsn defaults to 0. Full controller_unit_test suite: 1099 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED \u2014 deliberately deferred. task 7.4's own scope explicitly excluded redesigning trim_wal_segments's retention-floor awareness of apply-pump lag; documented via an accurate code comment instead. A future fix would give trim_wal_segments a floor input derived from the local apply cursor, not just standby-acked LSNs.", "tags": [ "ha", "persistence", @@ -12830,9 +12830,9 @@ { "id": "bug-1427", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job entities only — worker entities can diverge (worker_from_capnp()/restore_worker() genuinely creates a new worker on a WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry when absent) but this is currently invisible to the check.", + "error_message": "APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job entities only \u2014 worker entities can diverge (worker_from_capnp()/restore_worker() genuinely creates a new worker on a WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry when absent) but this is currently invisible to the check.", "file": "src/controller/persist.c (persist_verify_apply_correctness, count_jobmgr_entries)", - "root_cause": "Disclosed, accepted scope gap from raft-apply-decoupling task 5.1's own design: the entity-count check was scoped to jobs only (queues correctly excluded — config-derived, WAL_QUEUE_CTRL never creates a new one, verified by two review rounds). Workers were never brought in scope. Not a regression — a known, deliberate boundary, just not yet extended.", + "root_cause": "Disclosed, accepted scope gap from raft-apply-decoupling task 5.1's own design: the entity-count check was scoped to jobs only (queues correctly excluded \u2014 config-derived, WAL_QUEUE_CTRL never creates a new one, verified by two review rounds). Workers were never brought in scope. Not a regression \u2014 a known, deliberate boundary, just not yet extended.", "fix": "FIXED (TDD RED->GREEN). Extended persist_verify_apply_correctness() with a worker-count check mirroring the existing job-count check exactly: added count_workermgr_entries() (persist.c, mirrors count_jobmgr_entries()), a new expected_new_worker_count field on apply_expectation_t (persist.h), and a new workermgr_count_before_replay parameter threaded through persist_verify_apply_correctness()/persist_recover(). The harder part: apply_committed_entry()'s WAL_WORKER_CTRL case previously always returned 0 regardless of outcome (JOIN-creates-new, UPDATE-applies-to-existing, and idempotent-noop were all indistinguishable to callers), so replay_wal() had no signal to tally new worker joins from. Fixed by making worker_from_capnp() (worker.c) return 1 specifically for a genuine new-worker creation (discarding restore_worker()'s raw idx, which can legitimately be 0 and would have collided with the update-applied case) and propagating that rc through apply_committed_entry() instead of the hardcoded 0 -- verified no caller of either function depended on the exact prior return value (only sign and -EEXIST were ever checked). replay_wal() tallies apply_rc==1 for WAL_WORKER_CTRL exactly like it already does for WAL_JOB_SUBMIT. New regression tests: test_verify_apply_correctness_detects_worker_count_mismatch (test_apply_correctness_verification.c, mirrors the existing job-count-mismatch test) proves a worker-count divergence is now caught as APPLY_DIVERGENT -- RED before this fix (no worker check existed at all), GREEN after. Also updated test_apply_committed_entry.c's test_ace_worker_ctrl_join_restores_and_notifies, which asserted rc==0 for a genuine JOIN creation under the old (now-corrected) contract -- now asserts rc==1 with an explanatory message. Full controller_unit_test suite: 1097 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED. Extend persist_verify_apply_correctness() with a worker-count check mirroring count_jobmgr_entries() if/when a worker-creation divergence needs to be closed (task 5.1's own forward note).", "tags": [ "ha", @@ -12850,10 +12850,10 @@ { "id": "bug-1428", "timestamp": "2026-07-09T21:15:00.000Z", - "error_message": "Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix) — not reliably exercised by any current test.", + "error_message": "Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix) \u2014 not reliably exercised by any current test.", "file": "src/controller/persist.c (persist_register_pending_apply)", "root_cause": "Flagged during raft-apply-decoupling task 6.2 as a non-blocking fast-follow while closing bug-1415. Specific race window not fully documented in tasks.md's own notes; needs a fresh look at the function to characterize exactly what window remains and whether it's reachable in production timing.", - "fix": "FIXED (characterized then TDD RED->GREEN, per the entry's own 3-step plan). Characterized the exact window: persist_register_pending_apply()'s self-heal check (an UNLOCKED read of last_applied_lsn) ran BEFORE acquiring pending_applies_lock, separate from the locked insert that followed. Window: this thread reads last_applied_lsn (sees 'not yet applied') -> BEFORE it locks and inserts, the apply pump's thread advances last_applied_lsn past this lsn AND completes its own persist_take_pending_apply(lsn) call (finding nothing, since this thread hasn't inserted yet) -> this thread then inserts a node the pump will never revisit -- permanently orphaned, identical failure mode to bug-1415 (a hang, on_applied never fires) via a different, narrower trigger window. Production reachability: yes -- persist_register_pending_apply() is called from a different thread/vtask than the apply pump's walk (commit.c's continuation dispatch vs persist_apply_pump_run, per the function's own pre-existing doc comment), with no synchronization between them before this fix beyond the already-locked insert. Fix: moved the self-heal check inside pending_applies_lock, so the whole check-then-(insert-or-self-heal) sequence is one atomic critical section against persist_take_pending_apply's own locked scan-and-remove -- either this thread's check now runs before the pump's take (sees 'not yet applied', inserts, and the pump's later take, now forced to wait on the same lock, finds the freshly-inserted node) or after (sees 'already applied', self-heals instead of inserting, and the pump's take correctly finds nothing) -- never the interleaved gap that orphaned it. Regression test (test_apply_pump.c's test_register_pending_apply_race_against_concurrent_apply_self_heals): deterministic reproduction via controlled state (a main thread holding pending_applies_lock forces the register call to observe last_applied_lsn only after the lock's own update, removing dependence on real-time race timing) -- RED under the pre-fix code (on_applied never fires, node orphaned on pending_applies), GREEN under the fix (self-heals correctly, no orphan). Full controller_unit_test suite: 1098 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED — explicitly deferred as a fast-follow, not a blocker for task 6.2 or the spec's closure. Needs: (1) characterize the exact TOCTOU window, (2) determine production reachability, (3) add a regression test that exercises it before fixing.", + "fix": "FIXED (characterized then TDD RED->GREEN, per the entry's own 3-step plan). Characterized the exact window: persist_register_pending_apply()'s self-heal check (an UNLOCKED read of last_applied_lsn) ran BEFORE acquiring pending_applies_lock, separate from the locked insert that followed. Window: this thread reads last_applied_lsn (sees 'not yet applied') -> BEFORE it locks and inserts, the apply pump's thread advances last_applied_lsn past this lsn AND completes its own persist_take_pending_apply(lsn) call (finding nothing, since this thread hasn't inserted yet) -> this thread then inserts a node the pump will never revisit -- permanently orphaned, identical failure mode to bug-1415 (a hang, on_applied never fires) via a different, narrower trigger window. Production reachability: yes -- persist_register_pending_apply() is called from a different thread/vtask than the apply pump's walk (commit.c's continuation dispatch vs persist_apply_pump_run, per the function's own pre-existing doc comment), with no synchronization between them before this fix beyond the already-locked insert. Fix: moved the self-heal check inside pending_applies_lock, so the whole check-then-(insert-or-self-heal) sequence is one atomic critical section against persist_take_pending_apply's own locked scan-and-remove -- either this thread's check now runs before the pump's take (sees 'not yet applied', inserts, and the pump's later take, now forced to wait on the same lock, finds the freshly-inserted node) or after (sees 'already applied', self-heals instead of inserting, and the pump's take correctly finds nothing) -- never the interleaved gap that orphaned it. Regression test (test_apply_pump.c's test_register_pending_apply_race_against_concurrent_apply_self_heals): deterministic reproduction via controlled state (a main thread holding pending_applies_lock forces the register call to observe last_applied_lsn only after the lock's own update, removing dependence on real-time race timing) -- RED under the pre-fix code (on_applied never fires, node orphaned on pending_applies), GREEN under the fix (self-heals correctly, no orphan). Full controller_unit_test suite: 1098 tests, 2 pre-existing unrelated failures (bug-1561), no regressions.\n\n[Original entry]\nNOT FIXED \u2014 explicitly deferred as a fast-follow, not a blocker for task 6.2 or the spec's closure. Needs: (1) characterize the exact TOCTOU window, (2) determine production reachability, (3) add a regression test that exercises it before fixing.", "tags": [ "ha", "persistence", @@ -13132,10 +13132,10 @@ { "id": "bug-1439", "timestamp": "2026-07-14T13:15:00Z", - "error_message": "lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1: error: unterminated #ifndef — during ninja compile of lib/libxce.a.p/utils.c.o, cascading into ~440 bogus errors (glibc math.h storage-class errors, utils.h ectx_t redefinition) for the rest of that one translation unit", + "error_message": "lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1: error: unterminated #ifndef \u2014 during ninja compile of lib/libxce.a.p/utils.c.o, cascading into ~440 bogus errors (glibc math.h storage-class errors, utils.h ectx_t redefinition) for the rest of that one translation unit", "file": "src/xce.build/lib/controller.capnp.h (generated by [5/190] Generating lib/controllercapnp.[ch] custom command)", - "root_cause": "SUSPECTED (not confirmed): parallel-build race — gcc appears to have read lib/controller.capnp.h while the custom-command generation step was still writing it (non-atomic write, no write-to-tmp+rename). Evidence: re-inspecting the SAME on-disk file immediately after the failed build shows correct, well-formed content at line 4170 (get_LimitConfig(struct LimitConfig*, LimitConfig_list, int i);) and balanced #ifndef/#endif (9/9) — i.e. the finished file is NOT actually corrupt, only what gcc read mid-write was. Not yet reproduced a second time to confirm.", - "fix": "NONE APPLIED — this run was build+report only (nix develop ./src -- xce-build.sh controller), not a fix task. Build failed via set -euo pipefail before any tests ran; no controller/xce_controller, controller_unit_test, or xce_test binaries were produced. Next step if it recurs: check meson.build custom_target() for lib/controller.capnp.[ch] has correct output declarations / add explicit depends on all consumers, or file upstream against capnpc-c invocation for atomic output. RECURRENCE 2026-07-16 (2nd occurrence, CONFIRMS the race, different symptom): clean build (rm -rf xce.build) of `nix develop ./src --command bash src/tools/xce-build.sh controller asan` at HEAD 2a76940. This time gcc failed with `fatal error: controller.capnp.h: No such file or directory` (not corrupted content) for hook.c.o, threshold.c.o, xlist.c.o, utils.c.o, window.c.o, xceapi.c.o, wrap.c.o — all 7 consumers of utils.h#include controller.capnp.h. Critically, ninja's own completion log shows `[12/191] Generating lib/controllercapnp.[ch]` printed AFTER jobs [5..11] (the 7 failed compiles) had already completed/failed. This proves the custom_target that generates lib/controller.capnp.h has no dependency edge forcing it to run before these 7 specific compile units — confirms the suspected root cause in the original entry (missing depends()/output declaration in meson.build), not just a non-atomic-write timing fluke. Still NOT fixed (build+report only task, no source edits authorized). controller_unit_test and xce_test were never produced this run (clean build wiped the prior binaries and the new build failed before relinking). RECURRENCE 2026-07-17 (3rd occurrence, kiro-validate-impl smoke-boot build for job-array-single-wal-record): fresh (non-clean) build of `nix develop ./src --command bash src/tools/xce-build.sh controller` at HEAD ea8d9d3 failed on the FIRST invocation with yet another symptom of the same race: `fatal error: opening dependency file controller/controller_unit_test.p/ha.c.o.d: No such file or directory` AND simultaneously `fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory` for two unrelated targets at once ([187/192] and [188/192]) -- gcc could not open its OWN .o.d dependency-file output path, not an input header this time, reinforcing this is a general ninja/meson parallel-build filesystem race in this sandbox, not specific to controller.capnp.h generation. WORKAROUND CONFIRMED AGAIN: simply re-running the identical build command with no source changes succeeded cleanly (85/85 targets, controller_unit_test 3/3 OK), producing fresh xce_controller/xce_test binaries. Do-not-repeat: if xce-build.sh fails with any \"fatal error: opening dependency file ... .o.d: No such file or directory\" or \"controller.capnp.h: No such file or directory\", just retry the same command once before treating it as a real build break -- do not start debugging source code. RECURRENCE 2026-07-20 (4th occurrence, xce-build.sh all asan build+report task validating bug-1453/1454 C-side changes, HEAD after commit 619f41f): clean build failed on FIRST invocation with fatal error: controller.capnp.h: No such file or directory for xlist.c.o, htab.c.o, window.c.o, xstring.c.o, hook.c.o, wrap.c.o, threshold.c.o, xceapi.c.o, utils.c.o, zarray.c.o (10 consumers of utils.h#include controller.capnp.h) -- ninja log again shows [16/353] Generating lib/controllercapnp.[ch] printed AFTER the failed compiles. WORKAROUND CONFIRMED AGAIN (4th time): identical retry with zero source changes succeeded cleanly, 353/353 ninja targets, all 9 meson test suites OK (0 Fail), test_cluster (part of libxce test suite) and scheduler tests all passed. Still unfixed at the meson.build dependency-edge level -- do-not-repeat guidance holds: always retry once before treating this as a real regression. RECURRENCE 2026-07-26 (5th occurrence, build+report task validating bug-1474 repl.c fix via `nix develop ./src --command bash src/tools/xce-build.sh all asan`, HEAD c0b0c7d): TWO consecutive races in the same session, different symptoms each time. 1st invocation (clean build): classic capnp.h race -- hook.c.o etc. failed 'controller.capnp.h: No such file or directory' with the generator's status line ([6/361] Generating lib/controllercapnp.[ch]) printed AFTER the failed compiles -- identical to prior occurrences. 2nd invocation (retry, clean build): build progressed all the way to [361/361] (last job, frontend/apiserver's go_build) which FAILED with `/usr/bin/ld: cannot find -lxce: No such file or directory`; lib/libxce.a was confirmed absent from src/xce.build/lib/ immediately after the failure, and no 'Linking static target lib/libxce.a' line ever appeared in the ninja log for that invocation. NEW symptom variant: this is the first occurrence of the race hitting a Go custom_target (apiserver's CGO link against libxce.a) rather than a C compile/link step -- apiserver's CGO_LDFLAGS references @BUILD_ROOT@/lib/libxce.a only via an env-var string, which is suspected to carry no explicit ninja dependency edge on the lib archive target, letting the go_build job be eligible to run before the archive step completes. 3rd invocation (retry, clean build) succeeded cleanly: 361/361 ninja targets, meson test Ok:9 Fail:0, scheduler manual test 15/15 pass, frontend Go tests (apiserver + xcecli) all ok, xceadmin pytest 166 passed, producing fresh xce_controller/xce_test (confirmed containing the bug-1474 'reusing guard-holding slot' WARN string) and a fresh xceadmin/.venv/bin/xceadmin. WORKAROUND CONFIRMED AGAIN (do not debug source on first or second failure): simply retry the identical xce-build.sh command with no changes; this session needed up to 2 retries (3 total invocations) before a clean pass, still well within the established do-not-repeat guidance. Do-not-repeat addendum: the race is not limited to C compile/link edges -- it can also strand a Go custom_target waiting on a C static archive (apiserver <- lib/libxce.a) referenced only via env-var string interpolation, not a declared meson `depends:`/`link_with:` edge. Still unfixed at the meson.build dependency-edge level.", + "root_cause": "SUSPECTED (not confirmed): parallel-build race \u2014 gcc appears to have read lib/controller.capnp.h while the custom-command generation step was still writing it (non-atomic write, no write-to-tmp+rename). Evidence: re-inspecting the SAME on-disk file immediately after the failed build shows correct, well-formed content at line 4170 (get_LimitConfig(struct LimitConfig*, LimitConfig_list, int i);) and balanced #ifndef/#endif (9/9) \u2014 i.e. the finished file is NOT actually corrupt, only what gcc read mid-write was. Not yet reproduced a second time to confirm.", + "fix": "NONE APPLIED \u2014 this run was build+report only (nix develop ./src -- xce-build.sh controller), not a fix task. Build failed via set -euo pipefail before any tests ran; no controller/xce_controller, controller_unit_test, or xce_test binaries were produced. Next step if it recurs: check meson.build custom_target() for lib/controller.capnp.[ch] has correct output declarations / add explicit depends on all consumers, or file upstream against capnpc-c invocation for atomic output. RECURRENCE 2026-07-16 (2nd occurrence, CONFIRMS the race, different symptom): clean build (rm -rf xce.build) of `nix develop ./src --command bash src/tools/xce-build.sh controller asan` at HEAD 2a76940. This time gcc failed with `fatal error: controller.capnp.h: No such file or directory` (not corrupted content) for hook.c.o, threshold.c.o, xlist.c.o, utils.c.o, window.c.o, xceapi.c.o, wrap.c.o \u2014 all 7 consumers of utils.h#include controller.capnp.h. Critically, ninja's own completion log shows `[12/191] Generating lib/controllercapnp.[ch]` printed AFTER jobs [5..11] (the 7 failed compiles) had already completed/failed. This proves the custom_target that generates lib/controller.capnp.h has no dependency edge forcing it to run before these 7 specific compile units \u2014 confirms the suspected root cause in the original entry (missing depends()/output declaration in meson.build), not just a non-atomic-write timing fluke. Still NOT fixed (build+report only task, no source edits authorized). controller_unit_test and xce_test were never produced this run (clean build wiped the prior binaries and the new build failed before relinking). RECURRENCE 2026-07-17 (3rd occurrence, kiro-validate-impl smoke-boot build for job-array-single-wal-record): fresh (non-clean) build of `nix develop ./src --command bash src/tools/xce-build.sh controller` at HEAD ea8d9d3 failed on the FIRST invocation with yet another symptom of the same race: `fatal error: opening dependency file controller/controller_unit_test.p/ha.c.o.d: No such file or directory` AND simultaneously `fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory` for two unrelated targets at once ([187/192] and [188/192]) -- gcc could not open its OWN .o.d dependency-file output path, not an input header this time, reinforcing this is a general ninja/meson parallel-build filesystem race in this sandbox, not specific to controller.capnp.h generation. WORKAROUND CONFIRMED AGAIN: simply re-running the identical build command with no source changes succeeded cleanly (85/85 targets, controller_unit_test 3/3 OK), producing fresh xce_controller/xce_test binaries. Do-not-repeat: if xce-build.sh fails with any \"fatal error: opening dependency file ... .o.d: No such file or directory\" or \"controller.capnp.h: No such file or directory\", just retry the same command once before treating it as a real build break -- do not start debugging source code. RECURRENCE 2026-07-20 (4th occurrence, xce-build.sh all asan build+report task validating bug-1453/1454 C-side changes, HEAD after commit 619f41f): clean build failed on FIRST invocation with fatal error: controller.capnp.h: No such file or directory for xlist.c.o, htab.c.o, window.c.o, xstring.c.o, hook.c.o, wrap.c.o, threshold.c.o, xceapi.c.o, utils.c.o, zarray.c.o (10 consumers of utils.h#include controller.capnp.h) -- ninja log again shows [16/353] Generating lib/controllercapnp.[ch] printed AFTER the failed compiles. WORKAROUND CONFIRMED AGAIN (4th time): identical retry with zero source changes succeeded cleanly, 353/353 ninja targets, all 9 meson test suites OK (0 Fail), test_cluster (part of libxce test suite) and scheduler tests all passed. Still unfixed at the meson.build dependency-edge level -- do-not-repeat guidance holds: always retry once before treating this as a real regression. RECURRENCE 2026-07-26 (5th occurrence, build+report task validating bug-1474 repl.c fix via `nix develop ./src --command bash src/tools/xce-build.sh all asan`, HEAD c0b0c7d): TWO consecutive races in the same session, different symptoms each time. 1st invocation (clean build): classic capnp.h race -- hook.c.o etc. failed 'controller.capnp.h: No such file or directory' with the generator's status line ([6/361] Generating lib/controllercapnp.[ch]) printed AFTER the failed compiles -- identical to prior occurrences. 2nd invocation (retry, clean build): build progressed all the way to [361/361] (last job, frontend/apiserver's go_build) which FAILED with `/usr/bin/ld: cannot find -lxce: No such file or directory`; lib/libxce.a was confirmed absent from src/xce.build/lib/ immediately after the failure, and no 'Linking static target lib/libxce.a' line ever appeared in the ninja log for that invocation. NEW symptom variant: this is the first occurrence of the race hitting a Go custom_target (apiserver's CGO link against libxce.a) rather than a C compile/link step -- apiserver's CGO_LDFLAGS references @BUILD_ROOT@/lib/libxce.a only via an env-var string, which is suspected to carry no explicit ninja dependency edge on the lib archive target, letting the go_build job be eligible to run before the archive step completes. 3rd invocation (retry, clean build) succeeded cleanly: 361/361 ninja targets, meson test Ok:9 Fail:0, scheduler manual test 15/15 pass, frontend Go tests (apiserver + xcecli) all ok, xceadmin pytest 166 passed, producing fresh xce_controller/xce_test (confirmed containing the bug-1474 'reusing guard-holding slot' WARN string) and a fresh xceadmin/.venv/bin/xceadmin. WORKAROUND CONFIRMED AGAIN (do not debug source on first or second failure): simply retry the identical xce-build.sh command with no changes; this session needed up to 2 retries (3 total invocations) before a clean pass, still well within the established do-not-repeat guidance. Do-not-repeat addendum: the race is not limited to C compile/link edges -- it can also strand a Go custom_target waiting on a C static archive (apiserver <- lib/libxce.a) referenced only via env-var string interpolation, not a declared meson `depends:`/`link_with:` edge. Still unfixed at the meson.build dependency-edge level.", "tags": [ "ninja-race", "controller", @@ -13495,9 +13495,9 @@ { "id": "bug-1453", "timestamp": "2026-07-20T00:00:00.000Z", - "error_message": "Unnecessary CA certificate plumbing across C daemons and Go CLI — ca_cert_path fields, --ca-cert flags, and CACert config are dead or misleading; TLS verification is SSL_VERIFY_NONE everywhere", - "file": "multiple — see root_cause", - "root_cause": "XCE design uses server-side TLS only (encryption without peer certificate verification). The CA cert plumbing was intended for future P3 mTLS hardening but was never implemented: (1) C-side: zero SSL_CTX_load_verify_locations calls in entire codebase — ca_cert_path in xce/cluster.h:35 is set by controller/worker/scheduler but never loaded into any SSL_CTX; do_connect() production path in cluster.c:73 is a NULL-returning stub; (2) Go-side: HTTPS is disabled by default; xcecli uses trust-on-first-use pinning of the server leaf cert (not a CA) stored via a misleadingly-named CACert field; falls back to InsecureSkipVerify:true. Design docs (persistence-and-ha-design.md:1700) explicitly tag peer verification as optional P3 hardening, confirming current encryption-only posture is intentional.", + "error_message": "Unnecessary CA certificate plumbing across C daemons and Go CLI \u2014 ca_cert_path fields, --ca-cert flags, and CACert config are dead or misleading; TLS verification is SSL_VERIFY_NONE everywhere", + "file": "multiple \u2014 see root_cause", + "root_cause": "XCE design uses server-side TLS only (encryption without peer certificate verification). The CA cert plumbing was intended for future P3 mTLS hardening but was never implemented: (1) C-side: zero SSL_CTX_load_verify_locations calls in entire codebase \u2014 ca_cert_path in xce/cluster.h:35 is set by controller/worker/scheduler but never loaded into any SSL_CTX; do_connect() production path in cluster.c:73 is a NULL-returning stub; (2) Go-side: HTTPS is disabled by default; xcecli uses trust-on-first-use pinning of the server leaf cert (not a CA) stored via a misleadingly-named CACert field; falls back to InsecureSkipVerify:true. Design docs (persistence-and-ha-design.md:1700) explicitly tag peer verification as optional P3 hardening, confirming current encryption-only posture is intentional.", "fix": "DONE. C-side: removed ca_cert_path field from xce_cluster_config_t (src/include/xce/cluster.h); do_connect()/xce_connect_fn_t dropped the ca_cert param (src/lib/cluster.c); removed ca_cert_path non-empty check from xce_cluster_config_validate(); removed --ca-cert flag from worker (src/worker/src/main.c) and scheduler (src/scheduler/src/main.c); removed ca_cert_path population from src/controller/main.c:279-281; updated src/lib/testsuite/test_cluster.c mock_connect signature + fixture. Go-side: deleted xcecli/cmd/auth/cert/{cert.go,parser.go} (self-signed cert TOFU-pinning helper); removed CACert field from config.go APIServerInfo and models/logon.go LogonResponse; removed SetRootCAs/loadRootCAs from common/request/request.go and certPath param from Get/Post/DeleteRequestFromAPIServer; updated all ~18 xcecli cmd/* call sites; updated cmd/auth/logon.go to drop the https cert-fetch block; removed TestSetRootCAs_*/TestBuildTLSLogic from request_test.go. Verified: full C build (all components, ASAN) 353/353 targets, meson test suite 9/9 OK; xcecli go build/vet/test all PASS; grep -rn 'CACert|SetRootCAs|loadRootCAs|ca_cert' src/ clean (only historical design-doc mentions remain in src/docs/*.md, left as-is).", "tags": [ "dead-code", @@ -13521,7 +13521,7 @@ "timestamp": "2026-07-20T00:00:00.000Z", "error_message": "Scheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no EndpointRotator) and leaked ctx on HA path", "file": "src/scheduler/src/scheduler.c", - "root_cause": "VERIFICATION: --peers IS correctly parsed via xce_parse_peers() (main.c:129), stored via sched_set_cluster_cfg() (main.c:156), and iterated in sched_init() HA loop (scheduler.c:462-493) BEFORE sched_init — init ordering is correct. Narrowed findings: (1) nmasters=1 per attempt (line 468) — manual candidate loop instead of filling config->masters[] from all candidates so libxce EndpointRotator never engages. Functional but suboptimal. (2) Leaked apictx on HA path — xcecli_ctx_init at line 454 allocates sched->apictx, then HA loop at line 472 re-inits without xcecli_ctx_fini on the first allocation. On standby-failure branches (lines 480, 492) cleanup is correct. OUT OF SCOPE: runtime reconnect/rejoin after controller death belongs to .kiro/specs/component-failover-adoption/ (requirements + design approved, tasks 3.1/3.2 not yet approved).", + "root_cause": "VERIFICATION: --peers IS correctly parsed via xce_parse_peers() (main.c:129), stored via sched_set_cluster_cfg() (main.c:156), and iterated in sched_init() HA loop (scheduler.c:462-493) BEFORE sched_init \u2014 init ordering is correct. Narrowed findings: (1) nmasters=1 per attempt (line 468) \u2014 manual candidate loop instead of filling config->masters[] from all candidates so libxce EndpointRotator never engages. Functional but suboptimal. (2) Leaked apictx on HA path \u2014 xcecli_ctx_init at line 454 allocates sched->apictx, then HA loop at line 472 re-inits without xcecli_ctx_fini on the first allocation. On standby-failure branches (lines 480, 492) cleanup is correct. OUT OF SCOPE: runtime reconnect/rejoin after controller death belongs to .kiro/specs/component-failover-adoption/ (requirements + design approved, tasks 3.1/3.2 not yet approved).", "fix": "DONE (leak fix only). Fixed the confirmed defect: sched_init() called xcecli_ctx_init(&sched->apictx, config) unconditionally, then on the HA path re-called xcecli_ctx_init() inside the candidate loop without freeing the first ctx -> leaked one xcecli_ctx_t per sched_init() call. Fix: xcecli_ctx_fini(sched->apictx) added immediately before the HA loop starts (src/scheduler/src/scheduler.c), since the loop's first iteration always replaces it. Verified via full ASAN build: all 5 scheduler meson test suites + libxce suite pass, no regression. REJECTED the nmasters>1+EndpointRotator suggestion: memory obs 11825 (this session) confirms scheduler/worker deliberately use the candidates[]-loop path instead of filling config->masters[], so changing that would contradict the established design, not fix a bug. Runtime reconnect/rejoin gap remains out of scope, deferred to component-failover-adoption spec (unchanged from original triage).", "tags": [ "scheduler", @@ -13542,7 +13542,7 @@ "error_message": "Scheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that main.c populated via sched_set_cluster_cfg() just before calling sched_init(), so the HA candidate loop's candidates_count check always reads 0 and falls back to the hardcoded localhost:9990 single-host path regardless of --peers.", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/main.c, src/scheduler/include/scheduler.h", "root_cause": "sched_set_cluster_cfg(&sched, &cluster_cfg) at main.c was called before sched_init(&sched, &config, handler); sched_init's first action is memset(sched, 0, sizeof(*sched)), which zeroes the cluster_cfg field just written. sched_init's own internal HA-vs-single-host branch reads sched->cluster_cfg.candidates_count immediately after, so it always sees 0. User-reported regression found via code review after bug-1454's leak fix landed in the same function.", - "fix": "Restructured instead of patched around: sched_init() signature changed to take `const xce_cluster_config_t*` directly (assigned to sched->cluster_cfg AFTER the memset, inside sched_init, not before via a separate setter). Deleted sched_set_cluster_cfg() (no other callers). main.c hoists cluster_cfg/cluster_cfg_ptr to function scope and passes cluster_cfg_ptr (NULL or &cluster_cfg) straight into sched_init(). Also fixed an adjacent smell in the same HA loop: it used to mutate the caller's xce_client_config_t* config in place per candidate, leaving it in a surprising mutated state after sched_init returned — changed to a per-iteration local copy. Verified: full ASAN build (all components) 353/353 targets, meson 9/9 OK, manual scheduler testsuite 15/15, Go tests all pass. No test exercises sched_init's HA path with real --peers input (pre-existing gap, not introduced by this fix).", + "fix": "Restructured instead of patched around: sched_init() signature changed to take `const xce_cluster_config_t*` directly (assigned to sched->cluster_cfg AFTER the memset, inside sched_init, not before via a separate setter). Deleted sched_set_cluster_cfg() (no other callers). main.c hoists cluster_cfg/cluster_cfg_ptr to function scope and passes cluster_cfg_ptr (NULL or &cluster_cfg) straight into sched_init(). Also fixed an adjacent smell in the same HA loop: it used to mutate the caller's xce_client_config_t* config in place per candidate, leaving it in a surprising mutated state after sched_init returned \u2014 changed to a per-iteration local copy. Verified: full ASAN build (all components) 353/353 targets, meson 9/9 OK, manual scheduler testsuite 15/15, Go tests all pass. No test exercises sched_init's HA path with real --peers input (pre-existing gap, not introduced by this fix).", "tags": [ "scheduler", "peers", @@ -13563,7 +13563,7 @@ "timestamp": "2026-07-20T00:00:00.000Z", "error_message": "Dangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454) changed the loop to build a per-iteration stack-local xce_client_config_t candidate_config and pass &candidate_config into xcecli_ctx_init().", "file": "src/scheduler/src/scheduler.c", - "root_cause": "xcecli_ctx_init() (src/lib/client.c) stores the raw pointer it is given as ctx->conf — it does not copy the struct. That pointer is dereferenced later, long after sched_init() returns, by client_failover.c runtime reconnect/rotation logic (e.g. ac->conf->masters[next]). When the candidate loop used a stack-local xce_client_config_t per iteration, ctx->conf pointed at a stack frame that no longer existed once sched_init() returned — a classic dangling pointer that only manifests later during HA reconnect/rotation, making it easy to miss in a build+unit-test pass.", + "root_cause": "xcecli_ctx_init() (src/lib/client.c) stores the raw pointer it is given as ctx->conf \u2014 it does not copy the struct. That pointer is dereferenced later, long after sched_init() returns, by client_failover.c runtime reconnect/rotation logic (e.g. ac->conf->masters[next]). When the candidate loop used a stack-local xce_client_config_t per iteration, ctx->conf pointed at a stack frame that no longer existed once sched_init() returned \u2014 a classic dangling pointer that only manifests later during HA reconnect/rotation, making it easy to miss in a build+unit-test pass.", "fix": "Reverted to mutating the caller-supplied xce_client_config_t* config parameter in place (config->nmasters, config->masters[0], config->master_ports[0]) instead of a stack-local copy. config is owned by main()'s stack frame, which lives for the whole process, so storing &config-derived state in ctx->conf is lifetime-safe. Added a comment in scheduler.c explaining why the stack-local-copy pattern is unsafe here, to prevent it from being reintroduced.", "tags": [ "scheduler", @@ -13587,8 +13587,8 @@ "timestamp": "2026-07-20T22:56:00Z", "error_message": "xce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of starting as single-node primary", "file": "src/controller/myapp.c", - "root_cause": "Discovered while independently reviewing clusterconfig-schema-split task 5.3 by building and running the actual pre-spec baseline (commit 94f50ce) for comparison. Single-node startup (no --peers) was ALREADY broken before this spec began: myapp_run() calls cluster_open_listener() before the primary config (cluster_config) is loaded from disk, so the pre-spec fallback branch (cluster_config->port, only reachable when cluster_config != NULL) always hit its else branch and returned -EINVAL, unconditionally, for every zero-flag single-node start. This invalidates part of clusterconfig-schema-split's premise (requirements.md's Introduction claimed cluster_config->port was 'the sole live port-sourcing mechanism for single-node deployments' and 'single-node controller startup SHALL continue to work exactly as it does today' — in reality it did not work today). Not caused by, or a regression from, clusterconfig-schema-split's changes — task 2.1's port-resolution rewrite (commit 1229f9a) incidentally fixed this as a side effect, since its hardcoded-default fallback no longer depends on cluster_config being loaded.", - "fix": "RESOLVED -- incidentally repaired by clusterconfig-schema-split task 2.1 (cluster_open_listener now falls back to a hardcoded XCE_CONTROLLER_DEFAULT_PORT constant instead of an unloaded cluster_config->port), not as a dedicated fix for this entry. Live-verified post-fix: single-node startup with no --peers now correctly binds port 9990 and reaches CTRL_PRIMARY. No further action needed unless a future audit wants to correct the historical claim in clusterconfig-schema-split's requirements.md/design.md that pre-spec single-node startup was previously working — it was not, for an unrelated reason (myapp_run's listener-open/config-load ordering).", + "root_cause": "Discovered while independently reviewing clusterconfig-schema-split task 5.3 by building and running the actual pre-spec baseline (commit 94f50ce) for comparison. Single-node startup (no --peers) was ALREADY broken before this spec began: myapp_run() calls cluster_open_listener() before the primary config (cluster_config) is loaded from disk, so the pre-spec fallback branch (cluster_config->port, only reachable when cluster_config != NULL) always hit its else branch and returned -EINVAL, unconditionally, for every zero-flag single-node start. This invalidates part of clusterconfig-schema-split's premise (requirements.md's Introduction claimed cluster_config->port was 'the sole live port-sourcing mechanism for single-node deployments' and 'single-node controller startup SHALL continue to work exactly as it does today' \u2014 in reality it did not work today). Not caused by, or a regression from, clusterconfig-schema-split's changes \u2014 task 2.1's port-resolution rewrite (commit 1229f9a) incidentally fixed this as a side effect, since its hardcoded-default fallback no longer depends on cluster_config being loaded.", + "fix": "RESOLVED -- incidentally repaired by clusterconfig-schema-split task 2.1 (cluster_open_listener now falls back to a hardcoded XCE_CONTROLLER_DEFAULT_PORT constant instead of an unloaded cluster_config->port), not as a dedicated fix for this entry. Live-verified post-fix: single-node startup with no --peers now correctly binds port 9990 and reaches CTRL_PRIMARY. No further action needed unless a future audit wants to correct the historical claim in clusterconfig-schema-split's requirements.md/design.md that pre-spec single-node startup was previously working \u2014 it was not, for an unrelated reason (myapp_run's listener-open/config-load ordering).", "tags": [ "controller", "startup", @@ -13651,9 +13651,9 @@ { "id": "bug-1460", "timestamp": "2026-07-22T03:52:00Z", - "error_message": "Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon package in site-packages has no schema_files/ directory at all — controller.capnp/c.capnp/JSON schemas are silently dropped from the wheel", + "error_message": "Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon package in site-packages has no schema_files/ directory at all \u2014 controller.capnp/c.capnp/JSON schemas are silently dropped from the wheel", "file": "src/xceadmin/pyproject.toml", - "root_cause": "pyproject.toml had no [tool.setuptools.package-data] entry for xcecommon, so setuptools' default wheel build only includes *.py files. The xce-build.sh sync step correctly copies controller.capnp/c.capnp/*.schema.json into src/xcecommon/schema_files/ in the SOURCE tree, but `pip install .` (non-editable) builds a wheel from that source tree and, without package-data config, drops every non-.py file. An editable install (`pip install -e .`) never surfaces this because it resolves imports straight into the checked-out source tree, bypassing wheel packaging entirely — exactly the gap design.md's Testing Strategy 'Unit — schema bundling' note warned only a release-profile test run could catch (Requirement 5.5).", + "root_cause": "pyproject.toml had no [tool.setuptools.package-data] entry for xcecommon, so setuptools' default wheel build only includes *.py files. The xce-build.sh sync step correctly copies controller.capnp/c.capnp/*.schema.json into src/xcecommon/schema_files/ in the SOURCE tree, but `pip install .` (non-editable) builds a wheel from that source tree and, without package-data config, drops every non-.py file. An editable install (`pip install -e .`) never surfaces this because it resolves imports straight into the checked-out source tree, bypassing wheel packaging entirely \u2014 exactly the gap design.md's Testing Strategy 'Unit \u2014 schema bundling' note warned only a release-profile test run could catch (Requirement 5.5).", "fix": "Added [tool.setuptools.package-data] xcecommon = [\"schema_files/*.capnp\", \"schema_files/json/*.schema.json\"] to src/xceadmin/pyproject.toml. Verified: after `xce-build.sh xceadmin release`, site-packages/xcecommon/schema_files/ now contains controller.capnp, c.capnp, and json/*.schema.json, byte-identical to the synced source copies.", "tags": [ "xceadmin", @@ -14057,9 +14057,9 @@ { "id": "bug-1475", "timestamp": "2026-07-26T04:36:00Z", - "error_message": "Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify bug-1472's fix, FAILED — but NOT at bug-1472's own assertions (materialized-state checkpoint byte-identity and active-configuration identity both PASSED cleanly across all 3 members, confirming bug-1472's fix holds). Failure occurred one step later, inside the Requirement 6.4 host-admin-status-rotation check's own `force_elect_node(\"ctrl-1\")` maneuver: after 3 rounds of overlapping chaotic kills + a live config upgrade all completed and the cluster reconverged 3-of-3, `force_elect_node` killed the spoiler (ctrl-3) and tried to submit a job batch on the surviving {ctrl-2 primary, ctrl-1 standby} pair — 5 consecutive attempts all got 0/3 acked (\"waiting for write-quorum\"), and the round hard-failed: `[xce-test-ha] FAIL: force-elect ctrl-1: submit on {ctrl-2,ctrl-1} failed`. The Req 6.4 workerlist/admin-status assertion itself was never reached.", - "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby, ha_consume_append_entries, ha_evaluate_repl_action — HA_REPL_STREAM_WAL decision/dispatch loop); src/controller/config_repl.c (ha_stream_wal_to_standby catch-up streamer)", - "root_cause": "NOT data loss, NOT bug-1472 (worker/host admin-status replication is unaffected — unrelated subsystem, and its own checks passed). This is a standby catch-up LIVENESS stall: ctrl-1 (which had itself served as primary in term=2 before being killed in round 1 of the chaotic sequence, then restarted) got permanently stuck at applied_lsn=12 and never advanced again for the rest of the run. ctrl-1's own log (`ha_consume_append_entries`) shows a revealing pattern: a zero-entry PROBE AppendEntries at prevIdx=12,n=0 correctly succeeds with match=12 (confirming index 12 IS ctrl-1's true, correct log tip), but the very next entry-carrying AppendEntries from the SAME dispatch cycle uses prevIdx=13 (not 12) for n=1, which ctrl-1 correctly rejects with event=ae_reject_logmismatch (success=0 match=0) since it has no entry at index 13 yet to check a predecessor against. This 1-off prevIdx computation (using prevIdx=standby_applied_lsn+1 instead of prevIdx=standby_applied_lsn for the FIRST entry of a catch-up batch) means every retry of the SAME heartbeat-driven HA_REPL_STREAM_WAL decision (ha_evaluate_repl_action: standby_lsn(12) < committed_lsn(31) -> stream) recomputes and resends the IDENTICAL doomed range, forever, with zero backtrack/self-correction -- confirmed via ctrl-2's log: 'HA repl: standby[0] lsn=12 < committed_lsn=31 -> stream WAL batches' repeats verbatim every ~3s from 04:31:26 through 04:32:26 (60+ seconds, 20+ identical retries, zero progress) after ctrl-2 had already logged 'repl_broadcast: standby localhost:9991 evicted (pending_ack_count=17 > 16) -- triggering catch-up reconnect' and 'HA quorum lost: connected=0 required=1'. Once the soak's own force_elect_node killed the OTHER standby (ctrl-3) as its spoiler maneuver, ctrl-1 became the sole remaining standby, and its permanent inability to catch up meant the primary could never regain 2-of-3 write quorum again -- every subsequent job submission timed out (`commit: async continuation deadline expired`, `jobsubmit_commit_continuation: lsn=N commit failed/timed out (rc=-110)`), exactly matching the soak's observed '0/3 acked; waiting for write-quorum' x5 before hard-failing. Superficially similar symptom signature to already-FIXED bug-1028 (same 'ha_dispatch_repl_action_for_standby: standby[0] lsn=X < committed_lsn=Y -> stream WAL batches' log line looping indefinitely post-failover) but a DIFFERENT specific AE-reject reason -- bug-1028 was a logOk TERM-mapping mismatch (raw hdr.term vs persist_entry_term legacy mapping), fixed 2026-06-23; this is `ae_reject_logmismatch`, which reads like a prevLogIndex/off-by-one issue in how the catch-up streamer picks the FIRST entry's predecessor index, not a term problem -- so this is either a narrower unfixed variant of the same family, or a new regression in a related code path. NOT root-caused down to the exact defective line (this investigation was a live-soak VERIFICATION run for a DIFFERENT bug, not a fix task) -- flagging for whoever picks this up next to read ha_dispatch_repl_action_for_standby/ha_stream_wal_to_standby's prevIdx computation directly and compare against ctrl-1's actual persisted log at lsn=12/13 (waldump). Also notable: this specific soak step (force_elect_node's spoiler-kill-then-submit maneuver, previously only exercised by --sequential mode's post-loop 'leadership coverage' step against an otherwise-calm, fully-settled cluster) had never previously been run immediately after --config-upgrade's THREE rounds of aggressive overlapping chaotic kills plus a live config upgrade -- this specific combination/history may be what's needed to reproduce the stall, rather than it being config-upgrade-specific.", + "error_message": "Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify bug-1472's fix, FAILED \u2014 but NOT at bug-1472's own assertions (materialized-state checkpoint byte-identity and active-configuration identity both PASSED cleanly across all 3 members, confirming bug-1472's fix holds). Failure occurred one step later, inside the Requirement 6.4 host-admin-status-rotation check's own `force_elect_node(\"ctrl-1\")` maneuver: after 3 rounds of overlapping chaotic kills + a live config upgrade all completed and the cluster reconverged 3-of-3, `force_elect_node` killed the spoiler (ctrl-3) and tried to submit a job batch on the surviving {ctrl-2 primary, ctrl-1 standby} pair \u2014 5 consecutive attempts all got 0/3 acked (\"waiting for write-quorum\"), and the round hard-failed: `[xce-test-ha] FAIL: force-elect ctrl-1: submit on {ctrl-2,ctrl-1} failed`. The Req 6.4 workerlist/admin-status assertion itself was never reached.", + "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby, ha_consume_append_entries, ha_evaluate_repl_action \u2014 HA_REPL_STREAM_WAL decision/dispatch loop); src/controller/config_repl.c (ha_stream_wal_to_standby catch-up streamer)", + "root_cause": "NOT data loss, NOT bug-1472 (worker/host admin-status replication is unaffected \u2014 unrelated subsystem, and its own checks passed). This is a standby catch-up LIVENESS stall: ctrl-1 (which had itself served as primary in term=2 before being killed in round 1 of the chaotic sequence, then restarted) got permanently stuck at applied_lsn=12 and never advanced again for the rest of the run. ctrl-1's own log (`ha_consume_append_entries`) shows a revealing pattern: a zero-entry PROBE AppendEntries at prevIdx=12,n=0 correctly succeeds with match=12 (confirming index 12 IS ctrl-1's true, correct log tip), but the very next entry-carrying AppendEntries from the SAME dispatch cycle uses prevIdx=13 (not 12) for n=1, which ctrl-1 correctly rejects with event=ae_reject_logmismatch (success=0 match=0) since it has no entry at index 13 yet to check a predecessor against. This 1-off prevIdx computation (using prevIdx=standby_applied_lsn+1 instead of prevIdx=standby_applied_lsn for the FIRST entry of a catch-up batch) means every retry of the SAME heartbeat-driven HA_REPL_STREAM_WAL decision (ha_evaluate_repl_action: standby_lsn(12) < committed_lsn(31) -> stream) recomputes and resends the IDENTICAL doomed range, forever, with zero backtrack/self-correction -- confirmed via ctrl-2's log: 'HA repl: standby[0] lsn=12 < committed_lsn=31 -> stream WAL batches' repeats verbatim every ~3s from 04:31:26 through 04:32:26 (60+ seconds, 20+ identical retries, zero progress) after ctrl-2 had already logged 'repl_broadcast: standby localhost:9991 evicted (pending_ack_count=17 > 16) -- triggering catch-up reconnect' and 'HA quorum lost: connected=0 required=1'. Once the soak's own force_elect_node killed the OTHER standby (ctrl-3) as its spoiler maneuver, ctrl-1 became the sole remaining standby, and its permanent inability to catch up meant the primary could never regain 2-of-3 write quorum again -- every subsequent job submission timed out (`commit: async continuation deadline expired`, `jobsubmit_commit_continuation: lsn=N commit failed/timed out (rc=-110)`), exactly matching the soak's observed '0/3 acked; waiting for write-quorum' x5 before hard-failing. Superficially similar symptom signature to already-FIXED bug-1028 (same 'ha_dispatch_repl_action_for_standby: standby[0] lsn=X < committed_lsn=Y -> stream WAL batches' log line looping indefinitely post-failover) but a DIFFERENT specific AE-reject reason -- bug-1028 was a logOk TERM-mapping mismatch (raw hdr.term vs persist_entry_term legacy mapping), fixed 2026-06-23; this is `ae_reject_logmismatch`, which reads like a prevLogIndex/off-by-one issue in how the catch-up streamer picks the FIRST entry's predecessor index, not a term problem -- so this is either a narrower unfixed variant of the same family, or a new regression in a related code path. NOT root-caused down to the exact defective line (this investigation was a live-soak VERIFICATION run for a DIFFERENT bug, not a fix task) -- flagging for whoever picks this up next to read ha_dispatch_repl_action_for_standby/ha_stream_wal_to_standby's prevIdx computation directly and compare against ctrl-1's actual persisted log at lsn=12/13 (waldump). Also notable: this specific soak step (force_elect_node's spoiler-kill-then-submit maneuver, previously only exercised by --sequential mode's post-loop 'leadership coverage' step against an otherwise-calm, fully-settled cluster) had never previously been run immediately after --config-upgrade's THREE rounds of aggressive overlapping chaotic kills plus a live config upgrade -- this specific combination/history may be what's needed to reproduce the stall, rather than it being config-upgrade-specific.", "fix": "FIXED (2026-07-27/28, via a regression test + real-instrumentation debugging session that continued this entry's original live-soak finding). Landed as THREE coordinated changes, all required together -- the first two alone left the new regression test (test_bug1475_catchup_survives_midstream_checkpoint, test_config_repl.c) RED with applied_lsn stuck at 0 for the whole run:\n\n1. SEND side (config_repl.c, ha_stream_wal_to_standby): when scanning the WAL for entries to stream, a WAL_CHECKPOINT entry is still never built/sent as its own catchup entry, but the prev_log_index/prev_log_term tracker now advances across it like every other scanned entry, instead of freezing one entry behind. A standby that independently applied the same committed WAL_CONFIG_MARKER writes its OWN local checkpoint at the identical LSN (deterministic per-member trigger), so the checkpoint's LSN is a real, held position on both sides, not a gap to skip past chaining-wise.\n\n2. RECEIVE side (ha.c, ha_apply_term_entry_via_library): a JIT resync reads persist_durable_floor(p) and monotonically raises ha->raft.log_len to match before consuming each AppendEntries -- mirroring ha_sync_raft_log_from_recovery's boot-time computation, but applied live on the AE-receive path. This corrects for persist_write_checkpoint_marker (bug-1466, still not fixed at ITS root) advancing persist_t's current_lsn without going through raft_log_append, which otherwise leaves ha->raft.log_len stale and fails logOk's prev_log_index<=log_len bound forever.\n\n3. TERM-INDEX side (persist.c, record_entry_term/entry_term_locked) -- a THIRD, previously-undiscovered gap, found only via live fprintf instrumentation of the new regression test after fixes 1-2 above still left it RED. Fixes 1-2 both depend on persist_entry_term(checkpoint_lsn) reading back 'absent' (0) FOREVER on every member (the explicit, twice-documented design intent in both fixes' own comments and in persist_write_checkpoint_marker's comment). That invariant was actually FALSE: record_entry_term's geometric-growth realloc zero-filled any never-explicitly-recorded gap slot (like a checkpoint's own LSN, which is deliberately never passed to record_entry_term), and entry_term_locked's `lsn <= term_index_count` presence check then read that zero-fill back as WAL_LEGACY_TERM (1), not absent (0), the INSTANT any LATER lsn was recorded and grew term_index_count past the gap -- which happens immediately on a live leader (it appends real entries right after the checkpoint) but not yet on a receiver that has not itself appended past its own checkpoint. So the checkpoint LSN's 'term' was TIME-DEPENDENT: the sender (which streams AFTER its own later entries exist) computed prev_log_term=1 for the entry right after the checkpoint, while the receiver's honest, not-yet-grown term index still (correctly, per design) read 0 for the same LSN -- logOk's equality check spuriously mismatched (0 != 1) and rejected forever, exactly reproducing this entry's original ae_reject_logmismatch signature. Confirmed with real fprintf instrumentation at the streamer, the JIT resync, and raft.c's logOk check (added and removed during this investigation, not left in the tree): prev_log_term flipped from 0 to 1 in the SAME test run, immediately after the leader appended its 2nd post-checkpoint entry. Fixed with a TERM_INDEX_UNSET sentinel (UINT64_MAX) that record_entry_term's growth path now fills gap slots with (instead of 0), and entry_term_locked treats as absent -- so a never-explicitly-written slot reads absent PERMANENTLY, independent of later array growth. Confined to persist.c's two functions plus the term_index doc comment in persist.h; no other call site reads p->term_index[] directly.\n\nAlso completed as part of this fix: (a) audited whether WAL entry types OTHER than WAL_CHECKPOINT bypass record_entry_term the same way -- confirmed NO: WAL_CHECKPOINT (value 100) is the numeric max of wal_entry_type, persist_wal_async and persist_wal_write_replicated both explicitly reject type==WAL_CHECKPOINT up front (it cannot reach record_entry_term via either live path), and replay_wal's `hdr.type < WAL_CHECKPOINT` recording guard already covers every other currently-defined type (WAL_JOB_UPDATE=7, WAL_QUEUE_CTRL=20, WAL_WORKER_CTRL=30, WAL_NOOP=40, WAL_CONFIG_MARKER=50); (b) confirmed no interaction with the RequestVote/PreVote freshness gates (raft.c raft_log_ok, shared by raft_on_vote_request/raft_on_pre_vote_request): that gate also reads on_get_entry_term(s->log_len), which can legitimately land on a checkpoint LSN after the JIT resync in fix 2 -- the TERM_INDEX_UNSET fix makes that read deterministic (always absent) instead of time-dependent, which is a consistency IMPROVEMENT for vote freshness too, not a behavior change in the unsafe direction; full test_ha_prevote.c/test_ha_failover_prevote.c suites pass unchanged. Full regression suite: 1034 Tests, 0 Failures (was 1033/1034 with the new test failing before this fix). Files changed: src/controller/config_repl.c (REAL logic change, landed earlier in this same bug-1475 investigation: ha_stream_wal_to_standby's WAL_CHECKPOINT branch now falls through to the unconditional prev-tracker advance via `hdr.lsn > standby_lsn && hdr.type != WAL_CHECKPOINT`, instead of a hard `continue` that froze the tracker one entry behind -- plus cross-reference comments added in this later session); src/controller/ha.c (REAL logic change, also landed earlier in this investigation: a ~30-line JIT resync block added to the top of ha_apply_term_entry_via_library that raises ha->raft.log_len from persist_durable_floor(p) before consuming each AppendEntries -- plus cross-reference comments added in this later session); src/controller/persist.c (the TERM_INDEX_UNSET sentinel fix + doc comments, the new logic landed in THIS session); src/controller/testsuite/test_config_repl.c (test_bug1475_catchup_survives_midstream_checkpoint + updated doc comments describing all three fix parts). Correction (coordinator review 2026-07-27/28, Finding 1): an earlier version of this fix summary mischaracterized config_repl.c's and ha.c's changes as 'comment-only, no logic change' -- both carry genuine logic changes; they were simply written in an earlier part of this same bug-1475 investigation than the TERM_INDEX_UNSET sentinel (session 3 of the fix), not left unmodified by it.\n\nFinding 2 (coordinator review, 2026-07-27/28, CONFIRMED REAL and FIXED): after the TERM_INDEX_UNSET fix, the reviewer traced a plausible NEW instance of the identical permanent-wedge bug class, reachable via the SNAPSHOT-INSTALL catch-up path (HA_REPL_SEND_SNAPSHOT, dispatched whenever a standby has fallen too far behind for incremental WAL streaming -- a live, common path) instead of WAL-streaming catch-up. ha_snapshot_do_send (ha.c) computes checkpoint_term = persist_entry_term(leader, c->persist.checkpoint_lsn) and ships it on the wire; the receiver's ha_handle_snapshot_chunk calls persist_reconcile_checkpoint_term(p, checkpoint_lsn, checkpoint_term), which called record_entry_term(p, checkpoint_lsn, checkpoint_term) directly -- an EXPLICIT write, not a growth-fill, so it never got the TERM_INDEX_UNSET sentinel even when checkpoint_term==0. entry_term_locked's pre-existing legacy-0 rule then read that raw 0 back as WAL_LEGACY_TERM(1), forever -- permanently diverging from the leader's own permanently-absent(0) value for the SAME lsn whenever c->persist.checkpoint_lsn (== last_applied_lsn at general persist_checkpoint() time, NOT the WAL_CHECKPOINT entry's own physical lsn) happens to coincide with a checkpoint's own un-recorded lsn -- a real, reachable quiescent-state timing (the apply pump has walked, one lsn at a time including a no-op pass over checkpoint slots, all the way to exactly the checkpoint's own position before the next persist_checkpoint()+snapshot-send cycle runs, e.g. right after a config upgrade with no immediate follow-up activity) -- not the common case, but real and worth closing. Proven with a dedicated regression test (test_bug1475_finding2_snapshot_install_checkpoint_term_absent, test_config_repl.c) that drives the REAL ha_send_snapshot_to_standby -> ha_handle_snapshot_chunk -> persist_reconcile_checkpoint_term path (via the xce_snapshot_chunk_hook test seam) with a leader whose checkpoint_lsn is deliberately seeded (via last_applied_lsn) to coincide with its checkpoint's own physical lsn -- confirmed RED by temporarily reverting only persist_reconcile_checkpoint_term's fix (receiver read back term=1, not the expected absent=0) and GREEN after restoring it. Fixed with the SAME TERM_INDEX_UNSET sentinel: persist_reconcile_checkpoint_term now substitutes TERM_INDEX_UNSET for a wire checkpoint_term of exactly 0 before calling record_entry_term, instead of recording the raw 0 -- a wire value of 0 is unambiguous today (ha_snapshot_do_send is the ONLY sender and always computes it via persist_entry_term, whose public contract already returns a DIFFERENT value, WAL_LEGACY_TERM, for a present-but-legacy-0 entry -- so a public 0 can only mean genuinely, permanently absent). A genuinely non-zero checkpoint_term (any real historical term) is still recorded verbatim and still correctly overwrites a stale local term at that LSN, unchanged -- the pre-existing STALE-TERM reconcile test (test_ha_core_safety.c's test_3_1_snapshot_install_does_not_reconcile_boundary_term, CHECKPOINT_TERM=7, a synthetic non-zero value) is unaffected and still passes. Also fixed the stale 'zeroed-on-grow' comment at persist.c's truncate_term_index_locked (grown slots are TERM_INDEX_UNSET-filled since the original TERM_INDEX_UNSET fix, not zero-filled -- comment had not been updated). Re-verified RequestVote/PreVote freshness non-interference for this second fix too: persist_reconcile_checkpoint_term's caller (ha_handle_snapshot_chunk) never touches ha->raft.log_len or any vote-path state; the fix only changes what value entry_term_locked returns for a checkpoint-boundary LSN after a snapshot install (now consistently absent instead of inconsistently WAL_LEGACY_TERM), which is the SAME direction of improvement (deterministic absence vs time/path-dependent) the original TERM_INDEX_UNSET fix already established as safe for the freshness gate. Full suite: 1035 Tests, 0 Failures (was 1034 before this new test).", "tags": [ "ha.c", @@ -14093,7 +14093,7 @@ "error_message": "Live config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh --config-upgrade-wal-trim, run to confirm bug-1474's repl.c fix), deploy /home/claude/tmp/xce-ha-waltrim3: ctrl-3 ABORTed via AddressSanitizer heap-use-after-free during the round's live config-upgrade submission, well before TARGET(ctrl-1) was ever restarted. The round's own bug-1474 WAL-trim assertions (floor segment preserved, below-floor segments trimmed, active segment present) all PASSED cleanly before this crash was noticed -- ctrl-3's death was only discovered because it, combined with a second unrelated issue (bug-1475), prevented the round's final '3-of-3 reconvergence' assertion from ever succeeding.", "file": "src/controller/config_repl.c (load_primary_config ~line 656-666 free_ClusterConfig/free(myconf->cluster_config) call site; config_activate_committed ~line 360; handle_repl_config_push ~line 1280); lib/controller.capnp.c (decode_ClusterConfig_ptr ~line 17074 calloc site, decode_Resource_list ~line 4057 crash site)", "root_cause": "AddressSanitizer heap-use-after-free, WRITE of size 8, in decode_Resource_list (lib/controller.capnp.c:4057), thread T5. Full ASAN trace (ctrl-3.stderr):\nWRITE by T5: decode_Resource_list <- decode_ClusterConfig <- decode_ClusterConfig_ptr <- load_cluster_config_from_file (config_repl.c:606) <- load_primary_config (config_repl.c:664) <- config_activate_committed (config_repl.c:360) <- handle_repl_config_push (config_repl.c:1280) <- myreq_handler (cluster.c:654) <- vtask_entry (cluster.c:786)\nFREED by T6: load_primary_config (config_repl.c:659, the 'free any prior decoded config before overwriting the pointer' step: free_ClusterConfig(myconf->cluster_config); free(myconf->cluster_config)) <- config_activate_committed (config_repl.c:360) <- handle_repl_config_push (config_repl.c:1280) <- myreq_handler <- vtask_entry\nALLOCATED by T5: calloc in decode_ClusterConfig_ptr (lib/controller.capnp.c:17074) <- load_cluster_config_from_file (config_repl.c:606) <- load_primary_config (config_repl.c:664) <- config_activate_committed <- handle_repl_config_push\nThis is a genuine, unguarded data race: load_primary_config's sequence is (a) if myconf->cluster_config is non-NULL, free it and NULL the pointer, then (b) decode a fresh ClusterConfig into myconf->cluster_config via load_cluster_config_from_file/decode_ClusterConfig_ptr. decode_ClusterConfig_ptr appears to publish the new struct pointer into *config (i.e. myconf->cluster_config) before it has finished populating nested fields (e.g. the Resource_list), since T5 is later observed still writing into the struct via decode_Resource_list AFTER T6 already saw a non-NULL myconf->cluster_config, freed it (T6's own step (a)), and proceeded to decode its own replacement. No mutex protects myconf->cluster_config across this read-free-decode sequence, so two overlapping invocations of handle_repl_config_push -> config_activate_committed -> load_primary_config for the same node race on the shared myconf->cluster_config pointer. This matches the class of concern already flagged (but not as a crash) in bug-976/bug-981 (handle_repl_config_push / config_activate_committed activation-ordering races) -- this is a NEW, more severe manifestation: an actual heap-corruption crash, not just an ordering/verification gap. Plausible trigger in this specific soak: heavy duplicate/late-joiner reconnect churn was independently observed on ctrl-2 (the primary) around the same window (bug-1474's WARN 'reusing guard-holding slot' fired twice, confirming the still-open, out-of-scope ha.c heartbeat-reconnect-sweep defect -- bug-1474's un-addressed 'fix angle (a)' -- was actively spawning duplicate/late-joiner standby connections during this round). If the primary sent the same config-push payload to ctrl-3 twice (once per duplicate connection), ctrl-3 could dispatch two concurrent handle_repl_config_push invocations on separate VTASK_LIVE_BLOCK workers, which is consistent with the observed T5/T6 race. This causal link (duplicate-connection churn -> duplicate config-push delivery -> concurrent load_primary_config race) is a plausible hypothesis from timeline correlation, not confirmed by direct evidence of two inbound config-push messages -- flagged as a hypothesis for whoever investigates, not a proven mechanism.", - "fix": "FIXED (2026-07-28). Two-part fix in load_primary_config() (config_repl.c), independently reviewed and verified via a real pthread-based deterministic race test: (1) added myconf_t.lock (mylock_t, mirrors replicator_t.lock's idiom in repl.c) serializing the entire free-decode-publish sequence against any concurrent load_primary_config() call for the same myconf_t -- initialized in main.c before first use, finalized in myconf_fini(), and in the t43_attach_myconf/t43_free_myconf test fixture. (2) load_cluster_config_from_file() now decodes into a LOCAL temporary (cluster_config_t *tmp), never directly into the shared myconf->cluster_config field -- decode_ClusterConfig_ptr (lib/controller.capnp.c) calloc's and publishes its pointer via the out-parameter BEFORE decode_ClusterConfig populates nested fields (e.g. resources), so decoding into a local closes the early-publish exposure at its source; myconf->cluster_config is only swapped in after the new config is fully decoded and validated, still under myconf->lock. Part (2) is independently necessary because several other readers (cluster.c, queue.c, sched.c, worker.c, apiserver.c, ctrlstatus.c, reply.c) read c->myconf->cluster_config without ever taking myconf->lock, so the field itself must never be observably half-decoded. New regression test test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free (test_config_repl.c) drives two REAL concurrent pthreads through load_primary_config() for the same myconf_t via a deterministic test-only hook (load_primary_config_race_hook, XCE_TESTING-only, NULL/no-op in production) that parks the first thread exactly at the retire-old-config step the live crash raced -- pre-fix this reliably reproduces a genuine ASan-catchable double-free from the same missing-lock root cause; post-fix myconf->lock keeps the second thread blocked for the entire duration, verified both by a captured never-raced-through flag and by the final published config being fully valid, never torn. Full suite: 1035 -> 1036 tests, 0 failures.\n\n[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot resume] Prior fix landed with a comment block (config_repl.c:639-646) admitting it had been reverted to a RED-phase reproduction (no lock, decode-directly-into-shared-field) for test-construction purposes and NEVER restored to GREEN — the mylock_init/fini wiring (myconf.h, main.c, myconf.c) landed, but load_primary_config() itself still ran fully unguarded. Full suite run confirmed: test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free FAILED (\"thread B... Expected 0 Was 1\"), i.e. B raced through while A was parked -- the lock was never acquired. Fixed for real: mylock_lock(&myconf->lock) now wraps the ENTIRE load_primary_config() body (acquired first, released at every return including success), and Cap-n-Proto decode now targets a local cluster_config_t *new_config first -- myconf->cluster_config is only freed+swapped after new_config is fully decoded and validated (reserved-name check, generation default), never decoded directly into the shared field. Verified 4/4 clean full-suite runs (1036 Tests 0 Failures 0 Ignored, ASAN_OPTIONS=detect_leaks=0) after the fix, including the specific race test. Committed as part of config-upgrade-snapshot task 8.6/8.9 unblock resume.", + "fix": "FIXED (2026-07-28). Two-part fix in load_primary_config() (config_repl.c), independently reviewed and verified via a real pthread-based deterministic race test: (1) added myconf_t.lock (mylock_t, mirrors replicator_t.lock's idiom in repl.c) serializing the entire free-decode-publish sequence against any concurrent load_primary_config() call for the same myconf_t -- initialized in main.c before first use, finalized in myconf_fini(), and in the t43_attach_myconf/t43_free_myconf test fixture. (2) load_cluster_config_from_file() now decodes into a LOCAL temporary (cluster_config_t *tmp), never directly into the shared myconf->cluster_config field -- decode_ClusterConfig_ptr (lib/controller.capnp.c) calloc's and publishes its pointer via the out-parameter BEFORE decode_ClusterConfig populates nested fields (e.g. resources), so decoding into a local closes the early-publish exposure at its source; myconf->cluster_config is only swapped in after the new config is fully decoded and validated, still under myconf->lock. Part (2) is independently necessary because several other readers (cluster.c, queue.c, sched.c, worker.c, apiserver.c, ctrlstatus.c, reply.c) read c->myconf->cluster_config without ever taking myconf->lock, so the field itself must never be observably half-decoded. New regression test test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free (test_config_repl.c) drives two REAL concurrent pthreads through load_primary_config() for the same myconf_t via a deterministic test-only hook (load_primary_config_race_hook, XCE_TESTING-only, NULL/no-op in production) that parks the first thread exactly at the retire-old-config step the live crash raced -- pre-fix this reliably reproduces a genuine ASan-catchable double-free from the same missing-lock root cause; post-fix myconf->lock keeps the second thread blocked for the entire duration, verified both by a captured never-raced-through flag and by the final published config being fully valid, never torn. Full suite: 1035 -> 1036 tests, 0 failures.\n\n[UPDATE 2026-07-28, kiro-impl config-upgrade-snapshot resume] Prior fix landed with a comment block (config_repl.c:639-646) admitting it had been reverted to a RED-phase reproduction (no lock, decode-directly-into-shared-field) for test-construction purposes and NEVER restored to GREEN \u2014 the mylock_init/fini wiring (myconf.h, main.c, myconf.c) landed, but load_primary_config() itself still ran fully unguarded. Full suite run confirmed: test_bug1476_concurrent_load_primary_config_no_uaf_or_double_free FAILED (\"thread B... Expected 0 Was 1\"), i.e. B raced through while A was parked -- the lock was never acquired. Fixed for real: mylock_lock(&myconf->lock) now wraps the ENTIRE load_primary_config() body (acquired first, released at every return including success), and Cap-n-Proto decode now targets a local cluster_config_t *new_config first -- myconf->cluster_config is only freed+swapped after new_config is fully decoded and validated (reserved-name check, generation default), never decoded directly into the shared field. Verified 4/4 clean full-suite runs (1036 Tests 0 Failures 0 Ignored, ASAN_OPTIONS=detect_leaks=0) after the fix, including the specific race test. Committed as part of config-upgrade-snapshot task 8.6/8.9 unblock resume.", "tags": [ "config_repl.c", "load_primary_config", @@ -14202,9 +14202,9 @@ { "id": "bug-1479", "timestamp": "2026-07-28T13:25:00Z", - "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade2), run on top of bug-1477's fix (checkpoint-identity, config-identity, and host-admin-status-survival assertions ALL PASSED for the first time), FAILED later in the force-elect-rotation step: 5 consecutive job-submit attempts on {ctrl-1 (primary), ctrl-2 (standby)} all got 0/3 acked (\"waiting for write-quorum\"), then hard-failed. Investigated the ORIGINAL hypothesis (a leaked VTASK_LIVE_BLOCK/commit-admission slot from an earlier burst of 17 commit timeouts, lsn=15..31, rc=-110) and REFUTED it: commit_resolve()/wait_release_locked() correctly release the admission slot on every timeout (confirmed 17 'async continuation deadline expired' + 17 'commit failed/timed out' WARN pairs, 1:1, in ctrl-1.log) — no admission-pool leak exists. The REAL mechanism is unrelated to admission control entirely: a permanent Raft AppendEntries log-conflict wedge on standby ctrl-2, compounded by a NACK-handling gap that never triggers catch-up.", + "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade2), run on top of bug-1477's fix (checkpoint-identity, config-identity, and host-admin-status-survival assertions ALL PASSED for the first time), FAILED later in the force-elect-rotation step: 5 consecutive job-submit attempts on {ctrl-1 (primary), ctrl-2 (standby)} all got 0/3 acked (\"waiting for write-quorum\"), then hard-failed. Investigated the ORIGINAL hypothesis (a leaked VTASK_LIVE_BLOCK/commit-admission slot from an earlier burst of 17 commit timeouts, lsn=15..31, rc=-110) and REFUTED it: commit_resolve()/wait_release_locked() correctly release the admission slot on every timeout (confirmed 17 'async continuation deadline expired' + 17 'commit failed/timed out' WARN pairs, 1:1, in ctrl-1.log) \u2014 no admission-pool leak exists. The REAL mechanism is unrelated to admission control entirely: a permanent Raft AppendEntries log-conflict wedge on standby ctrl-2, compounded by a NACK-handling gap that never triggers catch-up.", "file": "src/controller/ha.c (repl_ack_decode_and_dispatch's WAL-ACK branch, ~6456-6644 -- the `!success` NACK path at ~6631-6642 only checks for a higher term and WARNs \"ignoring\", never calls repl_on_ack/repl_start_catchup/any retry); src/controller/repl.c (repl_broadcast ~295-380, the live single-entry send path -- fire-and-forget, no nextIndex backoff-and-retry on rejection; repl_on_ack ~250-271, the ONLY site that decrements pending_ack_count, never invoked on a NACK)", - "root_cause": "ROOT-CAUSED with byte-precise waldump evidence (xce_test's built-in `waldump` REPL command against the preserved deploy dir ~/tmp/xce-ha-configupgrade2), cross-correlated with both ctrl-1's and ctrl-2's logs line-by-line. During the soak's chaotic round-3 kill/restart churn, THREE leadership changes happened in rapid succession (term 3 -> 4 -> 5) within about 20 seconds (13:18:57-13:19:19): ctrl-2 briefly won term=3 and appended its own arm-own-term-gate NOOP, was killed before that entry could be widely replicated/committed, a short-lived term=4 leader (never directly observed in either surviving node's own become-leader lines, but proven to have existed by its WAL fingerprint) won next and wrote ITS OWN own-term NOOP at the SAME log index (14), and finally ctrl-1 won term=5 and became the lasting primary. `xce_test waldump` on ctrl-1's canonical (surviving, majority) WAL shows: lsn=12 CHECKPOINT term=2 (the marker-triggered checkpoint from config-upgrade-snapshot task 5, matching bug-1475/1477/1478's exact family), lsn=13 NOOP term=4 (the short-lived term-4 leader's noop), lsn=14 NOOP term=5 (ctrl-1's own winning noop). `xce_test waldump` on ctrl-2's own local pre-restart WAL shows: lsn=12 CHECKPOINT term=2 (agrees with ctrl-1), lsn=13 NOOP **term=3** (ctrl-2's OWN aborted, never-widely-replicated noop from its deposed term-3 stint) -- a genuine, CONFLICTING entry at the SAME index (13) with a DIFFERENT term (3 vs the canonical 4). This is an entirely NORMAL, expected Raft outcome of rapid leader churn (Raft's own §5.3 log-matching property explicitly anticipates followers accumulating stale, uncommitted tail entries from deposed leaders) -- ctrl-2 correctly rejects ctrl-1's AppendEntries at prevLogIndex=13 with logOk's standard conflicting-term check (confirmed via ha_consume_append_entries's own trace: `prevIdx=13 n=1 -> success=0 match=0`, `RAFT[p1] term=5 event=ae_reject_logmismatch`, repeating for EVERY subsequent new entry ctrl-1 tries to send: prevIdx=14,15,16,...29, one per newly-committed lsn, all rejected, spanning 13:19:19 through 13:19:52). The BUG is what happens (or rather, does not happen) next, confirmed via two independent, compounding gaps: (1) ctrl-1 NEVER backs off and retries with a DECREMENTED prevLogIndex to find the point of log agreement (the standard Raft nextIndex-conflict-resolution protocol, which would let ctrl-2 truncate its conflicting lsn=13 tail and adopt the canonical term=4 entry) -- repl_broadcast (repl.c:295-380) is a pure fire-and-forget per-new-entry send with zero synchronous handling of the reply; it always advances prevIdx forward to (next-new-lsn - 1), never backward. (2) repl_ack_decode_and_dispatch's WAL-ACK reply-dispatch branch in ha.c (~6631-6642) handles a NACK (success=0) by checking ONLY for a higher observed term (raft_on_higher_term_observed, correctly added for the DIFFERENT bug-1421 stale-term scenario) and then explicitly `WARN(\"...success=0 — ignoring\")` -- the comment says exactly what the code does: the rejection is discarded. Critically, `repl_on_ack` (repl.c:250-271) is the ONLY code path that decrements `standby_conn_t.pending_ack_count`, and it is called ONLY from the `success` branch -- so a standby that is stuck rejecting (for ANY reason) has pending_ack_count monotonically climb by 1 on every subsequent broadcast attempt with ZERO corresponding decrements, until repl_broadcast's own MAX_PENDING_ACKS=16 defensive eviction (repl.c:325-334, from ha-config-bootstrap task 7.2, commit 3969064e, 2026-05-28 -- a full month before config-upgrade-snapshot's first commit) fires and marks the standby in_catch_up=1 at 13:19:53 (confirmed: 'repl_broadcast: standby localhost:9992 evicted (pending_ack_count=17 > 16) — triggering catch-up reconnect'). This eviction correctly EXCLUDES the standby from repl_active_standby_count (hence from ha_has_write_quorum's majority calculation) -- but nothing in the codebase actually DRIVES a catch-up stream or snapshot-install for it afterward: ctrl-2's log shows ZERO entry-carrying AppendEntries for the remaining ~60 seconds of the run (13:19:53 through the log's end at 13:20:53) -- only bare heartbeat probes (prevIdx=13,n=0) that trivially succeed since a presence-only probe at the standby's own true log_len=13 doesn't hit the term-conflict check. With ctrl-2 permanently excluded (in_catch_up=1, never un-evicted) and ctrl-3 permanently failing to reconnect (separate, unrelated ECONNREFUSED loop, likely a real chaos-kill in this soak round), ctrl-1 has connected=0 standbys forever ('HA quorum lost: connected=0 required=1', logged once at the eviction and never restored) -- every subsequent job submission either times out waiting for write-quorum (0/3 acked, matching the force-elect step's failure verbatim) or gets an immediate -XCE_ERR_NOT_READY (-10013) fast-reject via ha_dispatch_gate's quorum check. This is a GENUINE, SERIOUS, and apparently PERMANENT availability defect: ANY standby whose log happens to diverge by even ONE entry at a single index during leader-election churn (an entirely normal, expected Raft occurrence, not itself a bug) can become irrecoverably stuck and permanently excluded from quorum, with no self-healing path -- in a 3-node cluster this is one bad-luck election sequence away from TOTAL, PERMANENT write-quorum loss if it hits 2 of 3 standbys. Same family as bug-1466 (open, 'WAL_CHECKPOINT breaks LSN/term co-tracking', NOT fixed at its root) and its now-3 fixed siblings (bug-1475, bug-1477, bug-1478) -- this is the family's 5th distinct member, but the trigger here (an ordinary post-election term-conflicted log tail, NOT specifically a checkpoint-boundary term-derivation issue) is narrower than bug-1466's checkpoint-specific framing and points instead at a genuinely MISSING piece of core Raft log-repair protocol (conflict-driven nextIndex backoff + NACK-triggered catch-up), which appears to have never been implemented for this replication path at all -- not a regression, a gap. git blame confirms both implicated mechanisms (repl_broadcast's pending_ack_count eviction, commit 3969064e, 2026-05-28; the NACK-handling 'ignoring' branch in repl_ack_decode_and_dispatch, present since at least bug-1421's fix, itself predating config-upgrade-snapshot) are pre-existing, core HA replication code untouched by any config-upgrade-snapshot task -- config-upgrade-snapshot's task 5 (marker-triggered per-member local checkpoint) merely adds one more entry into the WAL stream around the point where this pre-existing gap gets exercised; it does not create the gap itself.", + "root_cause": "ROOT-CAUSED with byte-precise waldump evidence (xce_test's built-in `waldump` REPL command against the preserved deploy dir ~/tmp/xce-ha-configupgrade2), cross-correlated with both ctrl-1's and ctrl-2's logs line-by-line. During the soak's chaotic round-3 kill/restart churn, THREE leadership changes happened in rapid succession (term 3 -> 4 -> 5) within about 20 seconds (13:18:57-13:19:19): ctrl-2 briefly won term=3 and appended its own arm-own-term-gate NOOP, was killed before that entry could be widely replicated/committed, a short-lived term=4 leader (never directly observed in either surviving node's own become-leader lines, but proven to have existed by its WAL fingerprint) won next and wrote ITS OWN own-term NOOP at the SAME log index (14), and finally ctrl-1 won term=5 and became the lasting primary. `xce_test waldump` on ctrl-1's canonical (surviving, majority) WAL shows: lsn=12 CHECKPOINT term=2 (the marker-triggered checkpoint from config-upgrade-snapshot task 5, matching bug-1475/1477/1478's exact family), lsn=13 NOOP term=4 (the short-lived term-4 leader's noop), lsn=14 NOOP term=5 (ctrl-1's own winning noop). `xce_test waldump` on ctrl-2's own local pre-restart WAL shows: lsn=12 CHECKPOINT term=2 (agrees with ctrl-1), lsn=13 NOOP **term=3** (ctrl-2's OWN aborted, never-widely-replicated noop from its deposed term-3 stint) -- a genuine, CONFLICTING entry at the SAME index (13) with a DIFFERENT term (3 vs the canonical 4). This is an entirely NORMAL, expected Raft outcome of rapid leader churn (Raft's own \u00a75.3 log-matching property explicitly anticipates followers accumulating stale, uncommitted tail entries from deposed leaders) -- ctrl-2 correctly rejects ctrl-1's AppendEntries at prevLogIndex=13 with logOk's standard conflicting-term check (confirmed via ha_consume_append_entries's own trace: `prevIdx=13 n=1 -> success=0 match=0`, `RAFT[p1] term=5 event=ae_reject_logmismatch`, repeating for EVERY subsequent new entry ctrl-1 tries to send: prevIdx=14,15,16,...29, one per newly-committed lsn, all rejected, spanning 13:19:19 through 13:19:52). The BUG is what happens (or rather, does not happen) next, confirmed via two independent, compounding gaps: (1) ctrl-1 NEVER backs off and retries with a DECREMENTED prevLogIndex to find the point of log agreement (the standard Raft nextIndex-conflict-resolution protocol, which would let ctrl-2 truncate its conflicting lsn=13 tail and adopt the canonical term=4 entry) -- repl_broadcast (repl.c:295-380) is a pure fire-and-forget per-new-entry send with zero synchronous handling of the reply; it always advances prevIdx forward to (next-new-lsn - 1), never backward. (2) repl_ack_decode_and_dispatch's WAL-ACK reply-dispatch branch in ha.c (~6631-6642) handles a NACK (success=0) by checking ONLY for a higher observed term (raft_on_higher_term_observed, correctly added for the DIFFERENT bug-1421 stale-term scenario) and then explicitly `WARN(\"...success=0 \u2014 ignoring\")` -- the comment says exactly what the code does: the rejection is discarded. Critically, `repl_on_ack` (repl.c:250-271) is the ONLY code path that decrements `standby_conn_t.pending_ack_count`, and it is called ONLY from the `success` branch -- so a standby that is stuck rejecting (for ANY reason) has pending_ack_count monotonically climb by 1 on every subsequent broadcast attempt with ZERO corresponding decrements, until repl_broadcast's own MAX_PENDING_ACKS=16 defensive eviction (repl.c:325-334, from ha-config-bootstrap task 7.2, commit 3969064e, 2026-05-28 -- a full month before config-upgrade-snapshot's first commit) fires and marks the standby in_catch_up=1 at 13:19:53 (confirmed: 'repl_broadcast: standby localhost:9992 evicted (pending_ack_count=17 > 16) \u2014 triggering catch-up reconnect'). This eviction correctly EXCLUDES the standby from repl_active_standby_count (hence from ha_has_write_quorum's majority calculation) -- but nothing in the codebase actually DRIVES a catch-up stream or snapshot-install for it afterward: ctrl-2's log shows ZERO entry-carrying AppendEntries for the remaining ~60 seconds of the run (13:19:53 through the log's end at 13:20:53) -- only bare heartbeat probes (prevIdx=13,n=0) that trivially succeed since a presence-only probe at the standby's own true log_len=13 doesn't hit the term-conflict check. With ctrl-2 permanently excluded (in_catch_up=1, never un-evicted) and ctrl-3 permanently failing to reconnect (separate, unrelated ECONNREFUSED loop, likely a real chaos-kill in this soak round), ctrl-1 has connected=0 standbys forever ('HA quorum lost: connected=0 required=1', logged once at the eviction and never restored) -- every subsequent job submission either times out waiting for write-quorum (0/3 acked, matching the force-elect step's failure verbatim) or gets an immediate -XCE_ERR_NOT_READY (-10013) fast-reject via ha_dispatch_gate's quorum check. This is a GENUINE, SERIOUS, and apparently PERMANENT availability defect: ANY standby whose log happens to diverge by even ONE entry at a single index during leader-election churn (an entirely normal, expected Raft occurrence, not itself a bug) can become irrecoverably stuck and permanently excluded from quorum, with no self-healing path -- in a 3-node cluster this is one bad-luck election sequence away from TOTAL, PERMANENT write-quorum loss if it hits 2 of 3 standbys. Same family as bug-1466 (open, 'WAL_CHECKPOINT breaks LSN/term co-tracking', NOT fixed at its root) and its now-3 fixed siblings (bug-1475, bug-1477, bug-1478) -- this is the family's 5th distinct member, but the trigger here (an ordinary post-election term-conflicted log tail, NOT specifically a checkpoint-boundary term-derivation issue) is narrower than bug-1466's checkpoint-specific framing and points instead at a genuinely MISSING piece of core Raft log-repair protocol (conflict-driven nextIndex backoff + NACK-triggered catch-up), which appears to have never been implemented for this replication path at all -- not a regression, a gap. git blame confirms both implicated mechanisms (repl_broadcast's pending_ack_count eviction, commit 3969064e, 2026-05-28; the NACK-handling 'ignoring' branch in repl_ack_decode_and_dispatch, present since at least bug-1421's fix, itself predating config-upgrade-snapshot) are pre-existing, core HA replication code untouched by any config-upgrade-snapshot task -- config-upgrade-snapshot's task 5 (marker-triggered per-member local checkpoint) merely adds one more entry into the WAL stream around the point where this pre-existing gap gets exercised; it does not create the gap itself.", "fix": "FIXED AND VERIFIED LIVE (2026-07-29, see UPDATE 2 below for the final closing fix) -- two compounding gaps were closed after the initial investigation: (1) the standby's own reject path sent no NACK reply at all (ha_send_wal_ack_reply hardcoded success=1 unconditionally), fixed via ha_send_wal_ack_reply_ex with success=0 on reject; (2) even with NACKs flowing, ha_stream_wal_to_standby's catch-up range was hard-capped at committed_lsn, which could never advance in this exact scenario (self-referential deadlock) -- fixed via ha_compute_repl_stream_ceiling_lsn widening the ceiling to the leader's own log tail, matching raft.tla's real AppendEntries semantics. Verified via three successive live run-ha-soak.sh --config-upgrade re-runs tracking each iteration (v1/v2 still failed, v3 with both fixes together PASSED end-to-end: 3/3 jobs acked on both force-elect rounds, zero materialized-state or config divergence). 1053/1053 controller_unit_test PASS. Full investigation history preserved below for context (including the earlier partial-fix iterations and root-causing).\n\n[Original investigation, 2026-07-28]\nNOT FIXED -- explicitly out of scope for a single safe pass, per this investigation's own conclusion (matches kiro-debug's guidance against forcing a fix when the true defect requires a protocol design decision, not a narrow patch). At least TWO separate, non-trivial design questions need to be resolved by whoever owns ha.c/repl.c's replication-catch-up protocol before a fix can be written safely: (a) should an ae_reject_logmismatch NACK immediately trigger repl_start_catchup (moving the standby to catch-up mode on the FIRST conflict, rather than silently accumulating up to 16 failed sends first)? This is the more targeted, root-cause-adjacent fix. (b) independently, once a standby IS marked in_catch_up (whether via this path or the existing MAX_PENDING_ACKS eviction), what mechanism is SUPPOSED to actually drive its catch-up (stream from join_applied_lsn, or snapshot-install if too far behind) and clear in_catch_up via repl_complete_catchup? Evidence here shows NOTHING does, for at least this eviction path -- config_repl.c's ACK-driven catchup_lane_fn/ha_walack_on_ack continuation (the only other repl_complete_catchup call site found) is itself ACK-driven, so it can never engage for a standby that never sends a genuine ACK in the first place; whether IT (or a periodic sweep independent of ACKs) is supposed to notice an in_catch_up standby and proactively start streaming it needs its own trace by the owning team. A correct fix likely needs both: NACK-triggered catch-up entry (closing gap (a)) AND a proactive (not purely ACK-reactive) catch-up driver that streams from join_applied_lsn or falls back to snapshot-install for any in_catch_up standby (closing gap (b)) -- implementing only one without the other would leave the defect only partially closed. Recommend routing to whoever owns ha.c/repl.c's HA replication/catch-up protocol (the same 'needs-owner' bucket as bug-1474's still-open fix angle (a) and bug-1472-remainder) for a proper design pass, THEN a TDD fix + live re-verification via `run-ha-soak.sh --config-upgrade` on a fresh deploy dir. Preserved evidence: ~/tmp/xce-ha-configupgrade2 (do not delete) -- `xce_test waldump ` (via the xce_test REPL: `printf 'waldump \\nquit\\n' | ASAN_OPTIONS=detect_leaks=0 timeout 60 /controller/xce_test`) reproduces the byte-precise lsn=13 term-3-vs-term-4 conflict evidence directly from the preserved WAL segments at any time.\n\nUPDATE (2026-07-29, this session): a fix WAS committed (uncommitted in the 'config' worktree at investigation time) implementing exactly gap (a) from this entry's own recommendation -- ha_raft_on_repl_nack() (ha.c/ha.h) routes a WAL-ACK-reply NACK into the raft library's nextIndex backoff and returns a corrected resume LSN; repl_ack_decode_and_dispatch's NACK branch calls it and immediately repl_start_catchup() with the corrected LSN; repl_get_catchup_join_lsn() (repl.c/repl.h) lets the heartbeat-driven ha_dispatch_repl_action_for_standby use the corrected position; repl_complete_catchup() now also resets pending_ack_count. 1048/1048 controller_unit_test PASS. HOWEVER: a fresh live-soak re-verification (run-ha-soak.sh --config-upgrade, deploy preserved at ~/tmp/xce_ha_bug1479_verify, NOT deleted) reproduced the ORIGINAL symptom verbatim -- force-elect ctrl-2 round: 5/5 submit_jobs attempts on {ctrl-1,ctrl-2} got 0/3 acked, hard FAIL. Root-caused: the fix's NACK-routing branch in repl_ack_decode_and_dispatch (ha.c ~6680-6720) is CORRECT but UNREACHABLE in production, because the standby's OWN reject path never sends a NACK (or any reply) in the first place. ha_apply_wal_catchup_entry has three reject paths (ha.c ~3230-3232 via ha_apply_term_entry_via_library for the term-carrying single/batch case -- the one actually hit live; ~3320-3321 for a batch sub-entry apply failure; ~3373-3374 for the legacy single-entry path) that all explicitly do 'no ACK' on a logOk reject/forward-gap. ha_send_wal_ack_reply (ha.c:3084-3106) is the ONLY function anywhere in the controller that ever constructs a WalAckReply, and it hardcodes success=1 unconditionally -- there is NO production code path that ever sends success=0. Live evidence: ctrl-2's own log shows repeating RAFT[p1] event=ae_reject_logmismatch / ha_consume_append_entries success=0 match=0 at growing prevIdx (40,41,...54) while stuck at match=20 forever; ctrl-1's log has ZERO 'WAL NACK from standby' or ha_raft_on_repl_nack lines the entire time -- the primary never learns of the rejection at all, so it never backs off nextIndex or re-arms catch-up, and committed_lsn stays frozen (the only alive standby can never re-sync, and the heartbeat-driven catch-up's own target is gated on the same frozen committed_lsn). STILL NOT FULLY FIXED. New follow-up spec .kiro/specs/raft-adapter-conformance/ created to (1) fix this concrete missing-NACK-reply defect at the three reject sites and (2) perform a broader raft.tla action-by-action conformance pass over the controller's Raft adapter (ha.c/repl.c/config_repl.c) -- distinct from raft-sim-conformance, which is scoped to src/lib/raft/raft_sim.c (library-only, in-process) and explicitly does not cover this adapter code.\n\nUPDATE 2 (2026-07-29, this session): fully FIXED and verified live end-to-end. Two more compounding gaps found and closed after the NACK-send fix (UPDATE 1) landed:\n(1) ha_apply_term_entry_via_library's logOk-reject path (ha.c, the single live-reachable reject site) sent NO reply at all on rejection -- ha_send_wal_ack_reply hardcoded success=1 unconditionally, so the primary-side ha_raft_on_repl_nack (correct since UPDATE 1) was unreachable dead code. Fixed by adding ha_send_wal_ack_reply_ex(ha, ep, c, lsn, success) and calling it with success=0 on reject, plus a new ha_wal_ack_send_hook (ha.h, XCE_TESTING) so a unit test can observe the NACK without a live endpoint (test_bug1479_ae_reject_sends_nack_not_silence, test_config_repl.c). Confirmed live in a 2nd soak run: ctrl-1's log showed real 'WAL NACK from standby ... success=0 -- marking catch-up' lines for the first time.\n(2) Even with (1) working, that 2nd soak run STILL failed at the identical force-elect ctrl-2 point: ha_stream_wal_to_standby's catch-up range was hard-bounded at committed_lsn, and committed_lsn can never advance in this exact scenario because advancing it requires an ACK from the very standby that's stuck catching up (2-of-3 alive, this standby is the only path to a new majority) -- a self-referential deadlock. Fixed by adding a new pure function ha_compute_repl_stream_ceiling_lsn(committed_lsn, last_written, checkpoint_lsn) (ha.c/ha.h), wired into ha_dispatch_repl_action_for_standby to widen the STREAM_WAL ceiling (and the ha_evaluate_repl_action convergence check that gates it) to the leader's own log tail instead of just committed_lsn -- matching raft.tla's real AppendEntries semantics (leaderCommit is advisory to the follower, never a cap on what gets sent) -- while still guarding against bug-1478's old trap (never widening to a trailing local-only checkpoint's own unstreamable LSN slot). 4 new pure-function tests in test_ha_role.c (test_compute_repl_stream_ceiling_lsn_*), RED-confirmed against a stubbed committed_lsn-only version before the real fix was restored.\nFull verification: 1053/1053 controller_unit_test PASS twice clean. THREE successive live run-ha-soak.sh --config-upgrade re-runs tracked each fix: v1 (NACK-send only) still FAILED at force-elect ctrl-2 (0/3 acked); v2 confirmed NACKs now reaching the primary but still FAILED (frozen committed_lsn=14, ha_stream_wal_to_standby target never advanced); v3 (both fixes together) PASSED end-to-end: all 3 force-elect rounds succeeded (ctrl-2: 3/3 jobs acked, term=6; ctrl-3: 3/3 jobs acked, term=7), final verdict 'PASS (config-upgrade): 12 jobs, zero materialized-state divergence, zero configuration divergence, host admin-status confirmed across all members'. Evidence preserved at ~/tmp/xce_ha_bug1479_verify, ~/tmp/xce_ha_bug1479_verify_v2, ~/tmp/xce_ha_bug1479_verify_v3 (all not deleted). config-upgrade-snapshot task 8.6 unblocked. The raft-adapter-conformance spec created mid-investigation had its concrete seed defect (missing NACK reply) fixed here; the broader raft.tla action-by-action conformance audit it also scopes remains valid, separate future work.", "tags": [ "ha.c", @@ -14905,7 +14905,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/design.md", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3→4 lines (2 removed)", + "fix": "Rewrote 3\u21924 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -14921,7 +14921,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/requirements.md", "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 21→7 lines (8 removed)", + "fix": "Rewrote 21\u21927 lines (8 removed)", "tags": [ "auto-detected", "refactor", @@ -14937,7 +14937,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/tasks.md", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6→6 lines (2 removed)", + "fix": "Rewrote 6\u21926 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -14995,7 +14995,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/testsuite/meson.build", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5→12 lines (2 removed)", + "fix": "Rewrote 5\u219212 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15011,7 +15011,7 @@ "error_message": "Wrong reference: true should be false", "file": "src/scheduler/src/scheduler.c", "root_cause": "Used \"true\" instead of \"false\"", - "fix": "Changed true → false", + "fix": "Changed true \u2192 false", "tags": [ "auto-detected", "wrong-reference", @@ -15027,7 +15027,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7→11 lines (2 removed) | Also: ERROR(\"%s: xcecli_join() failed across %d candidat; __func__, sched->cluster_cfg.candidates_count, rc)", + "fix": "Rewrote 7\u219211 lines (2 removed) | Also: ERROR(\"%s: xcecli_join() failed across %d candidat; __func__, sched->cluster_cfg.candidates_count, rc)", "tags": [ "auto-detected", "refactor", @@ -15064,7 +15064,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 12→10 lines (3 removed)", + "fix": "Rewrote 12\u219210 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15080,7 +15080,7 @@ "error_message": "Wrong reference: test_sched_init_no_peers_default_also_fails_cleanly should be test_all_candidates_unreachable_log_names_every_candidate_host_port", "file": "src/scheduler/testsuite/test_scheduler_connect.c", "root_cause": "Used \"test_sched_init_no_peers_default_also_fails_cleanly\" instead of \"test_all_candidates_unreachable_log_names_every_candidate_host_port\"", - "fix": "Changed test_sched_init_no_peers_default_also_fails_cleanly → test_all_candidates_unreachable_log_names_every_candidate_host_port", + "fix": "Changed test_sched_init_no_peers_default_also_fails_cleanly \u2192 test_all_candidates_unreachable_log_names_every_candidate_host_port", "tags": [ "auto-detected", "wrong-reference", @@ -15096,7 +15096,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/testsuite/test_scheduler_connect.c", "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11→21 lines (4 removed)", + "fix": "Rewrote 11\u219221 lines (4 removed)", "tags": [ "auto-detected", "refactor", @@ -15112,7 +15112,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/testsuite/test_scheduler_decision_redelivery.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6→7 lines (2 removed) | Also: * Fixtures call xce_sched_core_init()/xce_sched_co; * the full sched_init()) -- the same minimal-fixtu", + "fix": "Rewrote 6\u21927 lines (2 removed) | Also: * Fixtures call xce_sched_core_init()/xce_sched_co; * the full sched_init()) -- the same minimal-fixtu", "tags": [ "auto-detected", "refactor", @@ -15128,7 +15128,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 26→27 lines (5 removed) | Also: /* TEMPORARY RED-PHASE PROBE #2 (task 3.4 evidence; * immediately after use) -- core.stop checked only", + "fix": "Rewrote 26\u219227 lines (5 removed) | Also: /* TEMPORARY RED-PHASE PROBE #2 (task 3.4 evidence; * immediately after use) -- core.stop checked only", "tags": [ "auto-detected", "refactor", @@ -15144,7 +15144,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", "root_cause": "15 lines replaced/restructured", - "fix": "Rewrote 27→11 lines (15 removed)", + "fix": "Rewrote 27\u219211 lines (15 removed)", "tags": [ "auto-detected", "refactor", @@ -15159,7 +15159,7 @@ "timestamp": "2026-08-11T23:49:56.443Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block", "tags": [ "auto-detected", @@ -15176,7 +15176,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 21→26 lines (6 removed)", + "fix": "Rewrote 21\u219226 lines (6 removed)", "tags": [ "auto-detected", "refactor", @@ -15192,7 +15192,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/config.go", "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 5→8 lines (4 removed)", + "fix": "Rewrote 5\u21928 lines (4 removed)", "tags": [ "auto-detected", "refactor", @@ -15208,7 +15208,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/candidates.go", "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 40→74 lines (13 removed)", + "fix": "Rewrote 40\u219274 lines (13 removed)", "tags": [ "auto-detected", "refactor", @@ -15240,7 +15240,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", "root_cause": "40 lines replaced/restructured", - "fix": "Rewrote 83→229 lines (40 removed)", + "fix": "Rewrote 83\u2192229 lines (40 removed)", "tags": [ "auto-detected", "refactor", @@ -15275,7 +15275,7 @@ "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", "root_cause": "Had `scheduler.c`", - "fix": "Changed to 's two connect mechanisms into one — the deleted `", + "fix": "Changed to 's two connect mechanisms into one \u2014 the deleted `", "tags": [ "auto-detected", "wrong-value", @@ -15291,7 +15291,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/reconnect_test.go", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 9→14 lines (3 removed)", + "fix": "Rewrote 9\u219214 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15323,7 +15323,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/heartbeat.go", "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 17→62 lines (4 removed)", + "fix": "Rewrote 17\u219262 lines (4 removed)", "tags": [ "auto-detected", "refactor", @@ -15339,7 +15339,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7→14 lines (3 removed)", + "fix": "Rewrote 7\u219214 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15355,7 +15355,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/heartbeat_test.go", "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 48→214 lines (4 removed)", + "fix": "Rewrote 48\u2192214 lines (4 removed)", "tags": [ "auto-detected", "refactor", @@ -15387,7 +15387,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 23→95 lines (2 removed)", + "fix": "Rewrote 23\u219295 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15403,7 +15403,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/reconnect.go", "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 16→26 lines (4 removed)", + "fix": "Rewrote 16\u219226 lines (4 removed)", "tags": [ "auto-detected", "refactor", @@ -15436,7 +15436,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/reconnect_test.go", "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 17→30 lines (8 removed)", + "fix": "Rewrote 17\u219230 lines (8 removed)", "tags": [ "auto-detected", "refactor", @@ -15484,7 +15484,7 @@ "error_message": "Significant refactor of ", "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 11→13 lines (3 removed)", + "fix": "Rewrote 11\u219213 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15611,7 +15611,7 @@ "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 8→14 lines (7 removed)", + "fix": "Rewrote 8\u219214 lines (7 removed)", "tags": [ "auto-detected", "refactor", @@ -15627,7 +15627,7 @@ "error_message": "Significant refactor of ", "file": "src/xceadmin/tests/test_mapping.py", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3→5 lines (3 removed)", + "fix": "Rewrote 3\u21925 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15642,7 +15642,7 @@ "timestamp": "2026-08-13T07:25:51.662Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_worker_join.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block", "tags": [ "auto-detected", @@ -15659,7 +15659,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/worker.c", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 49→67 lines (3 removed)", + "fix": "Rewrote 49\u219267 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15713,7 +15713,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 30→41 lines (2 removed)", + "fix": "Rewrote 30\u219241 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15765,7 +15765,7 @@ "error_message": "Significant refactor of ", "file": "src/schema/controller.capnp", "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 14→3 lines (11 removed) | Also: # Display-unit hint copied from the resource's own; # at reply-build time -- empty/unset means \"raw nu", + "fix": "Rewrote 14\u21923 lines (11 removed) | Also: # Display-unit hint copied from the resource's own; # at reply-build time -- empty/unset means \"raw nu", "tags": [ "auto-detected", "refactor", @@ -15781,7 +15781,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/opget.c", "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 20→9 lines (7 removed)", + "fix": "Rewrote 20\u21929 lines (7 removed)", "tags": [ "auto-detected", "refactor", @@ -15813,7 +15813,7 @@ "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5→3 lines (2 removed)", + "fix": "Rewrote 5\u21923 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15829,7 +15829,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test.c", "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 8→37 lines (5 removed)", + "fix": "Rewrote 8\u219237 lines (5 removed)", "tags": [ "auto-detected", "refactor", @@ -15845,7 +15845,7 @@ "error_message": "Significant refactor of ", "file": "src/worker/src/xcecd.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3→7 lines (2 removed)", + "fix": "Rewrote 3\u21927 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15877,7 +15877,7 @@ "error_message": "Significant refactor of ", "file": "src/configs/resources.yaml", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7→7 lines (2 removed)", + "fix": "Rewrote 7\u21927 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15893,7 +15893,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test.c", "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 12→16 lines (6 removed) | Also: /* MaxMem is stored/wired as raw bytes (xce_resour; * for the maxmem resource definition) -- humanize ", + "fix": "Rewrote 12\u219216 lines (6 removed) | Also: /* MaxMem is stored/wired as raw bytes (xce_resour; * for the maxmem resource definition) -- humanize ", "tags": [ "auto-detected", "refactor", @@ -15909,7 +15909,7 @@ "error_message": "Significant refactor of ", "file": "src/configs/schema/hosts.schema.json", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10→17 lines (2 removed)", + "fix": "Rewrote 10\u219217 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15925,7 +15925,7 @@ "error_message": "Significant refactor of ", "file": "src/configs/schema/queues.schema.json", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10→17 lines (2 removed)", + "fix": "Rewrote 10\u219217 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -15941,7 +15941,7 @@ "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 16→58 lines (3 removed)", + "fix": "Rewrote 16\u219258 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -15957,7 +15957,7 @@ "error_message": "Significant refactor of ", "file": "src/tools/deploy-ha.sh", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 21→34 lines (2 removed) | Also: # NOT retroactively update an already-built snapsh; # priority order (as this used to be) silently pic", + "fix": "Rewrote 21\u219234 lines (2 removed) | Also: # NOT retroactively update an already-built snapsh; # priority order (as this used to be) silently pic", "tags": [ "auto-detected", "refactor", @@ -16008,7 +16008,7 @@ "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3→5 lines (3 removed)", + "fix": "Rewrote 3\u21925 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -16024,7 +16024,7 @@ "error_message": "Wrong reference: write_output should be dump_cluster_config", "file": "src/xceadmin/src/xceadmin/cli.py", "root_cause": "Used \"write_output\" instead of \"dump_cluster_config\"", - "fix": "Changed write_output → dump_cluster_config", + "fix": "Changed write_output \u2192 dump_cluster_config", "tags": [ "auto-detected", "wrong-reference", @@ -16087,7 +16087,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 14→30 lines (6 removed)", + "fix": "Rewrote 14\u219230 lines (6 removed)", "tags": [ "auto-detected", "refactor", @@ -16103,7 +16103,7 @@ "error_message": "Significant refactor of ", "file": "src/frontend/Makefile", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3→6 lines (3 removed)", + "fix": "Rewrote 3\u21926 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -16119,7 +16119,7 @@ "error_message": "Wrong reference: 039 should be 1548", "file": ".kiro/specs/component-failover-adoption/tasks.md", "root_cause": "Used \"039\" instead of \"1548\"", - "fix": "Changed 039 → 1548", + "fix": "Changed 039 \u2192 1548", "tags": [ "auto-detected", "wrong-reference", @@ -16151,7 +16151,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/spec.json", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 20→30 lines (2 removed)", + "fix": "Rewrote 20\u219230 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -16166,7 +16166,7 @@ "timestamp": "2026-08-15T14:20:32.090Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/component-failover-adoption/tasks.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block", "tags": [ "auto-detected", @@ -16199,7 +16199,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/spec.json", "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 11→17 lines (5 removed)", + "fix": "Rewrote 11\u219217 lines (5 removed)", "tags": [ "auto-detected", "refactor", @@ -16239,7 +16239,7 @@ "timestamp": "2026-08-16T03:42:14.435Z", "error_message": "Missing error handling in unknown", "file": "reports/aug08-aug15-2026.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block", "tags": [ "auto-detected", @@ -16272,7 +16272,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/config-upgrade-snapshot/spec.json", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 21→29 lines (3 removed) | Also: \"updated_at\": \"2026-07-29T09:20:00Z\",; \"phase\": \"tasks-generated\",", + "fix": "Rewrote 21\u219229 lines (3 removed) | Also: \"updated_at\": \"2026-07-29T09:20:00Z\",; \"phase\": \"tasks-generated\",", "tags": [ "auto-detected", "refactor", @@ -16288,7 +16288,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test_commit_barrier_reply.c", "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 11→16 lines (11 removed)", + "fix": "Rewrote 11\u219216 lines (11 removed)", "tags": [ "auto-detected", "refactor", @@ -16345,7 +16345,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/persist.c", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 37→86 lines (3 removed)", + "fix": "Rewrote 37\u219286 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -16360,7 +16360,7 @@ "timestamp": "2026-08-16T05:22:23.386Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block", "tags": [ "auto-detected", @@ -16377,7 +16377,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/persist.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6→17 lines (2 removed)", + "fix": "Rewrote 6\u219217 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -16409,7 +16409,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/worker.c", "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 12→23 lines (9 removed)", + "fix": "Rewrote 12\u219223 lines (9 removed)", "tags": [ "auto-detected", "refactor", @@ -16425,7 +16425,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/worker.c", "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 15→8 lines (7 removed)", + "fix": "Rewrote 15\u21928 lines (7 removed)", "tags": [ "auto-detected", "refactor", @@ -16440,7 +16440,7 @@ "timestamp": "2026-08-16T11:12:53.822Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/admin-config-tool/tasks.md", - "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", "fix": "Added try/catch block", "tags": [ "auto-detected", @@ -16473,7 +16473,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/specs/resval-id-decoupling/spec.json", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 21→22 lines (3 removed)", + "fix": "Rewrote 21\u219222 lines (3 removed)", "tags": [ "auto-detected", "refactor", @@ -16511,7 +16511,7 @@ "error_message": "Significant refactor of ", "file": "src/controller/ha.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6→8 lines (2 removed)", + "fix": "Rewrote 6\u21928 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -16582,7 +16582,7 @@ "error_message": "Significant refactor of ", "file": "src/tools/deploy-ha.sh", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 40→93 lines (2 removed)", + "fix": "Rewrote 40\u219293 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -16598,7 +16598,7 @@ "error_message": "Significant refactor of ", "file": ".claude/skills/xce-deploy-ha/SKILL.md", "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 65→132 lines (7 removed) | Also: # 1. Stop running controllers; bash \"$TOP/scripts/cleanup.sh\"", + "fix": "Rewrote 65\u2192132 lines (7 removed) | Also: # 1. Stop running controllers; bash \"$TOP/scripts/cleanup.sh\"", "tags": [ "auto-detected", "refactor", @@ -16614,7 +16614,7 @@ "error_message": "Significant refactor of ", "file": ".claude/skills/xce-build/SKILL.md", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3→26 lines (2 removed) | Also: does not need to run from inside this repo or the ; **not** bundle its dependencies (`click`, `pycapnp | Also: ## Installing the xceadmin wheel elsewhere; `/xce-build xceadmin` always leaves a fresh, teste", + "fix": "Rewrote 3\u219226 lines (2 removed) | Also: does not need to run from inside this repo or the ; **not** bundle its dependencies (`click`, `pycapnp | Also: ## Installing the xceadmin wheel elsewhere; `/xce-build xceadmin` always leaves a fresh, teste", "tags": [ "auto-detected", "refactor", @@ -16630,7 +16630,7 @@ "error_message": "Significant refactor of ", "file": "src/xceadmin/pyproject.toml", "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 6→9 lines (5 removed)", + "fix": "Rewrote 6\u21929 lines (5 removed)", "tags": [ "auto-detected", "refactor", @@ -16646,7 +16646,7 @@ "error_message": "Significant refactor of ", "file": "src/tools/deploy-ha.sh", "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 22→44 lines (13 removed)", + "fix": "Rewrote 22\u219244 lines (13 removed)", "tags": [ "auto-detected", "refactor", @@ -16694,7 +16694,7 @@ "error_message": "Wrong reference: fire_count should be g_durable_advance_fire_count", "file": "src/controller/testsuite/test_commit_durability.c", "root_cause": "Used \"fire_count\" instead of \"g_durable_advance_fire_count\"", - "fix": "Changed fire_count → g_durable_advance_fire_count", + "fix": "Changed fire_count \u2192 g_durable_advance_fire_count", "tags": [ "auto-detected", "wrong-reference", @@ -16766,7 +16766,7 @@ "error_message": "Significant refactor of ", "file": ".kiro/steering/build-deployment.md", "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 32→48 lines (7 removed)", + "fix": "Rewrote 32\u219248 lines (7 removed)", "tags": [ "auto-detected", "refactor", @@ -16782,7 +16782,7 @@ "error_message": "Significant refactor of ", "file": ".claude/skills/xce-build/SKILL.md", "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11→12 lines (4 removed)", + "fix": "Rewrote 11\u219212 lines (4 removed)", "tags": [ "auto-detected", "refactor", @@ -16798,7 +16798,7 @@ "error_message": "Wrong operator: && should be <<", "file": ".claude/skills/xce-build/SKILL.md", "root_cause": "Used \"&&\" instead of \"<<\"", - "fix": "Changed operator && → <<", + "fix": "Changed operator && \u2192 <<", "tags": [ "auto-detected", "operator-fix", @@ -16913,7 +16913,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/sim_loop.c", "root_cause": "47 lines replaced/restructured", - "fix": "Rewrote 52→12 lines (47 removed)", + "fix": "Rewrote 52\u219212 lines (47 removed)", "tags": [ "auto-detected", "refactor", @@ -16929,7 +16929,7 @@ "error_message": "Wrong reference: sim_register_default_picker should be sim_bootstrap_picker", "file": "src/scheduler/src/sim_loop.c", "root_cause": "Used \"sim_register_default_picker\" instead of \"sim_bootstrap_picker\"", - "fix": "Changed sim_register_default_picker → sim_bootstrap_picker", + "fix": "Changed sim_register_default_picker \u2192 sim_bootstrap_picker", "tags": [ "auto-detected", "wrong-reference", @@ -16945,7 +16945,7 @@ "error_message": "Wrong reference: test_job_rusage_applies_state_when_hook_is_null should be test_workers_push_twice_updates_existing_worker_in_place", "file": "src/scheduler/testsuite/test_core_events.c", "root_cause": "Used \"test_job_rusage_applies_state_when_hook_is_null\" instead of \"test_workers_push_twice_updates_existing_worker_in_place\"", - "fix": "Changed test_job_rusage_applies_state_when_hook_is_null → test_workers_push_twice_updates_existing_worker_in_place", + "fix": "Changed test_job_rusage_applies_state_when_hook_is_null \u2192 test_workers_push_twice_updates_existing_worker_in_place", "tags": [ "auto-detected", "wrong-reference", @@ -16961,7 +16961,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/core_events.c", "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 74→151 lines (3 removed) | Also: xce_sched_worker_t* worker; = core_build_worker(core, ev->u.workers_push.worke | Also: * WORKERS_PUSH core_build_worker() + sched_; * No picker hook (Req 1 AC 2:", + "fix": "Rewrote 74\u2192151 lines (3 removed) | Also: xce_sched_worker_t* worker; = core_build_worker(core, ev->u.workers_push.worke | Also: * WORKERS_PUSH core_build_worker() + sched_; * No picker hook (Req 1 AC 2:", "tags": [ "auto-detected", "refactor", @@ -16977,7 +16977,7 @@ "error_message": "Significant refactor of ", "file": "src/scheduler/src/repl_info.c", "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4→9 lines (2 removed)", + "fix": "Rewrote 4\u21929 lines (2 removed)", "tags": [ "auto-detected", "refactor", @@ -18463,7 +18463,7 @@ { "id": "bug-1673", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "Gitea issue #40 (\"job 调度不正确\"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined worker even though the host's real NCPUS (e.g. 16 or 22) is correctly reported and scheduling correctly dispatches up to that many concurrent jobs. Reproduced live on branch scheduler-framework: single-node deploy, worker joined as 'worker-1', `xcecli hosts list` -> NCPUS=22 MAX_SLOT=0. Submitted 9 single-slot sleep jobs -- all 9 eventually dispatched and ran (no hard cap observed under this test), so the reporter's secondary claim of a hard ~8-job concurrency ceiling did NOT reproduce here (most likely a mid-dispatch snapshot, dispatch is not instantaneous across all pending jobs on this branch; a queue/usergroup slot limit in limits.yaml, e.g. ug1_normal_slots=8, remains a plausible alternate explanation worth asking the original reporter about). Only the MAX_SLOT=0 display value itself is a confirmed, reproducible bug.", + "error_message": "Gitea issue #40 (\"job \u8c03\u5ea6\u4e0d\u6b63\u786e\"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined worker even though the host's real NCPUS (e.g. 16 or 22) is correctly reported and scheduling correctly dispatches up to that many concurrent jobs. Reproduced live on branch scheduler-framework: single-node deploy, worker joined as 'worker-1', `xcecli hosts list` -> NCPUS=22 MAX_SLOT=0. Submitted 9 single-slot sleep jobs -- all 9 eventually dispatched and ran (no hard cap observed under this test), so the reporter's secondary claim of a hard ~8-job concurrency ceiling did NOT reproduce here (most likely a mid-dispatch snapshot, dispatch is not instantaneous across all pending jobs on this branch; a queue/usergroup slot limit in limits.yaml, e.g. ug1_normal_slots=8, remains a plausible alternate explanation worth asking the original reporter about). Only the MAX_SLOT=0 display value itself is a confirmed, reproducible bug.", "file": "src/frontend/apiserver/backend/handler.go, src/configs/hosts.yaml", "root_cause": "OnHostConfigUpdated() (handler.go:60-70) only applies config-pushed fields (including MaxSlot) onto a host entry via `store.Hosts.Get(incoming.HostName)` -- an EXACT hostname match. hosts.yaml commonly defines only a wildcard `hostname: default` entry (plus a couple of literally-named ones, e.g. worker2/host3) -- confirmed via grep, no entry named after the actual dynamically-joined worker (e.g. 'worker-1'). So a worker that joins under its own hostname never receives a HostConfig push under that literal name, `store.Hosts.Get()` misses, OnHostConfigUpdated returns early, and MaxSlot is left at Go's zero-value (0) forever -- completely decoupled from the value actually used for scheduling. The C-side path is correct: worker.c's apply_join_state()/find_matched_hostconfig() falls back to the 'default' wildcard entry's mxj/maxcpus for real dispatch decisions, confirmed via a live controller log line showing the correct slots=22 resource value. This is purely a Go apiserver display gap, not a scheduling defect.", "fix": "FIXED (2026-08-24). src/frontend/apiserver/backend/handler.go -- OnHostConfigUpdated now caches the \"default\" pseudo-host's config-sourced fields (defaultHostConfig, package-level, mutex-guarded) instead of discarding them, and retroactively backfills that fallback onto any already-registered host with no explicit (non-default) config of its own; OnHostUpdated applies the same fallback immediately on a host's first-ever registration. The HOSTCONFIG_MXJ_AUTO_DETECT -1 sentinel (bug-1556) is further resolved to the host's real NumCPUs when applied via fallback, mirroring worker.c's C-side `if (nslots<0) nslots = maxcpus`. A host with its own real per-host hosts.yaml entry is never overridden. Added resetHostConfigCache() test hook plus 5 new tests (both arrival orderings, no-override-of-explicit-config, unresolvable-stays-negative) -- all pass, plus all 3 pre-existing OnHostConfigUpdated/OnHostUpdated tests pass unmodified (`make test-backend`). Live-verified: `xcecli hosts list` for the dynamically-joined worker-1 (no hosts.yaml entry) now shows MAX_SLOT=22, matching NCPUS=22, instead of 0.", @@ -18508,7 +18508,7 @@ { "id": "bug-1675", "timestamp": "2026-08-24T13:40:00Z", - "error_message": "Gitea issue #42 (\"刚启动的时候查询作业报的错误不太友好\"): `xcecli jobs list` run in the narrow window right after the apiserver (re)starts, before its listener is up, prints a raw `get the request failed: [connection refused]` instead of a user-friendly message. Reproduced live on branch scheduler-framework: killed and relaunched the apiserver, hammered `xcecli jobs list` in a tight loop from the instant of relaunch -- attempts 1-13 all returned the exact reported text; the listener came up by attempt 14.", + "error_message": "Gitea issue #42 (\"\u521a\u542f\u52a8\u7684\u65f6\u5019\u67e5\u8be2\u4f5c\u4e1a\u62a5\u7684\u9519\u8bef\u4e0d\u592a\u53cb\u597d\"): `xcecli jobs list` run in the narrow window right after the apiserver (re)starts, before its listener is up, prints a raw `get the request failed: [connection refused]` instead of a user-friendly message. Reproduced live on branch scheduler-framework: killed and relaunched the apiserver, hammered `xcecli jobs list` in a tight loop from the instant of relaunch -- attempts 1-13 all returned the exact reported text; the listener came up by attempt 14.", "file": "src/frontend/xcecli/common/request/request.go:97-98, src/frontend/xcecli/common/request/request.go:185-188", "root_cause": "ExtractErrorMessage() (request.go:185-188) explicitly special-cases the connection-refused case (`if strings.Contains(errStr, \"connection refused\") { return \"connection refused\" }`) but returns the raw OS-level string verbatim instead of a friendlier message (e.g. explaining the API server may still be starting up). GetRequestFromAPIServer() (request.go:97-98) then wraps it as `\"get the request failed: [%s]\"`, producing exactly the reported output.", "fix": "FIXED (2026-08-24). src/frontend/xcecli/common/request/request.go:187 -- ExtractErrorMessage()'s connection-refused branch now returns \"cannot reach the API server -- it may still be starting up, retry shortly\" instead of the raw \"connection refused\" string. Confirmed no other call site string-matches the old literal. Added TestExtractErrorMessage (request_test.go, table-driven) -- RED before, GREEN after. Full xcecli suite passes. Live-verified: killed+relaunched the apiserver, hit `xcecli jobs list` during the startup window -- new message confirmed both via the fresh test and independently by rebuilding xcecli and re-checking.", @@ -18570,7 +18570,7 @@ { "id": "bug-1678", "timestamp": "2026-08-24T13:55:00Z", - "error_message": "Gitea issue #43 (\"重启daemon后作业调度/信息不对\"): after restarting ALL daemons (controller, scheduler, worker, apiserver), previously-RUN jobs come back with STATUS=RUN correctly recovered but EXEC_HOST permanently blank (\"-\"), even though the underlying job is genuinely still running. CONFIRMED LIVE on branch scheduler-framework, corrected finding: an earlier verification pass on this same issue (2026-08-24, same session) restarted only controller+scheduler+worker and left the apiserver running, and did NOT reproduce the symptom -- that was a false negative, caused by the still-running apiserver simply continuing to display its stale-but-correct in-memory job cache from before the restart, never refreshed. Repeating the test with the apiserver ALSO restarted (matching the original report's \"restart all daemons\" exactly) reproduces the bug immediately and durably: 3 RUN jobs with EXEC_HOST=1*worker-1 before the restart; killed all 4 daemons; ran start-all.sh; EXEC_HOST was blank on every poll (15 polls over 30s, first poll ~6s after apiserver came back up) while STATUS stayed correctly RUN throughout. Controller log corroborates the mechanism: ctrl-1.log's recovery trace shows apply_job_update_transition logging 'duplicate UPDATE to state 0x2 -- no-op' for every job during WAL replay -- consistent with the WAL_JOB_DISPATCH fact applying SILENTLY first (PEND->RUN, no log line on the APPLIED path), immediately followed by each job's WAL_JOB_START fact (worker-confirmed start, also state=RUN) correctly detecting the now-redundant transition and logging the no-op. Neither replay step ever touches job.allocs.", + "error_message": "Gitea issue #43 (\"\u91cd\u542fdaemon\u540e\u4f5c\u4e1a\u8c03\u5ea6/\u4fe1\u606f\u4e0d\u5bf9\"): after restarting ALL daemons (controller, scheduler, worker, apiserver), previously-RUN jobs come back with STATUS=RUN correctly recovered but EXEC_HOST permanently blank (\"-\"), even though the underlying job is genuinely still running. CONFIRMED LIVE on branch scheduler-framework, corrected finding: an earlier verification pass on this same issue (2026-08-24, same session) restarted only controller+scheduler+worker and left the apiserver running, and did NOT reproduce the symptom -- that was a false negative, caused by the still-running apiserver simply continuing to display its stale-but-correct in-memory job cache from before the restart, never refreshed. Repeating the test with the apiserver ALSO restarted (matching the original report's \"restart all daemons\" exactly) reproduces the bug immediately and durably: 3 RUN jobs with EXEC_HOST=1*worker-1 before the restart; killed all 4 daemons; ran start-all.sh; EXEC_HOST was blank on every poll (15 polls over 30s, first poll ~6s after apiserver came back up) while STATUS stayed correctly RUN throughout. Controller log corroborates the mechanism: ctrl-1.log's recovery trace shows apply_job_update_transition logging 'duplicate UPDATE to state 0x2 -- no-op' for every job during WAL replay -- consistent with the WAL_JOB_DISPATCH fact applying SILENTLY first (PEND->RUN, no log line on the APPLIED path), immediately followed by each job's WAL_JOB_START fact (worker-confirmed start, also state=RUN) correctly detecting the now-redundant transition and logging the no-op. Neither replay step ever touches job.allocs.", "file": "src/controller/job.c:1369-1370, src/controller/persist.c:3708-3757, src/controller/job.c:288-351 (restore_job), src/controller/persist.c:6690-6713 (persist_submit_job_fact)", "root_cause": "dispatch_a_job_() (job.c:1343-1372) sets j->job.allocs directly in memory (lines 1346-1353, the host/resource assignment the scheduler decided) and transitions j->job.state = JOB_STAT_RUN (line 1355), but the durable record of this transition -- persist_submit_job_fact(&c->persist, WAL_JOB_DISPATCH, alloc->jobid, JOB_STAT_RUN) at line 1369-1370 -- only carries {jobid, state} (persist_submit_job_fact()'s own signature, persist.c:6690-6694, has no allocation parameter at all; its payload type wal_jobstate_t is exactly {jobid, state}). The resource allocation itself is never written to the WAL anywhere. On a full-process restart, job objects are reconstructed purely from replay: restore_job() (job.c:288-351) decodes a job's ONLY WAL_JOB_SUBMIT entry (written at initial submission, strictly before dispatch -- confirmed via persist_job_hook_fun's own comment that WAL_JOB_SUBMIT durability is written once, before any job object exists -- so its capnp payload's allocs field is genuinely empty/NULL at encode time). apply_committed_entry()'s WAL_JOB_DISPATCH case (persist.c:3708-3757) then replays the dispatch fact via apply_job_update_transition(), which by explicit design and its own doc comment 'only ever touches state' -- job->job.allocs is never set anywhere in the replay path. Result: after restart, job.state is correctly RUN (both WAL_JOB_DISPATCH and the later WAL_JOB_START fact carry state=RUN and replay correctly) but job.allocs is permanently NULL for the remaining life of that job -- nothing ever re-derives it, since scheduler-dispatch-commit's design treats a job's dispatch as a one-time PEND->RUN event with no re-dispatch/re-reconciliation path for an already-RUN job (see bug-1631's confirmation that no live caller ever re-adds a RUN job back to the scheduler's dispatch pool). EXEC_HOST is rendered by the apiserver from this now-NULL allocation once the recovered, allocation-less job is pushed to it -- which is why restarting the apiserver alongside the controller (forcing a fresh push) reveals the bug, while leaving the apiserver running (as an earlier verification pass on this same issue did) masks it behind stale cached data.", "fix": "FIXED (2026-08-25). No capnp schema change needed -- JobAlloc (xce_job_alloc_t) was already a full top-level capnp struct (the existing Job.allocs field) with generated encode/decode functions already present in the build. New glue only: src/controller/persist.c gains capnp_encode_joballoc()/decode_joballoc() (mirroring the existing capnp_encode_worker()/restore_job() patterns) and persist_submit_job_dispatch_fact(persist_t*, jobid, state, alloc) -- WAL_JOB_DISPATCH's payload is now a wal_jobstate_t header plus trailing capnp-encoded JobAlloc bytes, declared in src/include/persist.h. src/controller/job.c: dispatch_a_job_()'s success path now calls persist_submit_job_dispatch_fact(..., j->job.allocs) instead of persist_submit_job_fact(...), reading j->job.allocs (the freshly-constructed canonical struct) under the same write lock. Both apply seams updated: apply_committed_entry() (primary's own boot replay) gets one added else-if branch inside its existing WAL_JOB_START/WAL_JOB_FINISH stamp-chain (minimal diff, same shared plumbing) that decodes and adopts trailing allocation bytes when present. apply_wal_entry() (standby catch-up / replicated-apply -- also required, since a promoted standby would hit the exact same bug) has WAL_JOB_DISPATCH split into its own case, calling new restore_job_dispatch_state() (job.c, restore_job_state()'s sibling, takes ownership of an optional decoded allocation). A payload of exactly sizeof(wal_jobstate_t) (any WAL_JOB_DISPATCH entry written before this fix) decodes no allocation and behaves exactly as before -- full backward compatibility, verified by the 3 pre-existing test_wal_job_dispatch_fact.c tests passing completely unchanged.\n\nPrimary regression test: test_dispatch_commit.c's test_dc_success_appends_wal_job_dispatch_durably (already drove the REAL dispatch_a_job_() production path through a real WAL-write + persist_recover() restart cycle, previously only checking j2->job.state) extended to also assert j2->job.allocs matches what was dispatched (host, resource name, value) -- the user's exact acceptance criterion, 'after recover the job should get the same as it's state before shutdown.' RED->GREEN independently reproduced (not just narrated): git-stashed job.c/persist.c/persist.h with the test extension kept, rebuilt, confirmed FAIL at exactly this assertion; restored, rebuilt, confirmed PASS. Added 3 new tests to test_wal_job_dispatch_fact.c covering the richer payload directly on both apply seams plus a full restart-replay variant -- all pass. Full controller_unit_test suite independently re-run: 1136/1136 pass, zero regressions.\n\nLive-verified at the WAL level via xce_test's waldump: a real dispatch_a_job_() call produced a WAL_JOB_DISPATCH entry with PLEN=176 (vs the old bare wal_jobstate_t shape's PLEN=16), confirming the richer payload is genuinely written on the production path.\n\nIMPORTANT: full xcecli-visible E2E verification (submit -> restart -> check EXEC_HOST live) was blocked by discovering bug-1679 (see that entry) -- a separate, more severe, pre-existing bug where live job-status pushes to the apiserver never fire at all, restart or not. This fix's correctness rests on the unit-test evidence above (which inspects the controller's actual recovered in-memory state directly, with no dependency on the broken apiserver-notification path) and the direct WAL-payload inspection, not on an xcecli-visible live check.", @@ -20266,10 +20266,10 @@ "bug-1673", "bug-1556" ], - "error_message": "Gitea issue #46 (还需要一个slots的概念): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`, but of 9 submitted single-slot `sleep` jobs on queue `normal`, only 8 reach RUN and the 9th is stuck PEND -- dispatch stops at 8/16 host capacity. Secondary ask: `hosts list` should show slots used/free, not just NCPUS/MAX_SLOT.", + "error_message": "Gitea issue #46 (\u8fd8\u9700\u8981\u4e00\u4e2aslots\u7684\u6982\u5ff5): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`, but of 9 submitted single-slot `sleep` jobs on queue `normal`, only 8 reach RUN and the 9th is stuck PEND -- dispatch stops at 8/16 host capacity. Secondary ask: `hosts list` should show slots used/free, not just NCPUS/MAX_SLOT.", "file": "src/scheduler/src/resource.c (sched_check_host_satisfy_:328-377), src/scheduler/src/plugins/limit.c (build_vres_name/per-user-queue-project virtual resource accounting:183-227,312-376), src/configs/resources.yaml (slots resource:4, consumable:true), src/configs/limits.yaml (ug1_normal_slots example, value:8, lines 41-49), src/frontend/apiserver/models/hosts.go (HostInfo.NumRun:87)", "root_cause": "Two independent findings. (1) Dispatch-stops-at-8 is most likely NOT a slot-counting bug in the scheduler -- MAX_SLOT genuinely gates per-host admission in sched_check_host_satisfy_() the same way `mem` does (per .kiro/steering/resource-accounting.md, reservation==usage for virtual resources like slots), but there is a SEPARATE, independent admission gate: cluster-wide usergroup+queue slot limits enforced by the scheduler's limit plugin (limit.c), configured in limits.yaml. This repo's example limits.yaml defines `ug1_normal_slots`: users in group ug1 on queue `normal` share a pool capped at exactly 8 (limits.yaml:41-49) -- an exact match to the reporter's queue (`normal`) and the observed 8/16 cutoff. This exact scenario (a queue/usergroup slot limit as an alternate explanation to a host-capacity bug) was explicitly predicted-but-unconfirmed in bug-1673 (filed against the related earlier report Gitea #40) and appears to be that prediction now reproduced. CAVEAT: limits.yaml/users.yaml in this repo look like sample/demo config (users.yaml's stock ug1 group only contains user1/user2, not `admin`) -- confirming this requires checking the reporter's actual deployed limits.yaml/users.yaml or their job's pend-reason output, neither of which was available this session (static analysis only). (2) The feature-request half (show slot usage in hosts list) is NOT starting from scratch: models.HostInfo.NumRun (hosts.go:87) already exists in the schema and is already an output column in `xcecli hosts list` (matching the reporter's own output), but `grep -rn \"\\.NumRun\\s*=\" apiserver --include=*.go` returns zero hits anywhere in the apiserver -- the field is never assigned and is permanently stuck at Go's zero value, which is exactly why the reporter's own output shows NUM_RUN=0 while 8 jobs are actually running on that host. Same for NumSSusp/NumUSusp.", - "fix": "PARTIALLY FIXED (2026-09-01). Part 2 (hosts-list slot visibility) is FIXED: added computeHostSlotUsage() (controllers/hosts.go) which scans store.Jobs' live ExecHosts JSONB (the same {\"host_slots\":[{\"host_name\",\"num_slots\"}],...} shape backend/convert_pure.go populates) and JobStatus bits (JOBSTAT[\"RUN\"]/[\"SSUSP\"]/[\"USUSP\"]) to count occupied slots per host, per sub-state -- counting SLOTS (not just job count) so the numbers are directly comparable to max_slot, matching the reporter's actual ask ('host上需要体现用了多少slots'). processHostData() now overwrites row.Data[\"num_run\"/\"num_ssusp\"/\"num_ususp\"] with these live counts instead of leaving HostInfo's always-zero struct fields untouched. TDD: added TestProcessHostData_ComputesSlotUsageFromJobs (host_control_test.go) covering plain RUN, USUSP, a 2-slot SSUSP job (verifies slot-counting, not job-counting), a PEND job with no exec host (must not count), and a job on a different host (must not bleed across hosts). Confirmed red (num_run=0, want 6) before the fix, green after. Full apiserver suite passes with zero regressions.\n\nPart 1 (dispatch-stops-at-8 root cause) remains UNRESOLVED/NOT ACTIONABLE this session: confirming whether a real usergroup/queue limits.yaml entry is gating the reporter's deployment requires their actual live config or pend-reason output, neither available statically. No scheduler/limit.c code change was made pending that confirmation -- see root_cause for the open question to ask the reporter.", + "fix": "PARTIALLY FIXED (2026-09-01). Part 2 (hosts-list slot visibility) is FIXED: added computeHostSlotUsage() (controllers/hosts.go) which scans store.Jobs' live ExecHosts JSONB (the same {\"host_slots\":[{\"host_name\",\"num_slots\"}],...} shape backend/convert_pure.go populates) and JobStatus bits (JOBSTAT[\"RUN\"]/[\"SSUSP\"]/[\"USUSP\"]) to count occupied slots per host, per sub-state -- counting SLOTS (not just job count) so the numbers are directly comparable to max_slot, matching the reporter's actual ask ('host\u4e0a\u9700\u8981\u4f53\u73b0\u7528\u4e86\u591a\u5c11slots'). processHostData() now overwrites row.Data[\"num_run\"/\"num_ssusp\"/\"num_ususp\"] with these live counts instead of leaving HostInfo's always-zero struct fields untouched. TDD: added TestProcessHostData_ComputesSlotUsageFromJobs (host_control_test.go) covering plain RUN, USUSP, a 2-slot SSUSP job (verifies slot-counting, not job-counting), a PEND job with no exec host (must not count), and a job on a different host (must not bleed across hosts). Confirmed red (num_run=0, want 6) before the fix, green after. Full apiserver suite passes with zero regressions.\n\nPart 1 (dispatch-stops-at-8 root cause) remains UNRESOLVED/NOT ACTIONABLE this session: confirming whether a real usergroup/queue limits.yaml entry is gating the reporter's deployment requires their actual live config or pend-reason output, neither available statically. No scheduler/limit.c code change was made pending that confirmation -- see root_cause for the open question to ask the reporter.", "tags": [ "scheduler", "slots", @@ -20297,7 +20297,7 @@ "id": "bug-1744", "timestamp": "2026-09-01T05:45:00Z", "gitea_issue": 47, - "error_message": "Gitea issue #47 (rusage是否做了？): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed resreq string (trailing non-numeric \"a0\" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '乱写也可以提交' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent.", + "error_message": "Gitea issue #47 (rusage\u662f\u5426\u505a\u4e86\uff1f): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed resreq string (trailing non-numeric \"a0\" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '\u4e71\u5199\u4e5f\u53ef\u4ee5\u63d0\u4ea4' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent.", "file": "src/frontend/xcecli/cmd/jobs/submit.go (resReq flag, no value validation, just passed through: jobReq.Resreq = js.resReq), src/frontend/apiserver/controllers/jobs.go:1602-1606 (only length-checks Resreq, no syntax/grammar validation), src/lib/resreq.c (parse_resreq/parse_rusage -- the real PEG-generated grammar parser), src/lib/rusage.peg (rusage_param grammar, mem value must match `number <- [0-9]+('.'[0-9]*)?`, so \"100000000a0\" is a genuine grammar violation), src/scheduler/src/core_events.c:236-245 (parse_resreq call site inside core_build_job()) and :751-760 (JOBS_PUSH caller: on core_build_job()==NULL, WARNs server-side and `continue`s -- job silently dropped, never added to core->jobs)", "root_cause": "The ONLY resreq validation in the entire submission path is a length check (apiserver/controllers/jobs.go:1602-1606, `len(req.Resreq) > 4094`) -- there is no syntax/grammar validation anywhere client-side or in the apiserver before a job is accepted and persisted. The REAL validator, a PEG-generated grammar parser (parse_resreq()/parse_rusage(), src/lib/resreq.c, built from src/lib/rusage.peg + xce_resreq.peg via meson custom_target), exists but is only ever invoked deep inside the C scheduler at core_build_job() time (core_events.c:236), i.e. asynchronously, well AFTER the apiserver has already accepted the job, returned a submission-success response to the client, and persisted it. When parse_resreq() fails on malformed syntax (confirmed via rusage.peg's `number` rule, `[0-9]+('.'[0-9]*)?` -- \"100000000a0\" cannot match this and leaves \"a0\" unconsumed, a genuine PEG parse failure), core_build_job() logs an ERROR and returns NULL (core_events.c:238-245); its caller in the XCE_CORE_EVENT_JOBS_PUSH handler (core_events.c:756-760) only logs a server-side WARN and `continue`s to the next job in the batch -- the malformed job is silently dropped from the scheduler's core->jobs registry entirely. Net effect is worse than 'garbage accepted': the job is accepted, shows the user a success message, is almost certainly persisted and shown as PEND via `jobs list` (this session did not confirm the exact persisted/listed state live), and then sits PEND FOREVER with zero indication to the user that anything is wrong -- the only trace is a server-side log line the user never sees. This is architecturally the same class of gap as bug-1740 (a real, working validator/reconciler exists but is wired into the wrong -- or in this case, too-late -- point in the pipeline) and matches the maintainer's own comment that front-end validation is the intended design, not yet implemented.", "fix": "NOT YET FIXED -- investigated and design-confirmed this session; DEPRIORITIZED, deferred to later (explicit user decision, not urgent). Confirmed fix approach: validate resreq synchronously in the apiserver's submit handler (controllers/jobs.go:1602, alongside the existing length check), reusing libxce's existing PEG parser rather than reimplementing the grammar in Go. Specifically parse_rusage() (src/lib/resreq.c:154, declared resreq.h:301 -- `int parse_rusage(xlist_t*, char*, char**)`) needs NO resmgr/resource-manager context, unlike the fuller parse_resreq()/parse_select() (resreq.h:303,305 -- both take a `var resmgr` to validate resource NAMES in select[]/order[] clauses, which the apiserver doesn't currently construct) -- so a resreq-name-agnostic rusage[]-only syntax check is a small, low-risk CGo wrapper, not a larger resmgr-plumbing project. apiserver/backend already has the exact split pattern to follow: a `//go:build cgo` file with the real implementation (e.g. clib.go, which already #includes libxce headers and links -lxce) plus a `//go:build !cgo` stub (stub.go, returns an errNoCGO-style error) for the CGO-disabled test path. Full parse_resreq()/resmgr-backed select[]/order[] validation is explicitly out of scope for now -- would be a separate, larger follow-up if wanted.\n\nSCOPE DECISION (2026-09-01): will be implemented as a direct bugfix when picked up, NOT a kiro spec -- small, contained blast radius (one new CGo wrapper + stub in apiserver/backend, one call site in controllers/jobs.go, matching the size/shape of the already-shipped bug-1742/bug-1743 fixes), no HA/persistence/RBAC involvement, and no open design questions left (exact function, signature, and existing cgo/stub pattern to follow are already nailed down). Only the FULLER resmgr-backed select[]/order[] validation (out of scope here) would warrant at least a /kiro-spec-quick, since it needs new apiserver-side state (a resmgr context kept in sync with resource definitions) -- not needed for this rusage-only fix.", @@ -20738,6 +20738,43 @@ "related_bugs": [], "occurrences": 2, "last_seen": "2026-09-02T04:10:00Z" + }, + { + "id": "bug-1761", + "timestamp": "2026-09-02T04:30:00Z", + "error_message": "cli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion): repl_cmd_delete() (repl_break.c) parses its argument with atoi(), which silently returns 0 on non-numeric input instead of signalling an error -- `delete abc` reports \"[sim] delete: no breakpoint with id 0\" (a well-formed-but-missing-id message) rather than rejecting \"abc\" as not a number, masking an operator typo as a legitimate lookup miss.", + "file": "src/scheduler/src/repl_break.c (repl_cmd_delete)", + "root_cause": "CONFIRMED via source read: repl_cmd_delete() used `int id = atoi(tokens[1]);` with no validation of atoi()'s return, unlike this file family's own documented \"fail loud\" convention -- repl.c's repl_cmd_step() doc comment explicitly rejects the equivalent atoi()-style silent-coercion failure mode and uses strtol()+endptr instead specifically to catch non-numeric/trailing-garbage input. delete's atoi() call was never updated to match when that convention was established, so `delete abc`/`delete 1x` both silently coerced to some numeric prefix (or 0) instead of erroring.", + "fix": "FIXED. repl_cmd_delete() (repl_break.c) now uses strtol()+endptr, mirroring repl_cmd_step()'s exact validation shape: rejects a token with no valid leading integer, trailing non-numeric garbage, or a non-positive value (breakpoint ids are assigned via pre-incremented next_bp_id in repl_cmd_break(), so 1 is the smallest id that can ever legitimately exist), printing \"[sim] delete: '' is not a valid breakpoint id -- usage: delete |all\" instead of silently proceeding. Added test_delete_non_numeric_id_rejected (test_repl_breakpoints_inspection.c, registered via RUN_TEST) covering \"abc\", \"0\", \"-1\", and \"1x\", all asserting the existing breakpoint is left untouched. Verified via /xce-build scheduler: clean compile, test_delete_non_numeric_id_rejected PASSed (testlog.txt confirms all 4 cases -- \"abc\", \"0\", \"-1\", \"1x\" -- hit the new rejection path), sched_repl_breakpoints_inspection_test suite OK. One unrelated pre-existing failure in the same run (test_job_rusage_reconciles_mem_reservation_against_usage, test_core_events.c) -- different file, different subsystem (mem-reservation reconciliation, not REPL/breakpoint parsing), not touched by this fix.", + "tags": [ + "repl", + "scheduler-sim", + "cli-ux", + "input-validation", + "fail-loud" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-02T04:30:00Z" + }, + { + "id": "bug-1762", + "timestamp": "2026-09-02T07:50:00Z", + "error_message": "scheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's approved RED-phase meson.build target sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) lists ONLY test_repl_output_compression.c as its source, with no source providing the function's implementation -- so even after adding a correct repl_fmt_jobid_range() definition to src/scheduler/src/repl.c (as design.md's File Structure Plan literally directs: 'implements the new shared jobid-range-compression helper ... in repl.c'), the target still fails to link: undefined reference to repl_fmt_jobid_range.", + "file": "src/scheduler/testsuite/meson.build (sched_repl_output_compression_test), src/scheduler/src/repl.c", + "root_cause": "CONFIRMED empirically, but the FIRST fix attempt below drew the wrong conclusion from it. Adding repl.c itself as a bare extra SOURCE FILE to the test target's sources does not fix the link -- it trades one undefined reference for five: repl_dispatch() (a function in repl.c, never called by this test) references repl_cmd_break/repl_cmd_delete/repl_cmd_info/repl_cmd_snapshot/repl_cmd_set, implemented in sibling files repl_break.c/repl_info.c/repl_params.c that this deliberately-lightweight test target does not build. A directly-linked .o file (unlike an archive/static-library member) requires ALL of its undefined symbols resolved regardless of whether the calling code path is ever exercised at runtime -- there is no dead-code elimination at the object-file granularity without -ffunction-sections + -Wl,--gc-sections, which this build does not use. HOWEVER: this only rules out the narrow 'bare .c file in the source list' fix -- it does NOT mean repl_fmt_jobid_range() must live in its own translation unit. sched_scheduler_connect_test (src/scheduler/testsuite/meson.build) already establishes the correct alternative for exactly this shape of problem: link the target against the REAL scheduler_srcs object list (a `foreach f : scheduler_srcs` loop over the production source list, minus src/main.c) instead of hand-picking source files, which resolves repl_dispatch()'s cross-file externs for real (repl_break.c/repl_info.c/repl_params.c are simply also linked in) rather than needing them excluded. This alternative was available and untried at the time of the original fix below, which additionally violated design.md's explicit Architecture Integration commitment ('no new production .c file needed') by introducing repl_jobid_range.c.", + "fix": "REMEDIATED (superseding the original fix below, which was reviewed and rejected for violating design.md's 'no new production .c file needed' commitment). repl_fmt_jobid_range() and its private helpers now live in src/scheduler/src/repl.c itself, exactly as design.md's File Structure Plan directs, with only one new #include (, for the internal repl_jr_appendf() vararg helper) -- no new production .c file. sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) now links the real scheduler_srcs object list via a `foreach f : scheduler_srcs` loop (mirroring sched_scheduler_connect_test's own established technique) instead of hand-picking source files, so repl_dispatch()'s cross-file externs resolve for real; no `-Wl,--wrap=` or `-DXCE_SCHEDULER_C_PATH` needed since this test never calls sched_init() or reaches networking code. repl_jobid_range.c deleted; 'src/repl_jobid_range.c' removed from scheduler_srcs (src/scheduler/meson.build); repl_internal.h's doc comment on repl_fmt_jobid_range() updated to drop the now-false 'lives in its own translation unit' rationale. Verified: xce-build.sh scheduler --incremental succeeds with zero linker errors; sched_repl_output_compression_test's own binary output shows all 5 Unity tests PASS (5 Tests 0 Failures 0 Ignored); full scheduler suite still 55/56, the one failure being the pre-existing unrelated test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c, mem-reservation subsystem, untouched by this change); repl.c stays at 794/800 lines after the addition (up from 531 before this task).", + "tags": [ + "repl", + "scheduler-sim", + "build-wiring", + "meson", + "jobid-range-compression", + "test-infrastructure" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-09-02T07:50:00Z" } ] } diff --git a/src/scheduler/src/repl.c b/src/scheduler/src/repl.c index 878da86d..25f32052 100644 --- a/src/scheduler/src/repl.c +++ b/src/scheduler/src/repl.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -529,3 +530,265 @@ void xce_repl_on_replay_complete(xce_sched_sim_driver_t* sim) rs->replay_complete = true; repl_command_loop(sim); } + +/* + * ---- Job-identifier range compression (scheduler-sim-repl-verbosity spec, + * task 1.2, design.md's "Jobid Range Compressor" component, Req 4, ACs + * 4.1-4.5) ---- A pure, read-only formatter with zero dependency on + * xce_sched_sim_driver_t/xce_sched_core_t or any of this file's other REPL + * machinery, kept here per design.md's File Structure Plan ("... in repl.c, + * following the file's existing shared-helper convention" -- same + * convention repl_format_ts_human()/repl_split_tokens() above establish). + * Reuses fmt_jobid_ex()'s own jobid bit-layout macros (utils.h, already + * #included above) rather than re-deriving the layout. + */ + +/* + * xce_repl_jobid_group_t - one grouping unit while repl_fmt_jobid_range() + * walks its input: either a standalone non-array jobid (`is_array` false) + * or one array submission's (cluster, apiserver, timestamp, arraybase) + * identity plus every index seen for it so far. Kept in first-seen order + * (design.md: "distinct array bases appear in input order"). `idxs` grows + * by one realloc'd slot per newly-seen index, same idiom repl_break.c's + * repl_breakpoints_append() establishes for this file family. + */ +typedef struct { + bool is_array; + uint32_t cluster; + uint32_t apiserver; + uint32_t timestamp; + uint32_t arraybase; /* array groups only; already fmt_jobid_ex()-shifted */ + uint32_t plain_seq; /* non-array groups only */ + uint32_t* idxs; + int nidxs; +} xce_repl_jobid_group_t; + +static void repl_jobid_groups_free(xce_repl_jobid_group_t* groups, + int ngroups) +{ + int i; + + for (i = 0; i < ngroups; i++) { + free(groups[i].idxs); + } + free(groups); +} + +static bool repl_jobid_group_add_idx(xce_repl_jobid_group_t* g, uint32_t idx) +{ + uint32_t* grown = (uint32_t*)realloc( + g->idxs, (size_t)(g->nidxs + 1) * sizeof(*(g->idxs))); + + if (grown == NULL) { + return false; + } + + g->idxs = grown; + g->idxs[g->nidxs] = idx; + g->nidxs++; + + return true; +} + +static int repl_jobid_idx_cmp(const void* a, const void* b) +{ + uint32_t ua = *(const uint32_t*)a; + uint32_t ub = *(const uint32_t*)b; + + if (ua < ub) { + return -1; + } + if (ua > ub) { + return 1; + } + return 0; +} + +/* + * repl_jr_appendf - appends one snprintf()-formatted fragment onto a + * realloc'd dynamic string (`*out`/`*outlen` track the allocation and its + * used length; `*out` stays NUL-terminated), growing by exactly the + * fragment's length each time -- same realloc idiom as + * repl_jobid_group_add_idx() above. Every fragment this formats (a decimal + * number, dash, comma, or bracket) fits well inside the 63-byte local + * scratch below; exceeding that is a defensive failure (returns false, + * `*out`/`*outlen` unchanged), as is an allocation failure. + */ +static bool repl_jr_appendf(char** out, size_t* outlen, const char* fmt, ...) +{ + va_list ap; + char frag[64]; + int n; + char* grown; + + va_start(ap, fmt); + n = vsnprintf(frag, sizeof(frag), fmt, ap); + va_end(ap); + + if ((n < 0) || ((size_t)n >= sizeof(frag))) { + return false; + } + + grown = (char*)realloc(*out, *outlen + (size_t)n + 1); + if (grown == NULL) { + return false; + } + + memcpy(grown + *outlen, frag, (size_t)n + 1); /* fragment + its NUL */ + *out = grown; + *outlen += (size_t)n; + + return true; +} + +/* + * repl_fmt_jobid_range - design.md's "Jobid Range Compressor" (Req 4, ACs + * 4.1-4.5): formats `njobids` raw jobids into `buf` (bufsz-bounded), + * grouping by array submission (cluster, apiserver, timestamp, array base) + * and compressing contiguous [idx] runs into "[lo-hi]" form, comma-joining + * non-contiguous runs within one base; non-array jobids are listed + * individually. Distinct groups appear in the order each is FIRST seen in + * `jobids`; indices within a compressed range always ascend regardless of + * input order. Returns bytes written to `buf` (excluding the NUL), or + * negative if `buf` was too small -- mirroring fmt_jobid_ex()'s own pattern + * (src/lib/utils.c): the full rendering is built into a heap scratch buffer + * first, and `buf` is touched only once it is proven to fit -- never a + * partial/truncated write. + */ +int repl_fmt_jobid_range(char* buf, size_t bufsz, const uint64_t* jobids, + int njobids) +{ + xce_repl_jobid_group_t* groups = NULL; + int ngroups = 0; + char* out = NULL; + size_t outlen = 0; + bool ok = true; + int i; + int rc; + + if ((buf == NULL) || (bufsz == 0)) { + return -1; + } + if (njobids <= 0) { + buf[0] = '\0'; + return 0; + } + if (jobids == NULL) { + return -1; + } + + groups = (xce_repl_jobid_group_t*)calloc((size_t)njobids, + sizeof(*groups)); + if (groups == NULL) { + return -1; + } + + for (i = 0; (i < njobids) && ok; i++) { + uint64_t id = jobids[i]; + uint32_t tag = EXTRACT_ID_TAG(id); + uint32_t cluster = EXTRACT_ID_CLUSTER(id); + uint32_t apiserver = EXTRACT_ID_APISERVER(id); + uint32_t timestamp = EXTRACT_ID_TIMESTAMP(id); + + if ((tag & TAG_JOB_ARRAY) != 0) { + uint32_t arraybase = EXTRACT_ARRAY_JOB(id); + uint32_t idx = EXTRACT_ARRAY_IDX(id); + int g = -1; + int j; + + for (j = 0; j < ngroups; j++) { + if (groups[j].is_array && (groups[j].cluster == cluster) + && (groups[j].apiserver == apiserver) + && (groups[j].timestamp == timestamp) + && (groups[j].arraybase == arraybase)) { + g = j; + break; + } + } + if (g < 0) { + g = ngroups++; + groups[g].is_array = true; + groups[g].cluster = cluster; + groups[g].apiserver = apiserver; + groups[g].timestamp = timestamp; + groups[g].arraybase = arraybase; + } + ok = repl_jobid_group_add_idx(&groups[g], idx); + } else { + int g = ngroups++; + + groups[g].is_array = false; + groups[g].cluster = cluster; + groups[g].apiserver = apiserver; + groups[g].timestamp = timestamp; + groups[g].plain_seq = EXTRACT_ID_SEQUENCE(id); + } + } + + for (i = 0; (i < ngroups) && ok; i++) { + xce_repl_jobid_group_t* g = &groups[i]; + + if (i > 0) { + ok = repl_jr_appendf(&out, &outlen, ","); + if (!ok) { + break; + } + } + + if (!g->is_array) { + ok = repl_jr_appendf(&out, &outlen, "%u-%u-%u-%u", g->cluster, + g->apiserver, g->timestamp, g->plain_seq); + continue; + } + + qsort(g->idxs, (size_t)g->nidxs, sizeof(*(g->idxs)), + repl_jobid_idx_cmp); + + ok = repl_jr_appendf(&out, &outlen, "%u-%u-%u-%u[", g->cluster, + g->apiserver, g->timestamp, g->arraybase); + + { + int j = 0; + + while (ok && (j < g->nidxs)) { + uint32_t run_lo = g->idxs[j]; + uint32_t run_hi = run_lo; + int k = j + 1; + + while ((k < g->nidxs) && (g->idxs[k] == run_hi + 1)) { + run_hi = g->idxs[k]; + k++; + } + + if (j > 0) { + ok = repl_jr_appendf(&out, &outlen, ","); + } + if (ok && (run_lo == run_hi)) { + ok = repl_jr_appendf(&out, &outlen, "%u", run_lo); + } else if (ok) { + ok = repl_jr_appendf(&out, &outlen, "%u-%u", run_lo, + run_hi); + } + + j = k; + } + } + + if (ok) { + ok = repl_jr_appendf(&out, &outlen, "]"); + } + } + + repl_jobid_groups_free(groups, ngroups); + + if (!ok || (out == NULL) || ((outlen + 1) > bufsz)) { + free(out); + return -1; + } + + memcpy(buf, out, outlen + 1); + rc = (int)outlen; + free(out); + + return rc; +} diff --git a/src/scheduler/src/repl_internal.h b/src/scheduler/src/repl_internal.h index a0b79078..befe2682 100644 --- a/src/scheduler/src/repl_internal.h +++ b/src/scheduler/src/repl_internal.h @@ -234,4 +234,39 @@ extern bool repl_cmd_set(xce_sched_sim_driver_t* sim, const char* line); * a given state. */ extern const char* repl_job_state_name(uint32_t state); +/* + * repl_fmt_jobid_range (repl.c, scheduler-sim-repl-verbosity spec, task 1.2, + * Requirement 4 "Job Identifier Range Compression", ACs 4.1-4.5) - formats + * `njobids` raw jobids into `buf` (bufsz-bounded): groups by array + * submission (cluster, apiserver, timestamp, array base) and compresses + * contiguous [idx] runs into "[lo-hi]" form, comma-joining non-contiguous + * runs within one base; non-array jobids are listed individually, each at + * its own first-seen position. Distinct groups appear in the order each is + * first seen in `jobids`. Returns the number of bytes written to `buf` + * (excluding the NUL terminator), or a negative value if `buf` was too + * small to hold the full listing -- `buf` is left completely untouched in + * that case (never a partial/truncated write; see design.md's Error + * Handling section). Pure, read-only formatting -- never mutates `jobids` + * or any scheduler/job state. Shared by repl_print_event_summary()'s + * job-arrival listings, the Round Rollup's pend-jobid display, and + * repl_info_worker()'s assigned-job listing (design.md's Jobid Range + * Compressor component, Requirement 4.5) -- one helper, no per-caller + * variant. + * + * Lives in repl.c itself, exactly as design.md's File Structure Plan + * directs ("implements the new shared jobid-range-compression helper ... in + * repl.c"), even though it has zero dependency on + * xce_sched_sim_driver_t/xce_sched_core_t or any of repl_internal.h's OTHER + * extern cross-references (repl_cmd_break/repl_cmd_info/repl_cmd_set/... + * implemented in the repl_break.c/repl_info.c/repl_params.c siblings). + * test_repl_output_compression.c (task 1.1) still declares this function as + * a plain extern rather than #include-ing repl.c -- see that test file's + * own header comment -- and its meson target links the real scheduler_srcs + * object list instead (sched_scheduler_connect_test's own established + * technique), so it exercises this real implementation without pulling + * repl.c's other undefined cross-references into a directly-linked .o. + */ +extern int repl_fmt_jobid_range(char* buf, size_t bufsz, + const uint64_t* jobids, int njobids); + #endif diff --git a/src/scheduler/testsuite/meson.build b/src/scheduler/testsuite/meson.build index 08b91a58..6680db9a 100644 --- a/src/scheduler/testsuite/meson.build +++ b/src/scheduler/testsuite/meson.build @@ -974,21 +974,40 @@ sched_repl_equivalence_test = executable( dependencies: [scheduler_test_deps, libedit_dep], install: false) -# RED-phase contract test for the REPL's job-identifier range compressor -# (scheduler-sim-repl-verbosity spec, task 1.1, Requirement 4). Unlike every -# other sched_repl_*_test above, this is NOT a white-box #include of -# ../src/repl.c and friends -- repl_fmt_jobid_range() is a pure formatter -# with no xce_sched_sim_driver_t/xce_sched_core_t dependency, so this file -# declares it as a plain extern instead (see test_repl_output_compression.c's -# own header comment for the full rationale). As of task 1.1, the function -# has no definition anywhere in this codebase (task 1.2 adds it), so this -# target is EXPECTED to fail at link time -- that is task 1.1's own -# Observable evidence, not a defect in this build file. +# Contract test for the REPL's job-identifier range compressor +# (scheduler-sim-repl-verbosity spec, task 1.1/1.2, Requirement 4). +# repl_fmt_jobid_range() lives in ../src/repl.c (design.md's File Structure +# Plan: "implements the new shared jobid-range-compression helper ... in +# repl.c"), but repl.c's own dependency surface is too large for the lean +# "#include the one file under test" pattern the other lightweight REPL +# tests use (test_repl_output_compression.c declares repl_fmt_jobid_range() +# as a plain extern rather than #include-ing repl.c -- see that file's own +# header comment: repl.c's repl_dispatch() references repl_cmd_break/ +# repl_cmd_info/repl_cmd_set/... from sibling files repl_break.c/ +# repl_info.c/repl_params.c, and a directly-linked .o -- unlike an archive +# member -- requires ALL its undefined symbols resolved regardless of +# whether the calling code path is ever exercised). So instead this target +# links the real scheduler_srcs object list, the same technique +# sched_scheduler_connect_test below established for the identical "real +# production dependency surface, avoid textual inclusion" problem. +# scheduler_srcs (parent meson.build) minus src/main.c (this test file +# supplies its own main()) -- built via a loop, same as +# scheduler_connect_test_srcs below, so it can never drift from the +# production source list. No `-Wl,--wrap=` or `-DXCE_SCHEDULER_C_PATH` is +# needed here: this test never calls sched_init() or reaches networking +# code, it only exercises the pure repl_fmt_jobid_range() formatter. +repl_output_compression_test_srcs = ['test_repl_output_compression.c'] +foreach f : scheduler_srcs + if f != 'src/main.c' + repl_output_compression_test_srcs += ['../' + f] + endif +endforeach + sched_repl_output_compression_test = executable( 'sched_repl_output_compression_test', - ['test_repl_output_compression.c'], + repl_output_compression_test_srcs, include_directories:[incdir], - dependencies: [scheduler_test_deps], + dependencies: [scheduler_deps, unity_dep], install: false) # Tests for the scheduler's unified candidate-list connect/rejoin pipeline