From 2655edd2b4629c6f8e156f40a8423da641568f32 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 23 Jun 2026 07:04:07 +0000 Subject: [PATCH] =?UTF-8?q?log(bug-1028):=20post-failover=20availability?= =?UTF-8?q?=20=E2=80=94=20new=20primary=20stuck=20in=20catch-up=20loop,=20?= =?UTF-8?q?not=20serving=20(distinct=20from=20bug-828=20data=20safety=20/?= =?UTF-8?q?=20bug-854=20client=20discovery)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .wolf/buglog.json | 1863 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1863 insertions(+) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index bba2af6..4b1105b 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -15125,6 +15125,1869 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-06-22T03:45:00Z" + }, + { + "id": "bug-920", + "timestamp": "2026-06-22T03:58:08.538Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_term_safety_adapter.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T03:58:08.538Z" + }, + { + "id": "bug-921", + "timestamp": "2026-06-22T03:58:19.392Z", + "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_waldump\" instead of \"test_term_safety_adapter\"", + "fix": "Changed test_waldump \u2192 test_term_safety_adapter", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T03:58:28.433Z" + }, + { + "id": "bug-922", + "timestamp": "2026-06-22T03:59:43.099Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T03:59:43.099Z" + }, + { + "id": "bug-923", + "timestamp": "2026-06-22T04:01:37.467Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/persist.c", + "root_cause": "Condition was: if (hdr.lsn <= from_lsn)", + "fix": "Changed to: if (hdr.type < WAL_CHECKPOINT)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:01:37.467Z" + }, + { + "id": "bug-924", + "timestamp": "2026-06-22", + "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", + "file": "environment (/ overlay filesystem)", + "root_cause": "Disk filled to 100% (7466 leftover /tmp/xce_* WAL dirs from repeated ASAN test runs). make_fresh_persist's persist_init mkdir_p hit ENOSPC -> returned NULL -> cascade of 'Expected Non-NULL'. Also overlayfs returned fdatasync EIO under disk pressure. NOT a code regression: baseline full suite passes; test_config_repl catchup tests are order-dependent and fail in isolation on baseline too.", + "fix": "rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles.", + "tags": [ + "disk-space", + "enospc", + "overlayfs", + "fdatasync-eio", + "controller-tests", + "flaky", + "ha-term-safety" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22" + }, + { + "id": "bug-925", + "timestamp": "2026-06-22T04:42:28.897Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:42:28.897Z" + }, + { + "id": "bug-926", + "timestamp": "2026-06-22T04:45:07.018Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:45:07.018Z" + }, + { + "id": "bug-927", + "timestamp": "2026-06-22T04:46:36.310Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 19\u219225 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:46:36.310Z" + }, + { + "id": "bug-928", + "timestamp": "2026-06-22T04:55:58.789Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:55:58.789Z" + }, + { + "id": "bug-929", + "timestamp": "2026-06-22T04:57:03.605Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T04:57:15.567Z" + }, + { + "id": "bug-930", + "timestamp": "2026-06-22T04:57:25.141Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:57:25.141Z" + }, + { + "id": "bug-931", + "timestamp": "2026-06-22T05:45:09.373Z", + "error_message": "Function not marked async", + "file": "src/include/repl.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T05:45:09.373Z" + }, + { + "id": "bug-932", + "timestamp": "2026-06-22T05:47:00.864Z", + "error_message": "Missing error handling in xce_wal_batch_req_raw", + "file": "src/controller/repl.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T05:47:00.864Z" + }, + { + "id": "bug-933", + "timestamp": "2026-06-22T05:47:50.037Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !d", + "fix": "Added guard clause: if (!d)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T05:47:50.037Z" + }, + { + "id": "bug-934", + "timestamp": "2026-06-22T05:49:28.039Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 15\u219220 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T05:49:28.039Z" + }, + { + "id": "bug-935", + "timestamp": "2026-06-22T06:38:20.255Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T06:38:20.255Z" + }, + { + "id": "bug-936", + "timestamp": "2026-06-22T06:39:16.197Z", + "error_message": "Significant refactor of ", + "file": "src/controller/commit.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 10\u219225 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T06:39:16.197Z" + }, + { + "id": "bug-937", + "timestamp": "2026-06-22T06:39:51.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T06:39:51.351Z" + }, + { + "id": "bug-938", + "timestamp": "2026-06-22T07:19:14.726Z", + "error_message": "Significant refactor of ", + "file": "src/controller/commit.c", + "root_cause": "9 lines replaced/restructured", + "fix": "Rewrote 12\u219218 lines (9 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:19:14.726Z" + }, + { + "id": "bug-939", + "timestamp": "2026-06-22T07:19:47.218Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "14 lines replaced/restructured", + "fix": "Rewrote 24\u219233 lines (14 removed) | Also: /* Standby ACKs LSN 2 \u2192 durable majority covers LS; /* Fire the WIRED callback with commit_index=1 \u2192 t", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T07:22:08.898Z" + }, + { + "id": "bug-940", + "timestamp": "2026-06-22T07:39:01.964Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 17\u219247 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:39:01.964Z" + }, + { + "id": "bug-941", + "timestamp": "2026-06-22T07:53:06.457Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:53:06.457Z" + }, + { + "id": "bug-942", + "timestamp": "2026-06-22T07:54:52.956Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:54:52.956Z" + }, + { + "id": "bug-943", + "timestamp": "2026-06-22T07:55:30.714Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:55:30.714Z" + }, + { + "id": "bug-944", + "timestamp": "2026-06-22T07:56:08.623Z", + "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Used \"ha_arm_own_term_gate\" instead of \"ha_arm_own_term_gate_for_test\"", + "fix": "Changed ha_arm_own_term_gate \u2192 ha_arm_own_term_gate_for_test", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:56:08.623Z" + }, + { + "id": "bug-945", + "timestamp": "2026-06-22T08:09:41.508Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:09:41.508Z" + }, + { + "id": "bug-946", + "timestamp": "2026-06-22T08:16:54.023Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:16:54.023Z" + }, + { + "id": "bug-947", + "timestamp": "2026-06-22T08:19:44.572Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/request.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:19:44.572Z" + }, + { + "id": "bug-948", + "timestamp": "2026-06-22T08:21:57.062Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat ; * library as a 0-entry AppendEntries so the LIBRARY performs", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T08:23:05.006Z" + }, + { + "id": "bug-949", + "timestamp": "2026-06-22T08:22:36.807Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:22:36.807Z" + }, + { + "id": "bug-950", + "timestamp": "2026-06-22T08:24:31.832Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha_convergence.c", + "root_cause": "43 lines replaced/restructured", + "fix": "Rewrote 59\u219227 lines (43 removed) | Also: * ha_rejoin_reconcile_ctx_t \u2014 heap context for the; * Carries the clamped cutoff computed at trigger t", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T08:27:13.909Z" + }, + { + "id": "bug-951", + "timestamp": "2026-06-22T08:26:04.789Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "6 lines replaced/restructured", + "fix": "Rewrote 10\u219215 lines (6 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:26:04.789Z" + }, + { + "id": "bug-952", + "timestamp": "2026-06-22T08:42:30.744Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:42:30.744Z" + }, + { + "id": "bug-953", + "timestamp": "2026-06-22T08:48:45.792Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: rc == 0 && out_lsn", + "fix": "Added guard clause: if (rc == 0 && out_lsn)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:48:45.792Z" + }, + { + "id": "bug-954", + "timestamp": "2026-06-22T09:04:06.849Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "9 lines replaced/restructured", + "fix": "Rewrote 39\u219279 lines (9 removed) | Also: * path; neither library call performs I/O. */; ha_raft_on_repl_ack(c, vctx->peer_idx, acked_lsn, ", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T09:04:19.614Z" + }, + { + "id": "bug-955", + "timestamp": "2026-06-22T09:05:02.687Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.h", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 12\u219230 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:05:02.687Z" + }, + { + "id": "bug-956", + "timestamp": "2026-06-22T09:05:30.540Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 3\u21924 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:05:30.540Z" + }, + { + "id": "bug-957", + "timestamp": "2026-06-22T09:08:41.177Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ep_is_vote", + "fix": "Added guard clause: if (ep_is_vote)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T09:11:38.490Z" + }, + { + "id": "bug-960", + "timestamp": "2026-06-22T09:20:00.000Z", + "error_message": "ha_raft_on_repl_ack unconditionally remapped the ACK endpoint peer_idx (votes_idx->absolute); for heartbeat STANDBY endpoints peer_idx is ALREADY absolute, so it was double-mapped -> wrong-peer false quorum / out-of-bounds match_index[] write (memory corruption / data loss).", + "file": "src/controller/ha.c", + "root_cause": "The walAck call site (ha.c walAck branch) fires for BOTH promoted-vote eps (vote_ep_ctx_t, peer_idx relative) AND heartbeat standby eps (standby_ep_ctx_t, ctx==NULL, peer_idx absolute). ha_raft_on_repl_ack applied the self-excluded->absolute remap unconditionally, corrupting the heartbeat path. Library raft_on_append_entries_reply writes s->match_index[peer_idx] unchecked (raft.c:802).", + "fix": "Made ha_raft_on_repl_ack take an ALREADY-ABSOLUTE peer index + added a fail-loud [0,n_peers) self-exclusion bounds check (no assert in src/lib/raft). Added pure resolver ha_repl_ack_abs_peer_idx(ha, ep_is_vote, peer_idx) mirroring ha.c:6187-6189 (vote ep -> remap, standby ep -> passthrough). Call site resolves per endpoint type via vctx->ctx!=NULL before calling the seam. Added 3 tests: endpoint-type round-trip for my_priority 0 AND 1, heartbeat-standby absolute-slot end-to-end, and out-of-range rejection. The two heartbeat tests FAIL (Expected 1 Was 2) against the old blind remap.", + "tags": [ + "ha-term-safety", + "raft", + "peer-index", + "match_index", + "oob", + "false-quorum", + "task-3.3", + "remediation", + "c" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T09:20:00.000Z" + }, + { + "id": "bug-959", + "timestamp": "2026-06-22T09:31:11.227Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !sha256 || !buf || !out_len", + "fix": "Added guard clause: if (!sha256 || !buf || !out_len)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:31:11.227Z" + }, + { + "id": "bug-960", + "timestamp": "2026-06-22T09:43:23.309Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:43:23.309Z" + }, + { + "id": "bug-961", + "timestamp": "2026-06-22T09:45:31.779Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c || !sha256", + "fix": "Added guard clause: if (!c || !sha256)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:45:31.779Z" + }, + { + "id": "bug-962", + "timestamp": "2026-06-22T10:04:06.960Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: committed == 0", + "fix": "Added guard clause: if (committed == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:04:06.960Z" + }, + { + "id": "bug-963", + "timestamp": "2026-06-22T10:04:17.709Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 8\u219214 lines (2 removed) | Also: /* Sentinel header: magic (8 bytes) + checkpoint_l; char header[16];", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T10:04:29.639Z" + }, + { + "id": "bug-964", + "timestamp": "2026-06-22T10:05:22.007Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: acked_out", + "fix": "Added guard clause: if (acked_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:05:22.007Z" + }, + { + "id": "bug-965", + "timestamp": "2026-06-22T10:05:36.940Z", + "error_message": "Significant refactor of ", + "file": "src/controller/config_repl.c", + "root_cause": "9 lines replaced/restructured", + "fix": "Rewrote 22\u219226 lines (9 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:05:36.940Z" + }, + { + "id": "bug-966", + "timestamp": "2026-06-22T10:10:11.668Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:10:11.668Z" + }, + { + "id": "bug-967", + "timestamp": "2026-06-22T10:10:17.811Z", + "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"persist_roll_segment\" instead of \"persist_test_force_new_segment\"", + "fix": "Changed persist_roll_segment \u2192 persist_test_force_new_segment", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:10:17.811Z" + }, + { + "id": "bug-968", + "timestamp": "2026-06-22T10:16:44.417Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 12\u219225 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:16:44.417Z" + }, + { + "id": "bug-969", + "timestamp": "2026-06-22T10:19:46.334Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 25\u219234 lines (4 removed) | Also: /* Commit the marker's WAL LSN through the real co; * exactly as a majority WAL ack does in production", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T10:20:17.591Z" + }, + { + "id": "bug-970", + "timestamp": "2026-06-22T10:24:35.905518Z", + "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER \u2014 config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", + "file": "src/controller/config_repl.c, src/controller/persist.c", + "root_cause": "Finding1: do_upgrade_config_with_bytes still drove the term-blind committed_config_tag push/ack/advance and never called config_append_marker, so the derive primitive was dead code. Finding2: persist_checkpoint->trim_wal_segments unlinks older segments and the snapshot sentinel carried no config reference, so a committed marker in a trimmed segment vanished and config_committed_marker regressed/returned -ENOENT.", + "fix": "Added config_commit_change() wired into do_upgrade_config_with_bytes step 7: appends WAL_CONFIG_MARKER as the consensus record (config_append_marker), then delegates byte delivery + the existing majority barrier to push_config_to_standbys (mechanism unchanged); committed_config_tag is now only a cache, config_committed_marker (committed-prefix derive) is authoritative. Compaction safety: persist_checkpoint scans the committed prefix for the latest marker {version_tag,sha256} (scan_latest_committed_config_ref) BEFORE trim and stamps it into the snapshot sentinel (41-byte optional tail); load_snapshot reads it back into persist.cfgref_*; config_committed_marker falls back to it when no marker survives the live WAL. Added integration test (drives config_commit_change) + checkpoint-trim test + persist_test_force_new_segment seam. cb5 upgrade pins made persist-backed so the marker append succeeds. RED confirmed (Expected 1 Was 0; Expected 0 Was -110); GREEN 0 failures direct-run.", + "tags": [ + "ha-term-safety", + "task-4.2", + "config-marker", + "wal", + "checkpoint", + "compaction", + "review-remediation", + "controller" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T10:24:35.905518Z" + }, + { + "id": "bug-971", + "timestamp": "2026-06-22T10:39:44.072Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: !mc", + "fix": "Added guard clause: if (!mc)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:39:44.072Z" + }, + { + "id": "bug-972", + "timestamp": "2026-06-22T10:40:31.131Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:40:31.131Z" + }, + { + "id": "bug-973", + "timestamp": "2026-06-22T10:57:25.062Z", + "error_message": "Significant refactor of ", + "file": "src/controller/config_repl.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 7\u219218 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:57:25.062Z" + }, + { + "id": "bug-974", + "timestamp": "2026-06-22T10:58:21.742Z", + "error_message": "Significant refactor of ", + "file": "src/controller/myapp.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 8\u219226 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:58:21.742Z" + }, + { + "id": "bug-975", + "timestamp": "2026-06-22T11:00:42.661Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:00:42.661Z" + }, + { + "id": "bug-976", + "timestamp": "2026-06-22T11:15:00.000Z", + "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", + "file": "src/controller/config_repl.c, src/controller/ha.c, src/controller/myapp.c", + "root_cause": "config_activate_committed (verify-before-activate + fence) was unit-correct but not integrated: (A) handle_repl_config_push called load_primary_config UNCONDITIONALLY before config_activate_committed, so mismatched bytes went live before the gate; (B) myapp.c boot step 7 + on_commit_advanced never called the consumer, so a recovering member loaded config blind against the committed marker; (C) ha_dispatch_gate / pre-promotion gate never read config_serve_fence, so a fenced member still served + could promote", + "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY \u2014 removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", + "tags": [ + "ha-term-safety", + "config-replication", + "verify-before-activate", + "serve-fence", + "dispatch-gate", + "promotion-gate", + "safety", + "task-4.3", + "remediation" + ], + "related_bugs": [ + "bug-851", + "bug-924" + ], + "occurrences": 1, + "last_seen": "2026-06-22T11:15:00.000Z" + }, + { + "id": "bug-977", + "timestamp": "2026-06-22T11:21:09.742Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:21:09.742Z" + }, + { + "id": "bug-978", + "timestamp": "2026-06-22T11:21:58.604Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 17\u219233 lines (5 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:21:58.604Z" + }, + { + "id": "bug-979", + "timestamp": "2026-06-22T11:30:41.086Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "9 lines replaced/restructured", + "fix": "Rewrote 26\u219221 lines (9 removed) | Also: * commit barrier, which would advance committed_ls; * live (active tag is still the OLD marker's tag).", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T11:32:15.087Z" + }, + { + "id": "bug-980", + "timestamp": "2026-06-22T11:32:35.966Z", + "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"ClusterConfig\" instead of \"cluster_config_t\"", + "fix": "Changed ClusterConfig \u2192 cluster_config_t", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:32:35.966Z" + }, + { + "id": "bug-981", + "timestamp": "2026-06-22T11:35:00Z", + "error_message": "ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9.", + "file": "src/controller/config_repl.c, src/controller/ha.c", + "root_cause": "No post-commit retry trigger for a config serve-fence set under the push-before-commit ordering. config_committed_marker returns the older committed marker until the new marker is in the committed prefix; the two activation call sites (boot, handle_repl_config_push) both run BEFORE the new marker commits, and ha_cb_commit_advanced_stub deliberately did nothing (async-I/O invariant forbids disk reads on the iomgr ACK thread).", + "fix": "Added config_schedule_activate_retry(c) (non-blocking ordq_put via delay_task_ms, no-op unless serve-fenced AND delaymgr running) and config_activate_retry_fn(arg) (delaymgr task body that re-runs config_activate_committed off the iomgr thread). ha_cb_commit_advanced_stub now schedules the retry whenever config_serve_fenced(c) is set after commit advances. Once the NEW marker is in the committed prefix and the saved bytes hash-verify, the retry clears the fence and activates; -ENOENT still falls back without fencing; no busy-spin (no-op once unfenced). RED captured by stubbing config_activate_retry_fn to a no-op (fence stayed set, NEW config not loaded); GREEN with the real off-thread retry.", + "tags": [ + "ha-term-safety", + "config-repl", + "serve-fence", + "liveness", + "async-io", + "delaymgr", + "commit-advance", + "push-before-commit", + "R7.9" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:35:00Z" + }, + { + "id": "bug-982", + "timestamp": "2026-06-22T11:52:31.750Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "12 lines replaced/restructured", + "fix": "Rewrote 16\u21928 lines (12 removed) | Also: /* A standby's durable ACK covers LSN 2; route the; * The agreed entries are current-term (T), so the ", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T11:54:34.028Z" + }, + { + "id": "bug-983", + "timestamp": "2026-06-22T11:53:36.582Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: commit_index", + "fix": "Added guard clause: if (commit_index)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:53:36.582Z" + }, + { + "id": "bug-984", + "timestamp": "2026-06-22T12:03:15.865Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:03:15.865Z" + }, + { + "id": "bug-985", + "timestamp": "2026-06-22T12:06:43.530Z", + "error_message": "Wrong return value", + "file": "src/controller/testsuite/waldump.c", + "root_cause": "Was returning: -EINVAL;", + "fix": "Now returns: waldump_path_visit(path, out, stats, NULL, NULL);", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:06:43.530Z" + }, + { + "id": "bug-986", + "timestamp": "2026-06-22T12:08:51.529Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"waldump.h\"", + "fix": "Changed to \"walcompare.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:08:51.529Z" + }, + { + "id": "bug-987", + "timestamp": "2026-06-22T12:09:28.306Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test.c", + "root_cause": "No early return/throw for edge case: st.n_gaps > 0 || st.crc_broken", + "fix": "Added guard clause: if (st.n_gaps > 0 || st.crc_broken)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:09:28.306Z" + }, + { + "id": "bug-988", + "timestamp": "2026-06-22T13:05:15.064Z", + "error_message": "Missing error handling in degrades", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T13:05:15.064Z" + }, + { + "id": "bug-989", + "timestamp": "2026-06-22T13:19:16.414Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/clean_failover_preservation.sh", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 4\u219213 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T13:19:16.414Z" + }, + { + "id": "bug-990", + "timestamp": "2026-06-22T13:24:07.476480", + "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster.", + "file": "src/controller/repl.c (live send repl_broadcast persist.c:747) + src/controller/ha.c (ha_apply_wal_catchup_entry term!=0 legacy direct-apply)", + "root_cause": "The live replication SEND path was deferred from tasks 3.2/3.3 and still uses the legacy term-0 repl_broadcast; the standby applies via the legacy direct-apply path, NOT through raft_on_append_entries_request, so it never receives/persists the leader's term. The raft library send loop (on_send_append_entries / raft_send_append_entries) has ZERO live callers. on_persist_entry stamps the real term on the PRIMARY only.", + "fix": "NOT FIXED (out of task 5.2 testsuite-only boundary). Remaining integration work: retire the term-0 repl_broadcast live SEND and drive raft_send_append_entries/on_send_append_entries on the live primary so standbys persist the leader's term via raft_on_append_entries_request. Built clean_failover_preservation.sh (XCE_FAILOVER_STRICT_TERM=1) as the ready-to-run assertion this retirement must satisfy.", + "tags": [ + "ha-term-safety", + "task-5.2", + "live-send-retirement", + "term-0", + "repl_broadcast", + "committed-prefix-divergence", + "bug-851", + "BLOCKED" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T13:24:07.476480" + }, + { + "id": "bug-991", + "timestamp": "2026-06-22T16:30:00.000Z", + "error_message": "Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic.", + "file": "src/controller/ha.c (ha_apply_term_entry_via_library ~2115; batch ~2205; single-entry ~2225)", + "root_cause": "applied_lsn carried TWO conflicting semantics. Recovery (=durable_floor), checkpoint (=checkpoint_lsn), init (=0) and the leader's catch-up streamer + ha_evaluate_repl_action all treat it as 'last contiguously-held LSN' and stream (standby_lsn, committed]. But the three live/catch-up apply setters advanced to entry->lsn + 1 ('next-to-apply'). After a standby applied catch-up entry N it advertised N+1, so the leader's next stream began at N+2 and skipped N+1 -> logOk reject -> no WalAck.", + "fix": "fix-1: the three live-apply setters now advance applied_lsn to the LAST-HELD LSN (e->lsn / last_good_lsn / entry->lsn), not +1, so applied_lsn means 'last contiguously-held LSN' everywhere. Reader audit (all consistent with last-held, no +1 reader needed): state report (advertised LSN, ha.c:2461) now matches streamer contract; convergence reported_lsn>=committed_lsn (2536) correct; snapshot/stream boundary standby_lsn+1 routed through the library, so raft.log_len already == applied_lsn after fix-1; raft.log_len legitimately differs from applied_lsn on the legacy term-0 direct-apply and recovery paths, so re-sourcing would regress bug-848 incremental catch-up -> fix-1 alone delivers the value the streamer+logOk need. Unit RED->GREEN: test_catchup_applied_lsn_is_last_held_no_skip (Expected 1 Was 2 pre-fix, PASS post-fix). 7 legacy test_wal_catchup_* expected values updated lsn+1 -> last-held. NOTE: the LIVE clean_failover_preservation STRICT_TERM commit=5 target was NOT reachable in the sandbox: committed_lsn stalls at 1 IDENTICALLY with this fix stashed (baseline experiment) -> that stall is a SEPARATE upstream blocker (library raft.log_len / commit-advance not advancing past the own-term noop; ha-term-safety task 3.1 area), NOT the bug-991 applied_lsn skip. waldump confirms all 3 members hold LSN 1..5 byte-identical (term 1) -> the replication half (which this fix corrects) is sound.", + "tags": [ + "ha", + "controller", + "wal-catchup", + "applied_lsn", + "commit-stall", + "off-by-one", + "term-safety", + "bug-991", + "streamer-skip" + ], + "related_bugs": [ + "bug-851", + "bug-850", + "bug-848", + "bug-924", + "bug-960" + ], + "occurrences": 1, + "last_seen": "2026-06-22T16:30:00.000Z" + }, + { + "id": "bug-992", + "timestamp": "2026-06-22T15:09:54.490Z", + "error_message": "Significant refactor of ", + "file": "src/include/repl.h", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3\u219213 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:09:54.490Z" + }, + { + "id": "bug-993", + "timestamp": "2026-06-22T15:10:39.717Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:10:39.717Z" + }, + { + "id": "bug-994", + "timestamp": "2026-06-22T15:12:30.050Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "15 lines replaced/restructured", + "fix": "Rewrote 21\u21925 lines (15 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:12:30.050Z" + }, + { + "id": "bug-995", + "timestamp": "2026-06-22T15:13:02.631Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/request.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:13:02.631Z" + }, + { + "id": "bug-996", + "timestamp": "2026-06-22T15:14:29.189Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_repl_unit.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 15\u219223 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:14:29.189Z" + }, + { + "id": "bug-997", + "timestamp": "2026-06-22T15:26:52.514Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:26:52.514Z" + }, + { + "id": "bug-998", + "timestamp": "2026-06-22T15:27:10.703Z", + "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Used \"test_repl_ack_currentterm_quorum_advances_commit\" instead of \"test_live_broadcast_receive_persists_leader_term\"", + "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit \u2192 test_live_broadcast_receive_persists_leader_term", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:27:10.703Z" + }, + { + "id": "bug-990-fix", + "timestamp": "2026-06-22T16:00:00.000000", + "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990.", + "file": "src/controller/repl.c (repl_broadcast), src/controller/persist.c (persist_wal_async call site + entry_term_locked), src/controller/config_repl.c (ha_stream_wal_to_standby catch-up), src/controller/ha.c (ha_apply_wal_catchup_entry receive route), src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)", + "root_cause": "The LIVE replication SEND paths (repl_broadcast live append + ha_stream_wal_to_standby catch-up) built wal_catchup_entry_t via memset(0) and never set entry.term/prev_log_index/prev_log_term, so standbys received term=0 entries and applied them via the LEGACY term-0 direct-apply branch (persist_wal_write_replicated stamps the STANDBY's local p->term, not the leader's). The term-carrying library receive route (raft_on_append_entries_request, task 3.2) only fired for term!=0 entries, which the live send never produced. Additionally wal_entry_to_wire only encoded per-entry term, not the per-entry prev tuple, so multi-entry catch-up batches could not be routed through the library one-at-a-time.", + "fix": "Task 3.4 minimal cutover (chosen over driving raft_send_append_entries live, which would rip out the repl model and risk 848/849/850): (1) repl_broadcast gains a prev_log_term param and sets entry.term=hdr->term, entry.prev_log_index=hdr->lsn-1, entry.prev_log_term (sourced under p->lock via new entry_term_locked accessor in persist_wal_async). (2) ha_stream_wal_to_standby sets each catch-up entry's term + prev tuple (tracking the prior streamed entry's lsn/term in ascending order). (3) wal_entry_to_wire now encodes prevLogIndex/prevLogTerm/leaderCommit PER SUB-ENTRY so a multi-entry batch is library-routable. (4) ha_apply_wal_catchup_entry routes every term-carrying entry (single + each batch sub-entry) through new ha_apply_term_entry_via_library -> ha_consume_append_entries -> raft_on_append_entries_request -> on_persist_entry, which persist_set_term(leader_term) before persist_wal_write_replicated (bug-850 contiguity guard preserved beneath). The term-0 legacy direct-apply is retired for live/catch-up entries. ACK/commit path (3.3) unchanged.", + "tags": [ + "ha-term-safety", + "task-3.4", + "live-send-retirement", + "term-0", + "repl_broadcast", + "bug-990", + "FIXED", + "catch-up-term" + ], + "related_bugs": [ + "bug-990", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T16:00:00.000000", + "validation": "Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) \u2014 bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline)." + }, + { + "id": "bug-1000", + "timestamp": "2026-06-22T15:44:38.466Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:44:38.466Z" + }, + { + "id": "bug-1001", + "timestamp": "2026-06-22T15:44:54.113Z", + "error_message": "Missing error handling in errno", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - _Blocked (live portion only \u2014 build+unit gate CONFIRMED GR", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T15:48:39.777Z" + }, + { + "id": "bug-1002", + "timestamp": "2026-06-22T21:10:31.345Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Had 's term and standbys persist it, proven at the int", + "fix": "Changed to 's REAL term (2) for the committed prefix (pre-3.4", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T21:10:31.345Z" + }, + { + "id": "bug-1003", + "timestamp": "2026-06-22T22:09:23.064Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (agreed == 0)", + "fix": "Changed to: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T22:09:23.064Z" + }, + { + "id": "bug-1004", + "timestamp": "2026-06-22T22:09:40.202Z", + "error_message": "Significant refactor of ", + "file": "src/lib/raft/raft.c", + "root_cause": "12 lines replaced/restructured", + "fix": "Rewrote 16\u21923 lines (12 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T22:09:40.202Z" + }, + { + "id": "bug-1005", + "timestamp": "2026-06-22T22:31:03.315Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T22:31:08.791Z" + }, + { + "id": "bug-1006", + "timestamp": "2026-06-22T22:32:53.722Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 \u2014 see th", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T22:33:00.066Z" + }, + { + "id": "bug-1007", + "timestamp": "2026-06-22T22:33:09.231Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 7\u219210 lines (3 removed) | Also: * advance applied_lsn to last-successfully-app; * directly and advance applied_lsn to lsn+1 on", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T22:33:23.732Z" + }, + { + "id": "bug-1008", + "timestamp": "2026-06-22T22:34:27.478Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 13\u219216 lines (2 removed) | Also: /* applied_lsn must advance to 21+1=22 */; TEST_ASSERT_EQUAL_UINT64(22, | Also: /* applied_lsn must reach at least 31 (e0 applied); TEST_ASSERT_TRUE(final_lsn >= 31);", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-22T22:34:47.036Z" + }, + { + "id": "bug-1009", + "timestamp": "2026-06-22T22:44:50.365Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T22:44:50.365Z" + }, + { + "id": "bug-1010", + "timestamp": "2026-06-22T22:47:11.324Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "9 lines replaced/restructured", + "fix": "Rewrote 14\u21926 lines (9 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T22:47:11.324Z" + }, + { + "id": "bug-1011", + "timestamp": "2026-06-22T23:44:22.860Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/commit.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T23:44:22.860Z" + }, + { + "id": "bug-1012", + "timestamp": "2026-06-22T23:46:05.735Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "39 lines replaced/restructured", + "fix": "Rewrote 84\u2192147 lines (39 removed) | Also: RUN_TEST(test_commit_advanced_is_commit_source);; RUN_TEST(test_commit_advanced_ceiling_caps_durable | Also: * which raises the barrier ceiling. 3-peer cluster; * self holds LSN 1,2 at the current term; one stan | Also: /* Durable majority: the standby's durable positio; * barrier ceiling is not held below the library co | Also: /* (3) on_commit_advanced fired \u2192 the barrier ceil; * advanced to the library point under the durable ", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-22T23:48:10.091Z" + }, + { + "id": "bug-1013", + "timestamp": "2026-06-22T00:00:00Z", + "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes \u2014 STRICT_TERM live failover gate fails 'committed_lsn>=5'", + "file": "src/controller/commit.c", + "root_cause": "ha-term-safety task 2.4 added a min(durable-majority, raft_commit_index) ceiling in majority_candidate_locked. raft_commit_index is fed only via the library ACK path (ha_raft_on_repl_ack -> raft_advance_commit -> on_commit_advanced), which advances from match_index[] driven by the asynchronous catch-up streamer and routinely LAGS the durable majority. When delivery goes via catch-up, raft_commit_index stays low and PINS committed_lsn below the safe durable watermark (a redundant, starvation-prone gate). The durable-majority barrier was already current-term-safe under single-writer + the own-term NOOP writable gate, making the ceiling redundant.", + "fix": "Option A: removed the raft_commit_index fold from majority_candidate_locked (commit.c). The durable-majority barrier + own-term writable_lsn gate are now authoritative for commit-advance. commit_set_raft_commit_index / ce->raft_commit_index made INERT-ADVISORY (recorded for observability/conformance-oracle only; still fed by on_commit_advanced so the library half keeps a sink). Kept durable self-cap (persist_durable_lsn), persist-before-ACK, and assert(cand<=persist_durable_lsn). Verified live: commit advances 4->5 via durable barrier (commit.c:184); walcompare OK committed prefix byte-identical across 3 nodes up to LSN 5.", + "tags": [ + "ha", + "commit", + "raft", + "term-safety", + "starvation", + "commit-ceiling", + "optA", + "liveness", + "catch-up" + ], + "related_bugs": [ + "bug-991", + "bug-661", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T00:00:00Z" + }, + { + "id": "bug-1014", + "timestamp": "2026-06-23T00:22:40.544Z", + "error_message": "Missing error handling in can", + "file": "src/controller/testsuite/clean_failover_preservation.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T00:22:40.544Z" + }, + { + "id": "bug-1015", + "timestamp": "2026-06-23T01:30:18.896Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_election.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T01:30:18.896Z" + }, + { + "id": "bug-1016", + "timestamp": "2026-06-23T01:31:58.770Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 56\u2192117 lines (5 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T01:31:58.770Z" + }, + { + "id": "bug-828", + "timestamp": "2026-06-23T01:40:00.000Z", + "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", + "file": "src/controller/ha.c (vote_reply_dispatch), src/controller/persist.c (persist_wal_write_replicated)", + "root_cause": "vote_reply_decode_and_dispatch decoded the granting voter's advertised peerLastLogLsn (vr_lsn at ha.c ~4254) but vote_reply_dispatch DISCARDED it ('(void)peer_lsn; raft_on_vote_reply(...)'). raft_on_vote_reply only bumps s->votes_granted; it never records the peer LSN. The only writer of ctx->votes[idx].vote_granted/peer_lsn (handle_request_vote_reply) was, in production, called ONLY from the connect/error failure paths with (granted=0, lsn=0); the real-grant caller was gated behind a test-only hook (xce_vote_reply_hook, never set in production). So ctx->votes[*] stayed calloc'd zero -> ha_compute_safe_committed_lsn saw n_granted=1 (self) -> safe_committed_lsn = self_lsn. A standby promoting with its own durable_floor=1 then persist_truncate_wal_above(1) destroyed committed LSN 2..5.", + "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) \u2014 it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", + "tags": [ + "safety", + "failover", + "safe-committed-lsn", + "vote-reply", + "leader-completeness", + "ha-repl-hardening", + "PARTIAL-FIX", + "bug-795", + "c" + ], + "related_bugs": [ + "bug-795", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-23T01:40:00.000Z" + }, + { + "id": "bug-1018", + "timestamp": "2026-06-23T02:58:38.185Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T02:58:38.185Z" + }, + { + "id": "bug-828-part2", + "timestamp": "2026-06-23T03:05:00.000Z", + "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", + "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", + "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held \u2014 the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", + "fix": "No production code change required for part-2 \u2014 the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 \u2014 locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", + "tags": [ + "safety", + "failover", + "safe-committed-lsn", + "durable-tip", + "advertise", + "leader-completeness", + "bug-795", + "bug-661", + "commit-durability", + "INVESTIGATION", + "c" + ], + "related_bugs": [ + "bug-828", + "bug-795", + "bug-661", + "bug-851", + "bug-924" + ], + "occurrences": 1, + "last_seen": "2026-06-23T03:05:00.000Z" + }, + { + "id": "bug-1020", + "timestamp": "2026-06-23T03:27:59.030Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_repl_apply.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T03:27:59.030Z" + }, + { + "id": "bug-1021", + "timestamp": "2026-06-23T03:28:20.600Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "6 lines replaced/restructured", + "fix": "Rewrote 8\u219225 lines (6 removed) | Also: INFO(\"DBG828 lib_ack lsn=%\" PRIu64 \" e_term=%\" PRI; \" after_term=%\" PRIu64 \" before_term=%\" PRIu64,", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-23T03:28:37.106Z" + }, + { + "id": "bug-1022", + "timestamp": "2026-06-23T03:28:31.151Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 8\u21925 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T03:28:31.151Z" + }, + { + "id": "bug-1023", + "timestamp": "2026-06-23T03:45:00Z", + "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", + "file": "src/controller/ha.c", + "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp \u2014 NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", + "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning \u2014 it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step \u2014 committed jobs are physically present on disk (walcompare proves it).", + "tags": [ + "ha", + "raft", + "failover", + "leader-completeness", + "bug-828", + "bug-795", + "durability", + "truncate", + "election", + "durable_lsn" + ], + "related_bugs": [ + "bug-661", + "bug-795", + "bug-828", + "bug-851", + "bug-854", + "bug-339" + ], + "occurrences": 1, + "last_seen": "2026-06-23T03:45:00Z" + }, + { + "id": "bug-1024", + "timestamp": "2026-06-23T04:07:37.423Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T04:07:37.423Z" + }, + { + "id": "bug-1025", + "timestamp": "2026-06-23T04:09:25.647Z", + "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", + "file": "src/controller/testsuite/clean_failover_preservation.sh", + "root_cause": "Used \"$MASTERS\" instead of \"$MASTERS_LIVE\"", + "fix": "Changed $MASTERS \u2192 $MASTERS_LIVE", + "tags": [ + "auto-detected", + "wrong-reference", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T04:09:25.647Z" + }, + { + "id": "bug-1026", + "timestamp": "2026-06-23T04:47:06.139Z", + "error_message": "Significant refactor of ", + "file": ".kiro/steering/ha-async-io.md", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 10\u219210 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T04:47:06.139Z" + }, + { + "id": "bug-1027", + "timestamp": "2026-06-23T04:47:28.669Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/steering/ha-async-io.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T04:47:28.669Z" + }, + { + "id": "bug-1028", + "timestamp": "2026-06-23", + "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", + "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term), config_repl.c (ha_stream_wal_to_standby), persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK).", + "root_cause": "UNDER INVESTIGATION (provisional). After promotion the new primary appends its own-term NOOP (e.g. LSN 6 @ term 3) and repeatedly streams [committed-1+1..committed] to a standby stuck at LSN 5, which never converges -- a catch-up completion stall on the new term (possible (term,lsn)/logOk prev-tuple mismatch on the new-term entry, or an ack-accounting gap so the streamer re-sends forever). The new primary's client-serving readiness appears gated on standby convergence, so the stall keeps it non-serving. DISTINCT from bug-854 (client-side positional discovery) and from bug-828 (server-side committed-job-loss, FIXED). Orthogonal to data safety: walcompare shows the committed prefix byte-identical across all members with all jobs on disk. Live evidence: $HOME/tmp/xce-ha-deploy/final_green (kept).", + "fix": "NOT FIXED. Needs a dedicated investigation: (1) why the lagging standby does not converge to the new primary's own-term tip (trace the new-term catch-up apply + ACK + the streamer's re-send condition); (2) whether/why client-serving readiness is gated on full standby convergence and whether a freshly-promoted single-writer primary should serve as soon as its own-term NOOP is majority-durable. Owner: ha-commit-durability / failover-availability. Pairs with bug-854 (client discovery) for a fully-green live clean-failover harness; bug-828 (data safety) already fixed.", + "tags": [ + "failover", + "availability", + "liveness", + "catch-up", + "new-primary", + "not-serving", + "ha-commit-durability", + "NOT-FIXED" + ], + "related_bugs": [ + "bug-828", + "bug-854", + "bug-795", + "bug-991" + ], + "occurrences": 1, + "last_seen": "2026-06-23" } ] } \ No newline at end of file