diff --git a/.wolf/buglog.json b/.wolf/buglog.json index c95a572f..0e642b4f 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -10470,6 +10470,2559 @@ ], "occurrences": 1, "last_seen": "2026-06-17T15:00:00Z" + }, + { + "id": "bug-644", + "timestamp": "2026-06-17T08:15:04.978Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'testsuite/test_raft.c'", + "fix": "Changed to 'testsuite/test_netio_arm_failure.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T08:15:04.978Z" + }, + { + "id": "bug-645", + "timestamp": "2026-06-17T08:15:18.557Z", + "error_message": "Significant refactor of ", + "file": "src/lib/netio.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 25\u219239 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T08:15:18.557Z" + }, + { + "id": "bug-646", + "timestamp": "2026-06-17T08:21:32.099Z", + "error_message": "Wrong reference: 611 should be 646", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Used \"611\" instead of \"646\"", + "fix": "Changed 611 \u2192 646", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T08:21:32.099Z" + }, + { + "id": "bug-647", + "timestamp": "2026-06-17T08:21:56.735Z", + "error_message": "Wrong reference: 611 should be 646", + "file": "src/lib/netio.c", + "root_cause": "Used \"611\" instead of \"646\"", + "fix": "Changed 611 \u2192 646", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T08:21:56.735Z" + }, + { + "id": "bug-646", + "timestamp": "2026-06-17T16:30:00Z", + "error_message": "xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'.", + "file": "src/lib/netio.c", + "root_cause": "endpoint_update_epoll() swallowed iomgr_update_poll() failures: it WARN'd 'rc1=-1' but returned 0 and left ep->flags unchanged, so the poll interest was never armed. iomgr_update_poll uses EPOLL_CTL_MOD, which fails with ENOENT when the fd is no longer registered (e.g. after a failed reconnect: xce_ctx_reconnect endpoint_fini's the old fd but only iomgr_monitor/EPOLL_CTL_ADDs on a SUCCESSFUL connect). The next endpoint_enq() send then MOD-failed silently, request bytes were never flushed, no reply arrived, and the synchronous client blocked forever in cbuf_get (no timeout) with no socket-drop event to wake it (handle_disconnect only fires on a broken socket). Server side was a red herring: DEFAULT_SYNC_TIMEOUT_MS=500 bounds the commit barrier, and ha_dispatch_gate already replies STANDBY/NOT_READY for non-serving roles.", + "fix": "endpoint_update_epoll() captures errno and RETURNS -errno (not 0) on iomgr_update_poll failure. endpoint_enq(): on negative arm result, run handle_error(ep) (iomgr_unmonitor + handle_disconnect -> state=ERR + inject -ECONNRESET into wbuf, waking the blocked cbuf_get) and return the error so the failover loop rotates. handle_output(): propagates the arm failure so io_handler runs the error path. TDD: test_netio_arm_failure.c drives EPOLL_CTL_MOD ENOENT via endpoint_enq on an unregistered fd; asserts loud rc<0 + err_handler fired (verified red without fix, green with).", + "tags": [ + "libxce", + "failover", + "netio", + "epoll", + "EPOLL_CTL_MOD", + "ENOENT", + "silent-failure", + "hang", + "cbuf_get", + "reconnect" + ], + "related_bugs": [ + "bug-610", + "bug-609" + ], + "occurrences": 1, + "last_seen": "2026-06-17T16:30:00Z" + }, + { + "id": "bug-649", + "timestamp": "2026-06-17T09:26:05.914Z", + "error_message": "Significant refactor of ", + "file": "src/lib/netio.c", + "root_cause": "13 lines replaced/restructured", + "fix": "Rewrote 48\u219266 lines (13 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T09:26:05.914Z" + }, + { + "id": "bug-648", + "timestamp": "2026-06-17T17:30:00Z", + "error_message": "Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event.", + "file": "src/lib/netio.c", + "root_cause": "io_fsm_do() computed rc = -errno when a closed/reset connection was detected. By OpenSSL 3.x contract (verified against docs + a live probe on OpenSSL 3.6.0), a peer close makes SSL_read() return 0 and is reported via SSL_get_error(), NOT via the return value or errno: clean close_notify -> SSL_ERROR_ZERO_RETURN (errno 0); unexpected EOF (FIN, no close_notify) -> SSL_ERROR_SSL since OpenSSL 3.0 (pre-3.0 it was SSL_ERROR_SYSCALL with errno 0 -- documented); RST -> SSL_ERROR_SYSCALL errno=ECONNRESET. errno is only meaningful for SSL_ERROR_SYSCALL; for SSL_ERROR_SSL the error queue is authoritative and any errno (we observed EPIPE=32) is a leaked side effect of OpenSSL's internal alert-write (write() on a broken connection returns EPIPE; read() never does). So -errno is 0 for the documented EOF cases -> the close looked like success. NOTE: on this 3.6 build a process-death FIN surfaced as SSL_ERROR_SSL/errno=32, which the OLD code propagated -- so this swallow is NOT necessarily the cause of the observed controller-kill permanent hang (that still implicates the reconnect/SSL-handshake path); it is a real, separate, errno-dependent non-determinism.", + "fix": "Extracted pure io_classify_io_error(ssl_err, ecode, io_rc): returns 0 only for WANT_READ/WANT_WRITE and EAGAIN/EWOULDBLOCK syscall; otherwise returns a NEGATIVE error ((ecode!=0)?-ecode:-ECONNRESET) so a detected close NEVER returns 0 regardless of the unreliable errno. io_fsm_do() now sets FSM_STATE_ERROR iff the classifier returns <0. TDD: test_netio_close_propagation.c asserts ZERO_RETURN/errno0 and SSL_ERROR_SSL/EOF map to <0 (verified red with old -ecode, green with fix). libxce suite 168/0; controller_unit_test links clean.", + "tags": [ + "libxce", + "netio", + "ssl", + "SSL_read", + "EOF", + "close_notify", + "openssl-3.x", + "SSL_ERROR_ZERO_RETURN", + "SSL_ERROR_SSL", + "silent-failure", + "hang" + ], + "related_bugs": [ + "bug-646", + "bug-610" + ], + "occurrences": 1, + "last_seen": "2026-06-17T17:30:00Z" + }, + { + "id": "bug-651", + "timestamp": "2026-06-17T10:44:37.350Z", + "error_message": "Wrong return value", + "file": "src/lib/request.c", + "root_cause": "Was returning: 0;", + "fix": "Now returns: (enq_rc < 0) ? enq_rc : 0;", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T10:44:37.350Z" + }, + { + "id": "bug-650", + "timestamp": "2026-06-17T18:30:00Z", + "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket \u2014 the request was never transmitted yet the client waits for a reply forever.", + "file": "src/lib/request.c", + "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) \u2014 but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", + "fix": "All six request.c senders propagate the enq result: `int enq_rc = endpoint_enq(ep,&pkt->link); return (enq_rc < 0) ? enq_rc : 0;`. A failed send now returns <0 through xce_jobsubmit_req -> xce_jobsubmit, which returns before cbuf_get; the failover loop classifies the transport error and rotates instead of blocking on a reply that never arrives. TDD: test_request_send_propagation.c builds an OPCode_nop request (encodes OK) on a dead endpoint (EPOLL_CTL_MOD ENOENT) and asserts xce_request returns <0 (verified red with old `return 0`, green with fix). libxce 169/0; controller_unit_test links clean.", + "tags": [ + "libxce", + "request", + "xce_request", + "failover", + "silent-failure", + "hang", + "cbuf_get", + "endpoint_enq", + "send-propagation" + ], + "related_bugs": [ + "bug-646", + "bug-648" + ], + "occurrences": 1, + "last_seen": "2026-06-17T18:30:00Z" + }, + { + "id": "bug-653", + "timestamp": "2026-06-17T11:21:17.173Z", + "error_message": "Significant refactor of ", + "file": "src/lib/netio.c", + "root_cause": "8 lines replaced/restructured", + "fix": "Rewrote 14\u219219 lines (8 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T11:21:17.173Z" + }, + { + "id": "bug-654", + "timestamp": "2026-06-17T11:21:51.571Z", + "error_message": "Wrong return value", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Was returning: a negative error and (2) invoke the endpoint's err", + "fix": "Now returns: a negative, RETRIABLE error so the caller propagat", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T11:21:51.571Z" + }, + { + "id": "bug-655", + "timestamp": "2026-06-17T11:22:11.296Z", + "error_message": "Significant refactor of ", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 11\u219213 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T11:22:11.296Z" + }, + { + "id": "bug-656", + "timestamp": "2026-06-17T11:22:17.916Z", + "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Used \"test_enq_arm_failure_is_loud_and_fires_errhandler\" instead of \"test_enq_arm_failure_is_loud_no_errhandler\"", + "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler \u2192 test_enq_arm_failure_is_loud_no_errhandler", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T11:22:17.916Z" + }, + { + "id": "bug-652", + "timestamp": "2026-06-17T19:10:00Z", + "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", + "file": "src/lib/netio.c", + "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() \u2014 it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", + "fix": "endpoint_enq() no longer calls handle_error(); on arm failure it just returns a RETRIABLE -ECONNRESET. The caller propagates it (request.c senders return it -> client failover classifies ECONNRESET as RETRY_NOW and rotates; controller heartbeat path simply gets the error, no lock re-entry). The real teardown stays on the io_worker thread (handle_input/handle_output -> io_handler -> handle_error), which holds no caller locks. Updated test_netio_arm_failure.c: endpoint_enq must return <0 AND must NOT invoke the err_handler (g_err_count==0). libxce 169/0; controller_unit_test links clean.", + "tags": [ + "libxce", + "netio", + "endpoint_enq", + "deadlock", + "self-deadlock", + "ha-lock", + "reentrancy", + "regression", + "controller", + "vote_ep_err_handler" + ], + "related_bugs": [ + "bug-646", + "bug-650" + ], + "occurrences": 1, + "last_seen": "2026-06-17T19:10:00Z" + }, + { + "id": "bug-658", + "timestamp": "2026-06-17T14:46:54.135Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)", + "fix": "Changed to: if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T14:46:54.135Z" + }, + { + "id": "bug-659", + "timestamp": "2026-06-17T14:47:25.446Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (role == CTRL_STANDBY &&\n )", + "fix": "Changed to: if (role == CTRL_STANDBY &&\n )", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T14:47:25.446Z" + }, + { + "id": "bug-660", + "timestamp": "2026-06-17T22:00:00Z", + "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", + "file": "src/controller/ha.c", + "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` \u2014 checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", + "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) \u2014 so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", + "tags": [ + "controller", + "ha", + "election", + "prevote", + "convergence", + "fresh_heartbeat", + "raft_cb_vote_eligible", + "primary_ctrl_fd", + "slow-election" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T22:00:00Z" + }, + { + "id": "bug-661", + "timestamp": "2026-06-17T23:30:00Z", + "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", + "file": "src/controller/persist.c, src/controller/commit.c, src/controller/repl.c", + "root_cause": "'committed' does not imply 'durable'. WAL fsync is asynchronous (persist.c:173 fsync_worker, timer wal_sync_ms); persist_append and the standby WalAck path do NOT fdatasync before ACK. commit_on_ack (commit.c:303) advances committed_lsn on majority ACK = OS page cache only, durable on zero disks. On kill+restart, killed nodes lose the not-yet-fsynced tail; persist_recover hits CRC break at zeroed region (prev_crc=00000000, offset 40) and truncates replay to ~0 entries, yet the hollow node stays eligible, wins election under degraded required=1 quorum (PreVote+freshness gate cannot fence it because the committed entries were durable on nobody), self-promotes advertising committed_lsn=21 over an empty job table, and the node holding the real data converges DOWN. Violates Leader-Completeness assumption (majority-ACK != durable-on-majority) in src/controller/CLAUDE.md.", + "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 \u2014 matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro \u2014 this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", + "tags": [ + "controller", + "ha", + "durability", + "fsync", + "wal", + "commit-engine", + "data-loss", + "failover", + "leader-completeness", + "promotion-fence", + "async-fsync", + "deterministic-repro", + "wal-sync-ms" + ], + "related_bugs": [ + "bug-339", + "bug-447", + "bug-396" + ], + "occurrences": 2, + "last_seen": "2026-06-18T00:00:00Z" + }, + { + "id": "bug-662", + "timestamp": "2026-06-17T23:34:32.115Z", + "error_message": "Function not marked async", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T23:34:32.115Z" + }, + { + "id": "bug-663", + "timestamp": "2026-06-17T23:59:39.221Z", + "error_message": "Significant refactor of ", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 19\u219235 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T23:59:39.221Z" + }, + { + "id": "bug-664", + "timestamp": "2026-06-17T23:59:44.577Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"${2:-6}\"", + "fix": "Changed to \"${2:-8}\"", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-17T23:59:50.630Z" + }, + { + "id": "bug-665", + "timestamp": "2026-06-18T00:04:40.582Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-commit-durability/brief.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T00:04:40.582Z" + }, + { + "id": "bug-666", + "timestamp": "2026-06-18T02:32:07.670Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T02:32:07.670Z" + }, + { + "id": "bug-667", + "timestamp": "2026-06-18T02:35:08.157Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T02:35:08.157Z" + }, + { + "id": "bug-668", + "timestamp": "2026-06-18T03:12:26.259Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `persist.c:515`", + "fix": "Changed to `fsync_worker`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-18T03:15:00.270Z" + }, + { + "id": "bug-669", + "timestamp": "2026-06-18T03:13:32.314Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 6\u21926 lines (5 removed) | Also: - **Persistence & consistency:** `durable_lsn` adv; - **Concurrency strategy:** advanced under `p->loc", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T03:13:49.756Z" + }, + { + "id": "bug-670", + "timestamp": "2026-06-18T03:14:30.410Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:14:30.410Z" + }, + { + "id": "bug-671", + "timestamp": "2026-06-18T03:15:13.287Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `fsync_worker`", + "fix": "Changed to `/kiro-validate-design`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:15:13.287Z" + }, + { + "id": "bug-672", + "timestamp": "2026-06-18T03:35:27.530Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 18\u219225 lines (7 removed) | Also: - _Depends: 4.1, 3.3_; - Before a member becomes primary, refuse promotio", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T03:35:46.457Z" + }, + { + "id": "bug-673", + "timestamp": "2026-06-18T03:50:26.761Z", + "error_message": "Function not marked async", + "file": "src/controller/CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:50:26.761Z" + }, + { + "id": "bug-674", + "timestamp": "2026-06-18T03:50:55.752Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 5\u21926 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:50:55.752Z" + }, + { + "id": "bug-675", + "timestamp": "2026-06-18T03:59:55.411Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:59:55.411Z" + }, + { + "id": "bug-676", + "timestamp": "2026-06-18T04:00:06.872Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `src/include/persist.h`", + "fix": "Changed to `persist_wal_sync_append`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:00:06.872Z" + }, + { + "id": "bug-677", + "timestamp": "2026-06-18T04:39:24.460Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t \u2192 uint32_t", + "tags": [ + "auto-detected", + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:39:24.460Z" + }, + { + "id": "bug-678", + "timestamp": "2026-06-18T04:50:36.884Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_commit_durability.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:50:36.884Z" + }, + { + "id": "bug-679", + "timestamp": "2026-06-18T04:51:30.430Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:51:30.430Z" + }, + { + "id": "bug-680", + "timestamp": "2026-06-18T05:00:22.007Z", + "error_message": "Missing error handling in unknown", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:00:22.007Z" + }, + { + "id": "bug-681", + "timestamp": "2026-06-18T05:01:45.135Z", + "error_message": "Significant refactor of ", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 85\u2192202 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:01:45.135Z" + }, + { + "id": "bug-682", + "timestamp": "2026-06-18T05:02:07.556Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"$(mktemp /tmp/xce-durability-acked.XXXXXX)\"", + "fix": "Changed to \"$(mktemp \"", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:02:07.556Z" + }, + { + "id": "bug-683", + "timestamp": "2026-06-18T05:07:20.589Z", + "error_message": "Missing error handling in unknown", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:07:20.589Z" + }, + { + "id": "bug-684", + "timestamp": "2026-06-18T05:28:05.352Z", + "error_message": "Function not marked async", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "sh" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-18T05:29:29.811Z" + }, + { + "id": "bug-685", + "timestamp": "2026-06-18T05:42:21.676Z", + "error_message": "Missing error handling in advances", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:42:21.676Z" + }, + { + "id": "bug-686", + "timestamp": "2026-06-18T05:43:02.761Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T05:45:15.732Z" + }, + { + "id": "bug-687", + "timestamp": "2026-06-18T05:43:49.070Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 41\u2192152 lines (7 removed) | Also: /* Wake any WAIT caller blocked on the commit eval; * advanced, so a majority candidate may now cover ", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T05:44:05.035Z" + }, + { + "id": "bug-688", + "timestamp": "2026-06-18T05:54:46.908Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T05:54:55.010Z" + }, + { + "id": "bug-689", + "timestamp": "2026-06-18T05:55:28.140Z", + "error_message": "Missing error handling in FAILS", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:55:28.140Z" + }, + { + "id": "bug-690", + "timestamp": "2026-06-18T05:55:33.474Z", + "error_message": "Wrong reference: else should be RUN_TEST", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Used \"else\" instead of \"RUN_TEST\"", + "fix": "Changed else \u2192 RUN_TEST", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:55:33.474Z" + }, + { + "id": "bug-691", + "timestamp": "2026-06-18T05:59:05.965Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 24\u219252 lines (5 removed) | Also: * already joined and will not advance it again). *; fdatasync(p->wal_fd);", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T06:00:18.921Z" + }, + { + "id": "bug-692", + "timestamp": "2026-06-18T05:59:17.869Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-18T06:00:42.436Z" + }, + { + "id": "bug-693", + "timestamp": "2026-06-18T06:05:00.000Z", + "error_message": "durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority", + "file": "src/controller/persist.c", + "root_cause": "ha-commit-durability task 2.1 flush sites called fdatasync(p->wal_fd) without capturing/checking the return code, then advanced durable_lsn unconditionally. On sync error the bytes are not on stable storage, violating Req 1.1/1.2 and the fail-loud preference.", + "fix": "Capture int rc = fdatasync(...) at every durable_lsn advance site (flusher_flush_once gating + fallback paths, persist_stop) and advance durable_lsn ONLY when rc == 0; on error leave durable_lsn unchanged and log loudly via ERROR(). persist_flush already gated on rc==0 (reference). Added XCE_TESTING fault-injection seam (test_force_fsync_fail + persist_test_set_force_fsync_fail + flusher_fsync_forced_fail helper, constant-false in production) and error-path test test_durable_lsn_holds_on_fsync_failure proving durable_lsn holds at baseline under injected failure and recovers when cleared.", + "tags": [ + "ha-commit-durability", + "durability", + "fdatasync", + "fail-loud", + "remediation", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:05:00.000Z" + }, + { + "id": "bug-694", + "timestamp": "2026-06-18T06:09:08.298Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", + "root_cause": "Had `/kiro-spec-init ha-commit-durability`", + "fix": "Changed to 't fence the stale node because the entries were d", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:09:08.298Z" + }, + { + "id": "bug-695", + "timestamp": "2026-06-18T06:12:59.924Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:12:59.924Z" + }, + { + "id": "bug-696", + "timestamp": "2026-06-18T06:13:32.891Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:13:32.891Z" + }, + { + "id": "bug-697", + "timestamp": "2026-06-18T06:13:40.659Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:13:40.659Z" + }, + { + "id": "bug-698", + "timestamp": "2026-06-18T06:13:54.196Z", + "error_message": "Significant refactor of ", + "file": "src/controller/commit.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 12\u219224 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:13:54.196Z" + }, + { + "id": "bug-699", + "timestamp": "2026-06-18T06:14:16.158Z", + "error_message": "Function not marked async", + "file": "src/controller/commit.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:14:16.158Z" + }, + { + "id": "bug-700", + "timestamp": "2026-06-18T06:15:12.451Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:15:12.451Z" + }, + { + "id": "bug-701", + "timestamp": "2026-06-18T06:15:45.248Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Had \"config.h\"", + "fix": "Changed to \"commit.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:15:45.248Z" + }, + { + "id": "bug-702", + "timestamp": "2026-06-18T06:16:09.746Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:16:09.746Z" + }, + { + "id": "bug-703", + "timestamp": "2026-06-18T06:16:49.048Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "40 lines replaced/restructured", + "fix": "Rewrote 49\u21926 lines (40 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:16:49.048Z" + }, + { + "id": "bug-704", + "timestamp": "2026-06-18", + "error_message": "controller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0)", + "file": "src/controller/testsuite/{test_repl_majority,test_commit_barrier_rollout,test_coldstart_lsn0,test_demotion_convergence}.c", + "root_cause": "ha-commit-durability 2.2 changed majority_candidate_locked self-cap from highest_appended_lsn to persist_durable_lsn. Existing commit-path fixtures prime current_lsn (appended tip) but never set durable_lsn, so under the new cap self contributed 0 and committed_lsn never advanced.", + "fix": "In every commit-path stub/in-memory-primed fixture, set durable_lsn to the appended tip (appended_lsn / current_lsn-1) right after committed_lsn=0, preserving the pre-durability 'self covers every appended entry' model. The durability precondition itself is exercised by dedicated new tests.", + "tags": [ + "durable", + "commit", + "majority", + "self-cap", + "test-fixture", + "ha-commit-durability", + "task-2.2" + ], + "related_bugs": [ + "bug-661", + "bug-693" + ], + "occurrences": 1, + "last_seen": "2026-06-18" + }, + { + "id": "bug-705", + "timestamp": "2026-06-18T06:33:12.523Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T06:33:25.815Z" + }, + { + "id": "bug-706", + "timestamp": "2026-06-18T06:33:50.093Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3\u219222 lines (2 removed) | Also: int fsrc = fdatasync(p->wal_fd);; if (fsrc != 0 || flusher_fsync_forced_fail(p)) {", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T06:36:02.935Z" + }, + { + "id": "bug-707", + "timestamp": "2026-06-18T06:34:06.875Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p", + "fix": "Added guard clause: if (p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:34:06.875Z" + }, + { + "id": "bug-708", + "timestamp": "2026-06-18T06:34:36.356Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:34:36.356Z" + }, + { + "id": "bug-709", + "timestamp": "2026-06-18T06:34:53.620Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 7\u219210 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:34:53.620Z" + }, + { + "id": "bug-710", + "timestamp": "2026-06-18T06:50:13.989Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:50:13.989Z" + }, + { + "id": "bug-711", + "timestamp": "2026-06-18T06:50:49.987Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: written_lsn > durable", + "fix": "Added guard clause: if (written_lsn > durable)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:50:49.987Z" + }, + { + "id": "bug-712", + "timestamp": "2026-06-18T06:50:59.047Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 8\u219210 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:50:59.047Z" + }, + { + "id": "bug-713", + "timestamp": "2026-06-18T07:00:10.385Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 9\u21926 lines (5 removed) | Also: /* RED-PHASE: invariant assert DISABLED to let the; * the over-advance instead of aborting the binary.", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T07:00:44.663Z" + }, + { + "id": "bug-714", + "timestamp": "2026-06-18T07:00:36.008Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: written_lsn > durable", + "fix": "Added guard clause: if (written_lsn > durable)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T07:00:36.008Z" + }, + { + "id": "bug-715", + "timestamp": "2026-06-18T07:15:03.763Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T07:15:03.763Z" + }, + { + "id": "bug-716", + "timestamp": "2026-06-18T07:15:30.633Z", + "error_message": "Missing error handling in unknown", + "file": "src/include/persist.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T07:15:30.633Z" + }, + { + "id": "bug-717", + "timestamp": "2026-06-18T07:16:20.314Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 10\u219223 lines (2 removed) | Also: break; /* clean end of segment */; break; /* truncated or pre-allocated zeros */", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T07:16:37.162Z" + }, + { + "id": "bug-718", + "timestamp": "2026-06-18T07:19:17.397Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 12\u219214 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T07:19:17.397Z" + }, + { + "id": "bug-719", + "timestamp": "2026-06-18T07:51:32.985Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T07:51:32.985Z" + }, + { + "id": "bug-720", + "timestamp": "2026-06-18T07:52:20.595Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 3.2 \u2014 Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T07:52:29.952Z" + }, + { + "id": "bug-721", + "timestamp": "2026-06-18T08:04:18.857Z", + "error_message": "Missing error handling in enum", + "file": "src/include/persist.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Bounded catch-up window (ha-commit-durability task 3.3, R; * The FINITE deadline a recovery-truncated / behind member i", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T08:04:38.131Z" + }, + { + "id": "bug-722", + "timestamp": "2026-06-18T08:05:05.233Z", + "error_message": "Missing error handling in of", + "file": "src/controller/persist.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:05:05.233Z" + }, + { + "id": "bug-723", + "timestamp": "2026-06-18T08:06:25.738Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 7\u219251 lines (4 removed) | Also: TEST_IGNORE_MESSAGE(; \"row 4b (CLEAN + below + no peer + bounded catch-u", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T08:06:39.996Z" + }, + { + "id": "bug-724", + "timestamp": "2026-06-18T08:07:00.123Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:07:00.123Z" + }, + { + "id": "bug-725", + "timestamp": "2026-06-18T08:18:57.946Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:18:57.946Z" + }, + { + "id": "bug-726", + "timestamp": "2026-06-18T08:19:47.036Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:19:47.036Z" + }, + { + "id": "bug-727", + "timestamp": "2026-06-18T08:32:37.878Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:32:37.878Z" + }, + { + "id": "bug-728", + "timestamp": "2026-06-18T08:39:54.889Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `durable_floor \u2264 applied_lsn`", + "fix": "Changed to `applied_lsn`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:39:54.889Z" + }, + { + "id": "bug-729", + "timestamp": "2026-06-18T08:49:08.264Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 9\u219214 lines (3 removed) | Also: /* RED-DEMO (temporary): simulate the PRE-FIX bug ; * with a stale-high learned value so the candidate", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T08:52:37.412Z" + }, + { + "id": "bug-730", + "timestamp": "2026-06-18T08:49:30.666Z", + "error_message": "Significant refactor of ", + "file": "src/controller/myapp.c", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 11\u219217 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:49:30.666Z" + }, + { + "id": "bug-731", + "timestamp": "2026-06-18T08:49:41.212Z", + "error_message": "Significant refactor of ", + "file": "src/controller/cluster.c", + "root_cause": "6 lines replaced/restructured", + "fix": "Rewrote 10\u219215 lines (6 removed) | Also: uint64_t local_lsn = (uint64_t)atomic_load_explici; &c->ha.applied_lsn, memory_order_acquire);", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T08:49:49.852Z" + }, + { + "id": "bug-732", + "timestamp": "2026-06-18T08:51:34.635Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "No early return/throw for edge case: !mkdtemp(tmp", + "fix": "Added guard clause: if (!mkdtemp(tmp)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:51:34.635Z" + }, + { + "id": "bug-733", + "timestamp": "2026-06-18T08:52:12.840Z", + "error_message": "Wrong return value", + "file": "src/controller/ha.c", + "root_cause": "Was returning: persist_durable_floor(p);", + "fix": "Now returns: floor > 8ULL ? floor : 8ULL; /* BUG: pretends lear", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:52:12.840Z" + }, + { + "id": "bug-734", + "timestamp": "2026-06-18T09:07:33.096Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T09:07:33.096Z" + }, + { + "id": "bug-735", + "timestamp": "2026-06-18T09:08:09.215Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2500\u2500 ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:08:55.973Z" + }, + { + "id": "bug-736", + "timestamp": "2026-06-18T09:09:28.348Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fr; * member is not behind anything and must be promotable (no f", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:10:32.435Z" + }, + { + "id": "bug-737", + "timestamp": "2026-06-18T09:09:42.763Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 7\u219236 lines (4 removed) | Also: TEST_IGNORE_MESSAGE(; \"row 3 (TRUNCATED + peer + catch-up ok -> FENCE+CA | Also: TEST_IGNORE_MESSAGE(; \"row 5 (snapshot-sentinel CLEAN + below + peer -> ", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-18T09:10:12.609Z" + }, + { + "id": "bug-738", + "timestamp": "2026-06-18T09:45:40.163Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:50:15.505Z" + }, + { + "id": "bug-739", + "timestamp": "2026-06-18T09:46:18.019Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:46:36.395Z" + }, + { + "id": "bug-740", + "timestamp": "2026-06-18T09:46:46.814Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:47:04.913Z" + }, + { + "id": "bug-741", + "timestamp": "2026-06-18T09:49:58.840Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Test-injectable send hook (defined in repl.c under XCE_TE; * repl_broadcast from dereferencing the dummy endpoints as r", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:50:07.979Z" + }, + { + "id": "bug-742", + "timestamp": "2026-06-18T10:22:00.509Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:22:00.509Z" + }, + { + "id": "bug-743", + "timestamp": "2026-06-18T10:25:02.673Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ha_cancel_promotion_fence_timer: cancel a pending bounded; * (idempotent; no-op when none is armed). Called when the m", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T10:27:05.112Z" + }, + { + "id": "bug-744", + "timestamp": "2026-06-18T10:26:12.640Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:26:12.640Z" + }, + { + "id": "bug-745", + "timestamp": "2026-06-18T10:26:34.227Z", + "error_message": "Wrong operator: != should be ==", + "file": "src/controller/ha.c", + "root_cause": "Used \"!=\" instead of \"==\"", + "fix": "Changed operator != \u2192 ==", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:26:34.227Z" + }, + { + "id": "bug-746", + "timestamp": "2026-06-18T10:30:18.073Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 4.4 \u2014 bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T10:30:51.746Z" + }, + { + "id": "bug-747", + "timestamp": "2026-06-18T10:42:48.932Z", + "error_message": "Missing error handling in unknown", + "file": "src/include/persist.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:42:48.932Z" + }, + { + "id": "bug-748", + "timestamp": "2026-06-18T10:43:04.187Z", + "error_message": "Significant refactor of ", + "file": "src/include/persist.h", + "root_cause": "10 lines replaced/restructured", + "fix": "Rewrote 17\u219227 lines (10 removed)", + "tags": [ + "auto-detected", + "refactor", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:43:04.187Z" + }, + { + "id": "bug-749", + "timestamp": "2026-06-18T10:44:38.816Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/persist.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:44:38.816Z" + }, + { + "id": "bug-750", + "timestamp": "2026-06-18T10:45:30.263Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "11 lines replaced/restructured", + "fix": "Rewrote 15\u219222 lines (11 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:45:30.263Z" + }, + { + "id": "bug-751", + "timestamp": "2026-06-18T10:47:24.587Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:47:24.587Z" + }, + { + "id": "bug-752", + "timestamp": "2026-06-18T10:47:33.818Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.h", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 6\u21929 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:47:33.818Z" + }, + { + "id": "bug-753", + "timestamp": "2026-06-18T10:48:41.920Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:48:41.920Z" + }, + { + "id": "bug-754", + "timestamp": "2026-06-18T10:54:08.903Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 8\u21929 lines (2 removed) | Also: | Intent | Distinguish clean log end from corrupti; - FATAL policy: if a recovery-truncated member can | Also: > **Status:** DECISION recorded (task 3.2). No pro; **Decision (Req 2.2, 2.4):** A member that boots w", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-18T10:55:04.759Z" + }, + { + "id": "bug-755", + "timestamp": "2026-06-18T10:55:31.898Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had \"bounded window\"", + "fix": "Changed to `DEFAULT_RECOVERY_CATCHUP_WINDOW_MS`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:55:31.898Z" + }, + { + "id": "bug-756", + "timestamp": "2026-06-18T11:07:54.918Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/CLAUDE.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T11:07:54.918Z" + }, + { + "id": "bug-757", + "timestamp": "2026-06-18T11:08:39.844Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/steering/ha-async-io.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T11:08:39.844Z" + }, + { + "id": "bug-758", + "timestamp": "2026-06-18T11:14:25.437Z", + "error_message": "Missing error handling in gap", + "file": "src/controller/CLAUDE.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T11:14:25.437Z" + }, + { + "id": "bug-759", + "timestamp": "2026-06-18T11:21:44.844Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", + "root_cause": "Had `--wal-sync-ms`", + "fix": "Changed to `:FAIL:`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T11:21:44.844Z" + }, + { + "id": "bug-760", + "timestamp": "2026-06-18T13:00:00Z", + "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 \u2014 a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "ha-commit-durability task 2.2 capped the primary commit self-contribution at durable_lsn (advanced asynchronously by the demand-driven group-commit flusher) and wired a durable-advance commit-wake callback. This adjacent-spec (ha-commit-barrier-rollout) test uses a REAL persist+flusher and checks committed_lsn SYNCHRONOUSLY the instant after repl_on_ack(fact_lsn); the flusher has not yet fdatasync'd LSN1, so durable_lsn==0, self-cap==0, majority candidate==0, committed_lsn stays 0. Pre-spec the test passed (self counted the appended tip immediately). Task 2.2 primed durable_lsn in the in-memory stub fixtures (barrier_setup_vcpus) but did NOT update this real-flusher test. The implementation-note claim that this is a pre-existing /tmp-overlay fdatasync-EIO artifact is FALSE in this env: fdatasync returns rc=0 on /tmp here, and the test passed pre-spec.", + "fix": "FIXED \u2014 test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", + "tags": [ + "ha-commit-durability", + "commit-barrier-rollout", + "durable_lsn", + "group-commit", + "test-race", + "async-durability", + "validation", + "NO-GO" + ], + "related_bugs": [ + "bug-661", + "bug-704" + ], + "occurrences": 6, + "last_seen": "2026-06-18T13:30:00Z" + }, + { + "id": "bug-761", + "timestamp": "2026-06-18T13:13:49.968Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T13:13:49.968Z" + }, + { + "id": "bug-762", + "timestamp": "2026-06-18T13:18:43.638Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T13:18:43.638Z" + }, + { + "id": "bug-763", + "timestamp": "2026-06-18T14:54:25.478Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T14:54:25.478Z" + }, + { + "id": "bug-764", + "timestamp": "2026-06-18T14:55:00.167Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/steering/error-handling.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T14:55:00.167Z" + }, + { + "id": "bug-765", + "timestamp": "2026-06-18T14:30:00Z", + "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) \u2192 corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", + "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob \u2192 CREATE path)", + "root_cause": "Tasks were checked [x] without the dequeue/update path and rusage storage actually being wired; gap surfaced only by cross-task re-audit (per-task review missed it).", + "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation \u2014 sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 \u2014 enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", + "tags": [ + "scheduler", + "controller", + "msg-gap", + "validate-gap", + "not-implemented", + "HIGH", + "job-state-corruption" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T17:00:00Z" + }, + { + "id": "bug-766", + "timestamp": "2026-06-19T02:55:49.212Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "Was returning: `-ENOENT` if job not found", + "fix": "Now returns: `-ENOENT` if job not found. _Remediation (bug-765,", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T02:55:49.212Z" + }, + { + "id": "bug-767", + "timestamp": "2026-06-19T03:30:07.652Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/schedobj.c", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 18\u219212 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:30:07.652Z" + }, + { + "id": "bug-768", + "timestamp": "2026-06-19T03:30:28.056Z", + "error_message": "Wrong condition in logic", + "file": "src/scheduler/src/schedobj.c", + "root_cause": "Condition was: if (job->rusages != NULL && job->rusages != rusages)", + "fix": "Changed to: if (job->rusages != rusages)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:30:28.056Z" + }, + { + "id": "bug-769", + "timestamp": "2026-06-19T03:31:07.604Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 18\u219233 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:31:07.604Z" + }, + { + "id": "bug-770", + "timestamp": "2026-06-19T03:48:40.507Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 9\u219211 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:48:40.507Z" + }, + { + "id": "bug-771", + "timestamp": "2026-06-19T03:49:03.016Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3\u21925 lines (2 removed) | Also: #### New function: `xce_sched_op_name`; Signature: char* xce_sched_op_name(xce_sched_op_t ", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T03:49:12.246Z" + }, + { + "id": "bug-772", + "timestamp": "2026-06-19T03:49:23.624Z", + "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "Used \"xce_sched_op_name\" instead of \"sched_op_name\"", + "fix": "Changed xce_sched_op_name \u2192 sched_op_name", + "tags": [ + "auto-detected", + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:49:23.624Z" + }, + { + "id": "bug-773", + "timestamp": "2026-06-19T03:49:39.373Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "Had `xce_sched_op_name()`", + "fix": "Changed to `sched_op_name()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T03:49:46.161Z" + }, + { + "id": "bug-774", + "timestamp": "2026-06-19T03:50:12.460Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "Had `xce_sched_op_name()`", + "fix": "Changed to `sched_op_name()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T03:50:20.916Z" + }, + { + "id": "bug-775", + "timestamp": "2026-06-19T04:07:03.964Z", + "error_message": "Significant refactor of ", + "file": "src/scheduler/testsuite/test_handlers.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 5\u219211 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T04:07:03.964Z" + }, + { + "id": "bug-776", + "timestamp": "2026-06-19T04:13:42.107Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 7\u21927 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T04:13:42.107Z" + }, + { + "id": "bug-777", + "timestamp": "2026-06-19T13:00:00Z", + "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", + "file": "src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table", + "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up \u2014 so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", + "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain \u2014 no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking \u2014 the systematic trigger is gone.", + "tags": [ + "ha", + "convergence", + "promotion-fence", + "recovery-truncated", + "torn-tail", + "SIGKILL", + "availability", + "election-wedge", + "ha-commit-durability", + "ha-convergence-recovery", + "live-E2E", + "false-positive-data-loss" + ], + "related_bugs": [ + "bug-661", + "bug-339" + ], + "occurrences": 2, + "last_seen": "2026-06-19T14:30:00Z" + }, + { + "id": "bug-778", + "timestamp": "2026-06-19T13:40:00Z", + "error_message": "persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss.", + "file": "src/controller/persist.c:1336 (apply_wal_entry) + src/controller/ha.c ha_apply_single_entry (~1570-1588)", + "root_cause": "On the standby replicated-apply path, ha_apply_single_entry first calls persist_wal_write_replicated() which correctly sets p->last_crc to the standby's SELF-computed chain crc (the stream carries no crc; the on-disk CRC chain is node-local, prev_crc==last_crc). It THEN builds a synthetic wal_entry_hdr with hdr.crc=0 and calls apply_wal_entry(), whose line 1336 `p->last_crc = hdr->crc` OVERWRITES the correct chain link with 0. The next replicated write computes prev_crc=(seg_offset==0?WAL_CRC_NONE:last_crc)=0 and stamps prev_crc=0 on disk. So the on-disk CRC chain breaks at the 2nd replicated entry. Invisible while live (standby serves in-memory entities; never replays its own WAL), detonates on COLD RESTART where replay_wal validates the chain -> RECOVER_TRUNCATED -> recovers only the clean prefix -> committed jobs lost + the truncated verdict drives the promotion-fence availability wedge (the other half of bug-777). NOT an fd-position/offset race (pwrite+p->lock are correct); it is shared CRC-chain state (p->last_crc) written by two paths where the logical-apply step clobbers the physical-writer's link with 0.", + "fix": "FIXED 2026-06-19 (commit 64ab210). apply_wal_entry is now LOGICAL entity-restore only (signature (p,type,lsn,payload,payload_len); no header); it advances the current_lsn high-water but NEVER touches p->last_crc. The CRC chain has a single owner: persist_wal_write_replicated (live) + replay_wal/persist_recover (recovery). ha_apply_single_entry drops the fabricated crc=0 header. Verified: RED->GREEN test test_replicated_apply_chain_survives_cold_restart (real ha_apply_wal_catchup_entry path, cold-replay -> RECOVER_CLEAN); controller suite 0 failures; 3-node rolling-SIGKILL soak on real ext2/3 fs 5/5 (was wedge+loss at round 3); full cold restart recovers all 6 jobs (was 0), CLEAN verdicts, normal election. Test callers updated: test_repl_apply, test_ha_role_lifecycle, test_egress_filter.", + "tags": [ + "ha", + "wal", + "crc-chain", + "last_crc", + "apply_wal_entry", + "persist_wal_write_replicated", + "cold-restart", + "data-loss", + "bug-661", + "standby-replication", + "recovery", + "root-caused" + ], + "related_bugs": [ + "bug-777", + "bug-661" + ], + "occurrences": 2, + "last_seen": "2026-06-19T14:30:00Z" + }, + { + "id": "bug-779", + "timestamp": "2026-06-19T13:59:32.538Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T13:59:32.538Z" + }, + { + "id": "bug-780", + "timestamp": "2026-06-19T14:01:31.365Z", + "error_message": "Significant refactor of ", + "file": "src/include/persist.h", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 5\u219212 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T14:01:31.365Z" + }, + { + "id": "bug-781", + "timestamp": "2026-06-19T14:02:49.235Z", + "error_message": "Wrong reference: hdr should be payload_len", + "file": "src/controller/persist.c", + "root_cause": "Used \"hdr\" instead of \"payload_len\"", + "fix": "Changed hdr \u2192 payload_len", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T14:02:49.235Z" + }, + { + "id": "bug-782", + "timestamp": "2026-06-19T14:03:49.830Z", + "error_message": "Significant refactor of ", + "file": "src/controller/persist.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3\u21925 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T14:03:49.830Z" + }, + { + "id": "bug-783", + "timestamp": "2026-06-19T14:04:05.032Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "8 lines replaced/restructured", + "fix": "Rewrote 9\u21927 lines (8 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T14:04:05.032Z" + }, + { + "id": "bug-784", + "timestamp": "2026-06-19T14:07:12.930Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_repl_apply.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 11\u219213 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T14:07:12.930Z" + }, + { + "id": "bug-785", + "timestamp": "2026-06-19T14:08:12.866Z", + "error_message": "Wrong reference: NULL should be type", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Used \"NULL\" instead of \"type\"", + "fix": "Changed NULL \u2192 type", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T14:09:08.241Z" + }, + { + "id": "bug-786", + "timestamp": "2026-06-19T15:00:00Z", + "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", + "file": "src/controller/testsuite/test.c (cmd_jobsub)", + "root_cause": "On the 2nd+ submit (and during failover windows) the jobsub reply/jobid is not cleanly populated, leaving an uninitialized jobid struct; cmd_jobsub still prints 'jobsubmit ok' with the garbage id instead of verifying the controller reply status and failing. Forces E2E scripts (test-commit-durability.sh) to strict-regex-filter the 4-field id to avoid chasing phantom jobs.", + "fix": "PENDING \u2014 jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient \u2014 a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", + "tags": [ + "xce_test", + "jobsub", + "garbage-jobid", + "e2e", + "client", + "test-tooling", + "xce-test-e2e-primitives", + "stale-reply-union", + "live-soak" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-19" + }, + { + "id": "bug-787", + "timestamp": "2026-06-19T15:26:46.318Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "13 lines replaced/restructured", + "fix": "Rewrote 27\u219211 lines (13 removed) | Also: *; * XCE_TEST_VERDICT_FLAG gates the new exit-code/ab", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T15:26:54.160Z" + }, + { + "id": "bug-788", + "timestamp": "2026-06-19T15:34:41.623Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (active_ctx->parent != NULL)", + "fix": "Changed to: if (i > 1)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T15:34:41.623Z" + }, + { + "id": "bug-789", + "timestamp": "2026-06-19T15:41:13.871Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 9\u21923 lines (5 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T15:41:13.871Z" + }, + { + "id": "bug-790", + "timestamp": "2026-06-19T15:58:57.460Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "5 lines replaced/restructured", + "fix": "Rewrote 7\u21923 lines (5 removed) | Also: #if JOBSUB_RED_FORCE_SUCCESS; /* RED only: simulate a non-clean ack that surface", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T15:59:25.070Z" + }, + { + "id": "bug-791", + "timestamp": "2026-06-19T16:07:20.038Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 38\u219285 lines (2 removed) | Also: * g_last_jobcount most recent job count; -1 means ; * cmd_joblist (task 5.1).", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T16:07:27.736Z" + }, + { + "id": "bug-792", + "timestamp": "2026-06-19T16:13:02.715Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"shutdown\"", + "fix": "Changed to \"expect-jobs\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T16:13:02.715Z" + }, + { + "id": "bug-793", + "timestamp": "2026-06-19T16:32:47.571Z", + "error_message": "Significant refactor of ", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "31 lines replaced/restructured", + "fix": "Rewrote 63\u2192115 lines (31 removed) | Also: if ! wait_for_primary 40; then; fail \"cluster did not elect an initial primary wit | Also: # Wait until some live node is CTRL_PRIMARY and th; wait_for_primary() { | Also: script=$(printf 'set masters %s\\nclient\\n' \"$MASTE; script+=$(printf 'expect-job %s\\n' \"$jid\") | Also: # format-agnostically, instead of re-imposing a br; # submit did not cleanly ack, jobsub fails -> sess", + "tags": [ + "auto-detected", + "refactor", + "sh" + ], + "related_bugs": [], + "occurrences": 6, + "last_seen": "2026-06-19T16:37:04.187Z" + }, + { + "id": "bug-794", + "timestamp": "2026-06-19T16:50:45.275Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test.c", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 8\u219220 lines (7 removed) | Also: if (jobid == 0) {; /* Non-negative rc but no JobId was written: not a", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T16:50:54.261Z" + }, + { + "id": "bug-795", + "timestamp": "2026-06-19", + "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", + "file": "src/controller/persist.c (recovery/truncation) + commit/replication path \u2014 UPSTREAM (ha-commit-durability), NOT xce-test-e2e-primitives", + "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware.", + "fix": "NOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", + "tags": [ + "ha-commit-durability", + "commit-durability", + "cold-restart", + "total-outage", + "wal-truncate", + "upstream", + "xce-test-e2e-primitives", + "needs-real-fs" + ], + "related_bugs": [ + "bug-661", + "bug-778", + "bug-786" + ], + "occurrences": 1, + "last_seen": "2026-06-19" } ] } \ No newline at end of file