diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index b0a87d4a..976ee9c3 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -186,6 +186,19 @@ start_node() { local extra_args=() [ -n "$MAX_WAL_RECORDS" ] && extra_args+=(--max-wal-records "$MAX_WAL_RECORDS") [ -n "$WAL_SEGMENT_BYTES" ] && extra_args+=(--wal-segment-bytes "$WAL_SEGMENT_BYTES") + # bug-1376 debug seam: GDB_WRAP= launches that node directly + # under `gdb --batch` (gdb becomes its direct parent -> ptrace-legal under + # yama scope 1) instead of via start.sh, so a hang can be interrupted + # (SIGINT to gdb's own pid) for a live thread backtrace. Not for normal + # soak use -- temporary investigation-only seam. + if [ -n "${GDB_WRAP:-}" ] && [ "$GDB_WRAP" = "$1" ]; then + gdb --batch -ex run -ex "thread apply all bt full" -ex "kill inferior 1" -ex quit \ + --args "$BIN" --myname "$1" --peers "$PEERS" \ + --workdir "$TOP/controller" --certdir "$TOP/certs" \ + --logfile "$TOP/logs/$1.log" --vcpus 4 -vvv "${extra_args[@]}" \ + >>"$TOP/logs/$1.gdbwrap" 2>&1 & + return + fi if [ ${#extra_args[@]} -gt 0 ]; then bash "$TOP/scripts/start.sh" "$1" -vvv "${extra_args[@]}" \ >>"$TOP/logs/$1.stderr" 2>&1 & diff --git a/.kiro/specs/raft-drive-async/tasks.md b/.kiro/specs/raft-drive-async/tasks.md index 70a22b6c..797d114a 100644 --- a/.kiro/specs/raft-drive-async/tasks.md +++ b/.kiro/specs/raft-drive-async/tasks.md @@ -78,7 +78,7 @@ Tasks mirror the design's 4-phase migration. The slow lane is foundational (ever - _Depends: 2.2_ - [ ] 3. Phase 3 — Peer-control dispatch to vtask, coupled with long-op offload (receiver-side) -- [ ] 3.1 Dispatch peer-control message handling to the vtask pool +- [x] 3.1 Dispatch peer-control message handling to the vtask pool - Register peer endpoints so peer-control messages dispatch through the vtask pool instead of running inline on iomgr; convert the peer-control handler into a vtask entry that does only bounded protocol work inline. - Preserve per-connection processing order via the per-endpoint block primitive and preserve vote/ack/term semantics and persist-before-vote ordering; keep the steady-state heartbeat/vote send paths as non-blocking enqueues. - Observable: peer-control no longer runs on the iomgr loop; existing election/heartbeat/repl behavior is unchanged; messages on one connection are processed in arrival order. @@ -86,9 +86,9 @@ Tasks mirror the design's 4-phase migration. The slow lane is foundational (ever - _Requirements: 1.4, 4.1, 4.2, 4.5, 4.6, 5.1, 5.4, 8.1, 8.2_ - _Boundary: PeerCtrlDispatch_ - _Depends: 1.2, 1.6_ - - _2026-07-01 session (production flip attempt, both gate items closed but a NEW blocker found — flags stay OFF, task NOT complete): with 4.1+4.2 green, attempted the "full fix, defer prod flip" decision's flip. (1) Closed the snapshot-lane borrowed-endpoint hazard (task 3.2's Part A) properly: `ha_snapshot_send_dispatch` takes `endpoint_ref(ep)` before `slowlane_submit`, released on every non-lane terminal path and by `snapshot_lane_fn` on completion — reuses the general bug-1298 refcount system (standby-ep teardown already routes through `endpoint_unref` via `cluster.c`'s `endpoint_killer`, confirmed by reading it — this infra landed in raft-ha-core-safety task 5.8, AFTER this spec's 3.2/3.4 notes were written, so the hazard they describe is now closed for real). New unit test `test_snapshot_offload_holds_endpoint_ref_across_transfer` (RED→GREEN). (2) Flipped `g_drive_async_snapshot_offload` and `g_drive_async_peerctrl_dispatch_enabled` to default ON; full controller suite green; a 6-round `--chaotic` soak passed clean (zero divergence, zero job loss). (3) A `--force-snapshot` soak (genuinely empty node cold-start rejoin — the ONE scenario chaotic mode never exercises, since it restarts nodes with WAL intact) then surfaced a SEPARATE pre-existing bug, unmasked (not caused) by the flip's timing shift: `ha_stream_wal_to_standby` had no filter for `WAL_CHECKPOINT` markers (local-only, never a logical Raft entry — consumes a real LSN slot via `persist_write_checkpoint_marker`'s own `p->current_lsn++`) and shipped one to a catching-up standby, which correctly refused it and got stuck retrying the same LSN forever. FIXED (`config_repl.c`, skip `hdr.type == WAL_CHECKPOINT` entirely, mirroring the existing `replay_wal`/term-index precedent of treating checkpoint-marker LSNs as invisible gaps) with a RED→GREEN unit test (`test_wal_stream_skips_checkpoint_marker`, differential-verified: this specific soak scenario failed before the fix, passed after, with flags OFF as a control). (4) Re-ran `--force-snapshot` with BOTH flags ON and the checkpoint-skip fix in place — it STILL failed, this time with a THIRD, different, NOT-YET-ROOT-CAUSED issue (logged as **bug-1376**): the freshly-wiped empty node's own self-election appears to race `cluster_init_phase2`'s vtask-pool wiring (log shows `peer_ctrl_dispatch_to_pool: vtask pool not wired — running inline` immediately followed by `cluster_init_phase2` wiring the pool while `raft_cb_become_follower`'s deferred-teardown vtasks were being enqueued), after which the process went completely silent (no crash, no ASAN report) until the harness killed it — a genuine hang, not yet diagnosed. **Given a second real bug found from this flag flip in a row, reverted both flags to OFF (production behavior unchanged) rather than ship an unproven state.** The two logic fixes (endpoint-ref bracket, checkpoint-marker filter) are real, independently-verified, and shipped regardless of flag state — they are correct and safe on their own. Verified final state (flags OFF): full controller suite green, `--force-snapshot` PASS, 6-round `--chaotic` PASS — no regression from the two logic fixes. **Task 3.1/3.2/3.3 remain open.** Next step for a future session: reproduce the cold-start pool-wiring race deterministically (likely a unit/integration test driving `cluster_init_phase2` concurrently with an inbound vote reply on a freshly-constructed `cluster_t`, both flags ON) before attempting the flip again._ + - _2026-07-01/07-02 session (production flip — DONE, both flags now default ON): with 4.1+4.2 green, executed the "full fix, defer prod flip" decision's flip. (1) Closed the snapshot-lane borrowed-endpoint hazard (task 3.2's Part A) properly: `ha_snapshot_send_dispatch` takes `endpoint_ref(ep)` before `slowlane_submit`, released on every non-lane terminal path and by `snapshot_lane_fn` on completion — reuses the general bug-1298 refcount system (standby-ep teardown already routes through `endpoint_unref` via `cluster.c`'s `endpoint_killer`, confirmed by reading it — this infra landed in raft-ha-core-safety task 5.8, AFTER this spec's 3.2/3.4 notes were written, so the hazard they describe is now closed for real). New unit test `test_snapshot_offload_holds_endpoint_ref_across_transfer` (RED→GREEN). (2) Flipped `g_drive_async_snapshot_offload` and `g_drive_async_peerctrl_dispatch_enabled` to default ON; full controller suite green; a 6-round `--chaotic` soak passed clean (zero divergence, zero job loss). (3) A `--force-snapshot` soak (genuinely empty node cold-start rejoin — the ONE scenario chaotic mode never exercises, since it restarts nodes with WAL intact) then surfaced a SEPARATE pre-existing bug, unmasked (not caused) by the flip's timing shift: `ha_stream_wal_to_standby` had no filter for `WAL_CHECKPOINT` markers (local-only, never a logical Raft entry — consumes a real LSN slot via `persist_write_checkpoint_marker`'s own `p->current_lsn++`) and shipped one to a catching-up standby, which correctly refused it and got stuck retrying the same LSN forever. FIXED (`config_repl.c`, skip `hdr.type == WAL_CHECKPOINT` entirely, mirroring the existing `replay_wal`/term-index precedent of treating checkpoint-marker LSNs as invisible gaps) with a RED→GREEN unit test (`test_wal_stream_skips_checkpoint_marker`, differential-verified). (4) Re-ran `--force-snapshot` with BOTH flags ON and the checkpoint-skip fix in place — it STILL failed, this time with a THIRD, different issue logged as **bug-1376**: the freshly-wiped empty node's own self-election races `cluster_init_phase2`'s vtask-pool wiring, then the process hangs silently. Live-repro'd deterministically via a `GDB_WRAP` debug seam added to `run-ha-soak.sh` (launches a named node directly under `gdb --batch`, so it becomes ptrace-legal as gdb's direct child under yama scope 1 — bypasses a live-attach restriction) and a bounded `SIGINT`-to-gdb interrupt for a full ASan-free thread backtrace. **Root-caused and FIXED**: `peer_ctrl_msg_handler` held `ep->dispatch_inflight` incremented across the ENTIRE `peer_ctrl_dispatch_to_pool` call, including its inline fallback (the common cold-start shape when `ha->pool` isn't wired yet). That inline body can reach `raft_cb_become_follower`, which — now that `cluster_init_phase2` ran synchronously earlier in the SAME call chain and wired the pool — tears down its OWN loser vote-ep (the very ep whose reply triggered this call) via the deferred path (`vote_ep_teardown_async`), whose `endpoint_mark_tearing_down` spins waiting for `dispatch_inflight==0` on that SAME ep — a count only this exact call, on this exact thread, holds — a permanent self-deadlock. Fixed by moving the `dispatch_inflight` release into `peer_ctrl_dispatch_to_pool` itself, at the exact right point per exit path: released BEFORE calling the inline body on every fallback path (pool==NULL, OOM, enqueue failure) — safe because `io_worker` is single-threaded and the inline path never enqueues anything, so there is no pool-worker race left to protect against once inline execution is committed to — and only AFTER the (non-blocking) enqueue completes on the success path, preserving the original protection for the genuinely concurrent pool-worker case. New unit test `test_vep_self_teardown_does_not_deadlock` (`test_ha_vep_teardown.c`, bounded-timeout `pthread_timedjoin_np` join so a regression fails loud instead of hanging the suite; RED confirmed via a temporary manual revert — `Expected 0 Was 110` (ETIMEDOUT) — then GREEN restored). **Live-verified with both flags ON**: `--force-snapshot` soak now completes without hanging (2/3 runs confirmed a real snapshot install + `committed_divergence=0 committed_job_loss=0`; 1/3 hit the separate known-flaky "checkpoint didn't fire before the wipe this round" — a timing non-issue the harness itself already calls out, NOT a hang); a 6-round `--chaotic` soak with both flags ON also passed clean; full controller suite green throughout. **Both `g_drive_async_snapshot_offload` and `g_drive_async_peerctrl_dispatch_enabled` now default ON in production.** `g_drive_async_configpush_offload` (task 3.3) stays OFF — it has its own DISTINCT, still-open hazard (see task 3.3's note) unrelated to bug-1376._ -- [~] 3.2 Offload snapshot send to the slow lane +- [x] 3.2 Offload snapshot send to the slow lane - Trigger snapshot send + checkpoint to the lane from the vtask entry without holding the per-endpoint block across the transfer; dedup one snapshot per standby. - Enforce the snapshot/checkpoint concurrency invariant: snapshot reads a frozen term/lsn boundary; segment trim never deletes at/above any in-flight snapshot boundary or a standby catch-up cursor; at most one checkpoint in flight; bound the in-lane snapshot socket write so an unreachable-mid-transfer peer cannot pin a worker. - **Add the trim cursor-bound guard (bug-1147, stopgap):** `trim_wal_segments` (`persist.c:1374`) currently bounds deletion ONLY by the open segment and consults no standby cursor; before this task moves checkpoint/trim onto a concurrent lane worker, change it to compute `floor = min(all standby acked_lsn, in-flight snapshot boundaries)` under `p->lock` and refuse to delete at/above `floor`. This is strategy-A retention discipline; it does NOT fix empty/far-behind join (cursor=0) — that needs entity-bearing snapshots (separate spec). @@ -144,15 +144,15 @@ Tasks mirror the design's 4-phase migration. The slow lane is foundational (ever - **1.1**: `drive_async_liveness_reserved(n_peers, n_vcpus) = max(1, min(n_peers−1, n_vcpus−1))` in `ha.c` is the single source of truth for `VTASK_LIVENESS_RESERVED` (floor applied LAST). Consumed by 1.6. `myconf_t` CLI override fields for the new `DEFAULT_*` constants were deferred to the consumer tasks (1.2 connect-timeout, 1.3/1.6) per YAGNI — add each field with its reader. - **2.1**: reconnect offload is gated by a default-ON kill-switch `g_drive_async_reconnect_offload` (legacy inline path retained for the TDD-RED proof + operational fallback). Per-peer single-flight via `ha_state_t.reconnect_inflight[]` CAS + slowlane key; flag cleared on all 4 terminal paths (prep-fail, lane-fn success/-EAGAIN, -EEXIST, hard-error). Lane fn = `reconnect_lane_fn` → `ha_connect_finish_peer` (happy path: bounded connect + lock-guarded iomgr_monitor; 2.2 adds inbound/outbound race + failure hardening via the documented seam). **FOLLOW-UP (test hygiene, LOW)**: the GREEN reconnect tests in test_ha_liveness.c leak the prepped endpoint on the simulated-success hook path (production-clean; masked by detect_leaks=0) — free the un-adopted ep in teardown when convenient. - **1.6**: pool reservation lives in vtaskmgr (`vtask_class_t` + `blocking_inflight`/`liveness_reserved` + `vtaskmgr_blocking_acquire/release`); commit folds in via opt-in `commit_set_pool` (kept `commit_engine_init(n_vcpus)` signature — the `n_vcpus` local cap is the genuine pool==NULL fallback for unit fixtures). **Production wiring is in `cluster.c` phase2** (once-guarded, right after `commit_engine_init`): `vtaskmgr_set_liveness_reserved(R)` + `commit_set_pool` — DO NOT remove; without it the reservation is dead. Lock order `p->lock → ce->lock → blk_lock` (cycle-free). New-cap = `n_vcpus−R` (tighter than old `n_vcpus−2` for 5/7-node — intended). LOW deferred: `vtaskmgr_blocking_cap()` reads `liveness_reserved` without `blk_lock` (init-time set-once, benign). When 3.1 dispatches peer_ctrl, REQ_BLOCK slot is released in `run_task` — ensure no double-count vs the commit path. -- **3.1 (landed DORMANT — checkbox stays unchecked until 3.4 gate is green)**: `peer_ctrl_msg_handler` is now a shim gated by `g_drive_async_peerctrl_dispatch_enabled` (**DEFAULT 0 / OFF** — production behavior UNCHANGED; toggle via `ha_set_peerctrl_dispatch_for_test`). Flag OFF → `peer_ctrl_msg_handler_inline` runs on io_worker, byte-for-byte prior path. Flag ON → `peer_ctrl_dispatch_to_pool` builds a `VTASK_IO` task and `vtask_enq_classified`s it (per-endpoint order auto-enforced by `bind_vtask`→`endpoint_block_tasks`). The 3 routers (`vote_reply`/`pre_vote_reply`/`repl_ack`_decode_and_dispatch) take `pkt` as an explicit param now (no cross-thread `ep->fsm_in.pkt` re-read). **Packet ownership**: shim clears `ep->fsm_in.pkt` immediately (sole-owner transfer); `peer_ctrl_vtask_entry` NULLs `iod->pkt` before the body frees it once; new `vtask_t.owns_io_pkt` (default 0; controller sets 1) lets `taskq_item_killer` free an un-executed queued task's packet without double-freeing the libxce endpoint-owned path (bug-1199). `ha_state_t.pool` wired in `cluster.c` to `&c->vtaskmgr`. +- **3.1 (DONE — flag now DEFAULT ON, bug-1376 fixed 2026-07-02)**: `peer_ctrl_msg_handler` is now a shim gated by `g_drive_async_peerctrl_dispatch_enabled` (**DEFAULT 1 / ON**; toggle via `ha_set_peerctrl_dispatch_for_test` for a regression comparison). Flag OFF → `peer_ctrl_msg_handler_inline` runs on io_worker, byte-for-byte prior path. Flag ON → `peer_ctrl_dispatch_to_pool` builds a `VTASK_IO` task and `vtask_enq_classified`s it (per-endpoint order auto-enforced by `bind_vtask`→`endpoint_block_tasks`). The 3 routers (`vote_reply`/`pre_vote_reply`/`repl_ack`_decode_and_dispatch) take `pkt` as an explicit param now (no cross-thread `ep->fsm_in.pkt` re-read). **Packet ownership**: shim clears `ep->fsm_in.pkt` immediately (sole-owner transfer); `peer_ctrl_vtask_entry` NULLs `iod->pkt` before the body frees it once; new `vtask_t.owns_io_pkt` (default 0; controller sets 1) lets `taskq_item_killer` free an un-executed queued task's packet without double-freeing the libxce endpoint-owned path (bug-1199). `ha_state_t.pool` wired in `cluster.c` to `&c->vtaskmgr`. - **TWO FLAG-ON GATE ITEMS — MUST be resolved before 3.4 flips the flag ON** (documented with loud code comments; harmless while flag OFF): 1. **bug-1197 (CRITICAL UAF)**: `raft_cb_become_follower` `endpoint_fini()`s peer vote-eps from a pool worker → frees `ep->data` while the single io_worker can concurrently read `ep_j->data` in `peer_ctrl_msg_handler_inline` for a same-epoll-batch queued event. FIX (part of 3.4 enablement): `iomgr_unmonitor(ep)` BEFORE `endpoint_fini(ep)` in every pool-reachable teardown. 2. **bug-1198 (HIGH classification)**: `peer_ctrl_classify` returns `VTASK_LIVENESS` for all, but the CTRL_PRIMARY repl-ack path can run a blocking `HA_REPL_SEND_SNAPSHOT`→`persist_checkpoint` inline under the never-gated liveness reservation → breaks deadlock-freedom. The 3.2 SlowLane offload mechanism now EXISTS but is DORMANT (flag OFF, below), so bug-1198 stays OPEN until 3.4 flips it ON together with the peerctrl flag. -- **3.2 — Part B (trim guard, bug-1147) is LIVE; Part A (snapshot offload) landed DORMANT (checkbox `[~]` until 3.4 gate)**: +- **3.2 — Part B (trim guard, bug-1147) is LIVE; Part A (snapshot offload) is now DEFAULT ON (bug-1197 + bug-1376 fixed 2026-07-02)**: - **Part B (production-live, default-ON, no flag)**: `trim_wal_segments` (`persist.c`) now bounds deletion by `floor = repl_trim_lsn(&p->repl, active_seg_lsn, snap_interval_s)` = `min(active_seg_lsn, every active standby acked_lsn, every valid disconnected-slot last_disconnected_lsn)`, after a `repl_expire_disconnect_slots` call to age out stale slot guards. `floor ≤ active_seg_lsn` structurally (repl_trim_lsn caps at its `checkpoint_lsn` arg), so the new `seg_start_lsn < floor` check subsumes the old `< active_seg_lsn` guard and can only PRESERVE more segments — never delete one a live standby still needs (closes the bug-795/828 trim-vs-cursor race for connected/recently-disconnected standbys; does NOT fix empty/cursor=0 join — needs entity-bearing snapshots, separate spec). Lock-safe: `trim_wal_segments` runs with `p->lock` released; `repl_trim_lsn`/`repl_expire_disconnect_slots` take only `r->lock` — no inversion. Tests: `test_trim_guard_preserves_standby_segments`, `test_trim_without_standbys_proceeds_normally` (test_e2e_checkpoint.c). - **Part A (DORMANT, `g_drive_async_snapshot_offload` DEFAULT 0/OFF)**: `ha_snapshot_send_dispatch` routes `HA_REPL_SEND_SNAPSHOT` inline (flag OFF — byte-for-byte legacy, send on io_worker) or to `slowlane_submit`+`snapshot_lane_fn` (flag ON). Send body factored into `ha_snapshot_do_send` (shared by both paths, holds the XCE_TESTING hook). Per-peer single-flight via `ha_state_t.snapshot_inflight[MAX_HA_CANDIDATES]` CAS + slowlane key `peer_idx+1`; flag cleared on all 4 terminal paths (alloc-fail, -EEXIST, hard-error, lane-fn completion) exactly like reconnect (2.1). Tests: `test_snapshot_offload_runs_on_lane`, `test_snapshot_inline_when_flag_off`, `test_snapshot_offload_dedups_per_peer` (test_ha_liveness.c). - - **WHY DORMANT (flag-ON gate, owned by 3.4)**: `snapshot_lane_fn` captures a BORROWED standby endpoint (unlike reconnect, which owns its ep). With the flag ON the send runs on a lane thread while the io_worker may RDHUP-`endpoint_fini` that ep → cross-thread UAF (bug-1197 family). Flag OFF keeps send+teardown serialized on the io_worker. **3.4 is the single gate** that flips BOTH `g_drive_async_snapshot_offload` AND `g_drive_async_peerctrl_dispatch` ON, after establishing the endpoint-lifetime discipline (refcount or lane-serialized teardown) and proving liveness under pool saturation. Caught + fixed pre-commit: the offload flag was initially authored DEFAULT-ON (would have shipped the UAF) → corrected to OFF in both ha.c and the ha.h accessor doc (bug-1210). + - **WHY IT WAS DORMANT, AND HOW IT CLOSED**: `snapshot_lane_fn` captures a BORROWED standby endpoint (unlike reconnect, which owns its ep). With the flag ON the send runs on a lane thread while the io_worker may RDHUP-`endpoint_fini` that ep → cross-thread UAF (bug-1197 family). Flag OFF kept send+teardown serialized on the io_worker. Caught + fixed pre-commit early on: the offload flag was initially authored DEFAULT-ON (would have shipped the UAF) → corrected to OFF in both ha.c and the ha.h accessor doc (bug-1210). **Closed 2026-07-02**: `ha_snapshot_send_dispatch`/`snapshot_lane_fn` now bracket the lane dispatch with `endpoint_ref`/`endpoint_unref` (reusing the general bug-1298 refcount system), so the borrowed-endpoint hazard cannot recur regardless of thread timing — no lane-serialized-teardown redesign needed. `g_drive_async_snapshot_offload` and `g_drive_async_peerctrl_dispatch` both now default ON together (bug-1376, a distinct cold-start self-deadlock unmasked by flipping both, is also fixed). - **3.3 (landed DORMANT — checkbox `[~]` until 3.4 gate)**: requester-less config-push transport offload, flag `g_drive_async_configpush_offload` **DEFAULT 0/OFF** (toggle `ha_set_configpush_offload_for_test`). `ha_config_push_offloadable(c, bytes, blen, sha256, new_tag)` routes inline (flag OFF — verbatim `push_config_transport_only`, caller still owns+frees `bytes`) or to `slowlane_submit`+`configpush_lane_fn` (flag ON — DEEP-COPIES `bytes`+sha into a ctx the lane frees). Single cluster-wide single-flight via scalar `ha_state_t.configpush_inflight` CAS + lane key=1 (config push is a cluster-wide fan-out, not per-peer); flag cleared on all terminal paths (alloc-fail×2, -EEXIST, hard-error, lane completion). Send body factored into `ha_config_push_do_send` (production → `push_config_transport_only`; XCE_TESTING → `ha_configpush_probe` stub since cluster.c/transport isn't linked in test). THREE requester-less drive-thread call sites converted: `raft_cb_become_leader` (promotion, both bytes/NULL branches), the `HA_REPL_PUSH_CONFIG` heartbeat case, and the direct-CTRL_PRIMARY FSM promotion branch (~ha.c:4476). UNCHANGED: `push_config_to_standbys` admin-upgrade barrier (has a waiting requester) and `join.c`. Tests: `test_configpush_offload_runs_on_lane`, `test_configpush_inline_when_flag_off`, `test_configpush_offload_dedups` (test_ha_liveness.c). Same BORROWED-ENDPOINT/bug-1197 dormancy rationale as 3.2 (lane reads `c->ha.standby_endpoints`); **3.4 flips this ON together with snapshot + peerctrl flags**. cpp-review: no CRITICAL/HIGH/MEDIUM; controller suite EXIT=0. diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 88143c48..03651e1c 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -22813,15 +22813,14 @@ "timestamp": "2026-07-02T05:10:00.000Z", "error_message": "raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout.", "file": "src/controller/ha.c", - "root_cause": "NOT YET ROOT-CAUSED. Hypothesis: a cold-start race between a fresh node's own self-triggered election (which can fire before cluster_init_phase2 finishes wiring ha->pool) and cluster_init_phase2 wiring the vtask pool concurrently -- raft_cb_become_follower's deferred vote-ep teardown (bug-1197/task 3.4a) enqueues VTASK_IO teardown vtasks bound to eps, and if the pool wiring or vtaskmgr worker startup is not yet fully complete at that exact moment, the enqueued teardown (or a subsequent dispatch) may never run, wedging the node. Only reproduces via a genuinely EMPTY node's cold start (--force-snapshot's deliberate wipe) -- a 6-round --chaotic soak with both flags ON passed clean because it restarts nodes with WAL intact, never exercising a truly empty cold start.", - "fix": "NOT YET FIXED. Mitigated by reverting g_drive_async_snapshot_offload and g_drive_async_peerctrl_dispatch_enabled to their production-default OFF (commit f3b6e6a) -- production behavior unchanged, task 3.1/3.2/3.3 (raft-drive-async) stay open. Next step: a deterministic unit/integration test driving cluster_init_phase2 concurrently with an inbound vote reply on a freshly-constructed cluster_t, both flags ON, to reproduce the race without a full live soak.", + "root_cause": "peer_ctrl_msg_handler (ha.c) held ep->dispatch_inflight incremented across the ENTIRE peer_ctrl_dispatch_to_pool call, including its inline fallback (the common cold-start shape when ha->pool is not yet wired). That inline body can reach raft_cb_become_follower, which -- now that cluster_init_phase2 ran synchronously earlier in the SAME call chain and wired the pool -- tears down its own loser vote-ep (the very ep whose reply triggered this call) via the deferred path (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting for dispatch_inflight==0 on that SAME ep. That count is held only by this exact call, on this exact thread, and cannot reach 0 until the spin itself returns -- a permanent self-deadlock. Confirmed live via a gdb thread backtrace (GDB_WRAP debug seam added to run-ha-soak.sh, launching the target node directly under `gdb --batch` so it becomes ptrace-legal as gdb's own direct child, then SIGINT to gdb once the log went stale for a bounded interrupt + full backtrace): frame `peer_ctrl_dispatch_to_pool` showed pool=0x0 at dispatch time; the same ep pointer appeared in the nested vote_ep_teardown_async -> endpoint_mark_tearing_down frame, spinning on sched_yield().", + "fix": "Moved the ep->dispatch_inflight release into peer_ctrl_dispatch_to_pool itself, at the exact right point per exit path: released BEFORE calling the inline body on every fallback path (pool==NULL, OOM, enqueue failure) -- safe because io_worker is single-threaded and the inline path never enqueues anything, so there is no pool-worker race left to protect against once inline execution is committed to -- and only AFTER the (non-blocking) enqueue completes on the success path, preserving the original protection for the genuinely concurrent pool-worker case. New unit test test_vep_self_teardown_does_not_deadlock (test_ha_vep_teardown.c) uses a bounded-timeout pthread_timedjoin_np so a regression fails loud instead of hanging the suite; RED confirmed via a temporary manual revert (Expected 0 Was 110 == ETIMEDOUT) before restoring the fix. Live-verified with both g_drive_async_snapshot_offload and g_drive_async_peerctrl_dispatch_enabled flipped ON: --force-snapshot soak (the exact repro shape) completes without hanging across repeated runs, confirmed real snapshot installs, committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak with both flags ON also passed clean; full controller suite green throughout. Both flags now default ON in production (raft-drive-async tasks 3.1/3.2 closed).", "tags": [ "raft-drive-async", "cold-start", "vtask-pool", - "hang", - "deadlock-suspect", - "unresolved" + "self-deadlock", + "fixed" ], "related_bugs": [ "bug-1197", @@ -22829,7 +22828,71 @@ "bug-1310" ], "occurrences": 1, - "last_seen": "2026-07-02T05:10:00.000Z" + "last_seen": "2026-07-02T00:20:00.000Z" + }, + { + "id": "bug-1377", + "timestamp": "2026-07-02T07:03:54.861Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_vep_teardown.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T07:03:54.861Z" + }, + { + "id": "bug-1378", + "timestamp": "2026-07-02T07:05:07.924Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T07:05:07.924Z" + }, + { + "id": "bug-1379", + "timestamp": "2026-07-02T07:12:59.577Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "23 lines replaced/restructured", + "fix": "Rewrote 30\u219242 lines (23 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T07:12:59.577Z" + }, + { + "id": "bug-1380", + "timestamp": "2026-07-02T07:16:35.420Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `, differential-verified: this specific soak scena", + "fix": "Changed to `, differential-verified). (4) Re-ran `", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-02T07:17:36.171Z" } ] } diff --git a/src/controller/ha.c b/src/controller/ha.c index e55a3f86..9f050d9c 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -4292,30 +4292,29 @@ int ha_reconnect_offload_enabled(void) /* raft-drive-async task 3.2: SnapshotSend feature flag. * - * STILL DEFAULT OFF (0) — see below for why, even though the ORIGINAL - * borrowed-endpoint hazard this flag was gated on is now CLOSED. The - * bug-1197-family hazard (snapshot_lane_fn's `ep` is shared with the - * io_worker epoll loop, not lane-owned) is fixed: ha_snapshot_send_dispatch - * takes an endpoint_ref() before submit and snapshot_lane_fn releases it via - * endpoint_unref() on completion — the general bug-1298 refcount system - * (every heap-ep teardown site already routes through endpoint_unref, see - * cluster.c's endpoint_killer) means a concurrent RDHUP teardown can only - * drop the owner's reference while the transfer is in flight, never free the - * ep out from under it. Verified: dedicated unit test + * DEFAULT ON (1): the snapshot transfer runs on a slow-lane worker, off the + * caller (io_worker/drive) thread. The bug-1197-family borrowed-endpoint + * hazard (snapshot_lane_fn's `ep` is shared with the io_worker epoll loop, + * not lane-owned) is fixed: ha_snapshot_send_dispatch takes an endpoint_ref() + * before submit and snapshot_lane_fn releases it via endpoint_unref() on + * completion — the general bug-1298 refcount system (every heap-ep teardown + * site already routes through endpoint_unref, see cluster.c's endpoint_ + * killer) means a concurrent RDHUP teardown can only drop the owner's + * reference while the transfer is in flight, never free the ep out from + * under it. Verified: dedicated unit test * (test_snapshot_offload_holds_endpoint_ref_across_transfer), full controller * suite green. * - * STAYS OFF because this flag is coupled to g_drive_async_peerctrl_dispatch - * (bug-1198 requires both ON together) — see that flag's comment for the - * cold-start pool-wiring race discovered flipping BOTH ON together in a live - * force-snapshot soak (a freshly-wiped empty node's self-election can race - * cluster_init_phase2's vtask-pool wiring, then hang silently). Until that - * race is root-caused and fixed, this flag cannot flip ON either. + * Coupled to g_drive_async_peerctrl_dispatch (bug-1198 requires both ON + * together — see that flag's comment) and flips ON with it. The cold-start + * pool-wiring self-deadlock (bug-1376) found flipping both ON together in a + * live force-snapshot soak is root-caused and fixed — see + * g_drive_async_peerctrl_dispatch_enabled's comment. * * Keyed per standby (Req 1.5) so at most one snapshot per peer is in flight; * at most one persist_checkpoint in flight (existing atomic guard). Compiled * in ALL builds; the setter is the test seam to flip RED↔GREEN. */ -static _Atomic int g_drive_async_snapshot_offload = 0; +static _Atomic int g_drive_async_snapshot_offload = 1; void ha_set_snapshot_offload_for_test(int on) { @@ -4376,34 +4375,46 @@ int ha_configpush_offload_enabled(void) /* raft-drive-async task 3.1: PeerCtrlDispatch feature flag. * - * STILL DEFAULT OFF (0): the registered peer-control handler runs its - * routing body INLINE on the iomgr io_worker thread — byte-for-byte the - * pre-task behavior, so production behavior is unchanged. ON (1): the - * handler hands the decoded packet to the vtask pool and returns promptly. + * DEFAULT ON (1): the registered peer-control handler hands the decoded + * packet to the vtask pool and returns promptly, instead of running its + * routing body inline on the iomgr io_worker thread. * - * The originally-scoped gate items are closed: the task 3.4 deadlock-freedom - * proof (LivenessAdmission, task 1.6), the bug-1197 endpoint-quiescence fix - * (3.4a), and bug-1198 (closed by the snapshot-offload flag's endpoint_ref/ - * endpoint_unref fix, this session — see that flag's comment). A 6-round - * chaotic HA soak with BOTH flags ON passed clean (zero divergence, zero job - * loss, full controller suite green). + * Gate items closed: the task 3.4 deadlock-freedom proof (LivenessAdmission, + * task 1.6), the bug-1197 endpoint-quiescence fix (3.4a), bug-1198 (closed by + * the snapshot-offload flag's endpoint_ref/endpoint_unref fix — see that + * flag's comment), and bug-1376 (below). * - * BUT a live --force-snapshot soak (a genuinely EMPTY node cold-starting and - * rejoining, a scenario chaotic mode never exercises — it restarts nodes - * with WAL intact) with both flags ON HUNG: the freshly-wiped node's own - * self-election raced cluster_init_phase2's vtask-pool wiring (log shows - * peer_ctrl_dispatch_to_pool warning "vtask pool not wired — running inline" - * immediately followed by cluster_init_phase2 wiring the pool while - * raft_cb_become_follower's deferred-teardown vtasks were being enqueued), - * then the process went silent with no further log output until killed — no - * crash, no ASAN report, just a stall. NOT YET ROOT-CAUSED (logged as - * bug-1376). Flag stays OFF until this cold-start race is understood and - * fixed; do not flip ON based on the chaotic soak alone — it does not cover - * this startup shape. + * bug-1376 (self-deadlock, root-caused + fixed): a live --force-snapshot soak + * (a genuinely EMPTY node cold-starting and rejoining — the one scenario + * chaotic mode never exercises, since it restarts nodes with WAL intact) hung + * with both flags ON. Root cause: peer_ctrl_msg_handler held ep->dispatch_ + * inflight across the ENTIRE peer_ctrl_dispatch_to_pool call, including its + * inline fallback when ha->pool is not yet wired (the common cold-start + * shape — a fresh node's own self-election can resolve, via an inbound vote + * reply, before cluster_init_phase2 finishes wiring the pool). That inline + * body can reach raft_cb_become_follower, which — now that cluster_init_ + * phase2 has run synchronously earlier in the SAME call chain and wired the + * pool — tears down its own loser vote-eps via the deferred path + * (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting + * for dispatch_inflight==0 on THIS SAME ep. That count is held by this exact + * call, on this exact thread, and cannot reach 0 until the spin itself + * returns — a permanent self-deadlock. Fixed: peer_ctrl_dispatch_to_pool now + * releases dispatch_inflight at the exact right point for each of its 3 exit + * paths — before calling the inline body on every fallback path (pool==NULL, + * OOM, enqueue failure), since io_worker is single-threaded and the inline + * path never enqueues anything, so there is no pool-worker race left to + * protect against once inline execution is committed to; only after the + * (non-blocking) enqueue completes on the success path, preserving the + * original protection for the genuinely concurrent pool-worker case. Unit + * test: test_vep_self_teardown_does_not_deadlock (bounded-timeout join, + * RED->GREEN). Live-verified: --force-snapshot soak (the exact repro shape) + * now completes without hanging, confirmed a real snapshot install, + * committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak + * with both flags ON also passed clean. * * Compiled in ALL builds (production reads it on every peer-control message); - * the setter is the test seam to flip ON for testing. */ -static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 0; + * the setter is the test seam to flip OFF for a regression comparison. */ +static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 1; void ha_set_peerctrl_dispatch_for_test(int on) { @@ -5549,14 +5560,14 @@ static int peer_ctrl_vtask_entry(vtask_t *task) * │ occupy every liveness worker and starve the acks that advance │ * │ committed_lsn. │ * │ │ - * │ CLOSED (logic-wise): the borrowed-endpoint lifetime hazard (bug-1197) is │ - * │ fixed by an endpoint_ref/endpoint_unref bracket around the lane dispatch │ - * │ (see ha_snapshot_send_dispatch / snapshot_lane_fn), so WHENEVER both │ - * │ flags are ON together the repl-ack path's snapshot send is ALWAYS │ - * │ offloaded before peer_ctrl_classify ever tags it VTASK_LIVENESS on the │ - * │ pool — bug-1198 cannot recur. Both flags STILL default OFF, though — see │ - * │ g_drive_async_peerctrl_dispatch_enabled's comment for a SEPARATE, │ - * │ not-yet-fixed cold-start pool-wiring race found flipping both ON. │ + * │ CLOSED: the borrowed-endpoint lifetime hazard (bug-1197) is fixed by an │ + * │ endpoint_ref/endpoint_unref bracket around the lane dispatch (see │ + * │ ha_snapshot_send_dispatch / snapshot_lane_fn), so WHENEVER both flags are │ + * │ ON together the repl-ack path's snapshot send is ALWAYS offloaded before │ + * │ peer_ctrl_classify ever tags it VTASK_LIVENESS on the pool — bug-1198 │ + * │ cannot recur. Both flags now default ON (bug-1376, the cold-start │ + * │ pool-wiring self-deadlock found flipping both ON, is root-caused + fixed │ + * │ — see g_drive_async_peerctrl_dispatch_enabled's comment). │ * └──────────────────────────────────────────────────────────────────────────┘ */ static vtask_class_t peer_ctrl_classify(ha_state_t *ha, io_packet_t *pkt) @@ -5710,6 +5721,31 @@ void ha_vote_ep_teardown_for_test(ha_state_t *ha, endpoint_t *ep) * │ deferred vote-ep teardown block below and vote_ep_teardown_async. │ * └──────────────────────────────────────────────────────────────────────────┘ */ +/* + * bug-1376 (self-deadlock, found investigating the raft-drive-async flag + * flip): the caller (peer_ctrl_msg_handler) holds ep->dispatch_inflight + * incremented across this whole call. That counter exists to stop a POOL + * WORKER thread from touching a queued vtask's ep concurrently with a + * teardown — it protects the brief window between the tearing_down check + * and a (non-blocking) enqueue. It must NOT still be held while running the + * INLINE fallback body: peer_ctrl_msg_handler_inline can recursively reach + * raft_cb_become_follower -> vote_ep_teardown_async -> endpoint_mark_ + * tearing_down on THIS SAME ep (a node's own vote-request ep receiving the + * very reply that makes it step down), which spins waiting for + * dispatch_inflight==0 — a count only this call itself holds, on the same + * thread, unable to ever reach 0 until this call returns. Permanent hang + * (io_worker is single-threaded, so nothing else could ever be racing this + * ep while we run inline — dispatch_inflight has nothing left to protect + * once we've committed to the inline path). + * + * Fix: release dispatch_inflight at the exact right point for whichever of + * the 3 exit paths fires below, instead of the caller holding it across the + * whole call. The ENQUEUE-success path still releases only after the + * (non-blocking) enqueue completes, preserving the original protection for + * the genuinely concurrent pool-worker case; every INLINE-fallback path + * releases BEFORE calling the inline body, since no pool worker can ever be + * racing an ep whose message we are about to process synchronously + * ourselves. */ static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; @@ -5719,6 +5755,7 @@ static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt) if (pool == NULL) { WARN("peer_ctrl_dispatch_to_pool: vtask pool not wired — running " "inline (stay alive, never drop a peer-control message)"); + atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return peer_ctrl_msg_handler_inline(ep, pkt); } @@ -5729,6 +5766,7 @@ static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt) "inline (stay alive)"); free(data); free(task); + atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return peer_ctrl_msg_handler_inline(ep, pkt); } @@ -5755,9 +5793,15 @@ static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt) "— running inline (stay alive)", rc); free(data); free(task); + atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return peer_ctrl_msg_handler_inline(ep, pkt); } + /* Enqueued: release now that the (non-blocking) enqueue itself has + * completed. The queued vtask is what a pool worker will process; a + * concurrent teardown drain now correctly waits on the per-ep busy/FIFO + * gate (bind_vtask) for that vtask, not on this call. */ + atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return 0; /* returns promptly; the body runs on a pool worker */ } @@ -5818,15 +5862,11 @@ static int peer_ctrl_msg_handler(void *ep_raw) ha_peerctrl_predispatch_probe(ep); #endif - /* NOTE (drain liveness): this call runs while dispatch_inflight is held, so - * a concurrent endpoint_mark_tearing_down on this ep spins until it returns. - * peer_ctrl_dispatch_to_pool only ever enqueues (non-blocking) or, on OOM, - * runs peer_ctrl_msg_handler_inline — which must also stay non-blocking - * (the vtask-handler no-block invariant). Keep it bounded or the drain is. */ - int rc = peer_ctrl_dispatch_to_pool(ep, pkt); - - atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); - return rc; + /* bug-1376: peer_ctrl_dispatch_to_pool owns the matching fetch_sub now + * (released at the exact right point for whichever of its 3 exit paths + * fires — see its own comment) instead of a single unconditional + * decrement here after it returns. See that function for why. */ + return peer_ctrl_dispatch_to_pool(ep, pkt); } #ifdef XCE_TESTING diff --git a/src/controller/testsuite/test_ha_vep_teardown.c b/src/controller/testsuite/test_ha_vep_teardown.c index 874e6ebf..a27f8681 100644 --- a/src/controller/testsuite/test_ha_vep_teardown.c +++ b/src/controller/testsuite/test_ha_vep_teardown.c @@ -538,6 +538,91 @@ static void test_vep_mark_drains_inflight_dispatch(void) printf("test_vep_mark_drains_inflight_dispatch: PASS\n"); } +/* ── Test (f): bug-1376 SELF-TEARDOWN — must not self-deadlock ───────────── + * + * A node's OWN in-flight peer-control dispatch can, via its routing body, + * decide to tear down the VERY SAME ep the dispatch is currently processing + * (the live signature: a node's own vote-request ep receives the reply that + * makes it become follower, and the loser loop closes that same ep). Before + * the bug-1376 fix, peer_ctrl_msg_handler held ep->dispatch_inflight across + * the ENTIRE inline body call; vote_ep_teardown_async's endpoint_mark_ + * tearing_down then spun forever waiting for that same count — held by this + * same thread, unable to decrement until the spin itself returns — a + * permanent self-deadlock. Reproduced with a bounded-timeout join: the + * dispatch runs on a helper thread; the test FAILS (rather than hanging the + * whole suite) if it does not return within the timeout. + */ +static void *self_teardown_dispatch_thread(void *p) +{ + dispatch_arg_t *a = (dispatch_arg_t *)p; + io_packet_t *pkt = repl_catchup_done_pkt_for_test(); + peer_ctrl_msg_handler_for_test(a->ep, pkt); + atomic_store(&a->done, 1); + return NULL; +} + +static ha_state_t *g_self_teardown_ha = NULL; +static endpoint_t *g_self_teardown_ep = NULL; + +/* Fires at the top of the inline routing body (before role dispatch) — tears + * down the SAME ep this dispatch is currently processing, the self-referential + * case the fix must resolve. */ +static void self_teardown_body_probe(endpoint_t *ep) +{ + (void)ep; + ha_vote_ep_teardown_for_test(g_self_teardown_ha, g_self_teardown_ep); +} + +static void test_vep_self_teardown_does_not_deadlock(void) +{ + reset_oracles(); + + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1)); + /* CTRL_INIT → the inline body hits the default case (drop + free pkt) + * after the probe runs, so the stand-in vote_ep_ctx_t is sufficient. */ + atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); + ha.pool = NULL; /* matches the live cold-start repro (bug-1376) */ + + endpoint_t *ep = endpoint_get(); + TEST_ASSERT_NOT_NULL(ep); + ep->iomgr = NULL; + ep->fd = -1; + struct mini_vctx *v = calloc(1, sizeof(*v)); + TEST_ASSERT_NOT_NULL(v); + v->ha = &ha; + ep->data = v; + + ha_set_peerctrl_dispatch_for_test(1); + g_self_teardown_ha = &ha; + g_self_teardown_ep = ep; + ha_peerctrl_body_probe = self_teardown_body_probe; + + dispatch_arg_t darg = { .ep = ep }; + atomic_store(&darg.done, 0); + pthread_t t; + TEST_ASSERT_EQUAL_INT(0, + pthread_create(&t, NULL, self_teardown_dispatch_thread, &darg)); + + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += 5; + int jrc = pthread_timedjoin_np(t, NULL, &deadline); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, jrc, + "bug-1376: self-teardown (an ep's own dispatch tearing down the same " + "ep) must not deadlock — dispatch_inflight must be released before " + "the inline body runs, not held across it"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&darg.done), + "the dispatch thread must have completed"); + + ha_peerctrl_body_probe = NULL; + ha_set_peerctrl_dispatch_for_test(0); + g_self_teardown_ha = NULL; + g_self_teardown_ep = NULL; + ha_fini(&ha); + printf("test_vep_self_teardown_does_not_deadlock: PASS\n"); +} + /* ── Entry point ─────────────────────────────────────────────────────────── */ void test_ha_vep_teardown(void) @@ -547,4 +632,5 @@ void test_ha_vep_teardown(void) RUN_TEST(test_vep_teardown_vtask_disarms_unblock); RUN_TEST(test_vep_off_path_uses_plain_fini); RUN_TEST(test_vep_mark_drains_inflight_dispatch); + RUN_TEST(test_vep_self_teardown_does_not_deadlock); }