diff --git a/.wolf/buglog.json b/.wolf/buglog.json index ff8385b..df5e670 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -18758,6 +18758,1093 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-06-25T04:36:05.558Z" + }, + { + "id": "bug-1133", + "timestamp": "2026-06-25T07:12:01.309Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-log-replication/spec.json", + "root_cause": "Had \"2026-06-21T11:00:00Z\"", + "fix": "Changed to \"2026-06-25T00:00:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-25T07:12:25.418Z" + }, + { + "id": "bug-1134", + "timestamp": "2026-06-25T08:18:43.915Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T08:18:43.915Z" + }, + { + "id": "bug-1135", + "timestamp": "2026-06-25T09:26:49.441Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T09:26:49.441Z" + }, + { + "id": "bug-1136", + "timestamp": "2026-06-25T09:27:00.312Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T09:27:00.312Z" + }, + { + "id": "bug-1137", + "timestamp": "2026-06-25T11:01:21.356Z", + "error_message": "Significant refactor of ", + "file": ".kiro/steering/ha-async-io.md", + "root_cause": "7 lines replaced/restructured", + "fix": "Rewrote 11→19 lines (7 removed)", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T11:01:21.356Z" + }, + { + "id": "bug-1138", + "timestamp": "2026-06-25T11:36:52.942Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T11:36:52.942Z" + }, + { + "id": "bug-1139", + "timestamp": "2026-06-25T14:02:51.225Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/research.md", + "root_cause": "Had `VTASK_IO`", + "fix": "Changed to `endpoint_block_tasks()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-25T14:09:24.614Z" + }, + { + "id": "bug-1140", + "timestamp": "2026-06-25T14:03:12.607Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/raft-drive-async/requirements.md", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 4→6 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T14:03:12.607Z" + }, + { + "id": "bug-1141", + "timestamp": "2026-06-25T15:12:16.193Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T15:12:16.193Z" + }, + { + "id": "bug-1142", + "timestamp": "2026-06-25T15:13:00.445Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `.kiro/specs/raft-drive-async/brief.md`", + "fix": "Changed to `delay_task_ms`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T15:13:00.445Z" + }, + { + "id": "bug-1143", + "timestamp": "2026-06-25T23:37:38.579Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - Commit-advance under pool load (R4.5): with the vtask pool; - Snapshot/checkpoint concurrency (R1.1/1.3/1.5): concurrent", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-25T23:38:18.782Z" + }, + { + "id": "bug-1144", + "timestamp": "2026-06-25T23:37:49.697Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had `persist_checkpoint`", + "fix": "Changed to `wait_all_with_timeout`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T23:37:49.697Z" + }, + { + "id": "bug-1145", + "timestamp": "2026-06-25T23:38:08.746Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T23:38:08.746Z" + }, + { + "id": "bug-1146", + "timestamp": "2026-06-25T23:43:59.922Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T23:43:59.922Z" + }, + { + "id": "bug-1147", + "timestamp": "2026-06-26T03:46:38.904419+00:00", + "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor → lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", + "file": "src/controller/persist.c:1374 (trim_wal_segments)", + "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live — write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", + "fix": "NOT YET FIXED. Stopgap: bound trim_wal_segments by min(all standby acked_lsn, in-flight snapshot boundary) under p->lock. Full fix needs entity-bearing snapshots (separate spec) for empty/far-behind join. raft-drive-async task 3.2 moves checkpoint/trim onto a concurrent lane worker, making this latent race EASIER to hit — that spec must not regress it (add the stopgap guard or serialize trim vs in-flight catch-up).", + "tags": [ + "wal", + "trim", + "catch-up", + "durability", + "permanent-gap", + "bug-795", + "bug-828", + "raft-drive-async", + "snapshot", + "latent" + ], + "related_bugs": [ + "bug-795", + "bug-828", + "bug-1060" + ], + "occurrences": 1, + "last_seen": "2026-06-26T03:46:38.904419+00:00" + }, + { + "id": "bug-1148", + "timestamp": "2026-06-26T03:46:51.038Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had 's boundary or any standby'", + "fix": "Changed to `persist.c:1374`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T03:46:51.038Z" + }, + { + "id": "bug-1149", + "timestamp": "2026-06-26T03:48:48.222Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `/kiro-spec-design raft-drive-async`", + "fix": "Changed to `approved: false`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T03:49:01.467Z" + }, + { + "id": "bug-1150", + "timestamp": "2026-06-26T03:49:13.292Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Code path had no error handling — exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T03:49:13.292Z" + }, + { + "id": "bug-1151", + "timestamp": "2026-06-26T04:31:53.879Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had `commit_submit`", + "fix": "Changed to `commit.c`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T04:36:52.562Z" + }, + { + "id": "bug-1152", + "timestamp": "2026-06-26T04:32:20.960Z", + "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Used \"PeerCtrlDispatch\" instead of \"LivenessAdmission\"", + "fix": "Changed PeerCtrlDispatch → LivenessAdmission", + "tags": [ + "auto-detected", + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:32:20.960Z" + }, + { + "id": "bug-1153", + "timestamp": "2026-06-26T04:33:10.286Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `COMMIT_WAIT_RESERVED=2`", + "fix": "Changed to `commit.c:16`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:33:10.287Z" + }, + { + "id": "bug-1154", + "timestamp": "2026-06-26T04:37:18.052Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `VTASK_LIVENESS_RESERVED`", + "fix": "Changed to `R = clamp(n_peers − 1, 1, n_vcpus − 1)`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T04:42:01.556Z" + }, + { + "id": "bug-1155", + "timestamp": "2026-06-26T04:47:49.937Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_drive_async_config.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:47:49.937Z" + }, + { + "id": "bug-1156", + "timestamp": "2026-06-26T04:48:34.246Z", + "error_message": "Function not marked async", + "file": "src/include/config.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:48:34.246Z" + }, + { + "id": "bug-1157", + "timestamp": "2026-06-26T04:48:50.005Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:48:50.005Z" + }, + { + "id": "bug-1158", + "timestamp": "2026-06-26T04:49:04.568Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:49:04.568Z" + }, + { + "id": "bug-1159", + "timestamp": "2026-06-26T04:49:44.821Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:49:44.821Z" + }, + { + "id": "bug-1160", + "timestamp": "2026-06-26T04:57:57.877Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:57:57.877Z" + }, + { + "id": "bug-1161", + "timestamp": "2026-06-26T05:02:36.720Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_slowlane.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:02:36.720Z" + }, + { + "id": "bug-1162", + "timestamp": "2026-06-26T05:06:30.148Z", + "error_message": "Function not marked async", + "file": "src/include/xce/cluster_def.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:06:30.148Z" + }, + { + "id": "bug-1163", + "timestamp": "2026-06-26T05:06:42.680Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (ha_is_primary(&c->ha)", + "fix": "Changed to: if (c->slowlane.workers == NULL)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:06:42.680Z" + }, + { + "id": "bug-1164", + "timestamp": "2026-06-26T05:06:53.129Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:06:53.129Z" + }, + { + "id": "bug-1165", + "timestamp": "2026-06-26T06:51:00.808Z", + "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", + "file": "src/lib/testsuite/test.c", + "root_cause": "Used \"test_request_send_propagation\" instead of \"test_netio_bounded_connect\"", + "fix": "Changed test_request_send_propagation → test_netio_bounded_connect", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:51:00.808Z" + }, + { + "id": "bug-1166", + "timestamp": "2026-06-26T06:51:57.649Z", + "error_message": "Function not marked async", + "file": "src/lib/netio.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:51:57.649Z" + }, + { + "id": "bug-1167", + "timestamp": "2026-06-26T06:52:05.029Z", + "error_message": "Significant refactor of ", + "file": "src/lib/netio.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 9→10 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:52:05.029Z" + }, + { + "id": "bug-1168", + "timestamp": "2026-06-26T06:52:21.608Z", + "error_message": "Function not marked async", + "file": "src/include/netio.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:52:21.608Z" + }, + { + "id": "bug-1169", + "timestamp": "2026-06-26T07:16:36.377Z", + "error_message": "Function not marked async", + "file": "src/include/vtaskmgr.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T07:17:06.235Z" + }, + { + "id": "bug-1170", + "timestamp": "2026-06-26T07:17:42.916Z", + "error_message": "Significant refactor of ", + "file": "src/lib/vtaskmgr.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 6→10 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:17:42.916Z" + }, + { + "id": "bug-1171", + "timestamp": "2026-06-26T07:18:02.307Z", + "error_message": "Function not marked async", + "file": "src/lib/vtaskmgr.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:18:02.307Z" + }, + { + "id": "bug-1172", + "timestamp": "2026-06-26T07:19:14.857Z", + "error_message": "Function not marked async", + "file": "src/controller/include/commit.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T07:19:30.679Z" + }, + { + "id": "bug-1173", + "timestamp": "2026-06-26T07:19:45.059Z", + "error_message": "Function not marked async", + "file": "src/controller/commit.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:20:29.751Z" + }, + { + "id": "bug-1174", + "timestamp": "2026-06-26T07:20:07.233Z", + "error_message": "Wrong operator: >= should be !=", + "file": "src/controller/commit.c", + "root_cause": "Used \">=\" instead of \"!=\"", + "fix": "Changed operator >= → !=", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:20:07.233Z" + }, + { + "id": "bug-1175", + "timestamp": "2026-06-26T07:20:41.448Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:20:41.448Z" + }, + { + "id": "bug-1176", + "timestamp": "2026-06-26T07:28:30.067Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_commit_admission_pool.c", + "root_cause": "16 lines replaced/restructured", + "fix": "Rewrote 55→66 lines (16 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:28:30.067Z" + }, + { + "id": "bug-1177", + "timestamp": "2026-06-26T07:47:19.731Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:47:39.153Z" + }, + { + "id": "bug-1178", + "timestamp": "2026-06-26T07:48:00.209Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:48:57.917Z" + }, + { + "id": "bug-1179", + "timestamp": "2026-06-26T07:51:01.491Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: deadline.tv_nsec >= 1000000000L", + "fix": "Added guard clause: if (deadline.tv_nsec >= 1000000000L)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:51:01.491Z" + }, + { + "id": "bug-1180", + "timestamp": "2026-06-26T07:51:10.764Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:51:10.764Z" + }, + { + "id": "bug-1181", + "timestamp": "2026-06-26T07:56:06.307Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.h", + "root_cause": "10 lines replaced/restructured", + "fix": "Rewrote 16→4 lines (10 removed)", + "tags": [ + "auto-detected", + "refactor", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:56:06.307Z" + }, + { + "id": "bug-1182", + "timestamp": "2026-06-26T07:57:21.217Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "22 lines replaced/restructured", + "fix": "Rewrote 30→4 lines (22 removed) | Also: /* Lane already has a reconnect for this peer queu; * raced a not-yet-cleared flag). The existing tas", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:58:54.413Z" + }, + { + "id": "bug-1183", + "timestamp": "2026-06-26T07:57:35.484Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:57:35.484Z" + }, + { + "id": "bug-1184", + "timestamp": "2026-06-26T08:12:52.624Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T08:12:59.952Z" + }, + { + "id": "bug-1185", + "timestamp": "2026-06-26T08:13:19.511Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T08:15:04.588Z" + }, + { + "id": "bug-1186", + "timestamp": "2026-06-26T08:15:44.315Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx)", + "fix": "Changed to: if (0 && cur_sctx && cur_sctx->peer_idx == sctx->peer_)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T08:16:44.785Z" + }, + { + "id": "bug-1187", + "timestamp": "2026-06-26T08:21:29.383Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ha.c", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 14→20 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T08:21:29.383Z" + }, + { + "id": "bug-1188", + "timestamp": "2026-06-26T10:20:33.560Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T10:20:50.537Z" + }, + { + "id": "bug-1189", + "timestamp": "2026-06-26T10:21:12.319Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-26T10:23:40.982Z" + }, + { + "id": "bug-1190", + "timestamp": "2026-06-26T10:24:21.473Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:24:21.473Z" + }, + { + "id": "bug-1191", + "timestamp": "2026-06-26T10:27:00.557Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: atomic_load(&g_pc_gate_on", + "fix": "Added guard clause: if (atomic_load(&g_pc_gate_on)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:27:00.557Z" + }, + { + "id": "bug-1192", + "timestamp": "2026-06-26T10:27:14.358Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:27:14.358Z" + }, + { + "id": "bug-1193", + "timestamp": "2026-06-26T10:56:43.461Z", + "error_message": "Function not marked async", + "file": "src/include/vtaskmgr.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:56:43.461Z" + }, + { + "id": "bug-1194", + "timestamp": "2026-06-26T10:57:18.280Z", + "error_message": "Function not marked async", + "file": "src/lib/vtaskmgr.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:57:18.280Z" + }, + { + "id": "bug-1195", + "timestamp": "2026-06-26T10:57:44.750Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-26T10:59:54.657Z" + }, + { + "id": "bug-1196", + "timestamp": "2026-06-26T11:00:15.213Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T11:00:15.213Z" + }, + { + "id": "bug-1197", + "timestamp": "2026-06-26T11:04:56.177Z", + "error_message": "Wrong reference: 1100 should be 1197", + "file": "src/controller/ha.c", + "root_cause": "Used \"1100\" instead of \"1197\"", + "fix": "Changed 1100 → 1197", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T11:05:02.375Z" + }, + { + "id": "bug-1199", + "timestamp": "2026-06-26T00:00:00.000Z", + "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", + "file": "src/lib/vtaskmgr.c", + "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt — its pkt lifetime is the endpoint's, so an unconditional free would double-free.", + "fix": "Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan.", + "tags": ["raft-drive-async", "task-3.1", "memory-leak", "vtaskmgr", "packet-ownership", "review-finding"], + "related_bugs": ["bug-1077"], + "occurrences": 1, + "last_seen": "2026-06-26T00:00:00.000Z" + }, + { + "id": "bug-1197", + "timestamp": "2026-06-26T00:00:00.000Z", + "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF — raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", + "file": "src/controller/ha.c", + "root_cause": "Moving peer-control off the sequential io_worker onto the pool co-locates teardown (frees ep->data) and inbound delivery (reads ep->data) on different threads with no unmonitor-before-free ordering. Safe today only because g_drive_async_peerctrl_dispatch_enabled defaults OFF.", + "fix": "NOT FIXED — documented hazard. Before task 3.4 flips the flag ON, every pool-reachable teardown that frees a peer ep (raft_cb_become_follower's endpoint_fini loop) MUST iomgr_unmonitor(ep) BEFORE endpoint_fini(ep). Loud comment blocks added at peer_ctrl_dispatch_to_pool and the become_follower loop.", + "tags": ["raft-drive-async", "task-3.1", "use-after-free", "flag-on-hazard", "deferred", "review-finding"], + "related_bugs": ["bug-1198"], + "occurrences": 1, + "last_seen": "2026-06-26T00:00:00.000Z" + }, + { + "id": "bug-1198", + "timestamp": "2026-06-26T00:00:00.000Z", + "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline — a blocking op under the never-gated liveness reservation breaks deadlock-freedom", + "file": "src/controller/ha.c", + "root_cause": "repl_ack_decode_and_dispatch -> ha_dispatch_repl_action_for_standby can run the snapshot send inline. LIVENESS is only correct once that long op is offloaded off the pool.", + "fix": "NOT FIXED — documented hazard. Task 3.2 offloads HA_REPL_SEND_SNAPSHOT to the SlowLane, making the repl-ack body bounded and LIVENESS correct. Loud comment added in peer_ctrl_classify.", + "tags": ["raft-drive-async", "task-3.1", "deadlock-freedom", "liveness-admission", "flag-on-hazard", "deferred", "review-finding"], + "related_bugs": ["bug-1197"], + "occurrences": 1, + "last_seen": "2026-06-26T00:00:00.000Z" } ] } \ No newline at end of file diff --git a/src/controller/cluster.c b/src/controller/cluster.c index 1e1d5dd..8ca858d 100644 --- a/src/controller/cluster.c +++ b/src/controller/cluster.c @@ -1362,6 +1362,11 @@ int cluster_init_phase2(cluster_t* c) * until peer_ctrl dispatches classified tasks (task 3.1). */ vtaskmgr_set_liveness_reserved(&c->vtaskmgr, liveness_reserved); commit_set_pool(&c->commit, &c->vtaskmgr); + /* raft-drive-async task 3.1: give ha the pool handle so + * peer_ctrl_msg_handler can dispatch peer-control messages onto it when + * the dispatch flag is ON. Dormant until the flag flips (default OFF); + * stored as void* (ha.h does not pull vtaskmgr.h). */ + c->ha.pool = &c->vtaskmgr; if (c->vtaskmgr.n_vcpus < c->ha.n_peers) { /* Under-provisioned: the n_vcpus ceiling clamped the reservation * below the ideal n_peers-1. Still correct (deadlock-free via fast diff --git a/src/controller/ha.c b/src/controller/ha.c index 2c19efa..74116dd 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -1263,7 +1263,14 @@ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx) /* Close all outgoing vote endpoints (loser path). * The primary will push config and ctrlhello via promoted connections; - * the standby waits passively on its listener for the inbound connection. */ + * the standby waits passively on its listener for the inbound connection. + * + * FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197): when + * g_drive_async_peerctrl_dispatch_enabled is ON this loop runs on a POOL + * worker and frees ep->data while the io_worker may still deliver a queued + * peer-control event on another ep → UAF. Before task 3.4 flips the flag + * ON, each endpoint_fini below MUST be preceded by iomgr_unmonitor(ep). See + * the full hazard note at peer_ctrl_dispatch_to_pool. Safe today (flag OFF). */ { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); @@ -3795,6 +3802,35 @@ int ha_reconnect_offload_enabled(void) memory_order_acquire); } +/* raft-drive-async task 3.1: PeerCtrlDispatch feature flag. + * + * DEFAULT OFF (0): the registered peer-control handler runs its routing body + * INLINE on the iomgr io_worker thread — byte-for-byte the pre-task behavior, + * so THIS COMMIT DOES NOT CHANGE PRODUCTION BEHAVIOR. ON (1): the handler + * hands the decoded packet to the vtask pool and returns promptly. The flag + * stays OFF until the task 3.4 deadlock gate proves the dispatch path safe. + * + * Compiled in ALL builds (production reads it on every peer-control message); + * the setter is the test seam to flip RED<->GREEN. */ +static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 0; + +void ha_set_peerctrl_dispatch_for_test(int on) +{ + atomic_store_explicit(&g_drive_async_peerctrl_dispatch_enabled, on ? 1 : 0, + memory_order_release); +} + +int ha_peerctrl_dispatch_enabled(void) +{ + return atomic_load_explicit(&g_drive_async_peerctrl_dispatch_enabled, + memory_order_acquire); +} + +/* raft-drive-async task 3.1: test-only observation seam (plain global, NULL in + * production — zero cost, same idiom as ha_nop_req_hook). Invoked at the top + * of peer_ctrl_msg_handler_inline before any role routing. */ +void (*ha_peerctrl_body_probe)(endpoint_t *ep) = NULL; + /* ── Test-only hook shims (XCE_TESTING only) ────────────────────────────── * Type definitions (election_timeout_ctx_t), extern declarations * (cluster_init_phase2, push_config_transport_only, @@ -4259,11 +4295,13 @@ void handle_request_vote_reply(ha_state_t *ha, * and never reassigned — it routes by ha->role at dispatch time. */ static int peer_ctrl_msg_handler(void *ep_raw); -static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx); -static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx); +static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt); +static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx, + io_packet_t *pkt); /* ha-prevote task 5.1: preVoteReply decode/route; defined alongside the * real-vote decoder, feeds the later-defined pre_vote_reply_dispatch. */ -static int pre_vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx); +static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, + io_packet_t *pkt); static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c, int peer_idx, int granted); @@ -4433,22 +4471,23 @@ static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx, * vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch * to vote_reply_dispatch. Factored from vote_reply_msg_handler so that * peer_ctrl_msg_handler can invoke the same logic without code duplication. - * Packet ownership: caller passes ep->fsm_in.pkt; this function frees it. - * ep->fsm_in.pkt is cleared HERE, before dispatch — vote_reply_dispatch can - * reach raft_cb_become_follower → endpoint_fini(ep) (loser path), which - * would otherwise double-free the packet via the stale fsm_in.pkt pointer. + * Packet ownership (raft-drive-async task 3.1): the caller + * (peer_ctrl_msg_handler) has ALREADY taken `pkt` off ep->fsm_in.pkt and + * cleared ep->fsm_in.pkt to NULL — `pkt` is passed in explicitly and this + * function frees it on every path. Clearing in the caller (not here) is what + * makes the off-iomgr dispatch path race-free: the body never touches + * ep->fsm_in.pkt, so a concurrent io_worker read on the same endpoint cannot + * collide with the in-flight packet. vote_reply_dispatch can still reach + * raft_cb_become_follower → endpoint_fini(ep) (loser path) safely because + * ep->fsm_in.pkt is already NULL. */ -static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) +static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt) { ha_state_t *ha = vctx->ha; ha_elect_ctx_t *ctx = vctx->ctx; int idx = vctx->peer_idx; - io_packet_t *pkt = ep->fsm_in.pkt; int ok = 0; - ep->fsm_in.pkt = NULL; /* take ownership (same pattern as - * repl_ack_decode_and_dispatch) */ - uint64_t vr_term = 0; int vr_granted = 0; uint64_t vr_tag = 0; @@ -4527,20 +4566,19 @@ done: * real-vote tally, and deliberately performs NO term adoption (a PreVote reply * never changes our term; Req 1.2). vctx->peer_idx is the ABSOLUTE cluster * index that raft_on_pre_vote_reply expects (set in ha_send_pre_vote_request). - * Packet ownership: takes ep->fsm_in.pkt and frees it. + * Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly + * (already taken off ep->fsm_in.pkt by peer_ctrl_msg_handler); this function + * frees it on every path and never touches ep->fsm_in.pkt. */ -static int pre_vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) +static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, + io_packet_t *pkt) { ha_state_t *ha = vctx->ha; cluster_t *c = vctx->c; int idx = vctx->peer_idx; /* ABSOLUTE cluster index */ - io_packet_t *pkt = ep->fsm_in.pkt; int ok = 0; int vr_granted = 0; - ep->fsm_in.pkt = NULL; /* take ownership (same pattern as - * vote_reply_decode_and_dispatch) */ - if (!pkt || !pkt->data || pkt->sz == 0) goto done; @@ -4603,12 +4641,21 @@ done: * - OPCode_nop or OPCode_reply with any other body → silently ignored * - anything else → WARN + drop */ -static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) +static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx, + io_packet_t *pkt) { ha_state_t *ha = vctx->ha; cluster_t *c = vctx->c; - io_packet_t *pkt = ep->fsm_in.pkt; + /* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in + * explicitly (peer_ctrl_msg_handler already cleared ep->fsm_in.pkt); this + * function frees it and never touches ep->fsm_in.pkt. + * + * raft-drive-async tasks 3.2/3.3 (FUTURE): the long ops reachable from the + * stateReport branch below — ha_dispatch_repl_action_for_standby's + * snapshot send / config push — are where the SlowLane offload lands. For + * task 3.1 the body only relocates onto the pool and still runs bounded + * decode/route work inline in the vtask entry. */ if (!pkt || !pkt->data || pkt->sz == 0) { if (pkt) { free_packet(pkt); free(pkt); } return 0; @@ -4621,7 +4668,6 @@ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) (size_t)pkt->sz, 0); free_packet(pkt); free(pkt); - ep->fsm_in.pkt = NULL; if (capn_ok != 0) return 0; @@ -4795,13 +4841,24 @@ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) } /* - * peer_ctrl_msg_handler: state-aware iomgr handler set once in on_vote_connected. + * peer_ctrl_msg_handler_inline: the state-aware peer-control routing body. * Routes by ha->role at dispatch time — no explicit handler reassignment needed. * CTRL_ELECTING → vote_reply_decode_and_dispatch * CTRL_PRIMARY → repl_ack_decode_and_dispatch * CTRL_STANDBY + PreVote → pre_vote_reply_decode_and_dispatch (task 5.1) * any other role → drop packet (WARN) * + * Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly — + * the caller (peer_ctrl_msg_handler) has already taken it off ep->fsm_in.pkt and + * cleared that slot to NULL. This body and the routers it calls free `pkt` on + * every path and NEVER touch ep->fsm_in.pkt, which is what lets this body run + * off the iomgr thread (on the vtask pool) without colliding with a concurrent + * io_worker read on the same endpoint. + * + * This body is run either INLINE on the io_worker thread (dispatch flag OFF — + * the production default; identical to the pre-task behavior) or on a vtask + * pool worker (flag ON; via peer_ctrl_vtask_entry). + * * PreVote note (ha-prevote task 5.1): a preVoteReply arrives on the candidate's * OUTBOUND PreVote endpoint while the candidate's role is still CTRL_STANDBY * (a PreVote round performs no role change — Req 1.1). Without this branch the @@ -4809,33 +4866,199 @@ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) * times out. We gate on raft.pre_vote_in_progress so a STANDBY node with no * active round still drops unexpected traffic. */ -static int peer_ctrl_msg_handler(void *ep_raw) +static int peer_ctrl_msg_handler_inline(endpoint_t *ep, io_packet_t *pkt) { - endpoint_t *ep = (endpoint_t *)ep_raw; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; ha_state_t *ha = vctx->ha; + /* raft-drive-async task 3.1: test-only observation seam — fires before any + * routing so a test can count/timestamp/park the body. NULL in production. */ + if (ha_peerctrl_body_probe) + ha_peerctrl_body_probe(ep); + ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); /* PreVote replies land while CTRL_STANDBY with a round in progress. */ if (role == CTRL_STANDBY && ha->raft.pre_vote_in_progress) - return pre_vote_reply_decode_and_dispatch(ep, vctx); + return pre_vote_reply_decode_and_dispatch(vctx, pkt); switch (role) { case CTRL_ELECTING: - return vote_reply_decode_and_dispatch(ep, vctx); + return vote_reply_decode_and_dispatch(vctx, pkt); case CTRL_PRIMARY: - return repl_ack_decode_and_dispatch(ep, vctx); + return repl_ack_decode_and_dispatch(ep, vctx, pkt); - default: { - io_packet_t *pkt = ep->fsm_in.pkt; - if (pkt) { free_packet(pkt); free(pkt); ep->fsm_in.pkt = NULL; } + default: + if (pkt) { free_packet(pkt); free(pkt); } WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role); return 0; } +} + +/* + * peer_ctrl_vtask_entry (raft-drive-async task 3.1): vtask pool entry that runs + * the peer-control routing body off the iomgr thread. Extracts (ep, pkt) from + * the VTASK_IO task data — the SAME io_task_data_t shape the lib vtask path + * uses — takes ownership of pkt out of the task data (so vtask_fini, which only + * frees the data struct, cannot reach it) and hands it to the body, which frees + * it. No double-free: pkt is freed exactly once inside the body; run_task then + * frees the task + data struct. endpoint_unblock_tasks (per-endpoint + * serialization, Req 4.1/4.2) is driven by run_task after this returns. + */ +static int peer_ctrl_vtask_entry(vtask_t *task) +{ + io_task_data_t *iod = &task->data->u.iodata; + endpoint_t *ep = iod->ep; + io_packet_t *pkt = iod->pkt; + + iod->pkt = NULL; /* body owns pkt now; nothing else may reference it */ + + if (ep == NULL) { + if (pkt) { free_packet(pkt); free(pkt); } + return -EINVAL; } + return peer_ctrl_msg_handler_inline(ep, pkt); +} + +/* + * peer_ctrl_classify (raft-drive-async task 3.1): assign the LivenessAdmission + * task class for an inbound peer-control message. This handler decodes vote + * replies, PreVote replies and WAL/config ACKs — all RECEIPT decode + commit + * eval, which is bounded liveness work. Per the design's deadlock-freedom + * argument (R4.5), ack/commit-advance decode MUST be VTASK_LIVENESS so it is + * NEVER admission-gated: a quorum of acks can always find a reserved worker to + * advance committed_lsn even when the pool is saturated by *_BLOCK tasks. + * + * (The vote-GRANT term fsync — VTASK_LIVE_BLOCK, R4.6 — is handled on the + * INBOUND RequestVote handler, not on these reply/ack endpoints; and the long + * config-push/snapshot ops are offloaded to the SlowLane in tasks 3.2/3.3. So + * everything this handler dispatches is liveness work today.) + * + * ┌─ HIGH FLAG-ON HAZARD (raft-drive-async finding-2; bug-1198) ─────────────┐ + * │ LIVENESS is correct ONLY once task 3.2 offloads the snapshot send off the │ + * │ pool. TODAY the CTRL_PRIMARY repl-ack path can reach │ + * │ HA_REPL_SEND_SNAPSHOT → persist_checkpoint (a BLOCKING fdatasync) INLINE │ + * │ in repl_ack_decode_and_dispatch → ha_dispatch_repl_action_for_standby. │ + * │ Running that blocking op while tagged VTASK_LIVENESS (never gated) breaks │ + * │ the deadlock-freedom reservation — a burst of snapshot sends could occupy │ + * │ every liveness worker and starve the acks that advance committed_lsn. │ + * │ Task 3.2 offloads HA_REPL_SEND_SNAPSHOT to the SlowLane → the repl-ack │ + * │ body becomes bounded → LIVENESS becomes correct. │ + * │ DO NOT flip g_drive_async_peerctrl_dispatch_enabled ON until task 3.2 │ + * │ has landed (and task 3.4's deadlock gate is green). │ + * └──────────────────────────────────────────────────────────────────────────┘ + */ +static vtask_class_t peer_ctrl_classify(ha_state_t *ha, io_packet_t *pkt) +{ + (void)ha; + (void)pkt; + return VTASK_LIVENESS; +} + +/* + * peer_ctrl_dispatch_to_pool (raft-drive-async task 3.1): hand the decoded + * peer-control packet to the vtask pool and return promptly. Per-endpoint + * arrival-order serialization is automatic — bind_vtask calls + * endpoint_block_tasks(ep) and the dispatcher won't bind a second task for the + * same endpoint until this one completes (Req 4.1/4.2). + * + * Controllers NEVER self-terminate (CLAUDE.md): on any failure to enqueue + * (pool unwired, OOM, or admission reject) we fail loud and run the body INLINE + * rather than drop a consensus message — staying alive and correct. + * + * ┌─ CRITICAL FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197) ──────────┐ + * │ Enabling this dispatch (g_drive_async_peerctrl_dispatch_enabled == 1) │ + * │ introduces an endpoint use-after-free that does NOT exist while the body │ + * │ runs inline. raft_cb_become_follower (ha.c, see its endpoint_fini loop) │ + * │ runs on a POOL worker and calls endpoint_fini() on ALL peer vote-eps — │ + * │ freeing each ep->data (vctx) via FREEUP. Meanwhile the single io_worker │ + * │ thread can be concurrently inside peer_ctrl_msg_handler_inline(ep_j, ...) │ + * │ for a same-batch queued event on another ep, reading ep_j->data → races a │ + * │ freed vctx (null-deref / UAF). Safe TODAY only because the flag is OFF and │ + * │ the io_worker processes peer-control sequentially. │ + * │ FIX REQUIRED BEFORE task 3.4 FLIPS THE FLAG ON: every pool-reachable │ + * │ teardown that frees a peer ep (raft_cb_become_follower's endpoint_fini │ + * │ loop) MUST iomgr_unmonitor(ep) BEFORE endpoint_fini(ep) so the io_worker │ + * │ can never deliver a new event on a freed ep->data. Do NOT enable dispatch │ + * │ until that ordering is in place. │ + * └──────────────────────────────────────────────────────────────────────────┘ + */ +static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt) +{ + vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; + ha_state_t *ha = vctx->ha; + vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool; + + if (pool == NULL) { + WARN("peer_ctrl_dispatch_to_pool: vtask pool not wired — running " + "inline (stay alive, never drop a peer-control message)"); + return peer_ctrl_msg_handler_inline(ep, pkt); + } + + vtask_data_t *data = (vtask_data_t *)calloc(1, sizeof(*data)); + vtask_t *task = (vtask_t *)calloc(1, sizeof(*task)); + if (data == NULL || task == NULL) { + WARN("peer_ctrl_dispatch_to_pool: OOM allocating vtask — running " + "inline (stay alive)"); + free(data); + free(task); + return peer_ctrl_msg_handler_inline(ep, pkt); + } + + data->vtype = VTASK_IO; + data->u.iodata.ep = ep; /* drives per-endpoint serialization in bind_vtask */ + data->u.iodata.pkt = pkt; /* ownership transferred into the task */ + vtask_init(task, (vtask_func_t)peer_ctrl_vtask_entry, data); + /* raft-drive-async task 3.1: the task is the SOLE owner of pkt (the shim + * already cleared ep->fsm_in.pkt). Mark it so taskq_item_killer frees pkt + * if this task is destroyed while still queued (never executed) at + * vtaskmgr_fini — without this the pkt would leak on shutdown. The + * executed path frees pkt inside peer_ctrl_vtask_entry (and NULLs + * iod->pkt), so there is no double-free. */ + task->owns_io_pkt = 1; + + vtask_class_t cls = peer_ctrl_classify(ha, pkt); + int rc = vtask_enq_classified(pool, task, cls); + if (rc < 0) { + /* Task NOT enqueued — we still own task/data/pkt. LIVENESS is never + * gated so this should not happen; if it does, fail loud and run inline + * (never drop). Free the wrapper structs only — pkt is freed by the + * inline body below. */ + WARN("peer_ctrl_dispatch_to_pool: vtask_enq_classified failed (rc=%d) " + "— running inline (stay alive)", rc); + free(data); + free(task); + return peer_ctrl_msg_handler_inline(ep, pkt); + } + + return 0; /* returns promptly; the body runs on a pool worker */ +} + +/* + * peer_ctrl_msg_handler: the registered iomgr msg_handler for peer-control + * endpoints (set in on_vote_connected / on_pre_vote_connected / on_standby_ + * connected). Runs on the io_worker thread. + * + * raft-drive-async task 3.1: takes ownership of the inbound packet off + * ep->fsm_in.pkt and clears that slot IMMEDIATELY (so a subsequent io_worker + * read on this endpoint allocates a fresh packet and cannot collide with the + * one we are about to process), then either runs the routing body inline + * (dispatch flag OFF — production default, byte-for-byte the prior behavior) or + * hands it to the vtask pool and returns promptly (flag ON). + */ +static int peer_ctrl_msg_handler(void *ep_raw) +{ + endpoint_t *ep = (endpoint_t *)ep_raw; + io_packet_t *pkt = ep->fsm_in.pkt; + + ep->fsm_in.pkt = NULL; /* take ownership before any routing/dispatch */ + + if (!ha_peerctrl_dispatch_enabled()) + return peer_ctrl_msg_handler_inline(ep, pkt); + + return peer_ctrl_dispatch_to_pool(ep, pkt); } #ifdef XCE_TESTING diff --git a/src/controller/ha.h b/src/controller/ha.h index 65f60ee..369fe9b 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -173,6 +173,12 @@ typedef struct { _Atomic uint64_t committed_config_tag; /* Req 8.1: set on quorum ACK */ /* ── New fields added by election-iomgr-refactor spec (Task 2.1) ── */ void *client_ssl_ctx; /* SSL_CTX* (TLS_client_method); created in myapp_run */ + /* ── New field added by raft-drive-async spec (Task 3.1) ──────── */ + void *pool; /* vtaskmgr_t* (cast; avoids pulling vtaskmgr.h + * into every ha.h consumer). Set once at init + * (cluster_init_phase2 → &c->vtaskmgr). Used ONLY + * by peer_ctrl_msg_handler when peer-control + * dispatch is enabled; NULL otherwise (gate OFF). */ int phase2_done; /* 1 after cluster_init_phase2 runs */ int n_peers; /* total cluster size (including self), matches raft convention */ /* ── New fields added by raft-library spec (Task 6.1) ──────── */ @@ -1221,6 +1227,32 @@ void ha_start_primary_heartbeat(cluster_t *c); void ha_set_reconnect_offload_for_test(int on); int ha_reconnect_offload_enabled(void); +/* + * ha_set_peerctrl_dispatch_for_test / ha_peerctrl_dispatch_enabled: + * Feature-flag accessors (raft-drive-async Task 3.1). The dispatch flag gates + * whether the registered peer-control handler (peer_ctrl_msg_handler) runs its + * routing body INLINE on the iomgr io_worker thread (OFF — the production + * default for this commit; behavior is byte-for-byte identical to before the + * task) or hands the decoded packet to the vtask pool via vtask_enq_classified + * and returns promptly (ON — the target behavior, validated by the task 3.4 + * deadlock gate before it becomes the default). These accessors are compiled + * in ALL builds; the setter is used by tests to flip RED<->GREEN. + * + * DEFAULT: OFF (0). Production behavior is UNCHANGED by this commit. + */ +void ha_set_peerctrl_dispatch_for_test(int on); +int ha_peerctrl_dispatch_enabled(void); + +/* + * ha_peerctrl_body_probe: test-only observation seam (raft-drive-async Task + * 3.1). When non-NULL it is invoked at the TOP of the peer-control routing + * body (peer_ctrl_msg_handler_inline) with the endpoint, BEFORE any role + * routing. Tests use it to (a) count body invocations, (b) capture the thread + * the body ran on (proving inline-vs-pool execution), and (c) optionally park + * the body to prove per-endpoint arrival-order serialization. NULL in + * production (zero cost) — a plain global like ha_nop_req_hook. */ +extern void (*ha_peerctrl_body_probe)(endpoint_t *ep); + /* ── Task 2.4: Non-blocking async election initiator ────────────────────── */ /* diff --git a/src/controller/testsuite/test_ha_liveness.c b/src/controller/testsuite/test_ha_liveness.c index c85f8db..a7ba8bd 100644 --- a/src/controller/testsuite/test_ha_liveness.c +++ b/src/controller/testsuite/test_ha_liveness.c @@ -1480,6 +1480,271 @@ static void test_reconnect_preserves_other_peer_channel(void) printf("test_reconnect_preserves_other_peer_channel: PASS\n"); } +/* ═══════════════════════════════════════════════════════════════════════════ + * raft-drive-async Task 3.1 — PeerCtrlDispatch + * + * The registered peer-control handler (peer_ctrl_msg_handler) must, when the + * dispatch flag is ON, hand the decoded packet to the vtask pool and return + * PROMPTLY (so peer-control no longer runs on the iomgr io_worker thread), while + * preserving per-connection arrival order; with the flag OFF it must run the + * routing body INLINE exactly as before (no behavior change — the production + * default for this commit). The flag toggle is the RED↔GREEN driver. + * + * Test 18 (Req 1.4, 4.5, 8.1, 8.2): flag ON → handler returns promptly and + * the routing body runs on a POOL worker (a different thread). + * Test 19 (Req 6.x no-regression): flag OFF → body runs INLINE, synchronously, + * on the calling thread (identical to the pre-task behavior). + * Test 20 (Req 4.1, 4.2): two messages on ONE endpoint are processed in + * arrival order — the second cannot run until the first completes + * (per-endpoint endpoint_block_tasks serialization). + * ═══════════════════════════════════════════════════════════════════════════ */ + +extern int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt); +extern io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted, + uint64_t peer_tag, + uint64_t peer_lsn); + +/* Probe observation state (set via ha_peerctrl_body_probe). */ +static _Atomic int g_pc_runs = 0; /* body invocation count */ +/* _Atomic: written on the pool worker, read on main — avoid a formal C11 data + * race (raft-drive-async review finding-2). relaxed ordering is sufficient: + * g_pc_runs (acquire/release via fetch_add/load) already orders visibility. */ +static _Atomic pthread_t g_pc_last_thread; /* thread the body ran on */ +static _Atomic int g_pc_gate_on = 0; /* 1 → body parks at the gate */ +static pthread_mutex_t g_pc_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t g_pc_entered_cv = PTHREAD_COND_INITIALIZER; +static pthread_cond_t g_pc_release_cv = PTHREAD_COND_INITIALIZER; +static int g_pc_parked = 0; /* a body is parked at the gate */ +static int g_pc_release = 0; /* gate open → bodies may proceed */ + +static void pc_probe(endpoint_t *ep) +{ + (void)ep; + atomic_store_explicit(&g_pc_last_thread, pthread_self(), memory_order_relaxed); + atomic_fetch_add(&g_pc_runs, 1); + + if (atomic_load(&g_pc_gate_on)) { + pthread_mutex_lock(&g_pc_mu); + g_pc_parked = 1; + pthread_cond_broadcast(&g_pc_entered_cv); + while (!g_pc_release) + pthread_cond_wait(&g_pc_release_cv, &g_pc_mu); + g_pc_parked = 0; + pthread_mutex_unlock(&g_pc_mu); + } +} + +/* Wait (bounded) until a body has parked at the gate. */ +static int pc_wait_parked(int timeout_ms) +{ + struct timespec d; + clock_gettime(CLOCK_REALTIME, &d); + d.tv_sec += timeout_ms / 1000; + d.tv_nsec += (long)(timeout_ms % 1000) * 1000000L; + if (d.tv_nsec >= 1000000000L) { d.tv_sec++; d.tv_nsec -= 1000000000L; } + pthread_mutex_lock(&g_pc_mu); + int rc = 0; + while (!g_pc_parked) { + rc = pthread_cond_timedwait(&g_pc_entered_cv, &g_pc_mu, &d); + if (rc != 0) break; + } + int parked = g_pc_parked; + pthread_mutex_unlock(&g_pc_mu); + return parked ? 0 : -1; +} + +static void pc_release_gate(void) +{ + pthread_mutex_lock(&g_pc_mu); + g_pc_release = 1; + pthread_cond_broadcast(&g_pc_release_cv); + pthread_mutex_unlock(&g_pc_mu); +} + +/* Bounded poll until g_pc_runs >= want; returns the observed count. */ +static int pc_wait_runs(int want, int timeout_ms) +{ + for (int i = 0; i < timeout_ms; i++) { + if (atomic_load(&g_pc_runs) >= want) break; + struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; /* 1 ms */ + nanosleep(&ts, NULL); + } + return atomic_load(&g_pc_runs); +} + +static void pc_reset_probe_state(void) +{ + atomic_store(&g_pc_runs, 0); + atomic_store(&g_pc_gate_on, 0); + pthread_mutex_lock(&g_pc_mu); + g_pc_parked = 0; + g_pc_release = 0; + pthread_mutex_unlock(&g_pc_mu); +} + +/* + * Test 18 (Req 1.4, 4.5, 8.1, 8.2): with the dispatch flag ON, the registered + * handler enqueues the decoded packet onto the vtask pool and RETURNS PROMPTLY; + * the routing body then runs on a POOL worker — a DIFFERENT thread from the + * caller (proving peer-control no longer runs on the iomgr io_worker thread). + * The body (role=CTRL_INIT → bounded default-drop path) frees the packet. + */ +static void test_peerctrl_dispatch_runs_on_pool(void) +{ + vtaskmgr_t mgr; + TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, 4)); + TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&mgr)); + + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_init(&c.ha, &g_cfg, 0); + c.ha.pool = &mgr; + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_INIT, memory_order_release); + + endpoint_t ep; + memset(&ep, 0, sizeof(ep)); + standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0); + + pc_reset_probe_state(); + ha_peerctrl_body_probe = pc_probe; + ha_set_peerctrl_dispatch_for_test(1); + + pthread_t caller = pthread_self(); + io_packet_t *pkt = vote_reply_build_pkt_for_test(1, 1, 0, 0); + TEST_ASSERT_NOT_NULL(pkt); + + int rc = peer_ctrl_msg_handler_for_test(&ep, pkt); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "Req 8.1/8.2: dispatch handler must return promptly (0) after enqueue"); + + int runs = pc_wait_runs(1, 2000); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, runs, + "Req 1.4/4.5: the routing body must run on the pool exactly once"); + TEST_ASSERT_FALSE_MESSAGE( + pthread_equal(atomic_load_explicit(&g_pc_last_thread, memory_order_relaxed), + caller), + "Req 1.4: the body must run on a POOL worker, NOT the caller's " + "(io_worker) thread"); + + ha_set_peerctrl_dispatch_for_test(0); /* restore production default (OFF) */ + ha_peerctrl_body_probe = NULL; + vtaskmgr_fini(&mgr); + free(ep.data); + ha_fini(&c.ha); + printf("test_peerctrl_dispatch_runs_on_pool: PASS\n"); +} + +/* + * Test 19 (Req 6.x no-regression): with the dispatch flag OFF (the production + * default), the handler runs the routing body INLINE on the calling thread, + * synchronously — byte-for-byte the pre-task behavior. This is the RED side of + * the flag toggle: the body did NOT move to a pool thread. + */ +static void test_peerctrl_inline_when_flag_off(void) +{ + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_init(&c.ha, &g_cfg, 0); + c.ha.pool = NULL; /* OFF path must not need the pool */ + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_INIT, memory_order_release); + + endpoint_t ep; + memset(&ep, 0, sizeof(ep)); + standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0); + + pc_reset_probe_state(); + ha_peerctrl_body_probe = pc_probe; + ha_set_peerctrl_dispatch_for_test(0); /* explicit OFF */ + + pthread_t caller = pthread_self(); + io_packet_t *pkt = vote_reply_build_pkt_for_test(1, 1, 0, 0); + TEST_ASSERT_NOT_NULL(pkt); + + int rc = peer_ctrl_msg_handler_for_test(&ep, pkt); + TEST_ASSERT_EQUAL_INT(0, rc); + + /* Synchronous: the body already ran on THIS thread before the handler + * returned — no pool, no wait. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_pc_runs), + "Req 6.x: with the flag OFF the body must run inline (synchronously)"); + TEST_ASSERT_TRUE_MESSAGE( + pthread_equal(atomic_load_explicit(&g_pc_last_thread, memory_order_relaxed), + caller), + "Req 6.x: with the flag OFF the body must run on the caller's thread " + "(inline), unchanged from before the task"); + + ha_peerctrl_body_probe = NULL; + free(ep.data); + ha_fini(&c.ha); + printf("test_peerctrl_inline_when_flag_off: PASS\n"); +} + +/* + * Test 20 (Req 4.1, 4.2): two peer-control messages arriving on the SAME + * endpoint are processed in arrival order — the second body cannot start until + * the first completes. Proof: with the flag ON, the first body parks at the + * gate (holding its pool worker; the endpoint is blocked by bind_vtask's + * endpoint_block_tasks). While it is parked, dispatching a second message on + * the same endpoint does NOT run its body (run count stays 1). Releasing the + * first lets it complete (endpoint_unblock_tasks) and only THEN does the second + * run (count → 2). + */ +static void test_peerctrl_preserves_per_endpoint_order(void) +{ + vtaskmgr_t mgr; + TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, 4)); + TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&mgr)); + + cluster_t c; + memset(&c, 0, sizeof(c)); + ha_init(&c.ha, &g_cfg, 0); + c.ha.pool = &mgr; + atomic_store_explicit(&c.ha.role, (int32_t)CTRL_INIT, memory_order_release); + + endpoint_t ep; + memset(&ep, 0, sizeof(ep)); + standby_ep_ctx_build_for_test(&ep, &c.ha, &c, 0); + + pc_reset_probe_state(); + atomic_store(&g_pc_gate_on, 1); /* bodies park at the gate */ + ha_peerctrl_body_probe = pc_probe; + ha_set_peerctrl_dispatch_for_test(1); + + /* Message 1 — dispatched; its body parks at the gate. */ + io_packet_t *pkt1 = vote_reply_build_pkt_for_test(1, 1, 0, 0); + TEST_ASSERT_NOT_NULL(pkt1); + TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&ep, pkt1)); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, pc_wait_parked(2000), + "the first body must reach the gate (running on the pool)"); + + /* Message 2 — dispatched on the SAME endpoint while body 1 is parked. */ + io_packet_t *pkt2 = vote_reply_build_pkt_for_test(1, 1, 0, 0); + TEST_ASSERT_NOT_NULL(pkt2); + TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&ep, pkt2)); + + /* Give a generous window: body 2 must NOT run while body 1 holds the + * endpoint (per-connection serialization). */ + struct timespec ts = { .tv_sec = 0, .tv_nsec = 200 * 1000000L }; /* 200 ms */ + nanosleep(&ts, NULL); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_pc_runs), + "Req 4.1/4.2: the second message must NOT be processed while the first " + "is in flight on the same connection"); + + /* Release body 1; body 2 then becomes eligible and runs. */ + pc_release_gate(); + int runs = pc_wait_runs(2, 2000); + TEST_ASSERT_EQUAL_INT_MESSAGE(2, runs, + "Req 4.1/4.2: the second message runs only AFTER the first completes " + "(arrival order preserved)"); + + ha_set_peerctrl_dispatch_for_test(0); /* restore production default (OFF) */ + ha_peerctrl_body_probe = NULL; + vtaskmgr_fini(&mgr); + free(ep.data); + ha_fini(&c.ha); + printf("test_peerctrl_preserves_per_endpoint_order: PASS\n"); +} + /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_liveness(void) @@ -1513,5 +1778,9 @@ void test_ha_liveness(void) /* raft-drive-async Task 2.3 — reconnect-offload INTEGRATION (Req 2.1, 2.2, 2.5) */ RUN_TEST(test_reconnect_offload_sustains_survivor_heartbeats); RUN_TEST(test_reconnect_inline_stalls_survivor_heartbeats_RED); + /* raft-drive-async Task 3.1 — PeerCtrlDispatch (Req 1.4, 4.1, 4.2, 4.5, 6.x, 8.1, 8.2) */ + RUN_TEST(test_peerctrl_dispatch_runs_on_pool); + RUN_TEST(test_peerctrl_inline_when_flag_off); + RUN_TEST(test_peerctrl_preserves_per_endpoint_order); UNITY_END(); } diff --git a/src/include/vtaskmgr.h b/src/include/vtaskmgr.h index 482f2f1..a3daaf2 100644 --- a/src/include/vtaskmgr.h +++ b/src/include/vtaskmgr.h @@ -48,6 +48,16 @@ typedef struct { int vcpu; vtask_func_t entry; vtask_class_t cls; /* LivenessAdmission class (task 1.6); default LIVENESS */ + /* raft-drive-async task 3.1: when non-zero AND data->vtype == VTASK_IO, the + * carried io_packet_t (data->u.iodata.pkt) is OWNED by the task and must be + * freed by taskq_item_killer if the task is destroyed while still queued + * (never executed) at vtaskmgr_fini. Default 0 preserves the legacy + * endpoint-owned-packet model used by the libxce client path (xcectx.c's + * vtask_handle_msg does NOT clear ep->fsm_in.pkt, so the packet lifetime is + * tied to the endpoint, not the task — freeing it here would double-free). + * The controller's peer-control dispatch sets this to 1 because it clears + * ep->fsm_in.pkt and the task becomes the sole owner. */ + int owns_io_pkt; } vtask_t; #define ITPTR_TO_VTASK(p) ((vtask_t*)(((char*)p) - offsetof(vtask_t, link))) diff --git a/src/lib/vtaskmgr.c b/src/lib/vtaskmgr.c index 0565206..7c1f1ce 100644 --- a/src/lib/vtaskmgr.c +++ b/src/lib/vtaskmgr.c @@ -27,6 +27,7 @@ int vtask_init(vtask_t* task, vtask_func_t entry, void* data) task->data = data; task->owner = NULL; task->cls = VTASK_LIVENESS; /* never-gated by default; legacy enq paths */ + task->owns_io_pkt = 0; /* legacy: endpoint owns the packet (task 3.1) */ return 0; } @@ -166,6 +167,26 @@ static void taskq_item_killer(data_item_t* it) { if (it != NULL) { vtask_t* task = ITPTR_TO_VTASK(it); + + /* raft-drive-async task 3.1: a task destroyed while still queued (never + * executed) at vtaskmgr_fini would otherwise leak its carried packet — + * the ownership transfer (iod->pkt = NULL) only happens inside the + * executed entry (peer_ctrl_vtask_entry). Free the packet here ONLY + * when the producer marked the task as the packet's sole owner + * (owns_io_pkt); the legacy endpoint-owned model (owns_io_pkt == 0, + * e.g. the libxce client path) is left untouched so we never double-free + * a packet whose lifetime is the endpoint's. This runs only on + * un-executed tasks, so it cannot race the entry's free. vtask_fini + * frees only task->data (the wrapper struct), never the packet. */ + if (task->owns_io_pkt && task->data != NULL && + task->data->vtype == VTASK_IO && + task->data->u.iodata.pkt != NULL) { + io_packet_t* pkt = task->data->u.iodata.pkt; + free_packet(pkt); + free(pkt); + task->data->u.iodata.pkt = NULL; + } + vtask_fini(task); free(task); }