diff --git a/.kiro/specs/ha-commit-barrier-rollout/tasks.md b/.kiro/specs/ha-commit-barrier-rollout/tasks.md index 88d0f8d..27a2c96 100644 --- a/.kiro/specs/ha-commit-barrier-rollout/tasks.md +++ b/.kiro/specs/ha-commit-barrier-rollout/tasks.md @@ -50,7 +50,7 @@ - _Requirements: 2.3_ - [ ] 4. Fact admission during quorum loss and retention proof -- [ ] 4.1 Write RED tests for the dispatch-gate fact allowlist +- [x] 4.1 Write RED tests for the dispatch-gate fact allowlist - Tests against `ha_dispatch_gate` (ha.c:1527 anchor) with CTRL_PRIMARY and write quorum absent: `OPCode_notify`, `OPCode_update`, `OPCode_setpreason`, `OPCode_clrpreason` return 0; `OPCode_jobsub`, `OPCode_jobcontrol`, `OPCode_queuecontrol`, `OPCode_hostcontrol`, `OPCode_jobdispatch`, `OPCode_join` still return `-XCE_ERR_NOT_READY`; with quorum present everything returns 0 as today - Observable completion: the four fact-opcode cases FAIL (red) against the current allowlist - _Requirements: 3.2_ diff --git a/.wolf/memory.md b/.wolf/memory.md index 106f21a..8d73a40 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -948,3 +948,9 @@ | 11:06 | Task 3.3 receipt-ACK verdict pins: 4 GREEN pins (jobdispatch no-WAL receipt ACK, set/clr_preason, do_update_op rusage RETAIN) added to test_commit_barrier_reply.c; opupd.c added to test link | src/controller/testsuite/test_commit_barrier_reply.c, src/controller/meson.build | suite green 488 distinct PASS / 0 FAIL; discovered bug-451 (rusage hook va-arg type confusion) | ~95k | | 04:16 | Edited .kiro/specs/ha-commit-barrier-rollout/tasks.md | inline fix | ~14 | | 04:16 | Edited .kiro/specs/ha-commit-barrier-rollout/tasks.md | 1→2 lines | ~184 | +| 06:37 | Edited src/controller/testsuite/test_commit_barrier_rollout.c | modified cases() | ~312 | +| 06:37 | Edited src/controller/testsuite/test_commit_barrier_rollout.c | 6→11 lines | ~146 | +| 06:38 | Edited src/controller/testsuite/test_commit_barrier_rollout.c | added 1 condition(s) | ~2033 | +| 06:38 | Edited src/controller/testsuite/test_commit_barrier_rollout.c | expanded (+7 lines) | ~146 | +| 13:47 | ha-commit-barrier-rollout 4.1: added 6 dispatch-gate fact-allowlist tests (4 RED + 2 pins) | src/controller/testsuite/test_commit_barrier_rollout.c | RED verified: 4 FAIL (-XCE_ERR_NOT_READY), census 494=488+6, zero other failures | ~60k | +| 07:05 | Edited .kiro/specs/ha-commit-barrier-rollout/tasks.md | inline fix | ~17 | diff --git a/src/controller/testsuite/test_commit_barrier_rollout.c b/src/controller/testsuite/test_commit_barrier_rollout.c index 9d63a19..fd20d6b 100644 --- a/src/controller/testsuite/test_commit_barrier_rollout.c +++ b/src/controller/testsuite/test_commit_barrier_rollout.c @@ -60,7 +60,25 @@ * inert stub in persist.c (returns -ENOSYS — every case fails on its * assertions, not linking); green with the task-2.2 implementation. * - * Requirements: 1.1, 1.2, 1.3 (task 2.1); 2.1, 2.2, 3.1 (task 1.1) + * Task 4.1 adds the dispatch-gate fact-allowlist cases (Req 3.2 — facts + * accepted during quorum loss, the bug-396 silent-loss fix): with + * CTRL_PRIMARY and write quorum ABSENT, the fact-bearing opcodes + * OPCode_notify / OPCode_update / OPCode_setpreason / OPCode_clrpreason + * must pass ha_dispatch_gate (return 0) so reported facts reach the + * RETAIN path; the request-shaped opcodes (jobsub, jobcontrol, + * queuecontrol, hostcontrol, jobdispatch, join) stay rejected with + * -XCE_ERR_NOT_READY; with quorum present everything passes as today. + * The four fact cases FAIL (RED) against the current allowlist + * (ha.c CTRL_PRIMARY no-quorum switch — facts hit the default reject); + * the request-rejection and quorum-present cases are PINS that must stay + * green through task 4.2. Fixture copied from test_commit_engine.c + * gate_setup (ha_init + legacy standby_endpoints counting, repl + * back-pointer NULL; quorum-absent = empty endpoint registry). + * CTRL_PRIMARY_COMMITTING is intentionally NOT tested (deleted wholesale + * by ha-single-leader-role). + * + * Requirements: 1.1, 1.2, 1.3 (task 2.1); 2.1, 2.2, 3.1 (task 1.1); + * 3.2 (task 4.1) */ #include #include @@ -82,12 +100,17 @@ #include "xce/types.h" /* job_t */ #include "commit.h" +#include "controller.capnp.h" /* OPCode_* (task 4.1 gate cases) */ +#include "ha.h" /* ha_state_t, ha_config_t, zarray (task 4.1) */ #include "hook.h" /* HOOK_AFTER_EVENT */ #include "persist.h" /* wal_commit_mode_t, wal_classify (task 1.1 decls) */ #include "repl.h" #include "typemap.h" /* wal_catchup_entry_t (send hook signature) */ #include "wal_types.h" +/* ha_dispatch_gate — defined in ha.c (pattern from test_commit_engine.c) */ +int ha_dispatch_gate(ha_state_t *ha, int opcode); + /* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps * repl_broadcast from dereferencing the dummy endpoints. */ extern int (*repl_broadcast_send_hook)(void *ep, @@ -762,6 +785,208 @@ static void test_barrier_not_ready_passes_through_unmodified(void) printf("test_barrier_not_ready_passes_through_unmodified: PASS\n"); } +/* ════════════════════════════════════════════════════════════════════════ + * Task 4.1 — dispatch-gate fact allowlist (Req 3.2). + * (design.md §Fact Gate Admission — facts admitted during quorum loss) + * + * Fixture copied from test_commit_engine.c gate_setup: ha_init zeroes the + * struct (repl back-pointer NULL → legacy standby_endpoints counting), + * role forced to CTRL_PRIMARY, quorum driven purely by the number of + * endpoints in standby_endpoints (n_peers=3: 0 endpoints = quorum absent, + * 1 endpoint = quorum present). + * ════════════════════════════════════════════════════════════════════════ */ + +/* Statics survive Unity's longjmp on assertion failure. */ +static ha_state_t g_fg_ha; +static int g_fg_ep; /* dummy endpoint (zarray stores the pointer) */ + +static ha_config_t g_fg_cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 10000, + .election_timeout_ms = 2000, + .probe_connect_timeout_ms = 500, + .max_promotion_retries = 5, + .election_jitter_max_ms = 0, + .election_priority_step_ms = 500, + .committing_timeout_ms = 10000, +}; + +/* Arrange a CTRL_PRIMARY ha_state_t with n_endpoints connected standby + * endpoints (what raft_cb_become_leader / on_standby_connected populate). */ +static void fact_gate_setup(int n_peers, int n_endpoints) +{ + int i; + + TEST_ASSERT_EQUAL_INT(0, ha_init(&g_fg_ha, &g_fg_cfg, 0)); + g_fg_ha.n_peers = n_peers; + atomic_store_explicit(&g_fg_ha.role, (int32_t)CTRL_PRIMARY, + memory_order_release); + g_fg_ha.standby_endpoints = calloc(1, sizeof(zarray_t)); + TEST_ASSERT_NOT_NULL(g_fg_ha.standby_endpoints); + zarray_init(g_fg_ha.standby_endpoints, 4, NULL); + for (i = 0; i < n_endpoints; i++) { + zarray_store(g_fg_ha.standby_endpoints, &g_fg_ep); + } +} + +static void fact_gate_teardown(void) +{ + if (g_fg_ha.standby_endpoints) { + zarray_fini(g_fg_ha.standby_endpoints); + free(g_fg_ha.standby_endpoints); + g_fg_ha.standby_endpoints = NULL; + } + ha_fini(&g_fg_ha); +} + +/* ── OPCode_notify admitted without write quorum (Req 3.2) ──────────────── + * Worker-reported job state transitions arrive as OPCode_notify with no + * requester awaiting confirmation. Rejecting them during quorum loss is + * the bug-396 silent-loss hole: the worker never redelivers, so the fact + * is gone forever. The gate must admit it so the RETAIN path can append + * and retry until committed. RED against the current allowlist. */ +static void test_gate_no_quorum_admits_notify_fact(void) +{ + int rc; + + /* Arrange: 3-member primary, zero connected standbys — quorum absent */ + fact_gate_setup(3, 0); + + /* Act */ + rc = ha_dispatch_gate(&g_fg_ha, OPCode_notify); + + /* Assert: fact-bearing opcode must pass the gate during quorum loss */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "OPCode_notify (worker fact) must be admitted without write quorum"); + + fact_gate_teardown(); + printf("test_gate_no_quorum_admits_notify_fact: PASS\n"); +} + +/* ── OPCode_update admitted without write quorum (Req 3.2) ──────────────── + * Worker-reported resource usage (do_update_op rusage path) is a fact: + * receipt-style reply, RETAIN-registered append. RED. */ +static void test_gate_no_quorum_admits_update_fact(void) +{ + int rc; + + /* Arrange: quorum absent */ + fact_gate_setup(3, 0); + + /* Act */ + rc = ha_dispatch_gate(&g_fg_ha, OPCode_update); + + /* Assert */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "OPCode_update (worker rusage fact) must be admitted without write quorum"); + + fact_gate_teardown(); + printf("test_gate_no_quorum_admits_update_fact: PASS\n"); +} + +/* ── OPCode_setpreason admitted without write quorum (Req 3.2) ──────────── + * Scheduler-reported pending reasons are facts (do_set_preason, + * receipt-style reply). RED. */ +static void test_gate_no_quorum_admits_setpreason_fact(void) +{ + int rc; + + /* Arrange: quorum absent */ + fact_gate_setup(3, 0); + + /* Act */ + rc = ha_dispatch_gate(&g_fg_ha, OPCode_setpreason); + + /* Assert */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "OPCode_setpreason (scheduler fact) must be admitted without write quorum"); + + fact_gate_teardown(); + printf("test_gate_no_quorum_admits_setpreason_fact: PASS\n"); +} + +/* ── OPCode_clrpreason admitted without write quorum (Req 3.2) ──────────── + * Clearing a pending reason is the same scheduler-fact shape. RED. */ +static void test_gate_no_quorum_admits_clrpreason_fact(void) +{ + int rc; + + /* Arrange: quorum absent */ + fact_gate_setup(3, 0); + + /* Act */ + rc = ha_dispatch_gate(&g_fg_ha, OPCode_clrpreason); + + /* Assert */ + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "OPCode_clrpreason (scheduler fact) must be admitted without write quorum"); + + fact_gate_teardown(); + printf("test_gate_no_quorum_admits_clrpreason_fact: PASS\n"); +} + +/* ── PIN: request-shaped opcodes stay rejected without quorum (Req 3.2) ─── + * The allowlist is FACTS ONLY: every request-shaped opcode (a requester + * awaits confirmation, served by the task-3 handler barrier) must keep + * the -XCE_ERR_NOT_READY fast rejection. Green today; must remain green + * after task 4.2 widens the allowlist. */ +static void test_gate_no_quorum_still_rejects_request_opcodes(void) +{ + static const struct { int opcode; const char *name; } reqs[] = { + { OPCode_jobsub, "OPCode_jobsub" }, + { OPCode_jobcontrol, "OPCode_jobcontrol" }, + { OPCode_queuecontrol, "OPCode_queuecontrol" }, + { OPCode_hostcontrol, "OPCode_hostcontrol" }, + { OPCode_jobdispatch, "OPCode_jobdispatch" }, + { OPCode_join, "OPCode_join" }, + }; + size_t i; + + /* Arrange: quorum absent */ + fact_gate_setup(3, 0); + + /* Act + Assert: every request-shaped opcode keeps the fast reject */ + for (i = 0; i < sizeof(reqs) / sizeof(reqs[0]); i++) { + int rc = ha_dispatch_gate(&g_fg_ha, reqs[i].opcode); + TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc, reqs[i].name); + } + + fact_gate_teardown(); + printf("test_gate_no_quorum_still_rejects_request_opcodes: PASS\n"); +} + +/* ── PIN: with write quorum present everything passes as today (Req 3.2) ── + * The quorum-present path is `return 0` before any opcode inspection — + * facts AND requests alike. Green today; must remain green after 4.2. */ +static void test_gate_with_quorum_admits_all_opcodes(void) +{ + static const struct { int opcode; const char *name; } ops[] = { + { OPCode_notify, "OPCode_notify" }, + { OPCode_update, "OPCode_update" }, + { OPCode_setpreason, "OPCode_setpreason" }, + { OPCode_clrpreason, "OPCode_clrpreason" }, + { OPCode_jobsub, "OPCode_jobsub" }, + { OPCode_jobcontrol, "OPCode_jobcontrol" }, + { OPCode_queuecontrol, "OPCode_queuecontrol" }, + { OPCode_hostcontrol, "OPCode_hostcontrol" }, + { OPCode_jobdispatch, "OPCode_jobdispatch" }, + { OPCode_join, "OPCode_join" }, + }; + size_t i; + + /* Arrange: 3-member primary, one connected standby — quorum present */ + fact_gate_setup(3, 1); + + /* Act + Assert: quorum-present primary admits everything */ + for (i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { + int rc = ha_dispatch_gate(&g_fg_ha, ops[i].opcode); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, ops[i].name); + } + + fact_gate_teardown(); + printf("test_gate_with_quorum_admits_all_opcodes: PASS\n"); +} + /* ── Test group entry point ──────────────────────────────────────────────── */ void test_commit_barrier_rollout(void) @@ -784,5 +1009,12 @@ void test_commit_barrier_rollout(void) RUN_TEST(test_barrier_zero_timeout_falls_back_to_config_sync_timeout); RUN_TEST(test_barrier_single_member_returns_immediately); RUN_TEST(test_barrier_not_ready_passes_through_unmodified); + /* Task 4.1 — dispatch-gate fact allowlist (Req 3.2) */ + RUN_TEST(test_gate_no_quorum_admits_notify_fact); + RUN_TEST(test_gate_no_quorum_admits_update_fact); + RUN_TEST(test_gate_no_quorum_admits_setpreason_fact); + RUN_TEST(test_gate_no_quorum_admits_clrpreason_fact); + RUN_TEST(test_gate_no_quorum_still_rejects_request_opcodes); + RUN_TEST(test_gate_with_quorum_admits_all_opcodes); UNITY_END(); }