test(ha-commit-barrier-rollout): RED tests for dispatch-gate fact allowlist (task 4.1)
4 RED carriers (notify/update/setpreason/clrpreason must be admitted without quorum — currently NOT_READY, the bug-396 hole) + 2 pins (six request opcodes stay rejected; quorum-present admits all ten). Census 494 = 488 + 6; exactly 4 FAILs.
This commit is contained in:
parent
7103fa643d
commit
4fa38c6f88
3 changed files with 240 additions and 2 deletions
|
|
@ -60,7 +60,25 @@
|
|||
* inert stub in persist.c (returns -ENOSYS — every case fails on its
|
||||
* assertions, not linking); green with the task-2.2 implementation.
|
||||
*
|
||||
* Requirements: 1.1, 1.2, 1.3 (task 2.1); 2.1, 2.2, 3.1 (task 1.1)
|
||||
* Task 4.1 adds the dispatch-gate fact-allowlist cases (Req 3.2 — facts
|
||||
* accepted during quorum loss, the bug-396 silent-loss fix): with
|
||||
* CTRL_PRIMARY and write quorum ABSENT, the fact-bearing opcodes
|
||||
* OPCode_notify / OPCode_update / OPCode_setpreason / OPCode_clrpreason
|
||||
* must pass ha_dispatch_gate (return 0) so reported facts reach the
|
||||
* RETAIN path; the request-shaped opcodes (jobsub, jobcontrol,
|
||||
* queuecontrol, hostcontrol, jobdispatch, join) stay rejected with
|
||||
* -XCE_ERR_NOT_READY; with quorum present everything passes as today.
|
||||
* The four fact cases FAIL (RED) against the current allowlist
|
||||
* (ha.c CTRL_PRIMARY no-quorum switch — facts hit the default reject);
|
||||
* the request-rejection and quorum-present cases are PINS that must stay
|
||||
* green through task 4.2. Fixture copied from test_commit_engine.c
|
||||
* gate_setup (ha_init + legacy standby_endpoints counting, repl
|
||||
* back-pointer NULL; quorum-absent = empty endpoint registry).
|
||||
* CTRL_PRIMARY_COMMITTING is intentionally NOT tested (deleted wholesale
|
||||
* by ha-single-leader-role).
|
||||
*
|
||||
* Requirements: 1.1, 1.2, 1.3 (task 2.1); 2.1, 2.2, 3.1 (task 1.1);
|
||||
* 3.2 (task 4.1)
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
|
|
@ -82,12 +100,17 @@
|
|||
#include "xce/types.h" /* job_t */
|
||||
|
||||
#include "commit.h"
|
||||
#include "controller.capnp.h" /* OPCode_* (task 4.1 gate cases) */
|
||||
#include "ha.h" /* ha_state_t, ha_config_t, zarray (task 4.1) */
|
||||
#include "hook.h" /* HOOK_AFTER_EVENT */
|
||||
#include "persist.h" /* wal_commit_mode_t, wal_classify (task 1.1 decls) */
|
||||
#include "repl.h"
|
||||
#include "typemap.h" /* wal_catchup_entry_t (send hook signature) */
|
||||
#include "wal_types.h"
|
||||
|
||||
/* ha_dispatch_gate — defined in ha.c (pattern from test_commit_engine.c) */
|
||||
int ha_dispatch_gate(ha_state_t *ha, int opcode);
|
||||
|
||||
/* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps
|
||||
* repl_broadcast from dereferencing the dummy endpoints. */
|
||||
extern int (*repl_broadcast_send_hook)(void *ep,
|
||||
|
|
@ -762,6 +785,208 @@ static void test_barrier_not_ready_passes_through_unmodified(void)
|
|||
printf("test_barrier_not_ready_passes_through_unmodified: PASS\n");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Task 4.1 — dispatch-gate fact allowlist (Req 3.2).
|
||||
* (design.md §Fact Gate Admission — facts admitted during quorum loss)
|
||||
*
|
||||
* Fixture copied from test_commit_engine.c gate_setup: ha_init zeroes the
|
||||
* struct (repl back-pointer NULL → legacy standby_endpoints counting),
|
||||
* role forced to CTRL_PRIMARY, quorum driven purely by the number of
|
||||
* endpoints in standby_endpoints (n_peers=3: 0 endpoints = quorum absent,
|
||||
* 1 endpoint = quorum present).
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* Statics survive Unity's longjmp on assertion failure. */
|
||||
static ha_state_t g_fg_ha;
|
||||
static int g_fg_ep; /* dummy endpoint (zarray stores the pointer) */
|
||||
|
||||
static ha_config_t g_fg_cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
.election_jitter_max_ms = 0,
|
||||
.election_priority_step_ms = 500,
|
||||
.committing_timeout_ms = 10000,
|
||||
};
|
||||
|
||||
/* Arrange a CTRL_PRIMARY ha_state_t with n_endpoints connected standby
|
||||
* endpoints (what raft_cb_become_leader / on_standby_connected populate). */
|
||||
static void fact_gate_setup(int n_peers, int n_endpoints)
|
||||
{
|
||||
int i;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&g_fg_ha, &g_fg_cfg, 0));
|
||||
g_fg_ha.n_peers = n_peers;
|
||||
atomic_store_explicit(&g_fg_ha.role, (int32_t)CTRL_PRIMARY,
|
||||
memory_order_release);
|
||||
g_fg_ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
||||
TEST_ASSERT_NOT_NULL(g_fg_ha.standby_endpoints);
|
||||
zarray_init(g_fg_ha.standby_endpoints, 4, NULL);
|
||||
for (i = 0; i < n_endpoints; i++) {
|
||||
zarray_store(g_fg_ha.standby_endpoints, &g_fg_ep);
|
||||
}
|
||||
}
|
||||
|
||||
static void fact_gate_teardown(void)
|
||||
{
|
||||
if (g_fg_ha.standby_endpoints) {
|
||||
zarray_fini(g_fg_ha.standby_endpoints);
|
||||
free(g_fg_ha.standby_endpoints);
|
||||
g_fg_ha.standby_endpoints = NULL;
|
||||
}
|
||||
ha_fini(&g_fg_ha);
|
||||
}
|
||||
|
||||
/* ── OPCode_notify admitted without write quorum (Req 3.2) ────────────────
|
||||
* Worker-reported job state transitions arrive as OPCode_notify with no
|
||||
* requester awaiting confirmation. Rejecting them during quorum loss is
|
||||
* the bug-396 silent-loss hole: the worker never redelivers, so the fact
|
||||
* is gone forever. The gate must admit it so the RETAIN path can append
|
||||
* and retry until committed. RED against the current allowlist. */
|
||||
static void test_gate_no_quorum_admits_notify_fact(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* Arrange: 3-member primary, zero connected standbys — quorum absent */
|
||||
fact_gate_setup(3, 0);
|
||||
|
||||
/* Act */
|
||||
rc = ha_dispatch_gate(&g_fg_ha, OPCode_notify);
|
||||
|
||||
/* Assert: fact-bearing opcode must pass the gate during quorum loss */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"OPCode_notify (worker fact) must be admitted without write quorum");
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_no_quorum_admits_notify_fact: PASS\n");
|
||||
}
|
||||
|
||||
/* ── OPCode_update admitted without write quorum (Req 3.2) ────────────────
|
||||
* Worker-reported resource usage (do_update_op rusage path) is a fact:
|
||||
* receipt-style reply, RETAIN-registered append. RED. */
|
||||
static void test_gate_no_quorum_admits_update_fact(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* Arrange: quorum absent */
|
||||
fact_gate_setup(3, 0);
|
||||
|
||||
/* Act */
|
||||
rc = ha_dispatch_gate(&g_fg_ha, OPCode_update);
|
||||
|
||||
/* Assert */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"OPCode_update (worker rusage fact) must be admitted without write quorum");
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_no_quorum_admits_update_fact: PASS\n");
|
||||
}
|
||||
|
||||
/* ── OPCode_setpreason admitted without write quorum (Req 3.2) ────────────
|
||||
* Scheduler-reported pending reasons are facts (do_set_preason,
|
||||
* receipt-style reply). RED. */
|
||||
static void test_gate_no_quorum_admits_setpreason_fact(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* Arrange: quorum absent */
|
||||
fact_gate_setup(3, 0);
|
||||
|
||||
/* Act */
|
||||
rc = ha_dispatch_gate(&g_fg_ha, OPCode_setpreason);
|
||||
|
||||
/* Assert */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"OPCode_setpreason (scheduler fact) must be admitted without write quorum");
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_no_quorum_admits_setpreason_fact: PASS\n");
|
||||
}
|
||||
|
||||
/* ── OPCode_clrpreason admitted without write quorum (Req 3.2) ────────────
|
||||
* Clearing a pending reason is the same scheduler-fact shape. RED. */
|
||||
static void test_gate_no_quorum_admits_clrpreason_fact(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* Arrange: quorum absent */
|
||||
fact_gate_setup(3, 0);
|
||||
|
||||
/* Act */
|
||||
rc = ha_dispatch_gate(&g_fg_ha, OPCode_clrpreason);
|
||||
|
||||
/* Assert */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"OPCode_clrpreason (scheduler fact) must be admitted without write quorum");
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_no_quorum_admits_clrpreason_fact: PASS\n");
|
||||
}
|
||||
|
||||
/* ── PIN: request-shaped opcodes stay rejected without quorum (Req 3.2) ───
|
||||
* The allowlist is FACTS ONLY: every request-shaped opcode (a requester
|
||||
* awaits confirmation, served by the task-3 handler barrier) must keep
|
||||
* the -XCE_ERR_NOT_READY fast rejection. Green today; must remain green
|
||||
* after task 4.2 widens the allowlist. */
|
||||
static void test_gate_no_quorum_still_rejects_request_opcodes(void)
|
||||
{
|
||||
static const struct { int opcode; const char *name; } reqs[] = {
|
||||
{ OPCode_jobsub, "OPCode_jobsub" },
|
||||
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
|
||||
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
|
||||
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
|
||||
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
|
||||
{ OPCode_join, "OPCode_join" },
|
||||
};
|
||||
size_t i;
|
||||
|
||||
/* Arrange: quorum absent */
|
||||
fact_gate_setup(3, 0);
|
||||
|
||||
/* Act + Assert: every request-shaped opcode keeps the fast reject */
|
||||
for (i = 0; i < sizeof(reqs) / sizeof(reqs[0]); i++) {
|
||||
int rc = ha_dispatch_gate(&g_fg_ha, reqs[i].opcode);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc, reqs[i].name);
|
||||
}
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_no_quorum_still_rejects_request_opcodes: PASS\n");
|
||||
}
|
||||
|
||||
/* ── PIN: with write quorum present everything passes as today (Req 3.2) ──
|
||||
* The quorum-present path is `return 0` before any opcode inspection —
|
||||
* facts AND requests alike. Green today; must remain green after 4.2. */
|
||||
static void test_gate_with_quorum_admits_all_opcodes(void)
|
||||
{
|
||||
static const struct { int opcode; const char *name; } ops[] = {
|
||||
{ OPCode_notify, "OPCode_notify" },
|
||||
{ OPCode_update, "OPCode_update" },
|
||||
{ OPCode_setpreason, "OPCode_setpreason" },
|
||||
{ OPCode_clrpreason, "OPCode_clrpreason" },
|
||||
{ OPCode_jobsub, "OPCode_jobsub" },
|
||||
{ OPCode_jobcontrol, "OPCode_jobcontrol" },
|
||||
{ OPCode_queuecontrol, "OPCode_queuecontrol" },
|
||||
{ OPCode_hostcontrol, "OPCode_hostcontrol" },
|
||||
{ OPCode_jobdispatch, "OPCode_jobdispatch" },
|
||||
{ OPCode_join, "OPCode_join" },
|
||||
};
|
||||
size_t i;
|
||||
|
||||
/* Arrange: 3-member primary, one connected standby — quorum present */
|
||||
fact_gate_setup(3, 1);
|
||||
|
||||
/* Act + Assert: quorum-present primary admits everything */
|
||||
for (i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) {
|
||||
int rc = ha_dispatch_gate(&g_fg_ha, ops[i].opcode);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, ops[i].name);
|
||||
}
|
||||
|
||||
fact_gate_teardown();
|
||||
printf("test_gate_with_quorum_admits_all_opcodes: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test group entry point ──────────────────────────────────────────────── */
|
||||
|
||||
void test_commit_barrier_rollout(void)
|
||||
|
|
@ -784,5 +1009,12 @@ void test_commit_barrier_rollout(void)
|
|||
RUN_TEST(test_barrier_zero_timeout_falls_back_to_config_sync_timeout);
|
||||
RUN_TEST(test_barrier_single_member_returns_immediately);
|
||||
RUN_TEST(test_barrier_not_ready_passes_through_unmodified);
|
||||
/* Task 4.1 — dispatch-gate fact allowlist (Req 3.2) */
|
||||
RUN_TEST(test_gate_no_quorum_admits_notify_fact);
|
||||
RUN_TEST(test_gate_no_quorum_admits_update_fact);
|
||||
RUN_TEST(test_gate_no_quorum_admits_setpreason_fact);
|
||||
RUN_TEST(test_gate_no_quorum_admits_clrpreason_fact);
|
||||
RUN_TEST(test_gate_no_quorum_still_rejects_request_opcodes);
|
||||
RUN_TEST(test_gate_with_quorum_admits_all_opcodes);
|
||||
UNITY_END();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue