/* * test_commit_barrier_rollout.c — RED-phase tests for write-path commit * classification (ha-commit-barrier-rollout task 1.1). * * Covers Requirements 2.1 (every state-changing WAL type explicitly * classified WAIT/RETAIN/LOCAL), 2.2 (unclassified type fails fast under * XCE_TESTING), and 3.1 (fact-shaped appends are retained — appended, * non-blocking, pending purely by lsn > committed_lsn, committed later * by covering ACKs without redelivery). * * The classification table under test is the VERIFIED table from * .kiro/specs/ha-commit-barrier-rollout/design.md §Write-Path * Classification (it supersedes the umbrella draft table — research.md §2 * records the call-site-verified deviations: WAL_JOB_STOP/RESUME are * fact-origin only; WAL_JOB_UPDATE/WAL_JOB_FAIL are dual-origin and * classify RETAIN at the seam, with the request-shaped guarantee realized * by the handler barrier in do_jobcontrol_op — tasks 2.x/3.x): * * WAL_JOB_SUBMIT request WAIT * WAL_QUEUE_CTRL request WAIT * WAL_WORKER_CTRL + JOIN/UPDATE request WAIT * WAL_WORKER_CTRL + DEAD fact RETAIN * WAL_JOB_START/FINISH/FAIL/STOP/ * RESUME/UPDATE/RUSAGE/CKPT/ * PREASON fact RETAIN * WAL_CHECKPOINT local LOCAL (never replicated) * unknown — XCE_TESTING assert * * These cases FAIL (RED) against the task-1.1 inert wal_classify stub in * persist.c (returns an invalid sentinel mode) and go green with the * task-1.2 implementation. * * Honest-observable note for the RETAIN-registration case (task brief): * commit_submit(COMMIT_MODE_RETAIN) is registration-free bookkeeping * (commit.c — returns 0 immediately; commit state is derived entirely * from registry watermarks at ACK time), so RETAIN routing adds NO * engine-side observable distinct from the append itself. The fact-hook * test therefore asserts the full observable retention contract — * classified RETAIN (the RED carrier), hook returns only the local-append * rc without blocking, entry in the WAL with lsn > committed_lsn (pending, * never confirmed early), and committed_lsn advancing over the entry once * a covering ACK arrives (no redelivery) — rather than inventing a fake * registration observable. * * Fixture patterns from test_commit_engine.c (stub persist_t + replicator * with dummy endpoints; real tmpdir persist_t for the hook-driven append; * repl_broadcast_send_hook to drop broadcasts) and test_persist_hooks.c * (variadic trampoline driving persist_job_hook_fun with a real va_list). * * Task 2.1 adds the persist_commit_barrier(p, timeout_ms) semantics cases * (Req 1.1, 1.2, 1.3): the handler-callable WAIT barrier returns 0 without * blocking when committed_lsn already covers the appended watermark * (current_lsn - 1, the same snapshot persist_wal_sync_append reads — and * trivially when nothing was ever appended); -ETIMEDOUT when ACKs are * withheld past the deadline with committed_lsn unmoved; 0 once a covering * watermark ACK advances the commit index; timeout_ms = 0 falls back to * p->config.sync_timeout_ms; n_peers=1 succeeds immediately (engine * self-quorum, standalone preserved); and the engine's admission-cap * -XCE_ERR_NOT_READY passes through unmodified. RED against the task-2.1 * inert stub in persist.c (returns -ENOSYS — every case fails on its * assertions, not linking); green with the task-2.2 implementation. * * Task 4.1 adds the dispatch-gate fact-allowlist cases (Req 3.2 — facts * accepted during quorum loss, the bug-396 silent-loss fix): with * CTRL_PRIMARY and write quorum ABSENT, the fact-bearing opcodes * OPCode_notify / OPCode_update / OPCode_setpreason / OPCode_clrpreason * must pass ha_dispatch_gate (return 0) so reported facts reach the * RETAIN path; the request-shaped opcodes (jobsub, jobcontrol, * queuecontrol, hostcontrol, jobdispatch, join) stay rejected with * -XCE_ERR_NOT_READY; with quorum present everything passes as today. * The four fact cases FAIL (RED) against the current allowlist * (ha.c CTRL_PRIMARY no-quorum switch — facts hit the default reject); * the request-rejection and quorum-present cases are PINS that must stay * green through task 4.2. Fixture copied from test_commit_engine.c * gate_setup (ha_init + legacy standby_endpoints counting, repl * back-pointer NULL; quorum-absent = empty endpoint registry). * CTRL_PRIMARY_COMMITTING is intentionally NOT tested (deleted wholesale * by ha-single-leader-role). * * Task 5.1 adds the config-barrier cases (Req 4.1-4.4) — configAck * per-push arithmetic, push_config_to_standbys barrier-result contract, * and the upgradeConfig requester failure/success paths; see the task-5.1 * section banner below for the full RED rationale. * * Requirements: 1.1, 1.2, 1.3 (task 2.1); 2.1, 2.2, 3.1 (task 1.1); * 3.2 (task 4.1); 4.1, 4.2, 4.3, 4.4 (task 5.1) */ #include #include #include /* SHA256 (task 5.1 config blob hashing) */ #include #include #include #include #include #include #include #include #include /* mkdir (task 5.1 upgrade fixture) */ #include #include #include #include #include /* raw reply parsing (task 5.1) */ #include "xce/cluster_def.h" /* HOOK_DETAIL_*, XCE_CONTROLLER_HOOK_JOB */ #include "xce/errnox.h" /* XCE_ERR_NOT_READY (task 2.1 passthrough) */ #include "xce/types.h" /* job_t */ #include "commit.h" #include "config_repl.h" /* push_config_to_standbys, handle_upgrade_config * (+ netio.h, request.h, myconf.h, clusterconfig.h) */ #include "controller.capnp.h" /* OPCode_* (task 4.1 gate cases) */ #include "ha.h" /* ha_state_t, ha_config_t, zarray (task 4.1) */ #include "hook.h" /* HOOK_AFTER_EVENT */ #include "persist.h" /* wal_commit_mode_t, wal_classify (task 1.1 decls) */ #include "repl.h" #include "typemap.h" /* wal_catchup_entry_t (send hook signature) */ #include "wal_types.h" /* ha_dispatch_gate — defined in ha.c (pattern from test_commit_engine.c) */ int ha_dispatch_gate(ha_state_t *ha, int opcode); /* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps * repl_broadcast from dereferencing the dummy endpoints. */ extern int (*repl_broadcast_send_hook)(void *ep, wal_catchup_entry_t **entries, int n_entries); static int cbr_drop_send_hook(void *ep, wal_catchup_entry_t **entries, int n_entries) { (void)ep; (void)entries; (void)n_entries; return 0; /* pretend success, drop the data */ } /* ── Shared helpers ──────────────────────────────────────────────────────── */ static uint64_t mono_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u; } /* Valid-mode predicate for the totality sweep (Req 2.1). */ static int is_valid_mode(wal_commit_mode_t m) { return m == WAL_MODE_WAIT || m == WAL_MODE_RETAIN || m == WAL_MODE_LOCAL; } /* ── Classification table cases (Req 2.1) ───────────────────────────────── * All RED against the inert stub: it returns an out-of-enum sentinel, so * every equality below fails on assertion, not linking. */ static void test_classify_job_submit_is_wait(void) { /* Arrange / Act */ wal_commit_mode_t mode = wal_classify(WAL_JOB_SUBMIT, HOOK_DETAIL_JOB_NEW); /* Assert: client job submit is request-shaped — commit barrier before * the requester reply (reference impl keeps the in-hook sync path) */ TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_WAIT, mode, "WAL_JOB_SUBMIT must classify WAIT (request-shaped)"); printf("test_classify_job_submit_is_wait: PASS\n"); } static void test_classify_queue_ctrl_is_wait(void) { /* Arrange / Act */ wal_commit_mode_t mode = wal_classify(WAL_QUEUE_CTRL, HOOK_DETAIL_QUEUE_UPDATE); /* Assert: queue control is client-request-shaped */ TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_WAIT, mode, "WAL_QUEUE_CTRL must classify WAIT (request-shaped)"); printf("test_classify_queue_ctrl_is_wait: PASS\n"); } static void test_classify_worker_ctrl_detail_split(void) { /* Arrange / Act: WAL_WORKER_CTRL is the only detail-disambiguated type */ wal_commit_mode_t join_mode = wal_classify(WAL_WORKER_CTRL, HOOK_DETAIL_WORKER_JOIN); wal_commit_mode_t update_mode = wal_classify(WAL_WORKER_CTRL, HOOK_DETAIL_WORKER_UPDATE); wal_commit_mode_t dead_mode = wal_classify(WAL_WORKER_CTRL, HOOK_DETAIL_WORKER_DEAD); /* Assert: join/update have a requester awaiting confirmation (WAIT); * worker death is controller-detected — a reported fact (RETAIN) */ TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_WAIT, join_mode, "WAL_WORKER_CTRL + WORKER_JOIN must classify WAIT"); TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_WAIT, update_mode, "WAL_WORKER_CTRL + WORKER_UPDATE must classify WAIT"); TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_RETAIN, dead_mode, "WAL_WORKER_CTRL + WORKER_DEAD must classify RETAIN (fact)"); printf("test_classify_worker_ctrl_detail_split: PASS\n"); } static void test_classify_all_job_state_types_are_retain(void) { /* Arrange: every WAL_JOB_* state-transition type with its natural hook * detail (verified table: all fact-shaped — STOP/RESUME are fact-origin * only; FAIL/UPDATE dual-origin, request side covered by the handler * barrier in later tasks) */ static const struct { uint32_t wal_type; int detail; const char *name; } facts[] = { { WAL_JOB_START, HOOK_DETAIL_JOB_START, "WAL_JOB_START" }, { WAL_JOB_FINISH, HOOK_DETAIL_JOB_EXIT, "WAL_JOB_FINISH" }, { WAL_JOB_FAIL, HOOK_DETAIL_JOB_FAIL, "WAL_JOB_FAIL" }, { WAL_JOB_STOP, HOOK_DETAIL_JOB_STOP, "WAL_JOB_STOP" }, { WAL_JOB_RESUME, HOOK_DETAIL_JOB_RESUME, "WAL_JOB_RESUME" }, { WAL_JOB_UPDATE, HOOK_DETAIL_JOB_UPDATE, "WAL_JOB_UPDATE" }, { WAL_JOB_RUSAGE, HOOK_DETAIL_JOB_RUSAGE_UPDATE, "WAL_JOB_RUSAGE" }, { WAL_JOB_CKPT, HOOK_DETAIL_JOB_CKPT, "WAL_JOB_CKPT" }, { WAL_JOB_PREASON, HOOK_DETAIL_SETPREASON, "WAL_JOB_PREASON" }, }; size_t i; /* Act + Assert: each is a reported fact — retained until committed */ for (i = 0; i < sizeof(facts) / sizeof(facts[0]); i++) { char msg[96]; snprintf(msg, sizeof(msg), "%s must classify RETAIN (fact-shaped)", facts[i].name); TEST_ASSERT_EQUAL_INT_MESSAGE( WAL_MODE_RETAIN, wal_classify(facts[i].wal_type, facts[i].detail), msg); } printf("test_classify_all_job_state_types_are_retain: PASS\n"); } static void test_classify_checkpoint_is_local(void) { /* Arrange / Act: WAL_CHECKPOINT is a local WAL marker, never replicated */ wal_commit_mode_t mode = wal_classify(WAL_CHECKPOINT, 0); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(WAL_MODE_LOCAL, mode, "WAL_CHECKPOINT must classify LOCAL (never replicated)"); printf("test_classify_checkpoint_is_local: PASS\n"); } static void test_classify_is_total_over_wal_entry_type(void) { /* Arrange: EVERY value of wal_entry_type (wal_types.h:44-58, verified * 2026-06-11: 13 values, no WAL_NOOP yet). When a new type is added, * this sweep plus the unknown-type guard force it to be classified. */ static const struct { uint32_t wal_type; int detail; const char *name; } all_types[] = { { WAL_JOB_SUBMIT, HOOK_DETAIL_JOB_NEW, "WAL_JOB_SUBMIT" }, { WAL_JOB_START, HOOK_DETAIL_JOB_START, "WAL_JOB_START" }, { WAL_JOB_FINISH, HOOK_DETAIL_JOB_EXIT, "WAL_JOB_FINISH" }, { WAL_JOB_FAIL, HOOK_DETAIL_JOB_FAIL, "WAL_JOB_FAIL" }, { WAL_JOB_STOP, HOOK_DETAIL_JOB_STOP, "WAL_JOB_STOP" }, { WAL_JOB_RESUME, HOOK_DETAIL_JOB_RESUME, "WAL_JOB_RESUME" }, { WAL_JOB_UPDATE, HOOK_DETAIL_JOB_UPDATE, "WAL_JOB_UPDATE" }, { WAL_JOB_RUSAGE, HOOK_DETAIL_JOB_RUSAGE_UPDATE, "WAL_JOB_RUSAGE" }, { WAL_JOB_CKPT, HOOK_DETAIL_JOB_CKPT, "WAL_JOB_CKPT" }, { WAL_JOB_PREASON, HOOK_DETAIL_CLRPREASON, "WAL_JOB_PREASON" }, { WAL_QUEUE_CTRL, HOOK_DETAIL_QUEUE_UPDATE, "WAL_QUEUE_CTRL" }, { WAL_WORKER_CTRL, HOOK_DETAIL_WORKER_DEAD, "WAL_WORKER_CTRL" }, { WAL_CHECKPOINT, 0, "WAL_CHECKPOINT" }, }; size_t i; /* Act + Assert: classification is total — every enum value maps to a * valid mode (no silent barrier bypass for any state-changing type) */ for (i = 0; i < sizeof(all_types) / sizeof(all_types[0]); i++) { char msg[96]; snprintf(msg, sizeof(msg), "%s must map to a valid commit mode (totality)", all_types[i].name); TEST_ASSERT_TRUE_MESSAGE( is_valid_mode(wal_classify(all_types[i].wal_type, all_types[i].detail)), msg); } printf("test_classify_is_total_over_wal_entry_type: PASS\n"); } /* ── Unknown type fails fast under XCE_TESTING (Req 2.2) ────────────────── * The guard contract is assert(0) in testing builds (abort), production * ERROR + RETAIN fallback. The abort is observed from a forked child so * the test runner survives (fork pattern precedent: lib testsuite test.c). * RED: the inert stub returns its sentinel without asserting, so the child * exits normally and the SIGABRT assertion fails. */ static void test_classify_unknown_type_fails_fast_in_testing(void) { pid_t pid; int status = 0; /* Arrange: a WAL type far outside wal_entry_type */ const uint32_t unknown_type = 777u; /* Act: evaluate the unknown type in a child process */ pid = fork(); TEST_ASSERT_TRUE_MESSAGE(pid >= 0, "fork must succeed"); if (pid == 0) { /* Child: silence the expected assert/abort noise, then classify. * Reaching _exit(0) means the fail-fast path did NOT trigger. */ int devnull = open("/dev/null", O_WRONLY); if (devnull >= 0) { dup2(devnull, STDERR_FILENO); dup2(devnull, STDOUT_FILENO); } (void)wal_classify(unknown_type, 0); _exit(0); } /* Assert: the child must die on the XCE_TESTING fail-fast path */ TEST_ASSERT_EQUAL_INT_MESSAGE(pid, waitpid(pid, &status, 0), "waitpid must reap the classifier child"); TEST_ASSERT_TRUE_MESSAGE( WIFSIGNALED(status) && WTERMSIG(status) == SIGABRT, "unknown WAL type must trip the XCE_TESTING fail-fast (abort)"); printf("test_classify_unknown_type_fails_fast_in_testing: PASS\n"); } /* ── Fact-type append through the hook seam is RETAIN-routed (Req 3.1) ──── * Drives the production persist_job_hook_fun with a worker-reported job * finish (fact) against a real tmpdir persist_t with a wired commit engine * (n_peers=3) and ACKs withheld. Observable retention contract: * - wal_classify routes the type RETAIN (RED carrier — fails against the * inert stub; the behavioral assertions below pass on unmodified * persist.c and pin the contract task 1.2 must preserve) * - the hook returns the local-append rc (0) WITHOUT blocking on commit * progress (no timeout applies to facts) * - the entry is in the WAL and pending purely by lsn > committed_lsn * (never confirmed early) * - a later covering ACK advances committed_lsn over the entry with no * redelivery of the fact * commit_submit(RETAIN) itself is registration-free (commit.c) — see the * file header note; there is deliberately no fake "registered" observable. */ /* Variadic trampoline so the test passes typed args through a real va_list * (pattern from test_persist_hooks.c). */ static int call_job_hook(int type, int sit, int detail, persist_t *p, ...) { va_list ap; int rc; va_start(ap, p); rc = persist_job_hook_fun(type, sit, detail, p, ap); va_end(ap); return rc; } /* Statics survive Unity's longjmp on assertion failure. */ static persist_t *g_fp; static commit_engine_t g_fce; static job_t g_fact_job; static int g_fep_a, g_fep_b; /* dummy endpoints — replicator compares pointers */ static void test_fact_hook_append_is_retained_and_nonblocking(void) { persist_config_t cfg; char tmpdir[256]; uint64_t t0, elapsed_ms, fact_lsn; int rc; /* Arrange: real persist over a private tmpdir; sync timeout large so a * misrouted blocking wait is detectable as elapsed time */ cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; cfg.sync_timeout_ms = 3000; snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_cbr11_fact_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); g_fp = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(g_fp); persist_set_workdir(g_fp, tmpdir, "ctrl"); TEST_ASSERT_EQUAL_INT(0, persist_init(g_fp, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(g_fp)); /* 3-member cluster: 2 standbys registered, engine wired, ACKs withheld */ TEST_ASSERT_TRUE(repl_add_standby(&g_fp->repl, &g_fep_a, "sb_a", 7200) >= 0); TEST_ASSERT_TRUE(repl_add_standby(&g_fp->repl, &g_fep_b, "sb_b", 7201) >= 0); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_fce, g_fp, 3, 8)); repl_set_commit_engine(&g_fp->repl, &g_fce); repl_broadcast_send_hook = cbr_drop_send_hook; /* Priming async append so the fact entry lands at LSN 1 (the engine * rejects lsn == 0 — accepted interim LSN-0 edge, ha-commit-engine) */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(g_fp, WAL_JOB_UPDATE, "p", 1)); fact_lsn = g_fp->current_lsn; /* the LSN the fact append will get */ TEST_ASSERT_EQUAL_UINT64(1, fact_lsn); memset(&g_fact_job, 0, sizeof(g_fact_job)); g_fact_job.job.jobid = 42; g_fact_job.job.state = 0x10; /* Act: worker-reported job finish (fact) through the production hook. * JOB_EXIT AFTER_EVENT args: cluster_t*, xce_jobfinish_t*, job_t* */ t0 = mono_ms(); rc = call_job_hook(XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_EXIT, g_fp, (void *)NULL, (void *)NULL, &g_fact_job); elapsed_ms = mono_ms() - t0; /* Assert: the seam classifies the fact RETAIN (RED carrier — the routing * contract the behavioral assertions below depend on) */ TEST_ASSERT_EQUAL_INT_MESSAGE( WAL_MODE_RETAIN, wal_classify(WAL_JOB_FINISH, HOOK_DETAIL_JOB_EXIT), "fact-type WAL_JOB_FINISH must be RETAIN-routed at the hook seam"); /* ...the hook propagated only the local-append rc, without blocking on * commit progress (facts have no commit timeout) */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "fact hook must return the local-append rc only (no commit error)"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 1000, "fact append must not block on commit progress (no timeout applies)"); /* ...the entry is in the WAL, pending purely by lsn > committed_lsn */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(fact_lsn + 1, g_fp->current_lsn, "the fact entry must be appended to the WAL"); TEST_ASSERT_TRUE_MESSAGE( atomic_load(&g_fp->committed_lsn) < fact_lsn, "the uncommitted fact must be pending (lsn > committed_lsn), " "never confirmed early"); /* ...and a covering ACK later commits the retained entry — engine * accounting advances committed_lsn over it, no redelivery needed */ repl_on_ack(&g_fp->repl, &g_fep_a, fact_lsn); TEST_ASSERT_EQUAL_UINT64_MESSAGE( fact_lsn, atomic_load(&g_fp->committed_lsn), "a covering ACK must advance committed_lsn over the retained fact"); /* Teardown */ repl_broadcast_send_hook = NULL; commit_engine_fini(&g_fce); persist_stop(g_fp); persist_fini(g_fp); free(g_fp); g_fp = NULL; printf("test_fact_hook_append_is_retained_and_nonblocking: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.1 — persist_commit_barrier(p, timeout_ms) semantics * (Req 1.1, 1.2, 1.3). RED against the inert -ENOSYS stub in persist.c. * * Fixture: stub persist_t + replicator + wired commit engine (pattern from * test_commit_engine.c engine_setup_vcpus). The barrier reaches the engine * via p->repl.ce (the same wiring the RETAIN hook path uses), so * repl_set_commit_engine is part of the fixture. ACKs are driven through * repl_on_ack (registry watermark + engine evaluation, the production * path). current_lsn semantics: highest appended LSN = current_lsn - 1 * (persist.h invariant; persist.c:1021); current_lsn == 0 = nothing ever * appended. n_peers=3 unless noted. * ════════════════════════════════════════════════════════════════════════ */ /* Statics survive Unity's longjmp on assertion failure. */ static persist_t g_bp; /* stub persist for the barrier cases */ static commit_engine_t g_bce; static int g_bep_a, g_bep_b; /* dummy endpoints (pointer compares) */ static void barrier_setup_vcpus(int n_peers, int n_standbys, uint64_t current_lsn, int n_vcpus, uint32_t sync_timeout_ms) { memset(&g_bp, 0, sizeof(g_bp)); TEST_ASSERT_EQUAL_INT(0, pthread_mutex_init(&g_bp.lock, NULL)); g_bp.config = persist_config_defaults(); g_bp.config.sync_timeout_ms = sync_timeout_ms; TEST_ASSERT_EQUAL_INT(0, repl_init(&g_bp.repl)); if (n_standbys >= 1) TEST_ASSERT_TRUE( repl_add_standby(&g_bp.repl, &g_bep_a, "bsb_a", 7300) >= 0); if (n_standbys >= 2) TEST_ASSERT_TRUE( repl_add_standby(&g_bp.repl, &g_bep_b, "bsb_b", 7301) >= 0); g_bp.current_lsn = current_lsn; atomic_store(&g_bp.committed_lsn, 0); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_bce, &g_bp, n_peers, n_vcpus)); repl_set_commit_engine(&g_bp.repl, &g_bce); } static void barrier_setup(int n_peers, int n_standbys, uint64_t current_lsn) { barrier_setup_vcpus(n_peers, n_standbys, current_lsn, 8, 3000); } static void barrier_teardown(void) { commit_engine_fini(&g_bce); repl_fini(&g_bp.repl); pthread_mutex_destroy(&g_bp.lock); } static void cbr_sleep_ms(uint32_t ms) { struct timespec ts; ts.tv_sec = ms / 1000u; ts.tv_nsec = (long)(ms % 1000u) * 1000000L; nanosleep(&ts, NULL); } /* Barrier-caller thread context (covering-ACK case). */ typedef struct { uint32_t timeout_ms; int rc; uint64_t elapsed_ms; } barrier_arg_t; static barrier_arg_t g_barg; static pthread_t g_btid; static void *barrier_thread_fn(void *arg) { barrier_arg_t *a = arg; uint64_t t0 = mono_ms(); a->rc = persist_commit_barrier(&g_bp, a->timeout_ms); a->elapsed_ms = mono_ms() - t0; return NULL; } /* Direct engine waiter — saturates the WAIT admission slot for the * NOT_READY passthrough case (test_commit_engine.c test-13 pattern). */ typedef struct { uint64_t lsn; uint32_t timeout_ms; int rc; } engine_wait_arg_t; static engine_wait_arg_t g_ewarg; static pthread_t g_ewtid; static void *engine_wait_thread_fn(void *arg) { engine_wait_arg_t *a = arg; a->rc = commit_submit(&g_bce, a->lsn, COMMIT_MODE_WAIT, a->timeout_ms); return NULL; } /* Poll until `want` waiters are blocked inside the engine or the deadline * expires; returns the last observed inflight count. */ static int barrier_poll_inflight(int want, uint32_t deadline_ms) { uint64_t t0 = mono_ms(); int v; for (;;) { pthread_mutex_lock(&g_bce.lock); v = g_bce.inflight_waits; pthread_mutex_unlock(&g_bce.lock); if (v >= want || mono_ms() - t0 >= deadline_ms) return v; cbr_sleep_ms(5); } } /* ── Barrier short-circuits when already covered (Req 1.1) ──────────────── */ static void test_barrier_returns_zero_when_nothing_appended(void) { int rc; uint64_t t0, elapsed_ms; /* Arrange: 3-member cluster, NOTHING ever appended (current_lsn == 0, * fresh persist) — there are no pending entries to wait for; ACKs * withheld so any misrouted engine wait would block visibly */ barrier_setup(3, 2, 0); /* Act */ t0 = mono_ms(); rc = persist_commit_barrier(&g_bp, 5000); elapsed_ms = mono_ms() - t0; /* Assert: immediate success without consuming the timeout */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "barrier over an empty log must return 0 (nothing to commit)"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 1000, "barrier over an empty log must not block"); barrier_teardown(); printf("test_barrier_returns_zero_when_nothing_appended: PASS\n"); } static void test_barrier_returns_zero_when_committed_covers_appended(void) { int rc; uint64_t t0, elapsed_ms; /* Arrange: entries 1..2 appended (current_lsn = 3); a standby watermark * ACK has already committed both — committed_lsn covers the appended * watermark current_lsn - 1 before the barrier is called */ barrier_setup(3, 2, 3); repl_on_ack(&g_bp.repl, &g_bep_a, 2); TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, atomic_load(&g_bp.committed_lsn), "fixture: covering ACK must have committed the appended entries"); /* Act */ t0 = mono_ms(); rc = persist_commit_barrier(&g_bp, 5000); elapsed_ms = mono_ms() - t0; /* Assert: already covered → 0 immediately, no blocking */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "barrier must return 0 when committed_lsn already covers " "current_lsn - 1"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 1000, "already-covered barrier must not block"); barrier_teardown(); printf("test_barrier_returns_zero_when_committed_covers_appended: PASS\n"); } /* ── Withheld ACKs → -ETIMEDOUT, committed_lsn unmoved (Req 1.2) ────────── */ static void test_barrier_times_out_when_acks_withheld(void) { int rc; uint64_t t0, elapsed_ms; /* Arrange: entry 1 pending (current_lsn = 2), ACKs withheld forever */ barrier_setup(3, 2, 2); /* Act: short deadline, no ACK will ever arrive */ t0 = mono_ms(); rc = persist_commit_barrier(&g_bp, 250); elapsed_ms = mono_ms() - t0; /* Assert: timeout surfaced, deadline honored, commit index unmoved */ TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, rc, "withheld ACKs past the deadline must surface -ETIMEDOUT"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms >= 200, "barrier must hold until its deadline before timing out"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, atomic_load(&g_bp.committed_lsn), "committed_lsn must stay unmoved when no ACK ever covers the entry"); barrier_teardown(); printf("test_barrier_times_out_when_acks_withheld: PASS\n"); } /* ── Covering ACK while blocked → 0 (Req 1.1) ───────────────────────────── */ static void test_barrier_returns_zero_once_covering_ack_arrives(void) { /* Arrange: entry 1 pending (current_lsn = 2); the barrier call runs in * a thread so this thread can deliver the covering ACK */ barrier_setup(3, 2, 2); g_barg.timeout_ms = 2000; g_barg.rc = -9999; g_barg.elapsed_ms = 0; TEST_ASSERT_EQUAL_INT( 0, pthread_create(&g_btid, NULL, barrier_thread_fn, &g_barg)); /* Act: hold the barrier blocked >= 120 ms, then a covering watermark * ACK advances the commit index; join BEFORE any assertion */ cbr_sleep_ms(120); repl_on_ack(&g_bp.repl, &g_bep_a, 1); pthread_join(g_btid, NULL); /* Assert: success only after the ACK — the barrier really blocked */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_barg.rc, "barrier must return 0 once a covering ACK advances the commit index"); TEST_ASSERT_TRUE_MESSAGE(g_barg.elapsed_ms >= 100, "barrier must block until the covering ACK arrives, not before"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, atomic_load(&g_bp.committed_lsn), "the covering ACK must have advanced committed_lsn over the entry"); barrier_teardown(); printf("test_barrier_returns_zero_once_covering_ack_arrives: PASS\n"); } /* ── timeout_ms = 0 falls back to p->config.sync_timeout_ms (Req 1.2) ───── */ static void test_barrier_zero_timeout_falls_back_to_config_sync_timeout(void) { int rc; uint64_t t0, elapsed_ms; /* Arrange: entry 1 pending, ACKs withheld; config sync_timeout_ms set * to a short, recognizable deadline (250 ms) */ barrier_setup_vcpus(3, 2, 2, 8, 250); /* Act: explicit timeout 0 → must borrow the config deadline */ t0 = mono_ms(); rc = persist_commit_barrier(&g_bp, 0); elapsed_ms = mono_ms() - t0; /* Assert: timed out after ~the CONFIG deadline — neither an instant * zero-timeout reject nor an unbounded wait */ TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, rc, "timeout_ms=0 with withheld ACKs must time out via the config value"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms >= 200, "timeout_ms=0 must fall back to config.sync_timeout_ms, not 0"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 1500, "timeout_ms=0 must be bounded by config.sync_timeout_ms"); barrier_teardown(); printf("test_barrier_zero_timeout_falls_back_to_config_sync_timeout: " "PASS\n"); } /* ── n_peers=1 → immediate success (standalone preserved, Req 1.1) ──────── */ static void test_barrier_single_member_returns_immediately(void) { int rc; uint64_t t0, elapsed_ms; /* Arrange: single-member cluster, entry 1 pending (current_lsn = 2) — * engine self-quorum: the primary's own append is the majority */ barrier_setup(1, 0, 2); /* Act: large timeout that must NOT be consumed */ t0 = mono_ms(); rc = persist_commit_barrier(&g_bp, 5000); elapsed_ms = mono_ms() - t0; /* Assert: immediate local confirmation, standalone behavior preserved */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "n_peers=1 barrier must return 0 immediately (self-quorum)"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 1000, "n_peers=1 barrier must not block"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(1, atomic_load(&g_bp.committed_lsn), "self-quorum must advance committed_lsn over the entry"); barrier_teardown(); printf("test_barrier_single_member_returns_immediately: PASS\n"); } /* ── Engine -XCE_ERR_NOT_READY passes through unmodified (Req 1.3) ──────── * n_vcpus=2, COMMIT_WAIT_RESERVED=2 → admission cap = max(1, 0) = 1 * (test_commit_engine.c test-13 pattern): one direct engine WAIT saturates * the only slot, so the barrier's own commit_submit(WAIT) is rejected fast * with the retriable admission error — which the barrier must return * UNMODIFIED (no remapping, no success). */ static void test_barrier_not_ready_passes_through_unmodified(void) { int rc, observed_inflight; uint64_t t0, elapsed_ms; /* Arrange: entry 1 pending, cap = 1; a direct waiter occupies the slot */ barrier_setup_vcpus(3, 2, 2, 2, 3000); g_ewarg.lsn = 1; g_ewarg.timeout_ms = 5000; g_ewarg.rc = -9999; TEST_ASSERT_EQUAL_INT( 0, pthread_create(&g_ewtid, NULL, engine_wait_thread_fn, &g_ewarg)); observed_inflight = barrier_poll_inflight(1, 2000); /* Act: barrier call while the admission cap is saturated */ t0 = mono_ms(); rc = persist_commit_barrier(&g_bp, 5000); elapsed_ms = mono_ms() - t0; /* Release the blocked waiter, then join BEFORE any assertion */ repl_on_ack(&g_bp.repl, &g_bep_a, 1); pthread_join(g_ewtid, NULL); /* Assert: the slot was really saturated when the barrier ran... */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, observed_inflight, "the direct engine waiter must occupy the only admission slot"); /* ...the engine's admission rejection passed through unmodified... */ TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc, "barrier must return the engine's -XCE_ERR_NOT_READY unmodified"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 500, "admission-cap rejection must not block"); /* ...and the admitted waiter still completed via the covering ACK */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_ewarg.rc, "the admitted direct waiter must complete with success"); barrier_teardown(); printf("test_barrier_not_ready_passes_through_unmodified: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 4.1 — dispatch-gate fact allowlist (Req 3.2). * (design.md §Fact Gate Admission — facts admitted during quorum loss) * * Fixture copied from test_commit_engine.c gate_setup: ha_init zeroes the * struct (repl back-pointer NULL → legacy standby_endpoints counting), * role forced to CTRL_PRIMARY, quorum driven purely by the number of * endpoints in standby_endpoints (n_peers=3: 0 endpoints = quorum absent, * 1 endpoint = quorum present). * ════════════════════════════════════════════════════════════════════════ */ /* Statics survive Unity's longjmp on assertion failure. */ static ha_state_t g_fg_ha; static int g_fg_ep; /* dummy endpoint (zarray stores the pointer) */ static ha_config_t g_fg_cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .election_jitter_max_ms = 0, .election_priority_step_ms = 500, .committing_timeout_ms = 10000, }; /* Arrange a CTRL_PRIMARY ha_state_t with n_endpoints connected standby * endpoints (what raft_cb_become_leader / on_standby_connected populate). */ static void fact_gate_setup(int n_peers, int n_endpoints) { int i; TEST_ASSERT_EQUAL_INT(0, ha_init(&g_fg_ha, &g_fg_cfg, 0)); g_fg_ha.n_peers = n_peers; atomic_store_explicit(&g_fg_ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); g_fg_ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(g_fg_ha.standby_endpoints); zarray_init(g_fg_ha.standby_endpoints, 4, NULL); for (i = 0; i < n_endpoints; i++) { zarray_store(g_fg_ha.standby_endpoints, &g_fg_ep); } } static void fact_gate_teardown(void) { if (g_fg_ha.standby_endpoints) { zarray_fini(g_fg_ha.standby_endpoints); free(g_fg_ha.standby_endpoints); g_fg_ha.standby_endpoints = NULL; } ha_fini(&g_fg_ha); } /* ── OPCode_notify admitted without write quorum (Req 3.2) ──────────────── * Worker-reported job state transitions arrive as OPCode_notify with no * requester awaiting confirmation. Rejecting them during quorum loss is * the bug-396 silent-loss hole: the worker never redelivers, so the fact * is gone forever. The gate must admit it so the RETAIN path can append * and retry until committed. RED against the current allowlist. */ static void test_gate_no_quorum_admits_notify_fact(void) { int rc; /* Arrange: 3-member primary, zero connected standbys — quorum absent */ fact_gate_setup(3, 0); /* Act */ rc = ha_dispatch_gate(&g_fg_ha, OPCode_notify); /* Assert: fact-bearing opcode must pass the gate during quorum loss */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "OPCode_notify (worker fact) must be admitted without write quorum"); fact_gate_teardown(); printf("test_gate_no_quorum_admits_notify_fact: PASS\n"); } /* ── OPCode_update admitted without write quorum (Req 3.2) ──────────────── * Worker-reported resource usage (do_update_op rusage path) is a fact: * receipt-style reply, RETAIN-registered append. RED. */ static void test_gate_no_quorum_admits_update_fact(void) { int rc; /* Arrange: quorum absent */ fact_gate_setup(3, 0); /* Act */ rc = ha_dispatch_gate(&g_fg_ha, OPCode_update); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "OPCode_update (worker rusage fact) must be admitted without write quorum"); fact_gate_teardown(); printf("test_gate_no_quorum_admits_update_fact: PASS\n"); } /* ── OPCode_setpreason admitted without write quorum (Req 3.2) ──────────── * Scheduler-reported pending reasons are facts (do_set_preason, * receipt-style reply). RED. */ static void test_gate_no_quorum_admits_setpreason_fact(void) { int rc; /* Arrange: quorum absent */ fact_gate_setup(3, 0); /* Act */ rc = ha_dispatch_gate(&g_fg_ha, OPCode_setpreason); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "OPCode_setpreason (scheduler fact) must be admitted without write quorum"); fact_gate_teardown(); printf("test_gate_no_quorum_admits_setpreason_fact: PASS\n"); } /* ── OPCode_clrpreason admitted without write quorum (Req 3.2) ──────────── * Clearing a pending reason is the same scheduler-fact shape. RED. */ static void test_gate_no_quorum_admits_clrpreason_fact(void) { int rc; /* Arrange: quorum absent */ fact_gate_setup(3, 0); /* Act */ rc = ha_dispatch_gate(&g_fg_ha, OPCode_clrpreason); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "OPCode_clrpreason (scheduler fact) must be admitted without write quorum"); fact_gate_teardown(); printf("test_gate_no_quorum_admits_clrpreason_fact: PASS\n"); } /* ── PIN: request-shaped opcodes stay rejected without quorum (Req 3.2) ─── * The allowlist is FACTS ONLY: every request-shaped opcode (a requester * awaits confirmation, served by the task-3 handler barrier) must keep * the -XCE_ERR_NOT_READY fast rejection. Green today; must remain green * after task 4.2 widens the allowlist. */ static void test_gate_no_quorum_still_rejects_request_opcodes(void) { static const struct { int opcode; const char *name; } reqs[] = { { OPCode_jobsub, "OPCode_jobsub" }, { OPCode_jobcontrol, "OPCode_jobcontrol" }, { OPCode_queuecontrol, "OPCode_queuecontrol" }, { OPCode_hostcontrol, "OPCode_hostcontrol" }, { OPCode_jobdispatch, "OPCode_jobdispatch" }, { OPCode_join, "OPCode_join" }, }; size_t i; /* Arrange: quorum absent */ fact_gate_setup(3, 0); /* Act + Assert: every request-shaped opcode keeps the fast reject */ for (i = 0; i < sizeof(reqs) / sizeof(reqs[0]); i++) { int rc = ha_dispatch_gate(&g_fg_ha, reqs[i].opcode); TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc, reqs[i].name); } fact_gate_teardown(); printf("test_gate_no_quorum_still_rejects_request_opcodes: PASS\n"); } /* ── PIN: with write quorum present everything passes as today (Req 3.2) ── * The quorum-present path is `return 0` before any opcode inspection — * facts AND requests alike. Green today; must remain green after 4.2. */ static void test_gate_with_quorum_admits_all_opcodes(void) { static const struct { int opcode; const char *name; } ops[] = { { OPCode_notify, "OPCode_notify" }, { OPCode_update, "OPCode_update" }, { OPCode_setpreason, "OPCode_setpreason" }, { OPCode_clrpreason, "OPCode_clrpreason" }, { OPCode_jobsub, "OPCode_jobsub" }, { OPCode_jobcontrol, "OPCode_jobcontrol" }, { OPCode_queuecontrol, "OPCode_queuecontrol" }, { OPCode_hostcontrol, "OPCode_hostcontrol" }, { OPCode_jobdispatch, "OPCode_jobdispatch" }, { OPCode_join, "OPCode_join" }, }; size_t i; /* Arrange: 3-member primary, one connected standby — quorum present */ fact_gate_setup(3, 1); /* Act + Assert: quorum-present primary admits everything */ for (i = 0; i < sizeof(ops) / sizeof(ops[0]); i++) { int rc = ha_dispatch_gate(&g_fg_ha, ops[i].opcode); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, ops[i].name); } fact_gate_teardown(); printf("test_gate_with_quorum_admits_all_opcodes: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 5.1 — config barrier: configAck per-push arithmetic + upgrade * failure path (Req 4.1, 4.2, 4.3, 4.4; design §Config Barrier). * * UNIT half (repl.c configAck branch via repl_wait_majority_ack): the * hardened branch must take the in-flight push tag through the EXISTING * entry_lsn parameter (added by ha-commit-engine; no second parameter) * and count standbys with acked_config_lsn >= push_tag; majority = * 1 (self) + count > n_peers / 2 over configured members (engine * n_peers, the fixture's wired commit_engine_init value); n_peers <= 1 → * immediate success. RED carriers against the current branch * (repl.c:444-510, re-verified this session): it ignores entry_lsn, * counts acked_config_lsn > 0 ("ACKed anything ever"), uses denominator * n_active, and short-circuits n_active == 0 → success. * * INTEGRATION half (config_repl.c): push_config_to_standbys must return * the barrier result (-ETIMEDOUT with acks withheld / 0 with majority) * with the transport push count via the acked_out out-param (the task-5.1 * behavior-preserving signature change keeps the legacy push-count RETURN * until task 5.2 — both rc assertions are RED against it), and the * upgradeConfig requester must observe the barrier outcome: on failure an * error reply in the send_upgrade_error shape (status < 0, nodata) with * committed_config_tag unmoved and restart_requested clear (RED against * the unconditional success ACK + restart, config_repl.c:1073-1106 * post-edit anchors); on success the existing reply + restart behavior is * unchanged (PIN — both standbys pre-ack with UINT64_MAX so the case is * green under both the legacy counting and the per-push arithmetic). * * Fixtures: the unit cases reuse barrier_setup (stub persist_t + repl * registry + wired engine — the engine carries n_peers for the configAck * denominator). The integration cases use a minimal cluster_t with stack * standby endpoints (zeroed, no iomgr — pushes and replies stay on * ep->outq, the test_config_repl.c:2210 / test_commit_barrier_reply.c * capture precedent) and parse replies with the raw capnp reader. * ════════════════════════════════════════════════════════════════════════ */ /* ── Unit: configAck per-push arithmetic (Req 4.1, 4.2) ─────────────────── */ /* A standby's acked_config_lsn from a PREVIOUS push tag must not count * toward the current push: both standbys acked tag 0x50, the in-flight * push is 0x100 → no covering ack → -ETIMEDOUT. RED: the current branch * counts acked_config_lsn > 0, sees 2 >= quorum 2 and returns 0. */ static void test_configack_previous_tag_ack_does_not_count(void) { int rc; /* Arrange: n_peers=3, both standbys acked an OLDER push tag */ barrier_setup(3, 2, 0); repl_on_config_ack(&g_bp.repl, &g_bep_a, 0x50); repl_on_config_ack(&g_bp.repl, &g_bep_b, 0x50); /* Act: wait on push tag 0x100 (entry_lsn carries the push tag) */ rc = repl_wait_majority_ack(&g_bp.repl, 0x100, 150, (uint8_t)OPCode_upgradeConfigAck); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, rc, "a previous push's config ack must not count toward the current tag"); barrier_teardown(); printf("test_configack_previous_tag_ack_does_not_count: PASS\n"); } /* n_peers=3 with ZERO covering acks (here: zero connected standbys) must * time out — a 3-member cluster can never reach majority on self alone. * RED: the current `n_active == 0 → success` short-circuit returns 0. */ static void test_configack_no_active_standbys_times_out(void) { int rc; /* Arrange: n_peers=3, no standby connected at all */ barrier_setup(3, 0, 0); /* Act */ rc = repl_wait_majority_ack(&g_bp.repl, 0x100, 150, (uint8_t)OPCode_upgradeConfigAck); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, rc, "n_peers=3 with zero covering config acks must never succeed " "(the n_active == 0 success short-circuit must die on this branch)"); barrier_teardown(); printf("test_configack_no_active_standbys_times_out: PASS\n"); } /* Self + one covering standby is a majority of n_peers=3 (1 + 1 = 2 > 1). * RED: the current branch uses denominator n_active with NO self-count — * quorum (2+2)/2 = 2 vs acked 1 → it times out. */ static void test_configack_self_plus_one_covering_is_majority_of_three(void) { int rc; /* Arrange: n_peers=3, two standbys, one acks the in-flight tag */ barrier_setup(3, 2, 0); repl_on_config_ack(&g_bp.repl, &g_bep_a, 0x100); /* Act */ rc = repl_wait_majority_ack(&g_bp.repl, 0x100, 400, (uint8_t)OPCode_upgradeConfigAck); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "self + one covering standby must satisfy majority of n_peers=3"); barrier_teardown(); printf("test_configack_self_plus_one_covering_is_majority_of_three: PASS\n"); } /* n_peers=1: single-member cluster — immediate success without consuming * the timeout. PIN: green today (via the n_active == 0 short-circuit) and * green after task 5.2 (explicit n_peers <= 1 rule). */ static void test_configack_single_member_immediate_success(void) { int rc; uint64_t t0, elapsed_ms; /* Arrange */ barrier_setup(1, 0, 0); /* Act */ t0 = mono_ms(); rc = repl_wait_majority_ack(&g_bp.repl, 0x100, 500, (uint8_t)OPCode_upgradeConfigAck); elapsed_ms = mono_ms() - t0; /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "single-member cluster must confirm a config push immediately"); TEST_ASSERT_TRUE_MESSAGE(elapsed_ms < 200, "single-member success must not consume the ack timeout"); barrier_teardown(); printf("test_configack_single_member_immediate_success: PASS\n"); } /* ── Integration fixture: minimal cluster_t for the config push path ────── */ /* Statics survive Unity's longjmp on assertion failure. */ static cluster_t g_cb5_c; static commit_engine_t g_cb5_eng; static endpoint_t g_cb5_sb_ep[2]; /* standby push endpoints */ static ep_private_t g_cb5_sb_priv[2]; static endpoint_t g_cb5_req_ep; /* upgradeConfig requester */ static ep_private_t g_cb5_req_priv; static zarray_t g_cb5_standbys; static myconf_t g_cb5_myconf; static char g_cb5_tmpdir[256]; static int g_cb5_have_tmpdir; static void cb5_setup(void) { int i; memset(&g_cb5_c, 0, sizeof(g_cb5_c)); memset(g_cb5_sb_ep, 0, sizeof(g_cb5_sb_ep)); memset(g_cb5_sb_priv, 0, sizeof(g_cb5_sb_priv)); memset(&g_cb5_req_ep, 0, sizeof(g_cb5_req_ep)); memset(&g_cb5_req_priv, 0, sizeof(g_cb5_req_priv)); memset(&g_cb5_myconf, 0, sizeof(g_cb5_myconf)); g_cb5_have_tmpdir = 0; g_cb5_c.myhost = &g_cb5_c.myhost_info; g_cb5_c.myhost_info.name = (char *)"ctrl-cb5"; g_cb5_c.myhost_info.key = (char *)""; /* Zeroed stack endpoints, no iomgr: sends stay queued on ep->outq * (endpoint_update_epoll no-ops; zeroed pthread mutexes are valid). */ g_cb5_req_ep.fd = -1; g_cb5_req_ep.data = &g_cb5_req_priv; TEST_ASSERT_EQUAL_INT(0, zarray_init(&g_cb5_standbys, 4, NULL)); for (i = 0; i < 2; i++) { g_cb5_sb_ep[i].fd = -1; g_cb5_sb_ep[i].data = &g_cb5_sb_priv[i]; zarray_store(&g_cb5_standbys, &g_cb5_sb_ep[i]); } g_cb5_c.ha.standby_endpoints = &g_cb5_standbys; /* Replicator registry mirrors the push endpoints; wired engine carries * n_peers=3 for the configAck majority denominator. */ TEST_ASSERT_EQUAL_INT(0, repl_init(&g_cb5_c.persist.repl)); TEST_ASSERT_TRUE(repl_add_standby(&g_cb5_c.persist.repl, &g_cb5_sb_ep[0], "cb5_a", 7500) >= 0); TEST_ASSERT_TRUE(repl_add_standby(&g_cb5_c.persist.repl, &g_cb5_sb_ep[1], "cb5_b", 7501) >= 0); TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_cb5_eng, &g_cb5_c.persist, 3, 8)); repl_set_commit_engine(&g_cb5_c.persist.repl, &g_cb5_eng); /* The current do_upgrade_config_with_bytes restart path does * cbuf_put(&c->wbuf, …) — must be a real cbuf or it blocks forever. */ TEST_ASSERT_EQUAL_INT(0, cbuf_init(&g_cb5_c.wbuf, 4, sizeof(int))); } /* Upgrade-path extension: tmpdir-backed myconf so conf_save_config works * (it writes to //conf — the parent must exist). */ static void cb5_setup_upgrade(void) { char inst[300]; cb5_setup(); snprintf(g_cb5_tmpdir, sizeof(g_cb5_tmpdir), "/tmp/xce_cb51_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(g_cb5_tmpdir)); g_cb5_have_tmpdir = 1; snprintf(inst, sizeof(inst), "%s/ctrl", g_cb5_tmpdir); TEST_ASSERT_EQUAL_INT(0, mkdir(inst, 0700)); g_cb5_myconf.workdir = g_cb5_tmpdir; g_cb5_myconf.name = (char *)"ctrl"; g_cb5_myconf.conf_keep_n = 2; g_cb5_c.myconf = &g_cb5_myconf; } /* Drain and free every packet queued on an endpoint's outq. */ static void cb5_drain_ep(endpoint_t *ep) { data_item_t *it; while ((it = endpoint_deq(ep)) != NULL) { io_packet_t *pkt = ITPTR_TO_PACKET(it); free_packet(pkt); free(pkt); } } static void cb5_teardown(void) { cb5_drain_ep(&g_cb5_sb_ep[0]); cb5_drain_ep(&g_cb5_sb_ep[1]); cb5_drain_ep(&g_cb5_req_ep); commit_engine_fini(&g_cb5_eng); repl_fini(&g_cb5_c.persist.repl); zarray_fini(&g_cb5_standbys); cbuf_fini(&g_cb5_c.wbuf); free(g_cb5_myconf.conffile); g_cb5_myconf.conffile = NULL; if (g_cb5_have_tmpdir) { char cmd[300]; snprintf(cmd, sizeof(cmd), "rm -rf %s", g_cb5_tmpdir); system(cmd); g_cb5_have_tmpdir = 0; } } /* ── Reply observation (raw capnp parse of ep->outq packets) ────────────── */ typedef struct { int n_replies; /* replymsg packets seen */ int32_t last_status; /* status of the last replymsg observed */ int last_data_which; /* RequestReply_data_which discriminant */ } cb5_reply_obs_t; static void cb5_scan_replies(endpoint_t *ep, cb5_reply_obs_t *obs) { data_item_t *it; while ((it = endpoint_deq(ep)) != NULL) { io_packet_t *pkt = ITPTR_TO_PACKET(it); struct capn c; struct Request rq; Request_ptr rp; capn_init_mem(&c, (const uint8_t *)pkt->data, pkt->sz, 0); rp.p = capn_getp(capn_root(&c), 0, 1); read_Request(&rq, rp); if (rq.body_which == Request_body_replymsg) { struct RequestReply rr; read_RequestReply(&rr, rq.body.replymsg); obs->n_replies += 1; obs->last_status = rr.status; obs->last_data_which = (int)rr.data_which; } capn_free(&c); free_packet(pkt); free(pkt); } } /* Build a minimal valid serialised ClusterConfig blob (caller frees). */ static uint8_t *cb5_make_config_blob(size_t *len_out) { cluster_config_t cfg; struct capn cap; struct capn_segment *cs; ClusterConfig_ptr p; uint8_t *buf; size_t sz; memset(&cfg, 0, sizeof(cfg)); cfg.clustername = (char *)"cb5-upgrade"; cfg.port = 9990; cfg.config_generation = 1; capn_init_malloc(&cap); cs = capn_root(&cap).seg; encode_ClusterConfig_ptr(cs, &p, &cfg); capn_setp(capn_root(&cap), 0, p.p); sz = (size_t)capn_size(&cap); buf = (uint8_t *)malloc(sz); TEST_ASSERT_NOT_NULL(buf); capn_write_mem(&cap, buf, sz, 0); capn_free(&cap); *len_out = sz; return buf; } /* upgradeConfig request scaffolding (statics survive Unity longjmp). * handle_upgrade_config reads the raw_* fields xce_decode would have * populated; the test owns the buffers, so xce_free_request is NOT used. */ static upgrade_config_req_t g_cb5_ucr; static xce_request_t g_cb5_req; static void cb5_arrange_upgrade_req(uint8_t *blob, size_t blen, uint8_t sha[32]) { memset(&g_cb5_ucr, 0, sizeof(g_cb5_ucr)); memset(&g_cb5_req, 0, sizeof(g_cb5_req)); g_cb5_ucr.raw_sha256 = sha; g_cb5_ucr.raw_bytes = blob; g_cb5_ucr.raw_bytes_len = blen; g_cb5_req.opcode = OPCode_upgradeConfig; g_cb5_req.seq = 1; g_cb5_req.body_which = Request_body_upgradeconfigrequestmsg; g_cb5_req.body.upgradeconfigrequestmsg = &g_cb5_ucr; } /* ── push_config_to_standbys: barrier result + push count (Req 4.2, 4.3) ── */ /* ACKs withheld: the push barrier must report -ETIMEDOUT (return value), * the transport push count via acked_out, and the tag must never commit. * RED: the current return value IS the push count (2), and the internal * majority wait only logs its timeout. */ static void test_push_config_withheld_acks_returns_etimedout(void) { static const uint8_t blob[] = "cb5-push-blob"; uint8_t sha[32]; int pushed = -1; int rc; /* Arrange */ cb5_setup(); SHA256(blob, sizeof(blob) - 1, sha); /* Act: no standby ever acks tag 0xAB12 */ rc = push_config_to_standbys(&g_cb5_c, blob, sizeof(blob) - 1, sha, 0xAB12, &pushed); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, rc, "withheld config ACKs must surface as -ETIMEDOUT from the push (Req 4.2)"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, pushed, "acked_out must report the per-standby transport push count"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, atomic_load_explicit(&g_cb5_c.ha.committed_config_tag, memory_order_acquire), "an unacked config must never become committed (Req 4.3)"); cb5_teardown(); printf("test_push_config_withheld_acks_returns_etimedout: PASS\n"); } /* Majority acks (self + 1 covering standby of n_peers=3): the push must * return 0 and commit the tag. RED twice over: the return value is the * push count (2 != 0), and the legacy internal wait (quorum 2 over * n_active, no self-count) times out so the tag never advances. */ static void test_push_config_majority_acks_returns_zero_and_commits(void) { static const uint8_t blob[] = "cb5-push-blob"; uint8_t sha[32]; int pushed = -1; int rc; /* Arrange: one standby has already acked the in-flight tag */ cb5_setup(); SHA256(blob, sizeof(blob) - 1, sha); repl_on_config_ack(&g_cb5_c.persist.repl, &g_cb5_sb_ep[0], 0xAB12); /* Act */ rc = push_config_to_standbys(&g_cb5_c, blob, sizeof(blob) - 1, sha, 0xAB12, &pushed); /* Assert */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "majority config ACK must surface as success from the push (Req 4.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, pushed, "acked_out must report the per-standby transport push count"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0xAB12, atomic_load_explicit(&g_cb5_c.ha.committed_config_tag, memory_order_acquire), "majority-acked config must advance committed_config_tag (Req 4.1)"); cb5_teardown(); printf("test_push_config_majority_acks_returns_zero_and_commits: PASS\n"); } /* ── upgradeConfig requester: failure and success paths (Req 4.1-4.4) ───── */ /* Barrier failure (ACKs withheld) must reach the requester as an error * reply in the send_upgrade_error shape (status < 0, nodata), leave * committed_config_tag unmoved, and keep restart_requested clear. * RED: today the handler unconditionally ACKs status=0 with the * upgradeConfigReply payload and requests the restart. */ static void test_upgrade_config_barrier_failure_error_reply_no_restart(void) { uint8_t *blob; size_t blen = 0; uint8_t sha[32]; cb5_reply_obs_t obs = { 0 }; /* Arrange */ cb5_setup_upgrade(); blob = cb5_make_config_blob(&blen); SHA256(blob, blen, sha); cb5_arrange_upgrade_req(blob, blen, sha); /* Act: no standby ever acks the new tag */ (void)handle_upgrade_config(&g_cb5_c, &g_cb5_req_ep, &g_cb5_req); /* Assert: error reply, no commit, no restart */ cb5_scan_replies(&g_cb5_req_ep, &obs); TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n_replies, "the requester must receive exactly one reply"); TEST_ASSERT_TRUE_MESSAGE(obs.last_status < 0, "a failed config commit must reach the requester as an error reply" " (Req 4.2)"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RequestReply_data_nodata, obs.last_data_which, "the failure reply must use the send_upgrade_error shape (nodata)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, atomic_load_explicit(&g_cb5_c.ha.committed_config_tag, memory_order_acquire), "a failed upgrade must leave committed_config_tag unmoved (Req 4.3)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_cb5_c.restart_requested), "a failed upgrade must not request the activation restart (Req 4.4)"); free(blob); cb5_teardown(); printf("test_upgrade_config_barrier_failure_error_reply_no_restart: PASS\n"); } /* Success path PIN: with a full majority of covering ACKs the existing * behavior is unchanged — status-0 reply with the upgradeConfigReply * payload, committed_config_tag at the saved tag, restart requested. * Both standbys pre-ack UINT64_MAX (covers any tag — the new tag is only * known after conf_save_config hashes the re-encoded blob), so the case * is green under both the legacy counting and the per-push arithmetic. */ static void test_upgrade_config_majority_ack_success_unchanged(void) { uint8_t *blob; size_t blen = 0; uint8_t sha[32]; int rc; cb5_reply_obs_t obs = { 0 }; /* Arrange: full covering pre-ack from both standbys */ cb5_setup_upgrade(); blob = cb5_make_config_blob(&blen); SHA256(blob, blen, sha); cb5_arrange_upgrade_req(blob, blen, sha); repl_on_config_ack(&g_cb5_c.persist.repl, &g_cb5_sb_ep[0], UINT64_MAX); repl_on_config_ack(&g_cb5_c.persist.repl, &g_cb5_sb_ep[1], UINT64_MAX); /* Act */ rc = handle_upgrade_config(&g_cb5_c, &g_cb5_req_ep, &g_cb5_req); /* Assert: existing success reply + restart behavior unchanged */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "a majority-acked upgrade must succeed"); cb5_scan_replies(&g_cb5_req_ep, &obs); TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n_replies, "the requester must receive exactly one reply"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, obs.last_status, "the success ACK must carry status 0 (Req 4.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RequestReply_data_upgradeconfigreplymsg, obs.last_data_which, "the success ACK must carry the upgradeConfigReply payload"); TEST_ASSERT_TRUE_MESSAGE(g_cb5_myconf.config_version_tag != 0, "conf_save_config must have derived the new version tag"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(g_cb5_myconf.config_version_tag, atomic_load_explicit(&g_cb5_c.ha.committed_config_tag, memory_order_acquire), "a majority-acked config must be committed at the saved tag (Req 4.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cb5_c.restart_requested), "a successful upgrade must request the activation restart"); free(blob); cb5_teardown(); printf("test_upgrade_config_majority_ack_success_unchanged: PASS\n"); } /* ── Test group entry point ──────────────────────────────────────────────── */ void test_commit_barrier_rollout(void) { UNITY_BEGIN(); /* Task 1.1 — classification totality + RETAIN routing (Req 2.1, 2.2, 3.1) */ RUN_TEST(test_classify_job_submit_is_wait); RUN_TEST(test_classify_queue_ctrl_is_wait); RUN_TEST(test_classify_worker_ctrl_detail_split); RUN_TEST(test_classify_all_job_state_types_are_retain); RUN_TEST(test_classify_checkpoint_is_local); RUN_TEST(test_classify_is_total_over_wal_entry_type); RUN_TEST(test_classify_unknown_type_fails_fast_in_testing); RUN_TEST(test_fact_hook_append_is_retained_and_nonblocking); /* Task 2.1 — persist_commit_barrier semantics (Req 1.1, 1.2, 1.3) */ RUN_TEST(test_barrier_returns_zero_when_nothing_appended); RUN_TEST(test_barrier_returns_zero_when_committed_covers_appended); RUN_TEST(test_barrier_times_out_when_acks_withheld); RUN_TEST(test_barrier_returns_zero_once_covering_ack_arrives); RUN_TEST(test_barrier_zero_timeout_falls_back_to_config_sync_timeout); RUN_TEST(test_barrier_single_member_returns_immediately); RUN_TEST(test_barrier_not_ready_passes_through_unmodified); /* Task 4.1 — dispatch-gate fact allowlist (Req 3.2) */ RUN_TEST(test_gate_no_quorum_admits_notify_fact); RUN_TEST(test_gate_no_quorum_admits_update_fact); RUN_TEST(test_gate_no_quorum_admits_setpreason_fact); RUN_TEST(test_gate_no_quorum_admits_clrpreason_fact); RUN_TEST(test_gate_no_quorum_still_rejects_request_opcodes); RUN_TEST(test_gate_with_quorum_admits_all_opcodes); /* Task 5.1 — config barrier: configAck arithmetic + upgrade failure * path (Req 4.1, 4.2, 4.3, 4.4) */ RUN_TEST(test_configack_previous_tag_ack_does_not_count); RUN_TEST(test_configack_no_active_standbys_times_out); RUN_TEST(test_configack_self_plus_one_covering_is_majority_of_three); RUN_TEST(test_configack_single_member_immediate_success); RUN_TEST(test_push_config_withheld_acks_returns_etimedout); RUN_TEST(test_push_config_majority_acks_returns_zero_and_commits); RUN_TEST(test_upgrade_config_barrier_failure_error_reply_no_restart); RUN_TEST(test_upgrade_config_majority_ack_success_unchanged); UNITY_END(); }