#define _GNU_SOURCE /* * test_convergence_verification.c — RED-phase tests for ACK-driven catch-up * continuation, single-flight, and parity re-inclusion * (ha-convergence-recovery task 4.1). * * Requirements (this spec's IDs — see requirements.md): * 5.1 — when the primary receives a WAL ACK from a standby that remains BEHIND * the replication target, it continues streaming the missing entries to * that standby WITHOUT waiting for the next heartbeat exchange (the * continuation streams [acked_lsn+1, target] and dispatches no * heartbeat/stateReport replication action of its own). * 5.2 — while a catch-up continuation for a standby is already in flight, the * primary does NOT start a duplicate continuation for that standby * (per-standby single-flight guard); once the in-flight continuation * completes, the next lagging ACK schedules again. * 5.3 — the heartbeat-driven replication evaluation remains in place as the * fallback initiator of catch-up AND the initiator of snapshot-transfer * decisions: ha_evaluate_repl_action still returns HA_REPL_STREAM_WAL / * HA_REPL_SEND_SNAPSHOT for the same inputs (decision boundary pin). * 7.1 — when a lagging standby catches up to the replication boundary (a WAL * ACK at/above target), the primary re-includes it in live replication * via repl_complete_catchup, which clears the standby's in_catch_up. * * RED status against the current tree: the production walAck branch * (repl_ack_decode_and_dispatch, ha.c — the OPCode_reply / * RequestReply_data_walackreplymsg case) only calls repl_on_ack; it derives no * target, schedules no continuation, and never calls repl_complete_catchup * (which has NO production caller, repl.c). The task-4.1 test seam * ha_walack_continuation_for_test is a RED scaffold (ha_convergence.c) that * records the ACK only. The continuation / single-flight / parity cases below * therefore FAIL until task 4.2 lands the real D6 decision; the 5.3 decision-pin * case exercises existing behavior and PASSES now (pinning the boundary so 4.2 * cannot disturb the heartbeat path). * * Harness: a partial-but-real cluster_t (real persist over a private tmpdir, * a wired commit engine, stub standby registry, un-started delaymgr) mirroring * test_demotion_convergence.c's dc_setup, plus the catch-up streaming count seam * ha_repl_stream_wal_hook and a dispatch-observer seam to prove the continuation * takes no heartbeat path. * * AAA structure throughout. */ #include #include #include #include #include #include #include #include #include #include #include #include "commit.h" /* commit_engine_t, commit_engine_init */ #include "controller.capnp.h" /* OPCode_* (task 6.1 (c) dispatch-gate pin) */ #include "delay.h" /* delaymgr_t, delay_task_t */ #include "ha.h" /* ha_walack_continuation_for_test, ha_evaluate_repl_action, * ha_repl_stream_wal_hook, HA_REPL_* */ #include "ordq.h" /* ordq_item_t (drive the scheduled task) */ #include "persist.h" /* persist_t */ #include "repl.h" /* repl_add_standby / repl_remove_standby / * repl_active_standby_count / standby slots */ #include "xce/cluster_def.h" /* full cluster_t layout */ #include "xce/errnox.h" /* XCE_ERR_NOT_READY (task 6.1 (c) gate pin) */ #ifdef XCE_TESTING /* ── Broadcast drop-hook (precedent: test_demotion_convergence.c) — the stub int * endpoints registered via repl_add_standby are not real endpoint_t, so any * repl_broadcast triggered by priming appends must not dereference ep->peer. */ extern int (*repl_broadcast_send_hook)(void *ep, wal_catchup_entry_t **entries, int n_entries); static int cv_drop_send_hook(void *ep, wal_catchup_entry_t **entries, int n_entries) { (void)ep; (void)entries; (void)n_entries; return 0; } /* ── Catch-up streaming count seam (Req 5.1) ───────────────────────────────── * The ACK-driven continuation must stream the missing range through the existing * catch-up streaming path. ha_repl_stream_wal_hook is the established XCE_TESTING * seam for that path (config_repl.c ha_stream_wal_to_standby / ha.c * HA_REPL_STREAM_WAL). We count invocations and capture the [start, target] * range so the continuation's bounds can be asserted. */ static atomic_int g_stream_calls; static atomic_uint_least64_t g_stream_last_standby_lsn; static atomic_uint_least64_t g_stream_last_target_lsn; static int cv_stream_hook(void *ep, const char *wal_path, uint64_t standby_lsn, uint64_t committed_lsn) { (void)ep; (void)wal_path; atomic_fetch_add(&g_stream_calls, 1); atomic_store(&g_stream_last_standby_lsn, standby_lsn); atomic_store(&g_stream_last_target_lsn, committed_lsn); return 0; } /* ── Cluster fixture ─────────────────────────────────────────────────────── */ static cluster_t g_cv; /* large — must not live on the test stack */ static int g_cv_eps[MAX_STANDBYS]; #define CV_PERSIST_NAME "ctrl-test" /* ── Fan-out counting send-hook (task 6.1 (a)/(d), umbrella 6.1/6.4 pins) ───── * repl_broadcast routes each enqueue through repl_broadcast_send_hook under * XCE_TESTING; this counting variant records, per standby slot, how many entries * repl_broadcast tried to send to that slot's ep. An active, non-catch-up * standby must receive exactly one send; an empty slot or an in_catch_up slot * must receive ZERO. Indexed by matching ep against the g_cv_eps[] addresses. * (Declared after g_cv_eps so the slot match resolves.) */ static atomic_int g_bcast_total_calls; static atomic_int g_bcast_slot_calls[MAX_STANDBYS]; static int cv_count_send_hook(void *ep, wal_catchup_entry_t **entries, int n_entries) { (void)entries; (void)n_entries; atomic_fetch_add(&g_bcast_total_calls, 1); for (int i = 0; i < MAX_STANDBYS; i++) { if (ep == (void *)&g_cv_eps[i]) { atomic_fetch_add(&g_bcast_slot_calls[i], 1); break; } } return 0; } static void cv_bcast_reset_counters(void) { atomic_store(&g_bcast_total_calls, 0); for (int i = 0; i < MAX_STANDBYS; i++) atomic_store(&g_bcast_slot_calls[i], 0); } static ha_config_t cv_ha_cfg(void) { ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; return cfg; } /* Build g_cv: real persist over a fresh tmpdir, n_standbys stub standbys, a * wired commit engine, an initialised-but-UNSTARTED delaymgr (so any task in * tasks.head was armed by the code under test, not by a background runner). * Role left at INIT (the walAck continuation is a primary-side concern but the * decision under test needs only the persist watermark + repl registry, not the * full promotion sequence). Leaves committed_lsn at 0; callers set it. */ static void cv_setup(const char *suffix, int n_peers, int n_standbys, uint64_t appended_lsn, int n_vcpus) { char name[32]; char tmpdir[256]; memset(&g_cv, 0, sizeof(g_cv)); atomic_store(&g_stream_calls, 0); atomic_store(&g_stream_last_standby_lsn, 0); atomic_store(&g_stream_last_target_lsn, 0); ha_config_t cfg = cv_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&g_cv.ha, &cfg, 0)); g_cv.ha.phase2_done = 1; g_cv.ha.n_peers = n_peers; persist_config_t pcfg = persist_config_defaults(); pcfg.wal_sync_ms = 5000; /* slow background fsync — no races */ pcfg.sync_timeout_ms = 50; /* no real standby ACKs here */ snprintf(tmpdir, sizeof(tmpdir), "/tmp/xce_cv_%s_XXXXXX", suffix); TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir)); persist_set_workdir(&g_cv.persist, tmpdir, CV_PERSIST_NAME); TEST_ASSERT_EQUAL_INT(0, persist_init(&g_cv.persist, &pcfg, &g_cv)); TEST_ASSERT_EQUAL_INT(0, persist_start(&g_cv.persist)); /* Prime the appended watermark: current_lsn = appended_lsn + 1, so * last_written = appended_lsn (the replication target's upper bound). */ pthread_mutex_lock(&g_cv.persist.lock); g_cv.persist.current_lsn = appended_lsn + 1; pthread_mutex_unlock(&g_cv.persist.lock); atomic_store(&g_cv.persist.committed_lsn, 0); for (int i = 0; i < n_standbys; i++) { snprintf(name, sizeof(name), "cvsb%d", i); TEST_ASSERT_TRUE_MESSAGE( repl_add_standby(&g_cv.persist.repl, &g_cv_eps[i], name, 7600 + (uint32_t)i) >= 0, "repl_add_standby must return a slot index >= 0"); } TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_cv.commit, &g_cv.persist, n_peers, n_vcpus)); repl_set_commit_engine(&g_cv.persist.repl, &g_cv.commit); repl_broadcast_send_hook = cv_drop_send_hook; ha_repl_stream_wal_hook = cv_stream_hook; TEST_ASSERT_EQUAL_INT(0, delaymgr_init(&g_cv.delaymgr, 100, NULL)); } static void cv_teardown(void) { repl_broadcast_send_hook = NULL; ha_repl_stream_wal_hook = NULL; commit_engine_fini(&g_cv.commit); delaymgr_fini(&g_cv.delaymgr); persist_stop(&g_cv.persist); persist_fini(&g_cv.persist); repl_fini(&g_cv.persist.repl); ha_fini(&g_cv.ha); /* tmpdir left on disk — /tmp is ephemeral */ } /* Count pending delaymgr tasks by walking the ordq head chain (read-only; * single-threaded test, delaymgr never started). */ static int cv_task_count(const delaymgr_t *mgr) { int count = 0; const ordq_item_t *it = mgr->tasks.head; while (it != NULL) { count++; it = it->next; } return count; } /* Run the single pending delaymgr one-shot task synchronously by invoking its * stored entry(args), then return the entry's rc. Returns CV_NO_TASK if there * is no pending task (RED: nothing was scheduled). DEQUEUES + frees, mirroring * the real delaymgr worker (delay.c) and test_demotion_convergence.c's * dc_run_pending_task — required so a second call after a body that frees its own * ctx does not re-run a freed ctx (heap-UAF). */ #define CV_NO_TASK 0x7fffffff static int cv_run_pending_task(delaymgr_t *mgr) { ordq_item_t *it = ordq_get(&mgr->tasks, NULL); if (it == NULL || it->data == NULL) { if (it != NULL) free(it); return CV_NO_TASK; } delay_task_t *t = (delay_task_t *)it->data; if (t->entry == NULL) { free(t); free(it); return CV_NO_TASK; } int rc = t->entry(t->args); /* task body frees its own args ctx */ free(t); free(it); return rc; } /* Read the per-standby in_catch_up flag (repl registry slot identified by ep). */ static int cv_standby_in_catch_up(repl_ep_t ep) { replicator_t *r = &g_cv.persist.repl; int result = -1; mylock_lock(&r->lock); for (int i = 0; i < MAX_STANDBYS; i++) { if (r->standbys[i].ep == ep) { result = atomic_load_explicit(&r->standbys[i].in_catch_up, memory_order_acquire); break; } } mylock_unlock(&r->lock); return result; } /* ── Tests ──────────────────────────────────────────────────────────────── */ /* 5.1: a walAck with acked_lsn < target = max(committed_lsn, last_written) * schedules EXACTLY ONE continuation that streams [acked_lsn+1, target] without * any intervening heartbeat/stateReport dispatch. * RED: the walAck continuation scaffold schedules nothing → no task, the stream * hook is never called. */ static void test_walack_behind_target_schedules_one_continuation(void) { /* appended_lsn=9 → last_written=9; committed_lsn=4 → target=max(4,9)=9. */ cv_setup("behind", /*n_peers=*/3, /*n_standbys=*/1, /*appended_lsn=*/9, /*n_vcpus=*/8); atomic_store(&g_cv.persist.committed_lsn, 4); const int peer_idx = 0; const uint64_t acked_lsn = 3; /* behind target 9 */ const uint64_t target = 9; int tasks_before = cv_task_count(&g_cv.delaymgr); /* Act: standby acknowledges lsn=3 while target=9. */ int rc = ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx, acked_lsn); (void)rc; /* A continuation must have been scheduled OFF the ACK path (exactly one). */ int tasks_after = cv_task_count(&g_cv.delaymgr); TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after, "a walAck behind target must schedule exactly one catch-up continuation " "(Req 5.1)"); /* No streaming may have happened on the ACK path itself (it defers). */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_stream_calls), "the walAck branch must NOT stream inline — streaming runs in the " "deferred continuation (Req 5.1, iomgr-safe)"); /* Run the scheduled continuation; it must stream [acked_lsn+1, target]. */ int trc = cv_run_pending_task(&g_cv.delaymgr); TEST_ASSERT_NOT_EQUAL_MESSAGE(CV_NO_TASK, trc, "no continuation task was scheduled on the walAck-behind-target edge " "(Req 5.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_stream_calls), "the continuation must stream exactly once (Req 5.1)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(acked_lsn, atomic_load(&g_stream_last_standby_lsn), "the continuation must stream starting just above the ACKed LSN " "(ha_stream_wal_to_standby streams (standby_lsn, target] = " "[acked_lsn+1, target]) (Req 5.1)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(target, atomic_load(&g_stream_last_target_lsn), "the continuation must stream up to target = max(committed_lsn, " "last_written) (Req 5.1)"); cv_teardown(); printf("test_walack_behind_target_schedules_one_continuation: checked\n"); } /* 5.2 single-flight: a SECOND walAck while a continuation is already in flight * for the same standby schedules NOTHING. * RED: the scaffold schedules nothing for the first ACK either, so the * "exactly one continuation across two ACKs" assertion fails (0 != 1). */ static void test_walack_single_flight_no_duplicate_continuation(void) { cv_setup("single", /*n_peers=*/3, /*n_standbys=*/1, /*appended_lsn=*/9, /*n_vcpus=*/8); atomic_store(&g_cv.persist.committed_lsn, 4); const int peer_idx = 0; int tasks_before = cv_task_count(&g_cv.delaymgr); /* Act: first walAck (behind target) → schedules one continuation and marks * the standby's catchup_inflight. */ (void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx, /*acked_lsn=*/3); int tasks_after_first = cv_task_count(&g_cv.delaymgr); TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before + 1, tasks_after_first, "the first walAck behind target must schedule one continuation (Req 5.2 " "setup)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cv.ha.catchup_inflight[peer_idx]), "scheduling a continuation must set the per-standby single-flight flag " "(Req 5.2)"); /* Act: a SECOND walAck while the continuation is still in flight (the task * has NOT been drained). */ (void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx, /*acked_lsn=*/5); /* Assert (Req 5.2): no duplicate continuation — task count unchanged. */ TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_after_first, cv_task_count(&g_cv.delaymgr), "a second walAck while a continuation is in flight must schedule NOTHING " "(single-flight, Req 5.2)"); /* Drain the single in-flight continuation (clears the flag). */ (void)cv_run_pending_task(&g_cv.delaymgr); cv_teardown(); printf("test_walack_single_flight_no_duplicate_continuation: checked\n"); } /* 5.2 release: after the in-flight continuation completes (flag cleared), the * NEXT lagging walAck schedules a fresh continuation again. * RED: the scaffold never schedules/clears, so neither continuation appears. */ static void test_walack_reschedules_after_continuation_completes(void) { cv_setup("respawn", /*n_peers=*/3, /*n_standbys=*/1, /*appended_lsn=*/9, /*n_vcpus=*/8); atomic_store(&g_cv.persist.committed_lsn, 4); const int peer_idx = 0; /* Act 1: first walAck → continuation #1. */ (void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx, /*acked_lsn=*/3); int trc1 = cv_run_pending_task(&g_cv.delaymgr); TEST_ASSERT_NOT_EQUAL_MESSAGE(CV_NO_TASK, trc1, "first walAck must schedule a continuation (Req 5.2 release setup)"); /* After the continuation completes the single-flight flag must be clear. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_cv.ha.catchup_inflight[peer_idx]), "the continuation must clear the single-flight flag on completion " "(Req 5.2 release)"); int streams_after_first = atomic_load(&g_stream_calls); TEST_ASSERT_EQUAL_INT_MESSAGE(1, streams_after_first, "the first continuation must have streamed once"); /* Act 2: a later lagging walAck (still behind target 9) → continuation #2. */ (void)ha_walack_continuation_for_test(&g_cv, &g_cv_eps[peer_idx], peer_idx, /*acked_lsn=*/6); int trc2 = cv_run_pending_task(&g_cv.delaymgr); TEST_ASSERT_NOT_EQUAL_MESSAGE(CV_NO_TASK, trc2, "a lagging walAck after the prior continuation completed must schedule " "AGAIN (Req 5.2 release)"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_stream_calls), "the second continuation must stream once more after the first released " "(Req 5.2 release)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(6, atomic_load(&g_stream_last_standby_lsn), "the second continuation streams from the newly ACKed LSN (Req 5.2 " "release)"); cv_teardown(); printf("test_walack_reschedules_after_continuation_completes: checked\n"); } /* 7.1 parity: a walAck with acked_lsn >= target calls repl_complete_catchup, * which clears the standby's in_catch_up (re-inclusion in live replication). * RED: the scaffold never calls repl_complete_catchup, so in_catch_up — set here * via repl_start_catchup — stays 1. */ static void test_walack_at_parity_completes_catchup_and_clears_in_catch_up(void) { /* appended_lsn=9 → last_written=9; committed_lsn=9 → target=9. */ cv_setup("parity", /*n_peers=*/3, /*n_standbys=*/1, /*appended_lsn=*/9, /*n_vcpus=*/8); atomic_store(&g_cv.persist.committed_lsn, 9); const int peer_idx = 0; repl_ep_t ep = &g_cv_eps[peer_idx]; const uint64_t acked_lsn = 9; /* at parity with target 9 */ /* Put the standby into catch-up so re-inclusion is observable. */ repl_start_catchup(&g_cv.persist.repl, ep, /*join_applied_lsn=*/0); TEST_ASSERT_EQUAL_INT_MESSAGE(1, cv_standby_in_catch_up(ep), "fixture precondition: the standby must be in_catch_up before parity ACK"); int tasks_before = cv_task_count(&g_cv.delaymgr); /* Act: the standby acknowledges the target → parity. */ (void)ha_walack_continuation_for_test(&g_cv, ep, peer_idx, acked_lsn); /* Assert (Req 7.1): no continuation scheduled at parity, and the standby is * re-included — in_catch_up cleared by repl_complete_catchup. */ TEST_ASSERT_EQUAL_INT_MESSAGE(tasks_before, cv_task_count(&g_cv.delaymgr), "a parity walAck must NOT schedule a streaming continuation (Req 7.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_stream_calls), "a parity walAck must NOT stream (nothing left to send) (Req 7.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_standby_in_catch_up(ep), "a parity walAck must call repl_complete_catchup, clearing the standby's " "in_catch_up so it re-enters live replication (Req 7.1)"); cv_teardown(); printf("test_walack_at_parity_completes_catchup_and_clears_in_catch_up: checked\n"); } /* 5.3 decision pin: the heartbeat decision function ha_evaluate_repl_action is * the fallback initiator of catch-up and the SOLE owner of the snapshot * decision; the ACK-driven continuation must not change its verdicts. This pins * the existing boundary so task 4.2 cannot disturb the heartbeat path. * * This case exercises only existing pure-function behavior, so it PASSES against * the current tree (intended — it locks the boundary; the RED signal for task * 4.1 comes from the continuation/single-flight/parity cases above). */ static void test_heartbeat_decision_path_unchanged(void) { /* HA_REPL_STREAM_WAL: tag current, lsn within the WAL window * (oldest_wal_lsn <= standby_lsn < committed_lsn). */ ha_repl_action_t a_stream = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/3, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/1); TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)a_stream, "heartbeat decision: standby_lsn within the WAL window must remain " "HA_REPL_STREAM_WAL — the heartbeat path stays the fallback catch-up " "initiator (Req 5.3)"); /* HA_REPL_SEND_SNAPSHOT: standby_lsn genuinely GC'd behind the oldest * retained WAL (standby_lsn + 1 < oldest_wal_lsn → 2 + 1 = 3 < 5). */ ha_repl_action_t a_snap = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/2, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5); TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)a_snap, "heartbeat decision: standby GC'd behind oldest_wal_lsn must remain " "HA_REPL_SEND_SNAPSHOT — the heartbeat path remains the SOLE snapshot " "decision owner (Req 5.3); the ACK-driven continuation never decides " "snapshots"); /* Strict boundary (fix-snapshot-boundary): standby_lsn == oldest_wal_lsn - 1 * (next-needed entry retained) must STREAM, not snapshot — pins the * standby_lsn + 1 < oldest_wal_lsn comparison on the heartbeat path. */ ha_repl_action_t a_edge = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/4, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5); TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)a_edge, "heartbeat decision: standby_lsn (4) == oldest_wal_lsn (5) - 1 — the " "next-needed entry (5) is retained, so the heartbeat path must STREAM, " "not snapshot (strict standby_lsn + 1 < oldest_wal_lsn boundary)"); printf("test_heartbeat_decision_path_unchanged: checked\n"); } /* ── Quorum transition logging (task 5.1, Req 8.1-8.3) ─────────────────────── * * Observable pattern: there is NO log-capture harness in this testsuite, so * these tests use the emission-count seam ha_quorum_transition_log_hook (the * task permits "log-capture OR counter-hook pattern"). The real * ha_log_quorum_transition (task 5.2) invokes the hook exactly once on each * loss/restore EDGE it WARNs about, passing the new state and the * connected/required counts that go into the WARN line. We count invocations to * assert "logged once per transition" and inspect the args to assert the loss * line carries the right counts. * * RED status against the current tree: ha_log_quorum_transition is a no-op RED * scaffold (ha_convergence.c) — it performs no edge detection, flips no flag, * and never invokes the hook. Every assertion below that expects an emission * (or an edge-flag flip) therefore FAILS until task 5.2 lands the real edge * detector and wires the call sites. * * Quorum arithmetic for the fixture (ha_has_write_quorum, ha.c): * n_peers=3, ha->repl set → connected = repl_active_standby_count(repl); * quorum held iff connected >= n_peers/2 = 1. So 1 active standby = quorum, * 0 active standbys = quorum lost. The required count in the WARN line is the * threshold n_peers/2 = 1. */ /* Emission-count hook: records every transition WARN ha_log_quorum_transition * emits, capturing the latest (lost, connected, required) triple. */ static atomic_int g_qt_calls; static atomic_int g_qt_loss_calls; static atomic_int g_qt_restore_calls; static atomic_int g_qt_last_lost; static atomic_int g_qt_last_connected; static atomic_int g_qt_last_required; static void cv_quorum_log_hook(int lost, int connected, int required) { atomic_fetch_add(&g_qt_calls, 1); if (lost) atomic_fetch_add(&g_qt_loss_calls, 1); else atomic_fetch_add(&g_qt_restore_calls, 1); atomic_store(&g_qt_last_lost, lost); atomic_store(&g_qt_last_connected, connected); atomic_store(&g_qt_last_required, required); } static void cv_quorum_reset_counters(void) { atomic_store(&g_qt_calls, 0); atomic_store(&g_qt_loss_calls, 0); atomic_store(&g_qt_restore_calls, 0); atomic_store(&g_qt_last_lost, -1); atomic_store(&g_qt_last_connected, -1); atomic_store(&g_qt_last_required, -1); } /* Build a quorum fixture: n_peers=3, registry-truth quorum accounting wired * (ha.repl = &persist.repl) and exactly one active standby so the cluster STARTS * with write quorum held (quorum_lost == 0). Installs the emission-count hook. */ static void cv_quorum_setup(const char *suffix) { cv_setup(suffix, /*n_peers=*/3, /*n_standbys=*/1, /*appended_lsn=*/0, /*n_vcpus=*/8); /* Wire registry-truth quorum accounting (ha-commit-engine path): with * ha.repl set, ha_has_write_quorum counts repl_active_standby_count. */ g_cv.ha.repl = &g_cv.persist.repl; g_cv.ha.role = CTRL_PRIMARY; /* quorum gate is a CTRL_PRIMARY concern */ cv_quorum_reset_counters(); ha_quorum_transition_log_hook = cv_quorum_log_hook; /* Fixture precondition: one active standby → connected=1 >= required=1 → * quorum held at the start of every quorum test. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, repl_active_standby_count(&g_cv.persist.repl), "quorum fixture precondition: exactly one active standby (quorum held)"); } static void cv_quorum_teardown(void) { ha_quorum_transition_log_hook = NULL; cv_teardown(); } /* 8.1: dropping the only standby below majority logs the loss WARN EXACTLY ONCE, * carrying the connected/required counts; the edge flag flips to lost. * RED: the no-op stub never detects the edge → 0 emissions, flag stays 0. */ static void test_quorum_loss_logged_once_with_counts(void) { cv_quorum_setup("qloss"); /* Act: remove the only standby → connected drops 1→0 → quorum lost. */ repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]); TEST_ASSERT_EQUAL_INT_MESSAGE(0, repl_active_standby_count(&g_cv.persist.repl), "fixture: the cluster must have zero active standbys after removal"); ha_log_quorum_transition(&g_cv.ha); /* Assert (Req 8.1): exactly one loss WARN, no restore WARN. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls), "dropping below majority must log the quorum-loss WARN exactly once " "(Req 8.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_restore_calls), "a quorum-loss edge must not emit a restore WARN (Req 8.1)"); /* Assert (Req 8.1): the WARN carries the connected/required counts. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_last_connected), "the quorum-loss WARN must report connected=0 (Req 8.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_last_required), "the quorum-loss WARN must report required=1 (n_peers/2 threshold) " "(Req 8.1)"); /* The edge-detector flag must now read 'lost'. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cv.ha.quorum_lost), "a quorum-loss edge must set the ha_state_t.quorum_lost flag (Req 8.1)"); cv_quorum_teardown(); printf("test_quorum_loss_logged_once_with_counts: checked\n"); } /* 8.3: while quorum stays lost, repeated dispatch-gate rejections (modeled as * repeated ha_log_quorum_transition calls — the gate backstop calls it on every * rejected op) add NO further transition logs. * RED: the no-op stub never logs the first loss either, so the "exactly one loss * across many calls" assertion fails (0 != 1). */ static void test_quorum_loss_no_repeat_log_while_lost(void) { cv_quorum_setup("qstorm"); /* Cross the loss edge once. */ repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]); ha_log_quorum_transition(&g_cv.ha); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls), "setup: the first call after the loss edge must log once (Req 8.3 setup)"); /* Act: simulate many rejected operations re-invoking the backstop while * quorum stays lost (no membership change between calls). */ for (int i = 0; i < 5; i++) ha_log_quorum_transition(&g_cv.ha); /* Assert (Req 8.3): still exactly one loss WARN — no log storm. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls), "repeated quorum-transition calls while quorum stays lost must NOT log " "again — once per transition, no storms (Req 8.3)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_qt_restore_calls), "no restore WARN may appear while quorum remains lost (Req 8.3)"); cv_quorum_teardown(); printf("test_quorum_loss_no_repeat_log_while_lost: checked\n"); } /* 8.2: restoring a standby above majority logs the restore WARN ONCE; the edge * flag flips back to held. * RED: the no-op stub neither sets nor clears the flag and never emits. */ static void test_quorum_restore_logged_once(void) { cv_quorum_setup("qrestore"); /* Drive into the lost state first. */ repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]); ha_log_quorum_transition(&g_cv.ha); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls), "setup: must be in the lost state with one loss WARN (Req 8.2 setup)"); /* Act: re-register a standby → connected 0→1 → quorum restored. */ TEST_ASSERT_TRUE( repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0); TEST_ASSERT_EQUAL_INT_MESSAGE(1, repl_active_standby_count(&g_cv.persist.repl), "fixture: the re-registered standby must restore the active count to 1"); ha_log_quorum_transition(&g_cv.ha); /* Assert (Req 8.2): exactly one restore WARN, loss count unchanged. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_restore_calls), "restoring quorum must log the quorum-restore WARN exactly once " "(Req 8.2)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls), "the restore edge must not re-emit a loss WARN (Req 8.2)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_cv.ha.quorum_lost), "a quorum-restore edge must clear the ha_state_t.quorum_lost flag " "(Req 8.2)"); cv_quorum_teardown(); printf("test_quorum_restore_logged_once: checked\n"); } /* 8.1+8.2 edge detector (not a latch): a SECOND loss/restore cycle logs again. * RED: the no-op stub never logs any cycle. */ static void test_quorum_second_cycle_logs_again(void) { cv_quorum_setup("qcycle"); /* Cycle 1: lose then restore. */ repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]); ha_log_quorum_transition(&g_cv.ha); TEST_ASSERT_TRUE( repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0); ha_log_quorum_transition(&g_cv.ha); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_loss_calls), "cycle 1: exactly one loss WARN (edge-detector setup)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_qt_restore_calls), "cycle 1: exactly one restore WARN (edge-detector setup)"); /* Cycle 2: lose then restore AGAIN — the detector is not a one-shot latch. */ repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[0]); ha_log_quorum_transition(&g_cv.ha); TEST_ASSERT_TRUE( repl_add_standby(&g_cv.persist.repl, &g_cv_eps[0], "cvsb0", 7600) >= 0); ha_log_quorum_transition(&g_cv.ha); /* Assert: each transition logged again (totals = 2 loss + 2 restore). */ TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_qt_loss_calls), "a SECOND quorum-loss edge must log again — the transition logger is an " "edge detector, not a one-shot latch (Req 8.1)"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&g_qt_restore_calls), "a SECOND quorum-restore edge must log again (Req 8.2)"); cv_quorum_teardown(); printf("test_quorum_second_cycle_logs_again: checked\n"); } /* ════════════════════════════════════════════════════════════════════════════ * Task 6.1 — REGRESSION PINS (verification of existing behavior, NO production * change). These encode behavior already present in the current tree; each pin * PASSES against the code as-is. A failure here is a real regression, not a * RED-phase placeholder. * * (a) repl_broadcast full fan-out — umbrella 6.1 pin (Req 6.1) * (b) ha_evaluate_repl_action snapshot fallback boundary — umbrella 6.3 pin * (Req 6.1, decision boundary only) * (c) ha_dispatch_gate degraded-mode allowlist — umbrella 7.3 pin (Req 7.1) * (d) repl_complete_catchup re-inclusion mechanism — umbrella 6.4 pin * (Req 7.1, complements task 4's ACK-driven trigger test) * ════════════════════════════════════════════════════════════════════════════ */ /* Drive one repl_broadcast of a single WAL_NOOP entry (no payload). */ static int cv_broadcast_one(replicator_t *r, uint64_t lsn) { wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.lsn = lsn; hdr.type = WAL_NOOP; hdr.payload_len = 0; return repl_broadcast(r, &hdr, NULL, 0, 0 /* prev_log_term */); } /* (a) umbrella 6.1: repl_broadcast enqueues to EVERY registered active standby * and skips ONLY empty slots and in_catch_up standbys. * * Fixture: three standby slots — slot 0 active, slot 1 emptied (removed), slot 2 * active but in_catch_up. A single broadcast must hit slot 0 exactly once and * NEITHER slot 1 (empty) NOR slot 2 (catch-up). Pins existing repl.c:259-307. */ static void test_pin_repl_broadcast_full_fanout_skips_empty_and_catchup(void) { cv_setup("fanout", /*n_peers=*/3, /*n_standbys=*/3, /*appended_lsn=*/0, /*n_vcpus=*/8); /* Swap the drop hook for the counting hook and reset its tallies. */ cv_bcast_reset_counters(); repl_broadcast_send_hook = cv_count_send_hook; /* Arrange the three slots into the three states under test. */ repl_remove_standby(&g_cv.persist.repl, &g_cv_eps[1]); /* slot 1 → empty */ repl_start_catchup(&g_cv.persist.repl, &g_cv_eps[2], /*join_applied_lsn=*/0); /* slot 2 → catch-up */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, cv_standby_in_catch_up(&g_cv_eps[2]), "fixture precondition: slot 2 must be in_catch_up before broadcast"); /* Act: one broadcast. */ int rc = cv_broadcast_one(&g_cv.persist.repl, /*lsn=*/1); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "repl_broadcast must succeed"); /* Assert (Req 6.1): the active non-catch-up standby got exactly one send. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_bcast_slot_calls[0]), "repl_broadcast must enqueue to the active non-catch-up standby (slot 0) " "exactly once (umbrella 6.1 full fan-out)"); /* Assert: the EMPTY slot was skipped (no ep → no send). */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_bcast_slot_calls[1]), "repl_broadcast must skip the empty slot (slot 1) (umbrella 6.1)"); /* Assert: the in_catch_up standby was skipped. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_bcast_slot_calls[2]), "repl_broadcast must skip the in_catch_up standby (slot 2) " "(umbrella 6.1)"); /* Assert: exactly one send total — no over-fan-out. */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_bcast_total_calls), "repl_broadcast must enqueue exactly once across all slots in this " "fixture (one active, one empty, one catch-up) (umbrella 6.1)"); cv_teardown(); printf("test_pin_repl_broadcast_full_fanout_skips_empty_and_catchup: checked\n"); } /* (b) umbrella 6.3: ha_evaluate_repl_action returns HA_REPL_SEND_SNAPSHOT when * standby_lsn < oldest_wal_lsn (the standby predates the oldest retained WAL, so * streaming cannot reach it — a fresh snapshot is required). Decision boundary * only, per the task: at/above oldest_wal_lsn the verdict is NOT snapshot. * Pins existing ha.c:1274-1275. (The snapshot send path itself — * ha_send_snapshot_to_standby, config_repl.c:1857 — takes a fresh checkpoint * before sending; that send-internals behavior is out of scope for this * decision-boundary pin.) */ static void test_pin_evaluate_repl_action_snapshot_below_oldest_wal(void) { /* fix-snapshot-boundary: the snapshot fallback fires iff the standby's * NEXT-needed entry (standby_lsn + 1) is older than the oldest retained WAL * entry — i.e. standby_lsn + 1 < oldest_wal_lsn (genuinely GC'd behind). * tags equal (no config-push shortcut); committed_lsn high so the snapshot * boundary is the only deciding factor. */ /* GENUINELY GC'd behind: standby_lsn=2, oldest_wal_lsn=5 → entries 3,4 were * GC'd, the next-needed entry (3) is gone → SEND_SNAPSHOT. * (2 + 1 = 3 < 5 → snapshot.) */ ha_repl_action_t gcd_behind = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/2, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5); TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)gcd_behind, "standby_lsn (2) is GC'd behind oldest_wal_lsn (5) — its next-needed " "entry (3) is no longer retained, so it must decide SEND_SNAPSHOT " "(snapshot still fires when genuinely needed)"); /* BOUNDARY (streamable edge): standby_lsn == oldest_wal_lsn - 1, i.e. the * next-needed entry is EXACTLY the oldest retained entry → STREAM, not * snapshot. standby_lsn=4, oldest_wal_lsn=5 → 4 + 1 = 5 < 5 is false. This * is the strict boundary: a wrong comparison (standby_lsn < oldest_wal_lsn) * would (incorrectly) snapshot here, failing this assertion. */ ha_repl_action_t boundary = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/4, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5); TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)boundary, "standby_lsn (4) == oldest_wal_lsn (5) - 1: the next-needed entry (5) IS " "the oldest retained entry — streamable, must NOT decide SEND_SNAPSHOT " "(strict standby_lsn + 1 < oldest_wal_lsn boundary)"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)boundary, "at the snapshot boundary with lsn (4) below committed (9) the standby is " "streamable — HA_REPL_STREAM_WAL"); /* EMPTY standby on fresh cluster: standby_lsn=0, oldest_wal_lsn=1 → next * entry (1) retained → STREAM. 0 + 1 = 1 < 1 false. The core fix case. */ ha_repl_action_t empty_fresh = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/1); TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)empty_fresh, "empty standby (lsn=0), oldest_wal_lsn=1: next-needed entry (1) retained " "→ must STREAM, not snapshot (strict boundary)"); /* Above the oldest WAL and at committed → CONVERGED, never snapshot. */ ha_repl_action_t converged = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/9, /*committed_tag=*/5, /*committed_lsn=*/9, /*oldest_wal_lsn=*/5); TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)converged, "a fully caught-up standby must never decide SEND_SNAPSHOT " "(umbrella 6.3 boundary)"); printf("test_pin_evaluate_repl_action_snapshot_below_oldest_wal: checked\n"); } /* fix-snapshot-boundary: an EMPTY standby on a FRESH cluster must STREAM, not * demand a snapshot. * * After reserve-lsn-0 the first WAL entry is lsn 1, so a fresh cluster's oldest * retained entry is lsn 1. A freshly-joined empty standby reports applied_lsn=0. * Its NEXT-needed entry is lsn 1, which IS retained — so streaming can serve it; * no snapshot exists yet on a fresh cluster. The snapshot fallback must fire * only when the standby's next-needed entry (standby_lsn + 1) has been GC'd below * the oldest retained entry, i.e. standby_lsn + 1 < oldest_wal_lsn. * * RED before the fix: the old boundary (standby_lsn < oldest_wal_lsn) gives * 0 < 1 → SEND_SNAPSHOT for the empty standby. GREEN after: 0+1=1 < 1 is * false → STREAM_WAL streams lsn 1, the cold-start NOOP replicates + commits. */ static void test_empty_standby_fresh_cluster_streams_not_snapshot(void) { /* Empty standby (applied_lsn=0), fresh cluster (oldest_wal_lsn=1, NOOP at * lsn 1), WAL tip > 0 (committed_lsn=1). Tags equal so the config-push * shortcut does not mask the snapshot/stream decision. */ ha_repl_action_t a = ha_evaluate_repl_action( /*standby_tag=*/0, /*standby_lsn=*/0, /*committed_tag=*/0, /*committed_lsn=*/1, /*oldest_wal_lsn=*/1); TEST_ASSERT_NOT_EQUAL_MESSAGE((int)HA_REPL_SEND_SNAPSHOT, (int)a, "empty standby (lsn=0) on a fresh cluster (oldest_wal_lsn=1) needs lsn 1, " "which is retained — it must NOT demand a snapshot (there is none on a " "fresh cluster); the next-needed entry (lsn 1) is streamable"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)HA_REPL_STREAM_WAL, (int)a, "empty standby (lsn=0 < committed_lsn=1) with next-needed entry retained " "must STREAM_WAL so the cold-start NOOP (lsn 1) replicates and commits"); printf("test_empty_standby_fresh_cluster_streams_not_snapshot: checked\n"); } /* ── Degraded-mode dispatch-gate fixture (task 6.1 (c)) ────────────────────── * Self-contained ha_state_t with role=CTRL_PRIMARY and write quorum ABSENT * (n_peers=3, zero connected standby endpoints → ha_has_write_quorum false via * the standby_endpoints fallback count). Mirrors test_commit_barrier_rollout.c's * fact_gate_setup. No persist/repl needed — the gate reads role + quorum only. */ static ha_state_t g_cvg_ha; static int g_cvg_ep; /* dummy endpoint (zarray stores its address) */ static void cv_gate_setup(int n_peers, int n_endpoints) { ha_config_t cfg = cv_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&g_cvg_ha, &cfg, 0)); g_cvg_ha.n_peers = n_peers; atomic_store_explicit(&g_cvg_ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); g_cvg_ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(g_cvg_ha.standby_endpoints); zarray_init(g_cvg_ha.standby_endpoints, 4, NULL); for (int i = 0; i < n_endpoints; i++) zarray_store(g_cvg_ha.standby_endpoints, &g_cvg_ep); } static void cv_gate_teardown(void) { if (g_cvg_ha.standby_endpoints) { zarray_fini(g_cvg_ha.standby_endpoints); free(g_cvg_ha.standby_endpoints); g_cvg_ha.standby_endpoints = NULL; } ha_fini(&g_cvg_ha); } /* (c) umbrella 7.3: without write quorum, ha_dispatch_gate admits OPCode_get, * heartbeat, the replication-opcode allowlist, and the fact-shaped opcodes * (notify/update/setpreason/clrpreason — admitted per ha-commit-barrier-rollout's * fact allowlist), while rejecting request-shaped opcodes * (jobsub/jobcontrol/queuecontrol/hostcontrol/jobdispatch/join) with * -XCE_ERR_NOT_READY. Pins existing ha.c:1516-1551. */ static void test_pin_dispatch_gate_degraded_allowlist(void) { /* Arrange: CTRL_PRIMARY, 3-peer cluster, zero standbys → quorum absent. */ cv_gate_setup(/*n_peers=*/3, /*n_endpoints=*/0); /* ── ADMITTED (return 0) ───────────────────────────────────────────── */ static const struct { int opcode; const char *name; } admitted[] = { /* read-only query + probe */ { OPCode_get, "OPCode_get" }, { OPCode_probeConfigTag, "OPCode_probeConfigTag" }, /* heartbeat */ { OPCode_heartbeat, "OPCode_heartbeat" }, /* replication allowlist */ { OPCode_replConfig, "OPCode_replConfig" }, { OPCode_replConfigPush, "OPCode_replConfigPush" }, { OPCode_upgradeConfigAck, "OPCode_upgradeConfigAck" }, { OPCode_hello, "OPCode_hello" }, { OPCode_walAck, "OPCode_walAck" }, { OPCode_replEntry, "OPCode_replEntry" }, { OPCode_replAck, "OPCode_replAck" }, { OPCode_replCatchUpDone, "OPCode_replCatchUpDone" }, { OPCode_walCatchupEntry, "OPCode_walCatchupEntry" }, { OPCode_snapshotTransfer, "OPCode_snapshotTransfer" }, /* fact-shaped ops (admitted per the fact allowlist) */ { OPCode_notify, "OPCode_notify" }, { OPCode_update, "OPCode_update" }, { OPCode_setpreason, "OPCode_setpreason" }, { OPCode_clrpreason, "OPCode_clrpreason" }, }; for (size_t i = 0; i < sizeof(admitted) / sizeof(admitted[0]); i++) { int rc = ha_dispatch_gate(&g_cvg_ha, admitted[i].opcode); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, admitted[i].name); } /* ── REJECTED with -XCE_ERR_NOT_READY (request-shaped ops only) ────── */ static const struct { int opcode; const char *name; } rejected[] = { { OPCode_jobsub, "OPCode_jobsub" }, { OPCode_jobcontrol, "OPCode_jobcontrol" }, { OPCode_queuecontrol, "OPCode_queuecontrol" }, { OPCode_hostcontrol, "OPCode_hostcontrol" }, { OPCode_jobdispatch, "OPCode_jobdispatch" }, { OPCode_join, "OPCode_join" }, }; for (size_t i = 0; i < sizeof(rejected) / sizeof(rejected[0]); i++) { int rc = ha_dispatch_gate(&g_cvg_ha, rejected[i].opcode); TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc, rejected[i].name); } cv_gate_teardown(); printf("test_pin_dispatch_gate_degraded_allowlist: checked\n"); } /* (d) umbrella 6.4 mechanism: repl_complete_catchup clears in_catch_up and the * standby re-enters repl_broadcast iteration (i.e. it is sent to again). * Complements task 4's ACK-driven *trigger* test by pinning the *mechanism*. * Pins existing repl.c:602-628 + the broadcast re-inclusion at repl.c:265. */ static void test_pin_repl_complete_catchup_reincludes_in_broadcast(void) { cv_setup("reinc", /*n_peers=*/3, /*n_standbys=*/1, /*appended_lsn=*/0, /*n_vcpus=*/8); cv_bcast_reset_counters(); repl_broadcast_send_hook = cv_count_send_hook; repl_ep_t ep = &g_cv_eps[0]; /* Arrange: put the only standby into catch-up → excluded from broadcast. */ repl_start_catchup(&g_cv.persist.repl, ep, /*join_applied_lsn=*/0); TEST_ASSERT_EQUAL_INT_MESSAGE(1, cv_standby_in_catch_up(ep), "fixture precondition: the standby must be in_catch_up"); /* While in catch-up, a broadcast must NOT reach it. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_broadcast_one(&g_cv.persist.repl, 1), "repl_broadcast must succeed"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_bcast_slot_calls[0]), "an in_catch_up standby must be skipped by repl_broadcast (umbrella 6.4 " "precondition)"); /* Act: complete catch-up → re-inclusion. */ repl_complete_catchup(&g_cv.persist.repl, ep); /* Assert: in_catch_up cleared. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_standby_in_catch_up(ep), "repl_complete_catchup must clear in_catch_up (umbrella 6.4)"); /* Assert: the standby re-enters broadcast iteration — a fresh broadcast now * reaches it. */ cv_bcast_reset_counters(); TEST_ASSERT_EQUAL_INT_MESSAGE(0, cv_broadcast_one(&g_cv.persist.repl, 2), "repl_broadcast must succeed"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_bcast_slot_calls[0]), "after repl_complete_catchup the standby must re-enter repl_broadcast " "iteration and receive the next entry (umbrella 6.4 re-inclusion)"); cv_teardown(); printf("test_pin_repl_complete_catchup_reincludes_in_broadcast: checked\n"); } /* ── fix-promotion-standby-registration ───────────────────────────────────── * * Root cause pinned here: raft_cb_send_vote_request must set votes[i].peer * (the candidate pointer) alongside votes[i].ep, so that the promotion path * (raft_cb_become_leader) can call repl_add_standby for each surviving vote * endpoint. Before the fix, votes[i].peer was NEVER set, so the promotion * registration block (guarded by `votes[i].peer != NULL`) was skipped, the * standbys were never added to c->persist.repl.standbys[], commit/quorum saw * zero standbys, and every write failed with E_XCE_NOT_READY. * * These tests drive the REAL promotion path (raft_cb_become_leader via the * with-cluster seam) over a real persist/repl/commit fixture, then assert the * surviving vote endpoints are registered in the WAL replicator. * * The promotion tail is safe against this fixture: * - ha_register_persist_hooks: documented safe on this cluster_t (ha.c). * - ha_start_primary_heartbeat: no-ops because delaymgr is not started. * - ha_arm_own_term_gate: uses the real persist + commit engine (broadcast * drops through the stubbed repl_broadcast_send_hook). */ /* Build a vote endpoint entry on the election ctx: set BOTH .ep and .peer * (the post-fix vote-send contract). Uses g_cv_eps[] for the ep identity (so * repl_active_standby_count / broadcast slot-matching stay consistent with the * rest of this suite) and a caller-owned xce_candidate_t for .peer. */ static void cv_set_vote(ha_elect_ctx_t *ctx, int idx, xce_candidate_t *peer, const char *host, int port) { snprintf(peer->hostname, sizeof(peer->hostname), "%s", host); peer->port = port; ctx->votes[idx].ep = (struct endpoint_t *)&g_cv_eps[idx]; ctx->votes[idx].peer = peer; } /* (fix) Promotion with surviving vote endpoints carrying .peer registers every * one of them in the WAL replicator → repl_active_standby_count == n. * * RED before the fix: with the production vote-send leaving votes[i].peer NULL, * an election ctx built the SAME way the real path builds it would have * .peer == NULL, the promotion block would skip repl_add_standby, and the count * would be 0. This test sets .peer (simulating the fixed vote-send) and pins * that the promotion path then registers all surviving standbys. */ static void test_promotion_registers_standbys_when_peer_set(void) { /* n_standbys=0: the replicator starts EMPTY; only the promotion path may * populate it, so the count is attributable solely to registration. */ cv_setup("promreg", /*n_peers=*/2, /*n_standbys=*/0, /*appended_lsn=*/0, /*n_vcpus=*/8); /* Precondition: nothing registered yet. */ TEST_ASSERT_EQUAL_INT_MESSAGE(0, repl_active_standby_count(&g_cv.persist.repl), "fixture precondition: replicator must start empty (n_standbys=0)"); /* Build an election ctx with 2 surviving vote endpoints, each carrying a * peer pointer (the post-fix vote-send contract). */ ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0])); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = 2; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); xce_candidate_t peer0 = {0}, peer1 = {0}; cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700); cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701); /* standby_endpoints array must exist (promotion stores eps there too). */ g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints); zarray_init(g_cv.ha.standby_endpoints, 4, NULL); atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release); /* Act: real promotion with a live cluster. */ raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1); /* Assert (Req 1.1): both surviving vote endpoints are now registered in the * WAL replicator, so commit/quorum can advance committed_lsn. */ TEST_ASSERT_EQUAL_INT_MESSAGE(2, repl_active_standby_count(&g_cv.persist.repl), "promotion must register every surviving vote endpoint (votes[i].peer " "set) in the WAL replicator — repl_active_standby_count must equal the " "number of vote endpoints (fix-promotion-standby-registration Req 1.1)"); /* Cleanup */ atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release); if (g_cv.ha.standby_endpoints) { zarray_fini(g_cv.ha.standby_endpoints); free(g_cv.ha.standby_endpoints); g_cv.ha.standby_endpoints = NULL; } pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); cv_teardown(); printf("test_promotion_registers_standbys_when_peer_set: checked\n"); } /* RED-pin: with votes[i].peer == NULL (the PRE-fix vote-send behavior), the * promotion block must NOT register the endpoints — the count stays 0. This * pins the exact failure the fix addresses: the registration is conditional on * .peer being set, so leaving it NULL (as the buggy vote-send did) reproduces * the zero-standby / no-quorum / NOT_READY condition. GREEN: the production * vote-send now always sets .peer, so the real path takes the registered * branch above. */ static void test_promotion_skips_registration_when_peer_null(void) { cv_setup("promnull", /*n_peers=*/2, /*n_standbys=*/0, /*appended_lsn=*/0, /*n_vcpus=*/8); ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0])); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = 2; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); /* eps present (election connected), but peer left NULL (the BUG). */ ctx->votes[0].ep = (struct endpoint_t *)&g_cv_eps[0]; ctx->votes[0].peer = NULL; ctx->votes[1].ep = (struct endpoint_t *)&g_cv_eps[1]; ctx->votes[1].peer = NULL; g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints); zarray_init(g_cv.ha.standby_endpoints, 4, NULL); atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release); raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1); TEST_ASSERT_EQUAL_INT_MESSAGE(0, repl_active_standby_count(&g_cv.persist.repl), "with votes[i].peer NULL (pre-fix vote-send), promotion must NOT register " "standbys — this reproduces the zero-quorum / NOT_READY bug " "(fix-promotion-standby-registration RED pin)"); atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release); if (g_cv.ha.standby_endpoints) { zarray_fini(g_cv.ha.standby_endpoints); free(g_cv.ha.standby_endpoints); g_cv.ha.standby_endpoints = NULL; } pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); cv_teardown(); printf("test_promotion_skips_registration_when_peer_null: checked\n"); } /* ── raft-apply-decoupling task 5.3 (round-3 remediation, Req 8.4) ────────── * * COVERAGE GAP closed here: round-2 added an `!apply_divergent` conjunct to * BOTH the promotion_fence_timeout_fn re-alert timer (ha.c ~876) AND the * raft_cb_become_leader pre-promotion eligibility gate (ha.c ~1130, * `gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent`). * The round-2 reviewer proved the SECOND site had zero test coverage: reverting * ONLY that conjunct left the entire suite (298 tests across 13 groups, * including every test that drives raft_cb_become_leader / * raft_cb_become_leader_with_cluster_for_test) fully green. This test targets * exactly the scenario that conjunct alone must catch: a node that is * structurally ELIGIBLE (ha_promotion_gate_decision returns * RECOVER_DECISION_ELIGIBLE — not truncated, durable_floor >= the cluster * commit high-water mark) and NOT sticky-fenced (persist_recovery_unrecoverable * is explicitly false, i.e. the OTHER guard has nothing to contribute), whose * apply_verdict is nonetheless APPLY_DIVERGENT — a local materialized-state * defect that only a fresh, clean persist_recover() boot replay may clear * (task 5.1/5.2), never a peer/HWM condition. * * Fixture: identical to test_promotion_registers_standbys_when_peer_set above * (n_peers=2, n_standbys=0, appended_lsn=0) — that sibling test's own * assertion (repl_active_standby_count == 2 after promotion) is only * reachable if this exact fixture already drives ha_promotion_gate_decision to * RECOVER_DECISION_ELIGIBLE with unrecoverable == false, so reusing it here * isolates the apply_divergent conjunct as the ONLY variable changed. * * RED (site-2 guard reverted): the gate promotes anyway — role flips to * CTRL_PRIMARY and the vote endpoints get registered as standbys. * GREEN (current code): promotion is refused — role stays non-PRIMARY, the * member is fenced (in_catch_up=1, never-self-exit — it stays alive and keeps * retrying catch-up), and no standbys are registered. */ static void test_promotion_refused_when_apply_divergent_even_if_eligible(void) { /* n_standbys=0, appended_lsn=0: same structurally-healthy fixture as * test_promotion_registers_standbys_when_peer_set (durable_floor=0, * recovery_truncated=false, and — with vote_granted left at its * calloc-zero default — ha_compute_safe_committed_lsn's only counted * voter is self at self_lsn=0, so promotion_safe_lsn=0). durable_floor(0) * >= promotion_safe_lsn(0) and not truncated ⇒ genuinely ELIGIBLE. */ cv_setup("applydiv", /*n_peers=*/2, /*n_standbys=*/0, /*appended_lsn=*/0, /*n_vcpus=*/8); /* Precondition (Req 8.4 scenario, explicit per task spec): NOT fenced by * the sticky flag — isolates the apply_divergent conjunct from the * already-covered `unrecoverable` conjunct. */ TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(&g_cv.persist), "fixture precondition: sticky recovery_unrecoverable must be clear so " "this test isolates the apply_divergent guard, not the sticky-flag " "guard (Req 8.4)"); /* Force the apply-divergent verdict directly — exactly as task 5.1's * persist_recover() replay leaves it after detecting a materialized-state * mismatch, and exactly as * test_promotion_fence_apply_divergent_survives_structural_catchup * (test_commit_durability.c) sets it for the sibling (site-1) call site. */ g_cv.persist.apply_verdict = APPLY_DIVERGENT; ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0])); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = 2; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); xce_candidate_t peer0 = {0}, peer1 = {0}; cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700); cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701); g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints); zarray_init(g_cv.ha.standby_endpoints, 4, NULL); atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release); /* Act: real promotion attempt with a live, structurally-healthy cluster. */ raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1); /* Assert (Req 8.4): refused despite genuine structural ELIGIBLE + * unrecoverable==false — the apply_divergent conjunct alone must block * promotion. */ TEST_ASSERT_NOT_EQUAL_MESSAGE( (int32_t)CTRL_PRIMARY, atomic_load_explicit(&g_cv.ha.role, memory_order_acquire), "raft-apply-decoupling task 5.3 (round-3 coverage, Req 8.4): a " "structurally-eligible, non-sticky-fenced node whose apply_verdict is " "APPLY_DIVERGENT must NOT be promoted to CTRL_PRIMARY"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire), "a refused apply-divergent promotion must fence the member " "(in_catch_up=1) and keep it alive, retrying catch-up — never " "self-exit (Req 8.4)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, repl_active_standby_count(&g_cv.persist.repl), "promotion refusal must happen BEFORE the standby-registration block " "— no vote endpoint may be registered when the node never became " "primary"); /* Cleanup */ atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release); if (g_cv.ha.standby_endpoints) { zarray_fini(g_cv.ha.standby_endpoints); free(g_cv.ha.standby_endpoints); g_cv.ha.standby_endpoints = NULL; } pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); cv_teardown(); printf("test_promotion_refused_when_apply_divergent_even_if_eligible: " "checked\n"); } /* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ────────────────────── * Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the * Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced / * persist_mark_apply_resource_fenced) must gate this eligibility check the * same way apply_divergent/self_restart_pending already do. * * Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above * exactly — same structurally-healthy fixture (durable_floor(0) >= * promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision * == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the * sticky-flag conjunct contributes nothing) — swapping the trigger for the new * fence primitive so the new conjunct is isolated as the ONLY variable * changed, exactly as the round-3 apply_divergent test isolated its conjunct. * * RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced * conjunct, so a structurally-eligible node with the live fence engaged is * promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get * registered as standbys. GREEN (post-fix): promotion is refused — role stays * non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no * standbys are registered. */ static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void) { /* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible: * durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 => * genuinely ELIGIBLE on every OTHER conjunct. */ cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0, /*appended_lsn=*/0, /*n_vcpus=*/8); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(&g_cv.persist), "fixture precondition: sticky recovery_unrecoverable must be clear so " "this test isolates the apply-resource-fence guard, not the " "sticky-flag guard"); TEST_ASSERT_FALSE_MESSAGE( g_cv.persist.apply_verdict == APPLY_DIVERGENT, "fixture precondition: apply_verdict must not be APPLY_DIVERGENT so " "this test isolates the apply-resource-fence guard from the sibling " "apply_divergent guard"); /* Engage the Live Apply Resource Fence directly -- exactly as * persist_apply_pump_run does on a resource-exhaustion apply failure * (task 4.2), and exactly as the sibling site-1 test * (test_commit_durability.c) sets it for that call site. */ persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3); TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist)); ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0])); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = 2; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); xce_candidate_t peer0 = {0}, peer1 = {0}; cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700); cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701); g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints); zarray_init(g_cv.ha.standby_endpoints, 4, NULL); atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release); /* Act: real promotion attempt with a live, structurally-healthy cluster. */ raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1); /* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE + * unrecoverable==false + apply_verdict!=DIVERGENT -- the new * apply-resource-fence conjunct alone must block promotion. */ TEST_ASSERT_NOT_EQUAL_MESSAGE( (int32_t)CTRL_PRIMARY, atomic_load_explicit(&g_cv.ha.role, memory_order_acquire), "job-array-single-wal-record task 4.3 (Req 9.4/9.5): a " "structurally-eligible node with the Live Apply Resource Fence " "engaged must NOT be promoted to CTRL_PRIMARY"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire), "a refused apply-resource-fenced promotion must fence the member " "(in_catch_up=1) and keep it alive, retrying catch-up -- never " "self-exit"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, repl_active_standby_count(&g_cv.persist.repl), "promotion refusal must happen BEFORE the standby-registration block " "-- no vote endpoint may be registered when the node never became " "primary"); /* Now clear the fence (the apply pump's own self-heal, task 4.2) and * retry promotion: eligibility must return with no other action (task * 4.3's Observable clause). */ persist_clear_apply_resource_fence(&g_cv.persist); TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist)); atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release); xce_candidate_t peer0b = {0}, peer1b = {0}; ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0])); TEST_ASSERT_NOT_NULL(ctx2); ctx2->n_peers = 2; pthread_mutex_init(&ctx2->lock, NULL); sem_init(&ctx2->done, 0, 0); cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700); cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701); atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release); raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2); TEST_ASSERT_EQUAL_MESSAGE( (int32_t)CTRL_PRIMARY, atomic_load_explicit(&g_cv.ha.role, memory_order_acquire), "task 4.3: once the live apply resource fence clears, the node is " "eligible for promotion again with no other action"); /* Cleanup */ atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release); if (g_cv.ha.standby_endpoints) { zarray_fini(g_cv.ha.standby_endpoints); free(g_cv.ha.standby_endpoints); g_cv.ha.standby_endpoints = NULL; } pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); pthread_mutex_destroy(&ctx2->lock); sem_destroy(&ctx2->done); free(ctx2); cv_teardown(); printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: " "checked\n"); } #endif /* XCE_TESTING */ /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_convergence_verification(void) { UnitySetTestFile(__FILE__); #ifdef XCE_TESTING printf("\n>> ACK-driven catch-up continuation tests (RED, task 4.1)\n\n"); RUN_TEST(test_walack_behind_target_schedules_one_continuation); RUN_TEST(test_walack_single_flight_no_duplicate_continuation); RUN_TEST(test_walack_reschedules_after_continuation_completes); RUN_TEST(test_walack_at_parity_completes_catchup_and_clears_in_catch_up); RUN_TEST(test_heartbeat_decision_path_unchanged); printf("\n>> Quorum transition logging tests (RED, task 5.1)\n\n"); RUN_TEST(test_quorum_loss_logged_once_with_counts); RUN_TEST(test_quorum_loss_no_repeat_log_while_lost); RUN_TEST(test_quorum_restore_logged_once); RUN_TEST(test_quorum_second_cycle_logs_again); printf("\n>> Regression pins: fan-out / snapshot fallback / degraded gate / " "re-inclusion (task 6.1 — PASS against current code)\n\n"); RUN_TEST(test_pin_repl_broadcast_full_fanout_skips_empty_and_catchup); RUN_TEST(test_pin_evaluate_repl_action_snapshot_below_oldest_wal); RUN_TEST(test_empty_standby_fresh_cluster_streams_not_snapshot); RUN_TEST(test_pin_dispatch_gate_degraded_allowlist); RUN_TEST(test_pin_repl_complete_catchup_reincludes_in_broadcast); printf("\n>> Promotion standby registration " "(fix-promotion-standby-registration: votes[].peer → repl_add_standby)\n\n"); RUN_TEST(test_promotion_registers_standbys_when_peer_set); RUN_TEST(test_promotion_skips_registration_when_peer_null); printf("\n>> Promotion eligibility gate site-2 coverage " "(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n"); RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible); printf("\n>> Promotion eligibility gate site-2 coverage " "(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n"); RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible); #else printf("test_convergence_verification: SKIP (non-test build)\n"); #endif }