#include #include #include #include #include #include #include "ha.h" #include "myconf.h" #include "xce/cluster_def.h" static void test_ha_init_default_role_is_standby(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_ha_init_default_role_is_standby: PASS\n"); } static void test_ha_init_rejects_fast_heartbeat(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 1999, /* below 2000ms floor */ .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0)); printf("test_ha_init_rejects_fast_heartbeat: PASS\n"); } static void test_ha_init_rejects_short_election_timeout(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 499, /* below 500ms floor */ .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0)); printf("test_ha_init_rejects_short_election_timeout: PASS\n"); } static void test_ha_init_rejects_timeout_less_than_interval(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 5000, .heartbeat_timeout_ms = 4999, /* must be >= heartbeat_interval_ms */ .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0)); printf("test_ha_init_rejects_timeout_less_than_interval: PASS\n"); } static void test_ha_promote_sets_primary_role(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); /* Directly set role to PRIMARY (stub path — full promotion in P1-3a) */ atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_ha_promote_sets_primary_role: PASS\n"); } /* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by * ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */ /* ── P1-3a tests ─────────────────────────────────────────────── */ /* Term must increment on each promotion attempt */ static void test_promote_increments_term(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); ha_do_promote(&ha); /* internal promotion logic — no cluster_t needed */ uint64_t term_after = atomic_load_explicit(&ha.term, memory_order_acquire); TEST_ASSERT_TRUE(term_after > term_before); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_promote_increments_term: PASS\n"); } /* Demotion must set role back to STANDBY */ static void test_demote_sets_standby_role(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); ha_do_promote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_do_demote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_demote_sets_standby_role: PASS\n"); } /* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */ /* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */ /* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */ /* * Verify that ha_init zeroes the new fields added for election-iomgr-refactor: * phase2_done, n_peers, client_ssl_ctx * and that the surviving leader role values are distinct in ctrl_role_t. * * Requirements: 5.2, 8.1 (election-iomgr-refactor spec); 1.2 (ha-single-leader-role: * single leader role — no transitional leader state observable) */ static void test_ha_init_zeroes_new_fields(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* phase2_done must be explicitly zero after ha_init */ TEST_ASSERT_EQUAL_INT(0, ha.phase2_done); /* n_peers must be zero after ha_init */ TEST_ASSERT_EQUAL_INT(0, ha.n_peers); /* client_ssl_ctx must be NULL after ha_init */ TEST_ASSERT_NULL(ha.client_ssl_ctx); /* The four surviving roles are distinct: the single leader role is * CTRL_PRIMARY (no transitional leader sub-state, Req 1.2). */ TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_STANDBY); TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_INIT); TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY, (int)CTRL_ELECTING); ha_fini(&ha); printf("test_ha_init_zeroes_new_fields: PASS\n"); } /* * Task 1.4: per-standby convergence tracking fields zero-initialised by ha_init. * * Verifies: * - standby_reported_tag[i] == 0 for all i after ha_init() * - standby_reported_lsn[i] == 0 for all i after ha_init() * - standby_converged[i] == false for all i after ha_init() * - fields are writable from test context * * Requirements: 2.1 (ha-repl-hardening spec) */ static void test_ha_init_zeroes_standby_convergence_fields(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* All per-standby tracking arrays must be zero/false after ha_init */ for (int i = 0; i < MAX_HA_CANDIDATES; i++) { TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_tag[i]); TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_lsn[i]); TEST_ASSERT_FALSE(ha.standby_converged[i]); } /* Fields must be writable */ ha.standby_reported_tag[0] = 42ULL; ha.standby_reported_lsn[0] = 100ULL; ha.standby_converged[0] = true; TEST_ASSERT_EQUAL_UINT64(42ULL, ha.standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(100ULL, ha.standby_reported_lsn[0]); TEST_ASSERT_TRUE(ha.standby_converged[0]); ha_fini(&ha); printf("test_ha_init_zeroes_standby_convergence_fields: PASS\n"); } /* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */ /* * Declare the function under test. It is defined in ha.c and exposed only in * XCE_TESTING builds (forward declaration matches the implementation). */ #ifdef XCE_TESTING void ha_process_standby_state(ha_state_t *ha, int peer_idx, uint64_t reported_tag, uint64_t reported_lsn, uint64_t committed_tag, uint64_t committed_lsn); #endif /* Helper: build a ha_state_t with all convergence fields zeroed. */ static ha_state_t *build_ha_for_convergence_test(void) { static ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); return &ha; } /* * Test: standby with lsn == committed_lsn and tag == committed_tag → converged. * Fields standby_reported_tag[0] and standby_reported_lsn[0] updated correctly. * Requirements: 1.2, 2.2 (ha-repl-hardening) */ static void test_process_standby_state_converged(void) { ha_state_t *ha = build_ha_for_convergence_test(); ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]); TEST_ASSERT_TRUE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_converged: PASS\n"); } /* * Test: standby lsn < committed_lsn → not converged. * Requirements: 1.2, 2.2 */ static void test_process_standby_state_lsn_behind(void) { ha_state_t *ha = build_ha_for_convergence_test(); ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/50, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(50, ha->standby_reported_lsn[0]); TEST_ASSERT_FALSE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_lsn_behind: PASS\n"); } /* * Test: standby tag < committed_tag → not converged. * Requirements: 1.2, 2.2 */ static void test_process_standby_state_tag_behind(void) { ha_state_t *ha = build_ha_for_convergence_test(); ha_process_standby_state(ha, 0, /*reported_tag=*/3, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]); TEST_ASSERT_FALSE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_tag_behind: PASS\n"); } /* * Test: convergence regression — standby was converged; primary advances; * next update with old values should reset standby_converged to false. * Requirements: 2.5 (converged standby regresses → mark not-converged) */ static void test_process_standby_state_regression(void) { ha_state_t *ha = build_ha_for_convergence_test(); /* First update: converged at (tag=5, lsn=100) */ ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_TRUE(ha->standby_converged[0]); /* Primary advances committed_lsn to 200; standby still reports (5, 100) */ ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/200); TEST_ASSERT_FALSE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_regression: PASS\n"); } /* * Test: different standby indices updated independently. * Requirements: 2.2 */ static void test_process_standby_state_multiple_peers(void) { ha_state_t *ha = build_ha_for_convergence_test(); /* peer 0: converged */ ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); /* peer 1: not converged (tag behind) */ ha_process_standby_state(ha, 1, /*reported_tag=*/3, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_TRUE(ha->standby_converged[0]); TEST_ASSERT_FALSE(ha->standby_converged[1]); TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[1]); ha_fini(ha); printf("test_process_standby_state_multiple_peers: PASS\n"); } /* ── Task 5.2 convergence-checker tests REMOVED (ha-single-leader-role task 4.1) * * The majority-convergence checker was deleted in production (the convergence- * counting force-promotion path is gone — promotion is driven solely by the * heartbeat decision loop + per-entry majority ack). The machinery-subject * cases that exercised it (3node_none / 3node_one / 5node_threshold / * 3node_all / single_node_immediate) had nothing left to verify and were * deleted per design §"Transitional-State Deletions" (no replacements, no * shims). Their RUN_TEST registrations were removed as well. */ /* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ────── * * ha_dispatch_repl_action_for_standby reads the standby's stored * (standby_reported_tag, standby_reported_lsn), calls ha_evaluate_repl_action, * and if the result is HA_REPL_PUSH_CONFIG calls push_config_to_standbys (in * production) or ha_repl_push_config_for_standby_hook (in XCE_TESTING). * * Requirements: 1.4 (ha-repl-hardening), design §Unified Heartbeat-Driven Protocol */ #ifdef XCE_TESTING extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx); #endif /* Call-counting state for the hook */ static int g_push_config_for_standby_count = 0; static int g_push_config_for_standby_idx = -1; static int stub_repl_push_config_for_standby(void *c, int peer_idx) { (void)c; g_push_config_for_standby_count++; g_push_config_for_standby_idx = peer_idx; return 0; } /* * Test: when standby_reported_tag < committed_tag, ha_dispatch_repl_action_for_standby * must call push_config_to_standbys (via hook in test mode). * * Steps: * 1. Set ha->standby_reported_tag[0] = 3 (behind committed_tag=5) * 2. Set ha->standby_reported_lsn[0] = 100 (at committed_lsn) * 3. Call ha_dispatch_repl_action_for_standby(ha, NULL, 0, committed_tag=5, ...) * 4. Assert hook called once with peer_idx=0 * * Requirements: 1.4 (push_config when standby tag < committed_tag) */ static void test_dispatch_repl_action_push_config_when_tag_behind(void) { #ifdef XCE_TESTING ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby; g_push_config_for_standby_count = 0; g_push_config_for_standby_idx = -1; ha_state_t *ha = build_ha_for_convergence_test(); /* standby[0] has stale config tag */ ha->standby_reported_tag[0] = 3; ha->standby_reported_lsn[0] = 100; int rc = ha_dispatch_repl_action_for_standby( ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0); (void)rc; TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_push_config_for_standby_count, "PUSH_CONFIG: hook must fire once when standby_tag < committed_tag"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_idx, "PUSH_CONFIG: hook must receive correct peer_idx=0"); ha_fini(ha); ha_repl_push_config_for_standby_hook = NULL; printf("test_dispatch_repl_action_push_config_when_tag_behind: PASS\n"); #else printf("test_dispatch_repl_action_push_config_when_tag_behind: SKIP (non-test build)\n"); #endif } /* * Test: when standby is converged (tag == committed, lsn == committed), * ha_dispatch_repl_action_for_standby must NOT call push_config hook. * * Requirements: decision tree — CONVERGED does not trigger push */ static void test_dispatch_repl_action_no_push_when_converged(void) { #ifdef XCE_TESTING ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby; g_push_config_for_standby_count = 0; g_push_config_for_standby_idx = -1; ha_state_t *ha = build_ha_for_convergence_test(); /* standby[0] is fully converged */ ha->standby_reported_tag[0] = 5; ha->standby_reported_lsn[0] = 100; int rc = ha_dispatch_repl_action_for_standby( ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0); (void)rc; TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_count, "CONVERGED: hook must NOT fire when standby is already converged"); ha_fini(ha); ha_repl_push_config_for_standby_hook = NULL; printf("test_dispatch_repl_action_no_push_when_converged: PASS\n"); #else printf("test_dispatch_repl_action_no_push_when_converged: SKIP (non-test build)\n"); #endif } /* ── Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring ──── * * When standby_lsn < oldest_wal_lsn (standby predates WAL retention window), * ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby * (via ha_repl_send_snapshot_hook in XCE_TESTING builds). * * Requirements: 4.1 (ha-repl-hardening), design §Snapshot Transfer */ #ifdef XCE_TESTING extern int (*ha_repl_send_snapshot_hook)(int peer_idx); #endif /* Call-counting state for the snapshot hook */ static int g_send_snapshot_count = 0; static int g_send_snapshot_idx = -1; static int stub_repl_send_snapshot(int peer_idx) { g_send_snapshot_count++; g_send_snapshot_idx = peer_idx; return 0; } /* * Test: when standby_lsn < oldest_wal_lsn (and standby_tag == committed_tag), * ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby * (via hook in test mode). * * Steps: * 1. Set ha->standby_reported_tag[0] = committed_tag (tag is current) * 2. Set ha->standby_reported_lsn[0] = 10 (behind oldest_wal_lsn=50) * 3. Call ha_dispatch_repl_action_for_standby(..., oldest_wal_lsn=50) * 4. Assert hook called once with peer_idx=0 * * Requirements: 4.1 (snapshot when standby_lsn < oldest_wal_lsn) */ static void test_dispatch_repl_action_send_snapshot(void) { #ifdef XCE_TESTING ha_repl_send_snapshot_hook = stub_repl_send_snapshot; g_send_snapshot_count = 0; g_send_snapshot_idx = -1; ha_state_t *ha = build_ha_for_convergence_test(); /* standby[0] has current config tag but LSN predates WAL window */ ha->standby_reported_tag[0] = 5; ha->standby_reported_lsn[0] = 10; /* < oldest_wal_lsn=50 */ int rc = ha_dispatch_repl_action_for_standby( ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); (void)rc; TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_send_snapshot_count, "SEND_SNAPSHOT: hook must fire once when standby_lsn < oldest_wal_lsn"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_send_snapshot_idx, "SEND_SNAPSHOT: hook must receive correct peer_idx=0"); ha_fini(ha); ha_repl_send_snapshot_hook = NULL; printf("test_dispatch_repl_action_send_snapshot: PASS\n"); #else printf("test_dispatch_repl_action_send_snapshot: SKIP (non-test build)\n"); #endif } /* ── Task 3.2: ha_evaluate_repl_action unified decision tree ─────────────── */ /* * Declare the function under test. It is defined in ha.c. * Pure function — no side effects — so no XCE_TESTING guard needed. */ ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag, uint64_t standby_lsn, uint64_t committed_tag, uint64_t committed_lsn, uint64_t oldest_wal_lsn); /* * Test: (tag=0, lsn=0) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → PUSH_CONFIG (tag behind). * Requirements: 2.2, 2.3 (ha-repl-hardening) */ static void test_evaluate_repl_action_push_config(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/0, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_PUSH_CONFIG, action); printf("test_evaluate_repl_action_push_config: PASS\n"); } /* * Test: (tag=5, lsn=0) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn=0 is below WAL retention window). * Requirements: 2.2, 2.3 */ static void test_evaluate_repl_action_send_snapshot_lsn_zero(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); /* lsn=0 < oldest_wal_lsn=50 → SEND_SNAPSHOT */ TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action); printf("test_evaluate_repl_action_send_snapshot_lsn_zero: PASS\n"); } /* * Test: (tag=5, lsn=100) with committed_tag=5, committed_lsn=100 → CONVERGED. * Requirements: 2.2, 2.4 */ static void test_evaluate_repl_action_converged(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, action); printf("test_evaluate_repl_action_converged: PASS\n"); } /* * Test: (tag=5, lsn=10) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn < oldest_wal). * Requirements: 2.2, 2.3 */ static void test_evaluate_repl_action_send_snapshot(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/10, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action); printf("test_evaluate_repl_action_send_snapshot: PASS\n"); } /* * Test: (tag=5, lsn=50) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → STREAM_WAL (lsn >= oldest_wal but < committed). * Requirements: 2.2, 2.3 */ static void test_evaluate_repl_action_stream_wal(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/50, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, action); printf("test_evaluate_repl_action_stream_wal: PASS\n"); } /* * Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet) * → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory; * lsn=0 < committed_lsn → stream WAL from the beginning). * * This is the canonical Task 6.1 case: a standby that has caught up on config * (tag matches) but has no WAL applied yet on a fresh system. When there is no * WAL retention window to worry about (oldest_wal_lsn == 0), the correct action * is STREAM_WAL, not SEND_SNAPSHOT. * * Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening) */ static void test_evaluate_repl_action_stream_wal_fresh_system(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0); /* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */ TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action, "(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)"); printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n"); } /* ── bug-1478: ha_compute_repl_target_lsn ───────────────────────────────── * * repl_ack_decode_and_dispatch (ha.c) computes the replication/convergence * target passed into ha_dispatch_repl_action_for_standby as * max(committed_lsn, last_written), where last_written is the raw locally- * appended WAL tip (persist.current_lsn - 1). A live 3-node HA soak * (config-upgrade-snapshot task 8.9, --config-upgrade-wal-trim) caught this * unconditional max() permanently wedging catch-up: once a marker-triggered * checkpoint (a local-only, NEVER-replicated WAL_CHECKPOINT entry — * persist_write_checkpoint_marker bumps current_lsn directly, bypassing * raft_log_append/the commit engine, bug-1466) trails an already-advancing * committed_lsn, last_written sits permanently one LSN past it — a position * no standby's applied_lsn can EVER legitimately reach (the checkpoint's own * slot is, by design, never streamed or applied). ha_evaluate_repl_action's * standby_lsn < committed_lsn check then never reaches HA_REPL_CONVERGED, * and the primary re-streams the same empty range every heartbeat, forever. * Ground-truthed via `xce_test waldump` on the live soak's preserved WAL * (ctrl-2: lsn 21 = the WAL_CONFIG_MARKER, lsn 22 = WAL_CHECKPOINT, 0 bytes) * and via ctrl-2's own commit_advance_locked log (the REAL, majority-gated * committed_lsn never advanced past 21 — only the target_lsn fallback * leaked 22 into the heartbeat decision). */ static void test_compute_repl_target_lsn_bootstrap_fallback(void) { /* committed_lsn genuinely stuck at 0 (no standby has ever acked) — the * documented bootstrap exception: fall back to last_written so a * first-ever connecting standby can catch up to real data. */ TEST_ASSERT_EQUAL_UINT64(42, ha_compute_repl_target_lsn(0, 42)); printf("test_compute_repl_target_lsn_bootstrap_fallback: PASS\n"); } static void test_compute_repl_target_lsn_zero_zero(void) { /* Genuinely fresh cluster, nothing appended yet. */ TEST_ASSERT_EQUAL_UINT64(0, ha_compute_repl_target_lsn(0, 0)); printf("test_compute_repl_target_lsn_zero_zero: PASS\n"); } static void test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn(void) { /* bug-1478's exact reproduction shape: committed_lsn=21 is already * advancing (non-zero, real, majority-gated), but last_written=22 * reflects a local-only WAL_CHECKPOINT that trails it. The bootstrap * fallback must NOT fire here — the target must stay at the real * committed_lsn (21), never the unstreamable checkpoint slot (22). */ TEST_ASSERT_EQUAL_UINT64(21, ha_compute_repl_target_lsn(21, 22)); printf("test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn: PASS\n"); } static void test_compute_repl_target_lsn_committed_already_ahead(void) { /* Normal steady-state case: committed_lsn is already the higher/current * value; last_written (a possibly-stale, lock-free read) trails it. * The target must simply be committed_lsn. */ TEST_ASSERT_EQUAL_UINT64(100, ha_compute_repl_target_lsn(100, 50)); printf("test_compute_repl_target_lsn_committed_already_ahead: PASS\n"); } /* The transitional-leader timeout default test was removed: the config knob * and its default macro are deleted by ha-single-leader-role (the wall-clock * force-promotion watchdog it configured is gone — Req 1.5). */ /* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry * in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT. * * Requirements: ha-wal-replication Req 3.3 (standby role guard). * Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0 * (allow) for the three roles where a standby legitimately receives WAL * catch-up entries from the primary. CTRL_PRIMARY must NOT allow it * (the leader never receives WAL pushes). */ static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* CTRL_STANDBY — must allow */ atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_walCatchupEntry), "CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)"); /* CTRL_ELECTING — must allow */ atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_walCatchupEntry), "CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)"); /* CTRL_INIT — must allow */ atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_walCatchupEntry), "CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)"); ha_fini(&ha); printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n"); } /* ── ha-prevote: dispatch gate must admit OPCode_preVoteRequest wherever it * admits OPCode_requestVote (CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT). * * Regression (bug-605): the PreVote opcode + handler were added but never * wired into ha_dispatch_gate, so a STANDBY voter rejected incoming PreVote * straw-polls with -XCE_ERR_STANDBY before the handler ran. No preVoteReply * was ever sent; the candidate's failed exchange was tallied as a denial, * so a cold cluster looped PreVote LOST forever and never elected a primary. * * PreVote is strictly non-mutating, so it must be at least as admissible as a * real requestVote in every role. This test asserts that parity. */ static void test_dispatch_gate_allows_prevote_in_voter_roles(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); const int32_t voter_roles[] = { CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT }; for (size_t i = 0; i < sizeof(voter_roles) / sizeof(voter_roles[0]); i++) { atomic_store_explicit(&ha.role, voter_roles[i], memory_order_release); /* PreVote must be admitted exactly where the real vote is. */ TEST_ASSERT_EQUAL_INT_MESSAGE( ha_dispatch_gate(&ha, OPCode_requestVote), ha_dispatch_gate(&ha, OPCode_preVoteRequest), "ha_dispatch_gate must admit OPCode_preVoteRequest wherever it " "admits OPCode_requestVote (bug-605)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_preVoteRequest), "voter role must admit OPCode_preVoteRequest (bug-605)"); } ha_fini(&ha); printf("test_dispatch_gate_allows_prevote_in_voter_roles: PASS\n"); } /* ── ha-prevote/raft (bug-606): failover election stagger desyncs survivors. * * After a primary dies, two surviving standbys must NOT campaign in lock-step * (split-vote livelock, term climbing unbounded). ha_election_stagger_ms gives * each node a priority-ordered, jitter-bounded delay before starting its * failover PreVote so one survivor wins before the other begins. Asserts: * - bounds: stagger ∈ [my_priority*step, my_priority*step + jitter_max] * - ordering: a higher-priority-index node's MINIMUM stagger exceeds a * lower-index node's MAXIMUM (requires step > jitter_max — the desync * guarantee), so the lower index always campaigns first. */ static void test_election_stagger_desyncs_by_priority(void) { const uint32_t step = 500, jmax = 200; /* step > jmax → strict ordering */ ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 5000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .election_priority_step_ms = step, .election_jitter_max_ms = jmax, }; for (int prio = 0; prio < 3; prio++) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, prio)); uint32_t lo = (uint32_t)prio * step; uint32_t hi = lo + jmax; /* Sample several times — jitter is re-randomized each call. */ for (int i = 0; i < 16; i++) { uint32_t d = ha_election_stagger_ms(&ha); TEST_ASSERT_TRUE_MESSAGE(d >= lo && d <= hi, "stagger must be within [prio*step, prio*step+jitter_max] (bug-606)"); } ha_fini(&ha); } /* Desync guarantee: prio 1's max (step+jmax) < prio 2's min (2*step). */ TEST_ASSERT_TRUE_MESSAGE((step + jmax) < (2u * step), "election_priority_step_ms must exceed election_jitter_max_ms so " "adjacent priorities never overlap (bug-606 desync guarantee)"); printf("test_election_stagger_desyncs_by_priority: PASS\n"); } /* ── controller-status task 2.2: dispatch gate admits OPCode_ctrlstatus in * EVERY role — STANDBY, ELECTING, INIT, and PRIMARY-without-quorum — so the * read-only status query always answers (even fenced / without quorum), * rather than being rejected as -XCE_ERR_STANDBY / -XCE_ERR_NOT_READY. * * Requirements: controller-status 7.1, 7.2 (failover-aware, any-node delivery); * design §"Dispatch-gate allow-lists — Service contract". * * Scope guard: the change must be strictly scoped to OPCode_ctrlstatus — no * other opcode's gating may widen. This test also asserts that a previously * primary-only-when-serving read (OPCode_clusterconf) is STILL rejected on a * STANDBY after the change, proving the allow-list was not broadened. * * Note on PRIMARY-without-quorum: ha_has_write_quorum() returns false when the * connected-standby count is below n_peers/2. With n_peers == 0 (default * ha_init) the threshold is 0 and quorum is trivially satisfied, so to exercise * the no-quorum branch we set n_peers high enough that 0 connected standbys * fail the >= n_peers/2 test. */ static void test_dispatch_gate_allows_ctrlstatus_in_every_role(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* CTRL_STANDBY — must allow ctrlstatus (Req 7.1, 7.2) */ atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_ctrlstatus), "CTRL_STANDBY must allow OPCode_ctrlstatus (Req 7.1, 7.2)"); /* CTRL_ELECTING — must allow ctrlstatus */ atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_ctrlstatus), "CTRL_ELECTING must allow OPCode_ctrlstatus (Req 7.1, 7.2)"); /* CTRL_INIT — must allow ctrlstatus */ atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_ctrlstatus), "CTRL_INIT must allow OPCode_ctrlstatus (Req 7.1, 7.2)"); /* CTRL_PRIMARY without write quorum — must allow ctrlstatus. * Force the no-quorum branch: with no replicator and zero connected * standby_endpoints, ha_has_write_quorum() returns false once n_peers > 1 * (connected 0 < n_peers/2). */ ha.n_peers = 3; /* threshold = n_peers/2 = 1; connected = 0 → no quorum */ atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_ctrlstatus), "CTRL_PRIMARY without quorum must allow OPCode_ctrlstatus (Req 7.1, 7.2)"); /* Scope guard (Req: change strictly scoped to ctrlstatus): a read that is * primary-only-when-serving (OPCode_clusterconf) must STILL be rejected on * a STANDBY — proving the allow-list was not widened for any other opcode. */ atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_STANDBY, ha_dispatch_gate(&ha, OPCode_clusterconf), "CTRL_STANDBY must STILL reject OPCode_clusterconf — gate change must " "be scoped strictly to OPCode_ctrlstatus"); ha_fini(&ha); printf("test_dispatch_gate_allows_ctrlstatus_in_every_role: PASS\n"); } void test_ha_role(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_ha_init_default_role_is_standby); RUN_TEST(test_ha_init_rejects_fast_heartbeat); RUN_TEST(test_ha_init_rejects_short_election_timeout); RUN_TEST(test_ha_init_rejects_timeout_less_than_interval); RUN_TEST(test_ha_promote_sets_primary_role); /* P1-3a */ RUN_TEST(test_promote_increments_term); RUN_TEST(test_demote_sets_standby_role); /* Watchdog / SIGUSR1 tests deleted */ /* Task 2.1: election-iomgr-refactor new fields */ RUN_TEST(test_ha_init_zeroes_new_fields); /* Task 1.4: ha-repl-hardening per-standby convergence fields */ RUN_TEST(test_ha_init_zeroes_standby_convergence_fields); /* Task 3.1: ha_process_standby_state convergence tracking */ RUN_TEST(test_process_standby_state_converged); RUN_TEST(test_process_standby_state_lsn_behind); RUN_TEST(test_process_standby_state_tag_behind); RUN_TEST(test_process_standby_state_regression); RUN_TEST(test_process_standby_state_multiple_peers); /* Task 3.2: ha_evaluate_repl_action unified decision tree */ RUN_TEST(test_evaluate_repl_action_push_config); RUN_TEST(test_evaluate_repl_action_send_snapshot_lsn_zero); RUN_TEST(test_evaluate_repl_action_converged); RUN_TEST(test_evaluate_repl_action_send_snapshot); RUN_TEST(test_evaluate_repl_action_stream_wal); /* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */ RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system); /* bug-1478: ha_compute_repl_target_lsn */ RUN_TEST(test_compute_repl_target_lsn_bootstrap_fallback); RUN_TEST(test_compute_repl_target_lsn_zero_zero); RUN_TEST(test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn); RUN_TEST(test_compute_repl_target_lsn_committed_already_ahead); /* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */ RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind); RUN_TEST(test_dispatch_repl_action_no_push_when_converged); /* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */ RUN_TEST(test_dispatch_repl_action_send_snapshot); /* Task 5.2 convergence-checker cases deleted (ha-single-leader-role 4.1: * the majority-convergence checker removed — no shims, machinery-subject) */ /* transitional-leader timeout default test removed (ha-single-leader-role: * config knob + wall-clock watchdog deleted, Req 1.5) */ /* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */ RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles); /* ha-prevote (bug-605): dispatch gate must admit PreVote like the real vote */ RUN_TEST(test_dispatch_gate_allows_prevote_in_voter_roles); /* raft/ha-prevote (bug-606): failover election stagger desyncs survivors */ RUN_TEST(test_election_stagger_desyncs_by_priority); /* controller-status task 2.2: dispatch gate admits ctrlstatus in every role */ RUN_TEST(test_dispatch_gate_allows_ctrlstatus_in_every_role); } /* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */