#include #include #include #include #include #include #include "ha.h" #include "myconf.h" #include "xce/cluster_def.h" static void test_ha_init_default_role_is_standby(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_ha_init_default_role_is_standby: PASS\n"); } static void test_ha_init_rejects_fast_heartbeat(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 1999, /* below 2000ms floor */ .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0)); printf("test_ha_init_rejects_fast_heartbeat: PASS\n"); } static void test_ha_init_rejects_short_election_timeout(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 499, /* below 500ms floor */ .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0)); printf("test_ha_init_rejects_short_election_timeout: PASS\n"); } static void test_ha_init_rejects_timeout_less_than_interval(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 5000, .heartbeat_timeout_ms = 4999, /* must be >= heartbeat_interval_ms */ .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; TEST_ASSERT_NOT_EQUAL(0, ha_init(&ha, &cfg, 0)); printf("test_ha_init_rejects_timeout_less_than_interval: PASS\n"); } static void test_ha_promote_sets_primary_role(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); /* Directly set role to PRIMARY (stub path — full promotion in P1-3a) */ atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_ha_promote_sets_primary_role: PASS\n"); } /* Watchdog / SIGUSR1 tests deleted — watchdog pthread replaced by * ha_arm_heartbeat_timeout_vtask (delaymgr); failover_requested removed */ /* ── P1-3a tests ─────────────────────────────────────────────── */ /* Term must increment on each promotion attempt */ static void test_promote_increments_term(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); ha_do_promote(&ha); /* internal promotion logic — no cluster_t needed */ uint64_t term_after = atomic_load_explicit(&ha.term, memory_order_acquire); TEST_ASSERT_TRUE(term_after > term_before); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_promote_increments_term: PASS\n"); } /* Demotion must set role back to STANDBY */ static void test_demote_sets_standby_role(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; ha_init(&ha, &cfg, 0); ha_do_promote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_do_demote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_demote_sets_standby_role: PASS\n"); } /* test_sigusr1_sets_failover_requested deleted — SIGUSR1 / failover_requested removed */ /* am_i_primary() tests deleted — function removed (election-iomgr-refactor) */ /* ── Task 2.1: ha_state_t new fields zero-initialised by ha_init ──────── */ /* * Verify that ha_init zeroes the new fields added for election-iomgr-refactor: * phase2_done, repl_ack_count, n_peers, client_ssl_ctx * and that CTRL_PRIMARY_COMMITTING is a distinct enum value in ctrl_role_t. * Also verifies ha_config_t.committing_timeout_ms default macro exists. * * Requirements: 5.2, 8.1 (election-iomgr-refactor spec) */ static void test_ha_init_zeroes_new_fields(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* phase2_done must be explicitly zero after ha_init */ TEST_ASSERT_EQUAL_INT(0, ha.phase2_done); /* repl_ack_count is _Atomic int; must be zero after ha_init */ TEST_ASSERT_EQUAL_INT(0, atomic_load_explicit(&ha.repl_ack_count, memory_order_relaxed)); /* n_peers must be zero after ha_init */ TEST_ASSERT_EQUAL_INT(0, ha.n_peers); /* client_ssl_ctx must be NULL after ha_init */ TEST_ASSERT_NULL(ha.client_ssl_ctx); /* committing_timeout_ms stored in config */ TEST_ASSERT_EQUAL_UINT32(HA_DEFAULT_COMMITTING_TIMEOUT_MS, ha.config.committing_timeout_ms); /* CTRL_PRIMARY_COMMITTING must be a distinct enum value */ TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_PRIMARY); TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_STANDBY); TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_INIT); TEST_ASSERT_NOT_EQUAL((int)CTRL_PRIMARY_COMMITTING, (int)CTRL_ELECTING); ha_fini(&ha); printf("test_ha_init_zeroes_new_fields: PASS\n"); } /* * Task 1.4: per-standby convergence tracking fields zero-initialised by ha_init. * * Verifies: * - standby_reported_tag[i] == 0 for all i after ha_init() * - standby_reported_lsn[i] == 0 for all i after ha_init() * - standby_converged[i] == false for all i after ha_init() * - fields are writable from test context * * Requirements: 2.1 (ha-repl-hardening spec) */ static void test_ha_init_zeroes_standby_convergence_fields(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* All per-standby tracking arrays must be zero/false after ha_init */ for (int i = 0; i < MAX_HA_CANDIDATES; i++) { TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_tag[i]); TEST_ASSERT_EQUAL_UINT64(0, ha.standby_reported_lsn[i]); TEST_ASSERT_FALSE(ha.standby_converged[i]); } /* Fields must be writable */ ha.standby_reported_tag[0] = 42ULL; ha.standby_reported_lsn[0] = 100ULL; ha.standby_converged[0] = true; TEST_ASSERT_EQUAL_UINT64(42ULL, ha.standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(100ULL, ha.standby_reported_lsn[0]); TEST_ASSERT_TRUE(ha.standby_converged[0]); ha_fini(&ha); printf("test_ha_init_zeroes_standby_convergence_fields: PASS\n"); } /* ── Task 3.1: ha_process_standby_state convergence tracking ─────────────── */ /* * Declare the function under test. It is defined in ha.c and exposed only in * XCE_TESTING builds (forward declaration matches the implementation). */ #ifdef XCE_TESTING void ha_process_standby_state(ha_state_t *ha, int peer_idx, uint64_t reported_tag, uint64_t reported_lsn, uint64_t committed_tag, uint64_t committed_lsn); #endif /* Helper: build a ha_state_t with all convergence fields zeroed. */ static ha_state_t *build_ha_for_convergence_test(void) { static ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&ha, &cfg, 0); return &ha; } /* * Test: standby with lsn == committed_lsn and tag == committed_tag → converged. * Fields standby_reported_tag[0] and standby_reported_lsn[0] updated correctly. * Requirements: 1.2, 2.2 (ha-repl-hardening) */ static void test_process_standby_state_converged(void) { ha_state_t *ha = build_ha_for_convergence_test(); ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]); TEST_ASSERT_TRUE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_converged: PASS\n"); } /* * Test: standby lsn < committed_lsn → not converged. * Requirements: 1.2, 2.2 */ static void test_process_standby_state_lsn_behind(void) { ha_state_t *ha = build_ha_for_convergence_test(); ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/50, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(50, ha->standby_reported_lsn[0]); TEST_ASSERT_FALSE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_lsn_behind: PASS\n"); } /* * Test: standby tag < committed_tag → not converged. * Requirements: 1.2, 2.2 */ static void test_process_standby_state_tag_behind(void) { ha_state_t *ha = build_ha_for_convergence_test(); ha_process_standby_state(ha, 0, /*reported_tag=*/3, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(100, ha->standby_reported_lsn[0]); TEST_ASSERT_FALSE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_tag_behind: PASS\n"); } /* * Test: convergence regression — standby was converged; primary advances; * next update with old values should reset standby_converged to false. * Requirements: 2.5 (converged standby regresses → mark not-converged) */ static void test_process_standby_state_regression(void) { ha_state_t *ha = build_ha_for_convergence_test(); /* First update: converged at (tag=5, lsn=100) */ ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_TRUE(ha->standby_converged[0]); /* Primary advances committed_lsn to 200; standby still reports (5, 100) */ ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/200); TEST_ASSERT_FALSE(ha->standby_converged[0]); ha_fini(ha); printf("test_process_standby_state_regression: PASS\n"); } /* * Test: different standby indices updated independently. * Requirements: 2.2 */ static void test_process_standby_state_multiple_peers(void) { ha_state_t *ha = build_ha_for_convergence_test(); /* peer 0: converged */ ha_process_standby_state(ha, 0, /*reported_tag=*/5, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); /* peer 1: not converged (tag behind) */ ha_process_standby_state(ha, 1, /*reported_tag=*/3, /*reported_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100); TEST_ASSERT_TRUE(ha->standby_converged[0]); TEST_ASSERT_FALSE(ha->standby_converged[1]); TEST_ASSERT_EQUAL_UINT64(5, ha->standby_reported_tag[0]); TEST_ASSERT_EQUAL_UINT64(3, ha->standby_reported_tag[1]); ha_fini(ha); printf("test_process_standby_state_multiple_peers: PASS\n"); } /* ── Task 5.2: ha_check_majority_convergence → PRIMARY transition ──────────── * * After ha_process_standby_state updates standby_converged[], the primary must * count the number of converged standbys and promote to CTRL_PRIMARY when the * majority threshold is met and the current role is CTRL_PRIMARY_COMMITTING. * * Threshold formula (Req 2.3): ceil(n_peers/2) - 1 converged standbys — * i.e. primary + converged standbys form a majority of the cluster. * Integer form: n_peers / 2 standbys (n=3 → 1, n=5 → 2). * Single-node (n_peers<=1): threshold=0 → satisfied immediately. * n_peers = total cluster size including self. * * Requirements: 2.3, 2.4, 6.1, 6.3, 6.5 (ha-repl-hardening) */ #ifdef XCE_TESTING void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c); #endif /* * Test: 3-node cluster; no standbys converged (0/2) * → role must remain CTRL_PRIMARY_COMMITTING. * Requirements: 2.3, 6.3 (ha-repl-hardening) */ static void test_majority_convergence_3node_none_stays_committing(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 3; /* 3-node cluster: primary + 2 standbys */ atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* No standby converged */ c.ha.standby_converged[0] = false; c.ha.standby_converged[1] = false; ha_check_majority_convergence(&c.ha, &c); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role, "3-node: 0/2 standbys converged → must remain CTRL_PRIMARY_COMMITTING (Req 2.3)"); ha_fini(&c.ha); printf("test_majority_convergence_3node_none_stays_committing: PASS\n"); #else printf("test_majority_convergence_3node_none_stays_committing: SKIP\n"); #endif } /* * Test: 3-node cluster; 1/2 standbys converged → primary + 1 standby = 2/3 * nodes hold the state = majority → must promote to CTRL_PRIMARY. * (Requirement 2.3 formula: ceil(3/2)-1 = 1 converged standby suffices. * A dead minority standby must not force the degraded-mode timeout.) * Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening) */ static void test_majority_convergence_3node_one_standby_promotes(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 3; atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* Only standby[0] converged; standby[1] not converged */ c.ha.standby_converged[0] = true; c.ha.standby_converged[1] = false; ha_check_majority_convergence(&c.ha, &c); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "3-node: 1/2 standbys converged → majority → CTRL_PRIMARY (Req 2.3)"); ha_fini(&c.ha); printf("test_majority_convergence_3node_one_standby_promotes: PASS\n"); #else printf("test_majority_convergence_3node_one_standby_promotes: SKIP\n"); #endif } /* * Test: 5-node cluster; threshold = 5/2 = 2 converged standbys. * 1/4 converged → stays COMMITTING; 2/4 converged → promotes. * Requirements: 2.3, 6.3 (ha-repl-hardening) */ static void test_majority_convergence_5node_threshold(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 5; atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* 1/4 converged: primary + 1 = 2 of 5 — not a majority → stays */ c.ha.standby_converged[0] = true; ha_check_majority_convergence(&c.ha, &c); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, role, "5-node: 1/4 standbys converged → must remain COMMITTING (Req 2.3)"); /* 2/4 converged: primary + 2 = 3 of 5 — majority → promotes */ c.ha.standby_converged[1] = true; ha_check_majority_convergence(&c.ha, &c); role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "5-node: 2/4 standbys converged → majority → CTRL_PRIMARY (Req 2.3)"); ha_fini(&c.ha); printf("test_majority_convergence_5node_threshold: PASS\n"); #else printf("test_majority_convergence_5node_threshold: SKIP\n"); #endif } /* * Test: 3-node cluster (n_peers=2); both standbys converged (2/2) * → role must transition to CTRL_PRIMARY. * Requirements: 2.3, 2.4, 6.3 (ha-repl-hardening) */ static void test_majority_convergence_3node_all_standbys_promotes(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 3; /* 3-node cluster */ atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* Both standbys converged */ c.ha.standby_converged[0] = true; c.ha.standby_converged[1] = true; ha_check_majority_convergence(&c.ha, &c); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "3-node: 2/2 standbys converged → must transition to CTRL_PRIMARY (Req 2.3, 2.4)"); ha_fini(&c.ha); printf("test_majority_convergence_3node_all_standbys_promotes: PASS\n"); #else printf("test_majority_convergence_3node_all_standbys_promotes: SKIP\n"); #endif } /* * Test: single-node cluster (n_peers=0) * → threshold=0, satisfied immediately → must promote to CTRL_PRIMARY. * Requirements: 6.5, 2.3 (ha-repl-hardening): single-node is always satisfied. */ static void test_majority_convergence_single_node_immediate(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 1; /* single-node: just self */ atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* No standbys to converge — threshold=0 must be satisfied immediately */ ha_check_majority_convergence(&c.ha, &c); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "single-node: n_peers=1 → threshold=0 satisfied → must promote to CTRL_PRIMARY (Req 6.5)"); ha_fini(&c.ha); printf("test_majority_convergence_single_node_immediate: PASS\n"); #else printf("test_majority_convergence_single_node_immediate: SKIP\n"); #endif } /* ── Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring ────── * * ha_dispatch_repl_action_for_standby reads the standby's stored * (standby_reported_tag, standby_reported_lsn), calls ha_evaluate_repl_action, * and if the result is HA_REPL_PUSH_CONFIG calls push_config_to_standbys (in * production) or ha_repl_push_config_for_standby_hook (in XCE_TESTING). * * Requirements: 1.4 (ha-repl-hardening), design §Unified Heartbeat-Driven Protocol */ #ifdef XCE_TESTING extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx); #endif /* Call-counting state for the hook */ static int g_push_config_for_standby_count = 0; static int g_push_config_for_standby_idx = -1; static int stub_repl_push_config_for_standby(void *c, int peer_idx) { (void)c; g_push_config_for_standby_count++; g_push_config_for_standby_idx = peer_idx; return 0; } /* * Test: when standby_reported_tag < committed_tag, ha_dispatch_repl_action_for_standby * must call push_config_to_standbys (via hook in test mode). * * Steps: * 1. Set ha->standby_reported_tag[0] = 3 (behind committed_tag=5) * 2. Set ha->standby_reported_lsn[0] = 100 (at committed_lsn) * 3. Call ha_dispatch_repl_action_for_standby(ha, NULL, 0, committed_tag=5, ...) * 4. Assert hook called once with peer_idx=0 * * Requirements: 1.4 (push_config when standby tag < committed_tag) */ static void test_dispatch_repl_action_push_config_when_tag_behind(void) { #ifdef XCE_TESTING ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby; g_push_config_for_standby_count = 0; g_push_config_for_standby_idx = -1; ha_state_t *ha = build_ha_for_convergence_test(); /* standby[0] has stale config tag */ ha->standby_reported_tag[0] = 3; ha->standby_reported_lsn[0] = 100; int rc = ha_dispatch_repl_action_for_standby( ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0); (void)rc; TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_push_config_for_standby_count, "PUSH_CONFIG: hook must fire once when standby_tag < committed_tag"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_idx, "PUSH_CONFIG: hook must receive correct peer_idx=0"); ha_fini(ha); ha_repl_push_config_for_standby_hook = NULL; printf("test_dispatch_repl_action_push_config_when_tag_behind: PASS\n"); #else printf("test_dispatch_repl_action_push_config_when_tag_behind: SKIP (non-test build)\n"); #endif } /* * Test: when standby is converged (tag == committed, lsn == committed), * ha_dispatch_repl_action_for_standby must NOT call push_config hook. * * Requirements: decision tree — CONVERGED does not trigger push */ static void test_dispatch_repl_action_no_push_when_converged(void) { #ifdef XCE_TESTING ha_repl_push_config_for_standby_hook = stub_repl_push_config_for_standby; g_push_config_for_standby_count = 0; g_push_config_for_standby_idx = -1; ha_state_t *ha = build_ha_for_convergence_test(); /* standby[0] is fully converged */ ha->standby_reported_tag[0] = 5; ha->standby_reported_lsn[0] = 100; int rc = ha_dispatch_repl_action_for_standby( ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0); (void)rc; TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_push_config_for_standby_count, "CONVERGED: hook must NOT fire when standby is already converged"); ha_fini(ha); ha_repl_push_config_for_standby_hook = NULL; printf("test_dispatch_repl_action_no_push_when_converged: PASS\n"); #else printf("test_dispatch_repl_action_no_push_when_converged: SKIP (non-test build)\n"); #endif } /* ── Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring ──── * * When standby_lsn < oldest_wal_lsn (standby predates WAL retention window), * ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby * (via ha_repl_send_snapshot_hook in XCE_TESTING builds). * * Requirements: 4.1 (ha-repl-hardening), design §Snapshot Transfer */ #ifdef XCE_TESTING extern int (*ha_repl_send_snapshot_hook)(int peer_idx); #endif /* Call-counting state for the snapshot hook */ static int g_send_snapshot_count = 0; static int g_send_snapshot_idx = -1; static int stub_repl_send_snapshot(int peer_idx) { g_send_snapshot_count++; g_send_snapshot_idx = peer_idx; return 0; } /* * Test: when standby_lsn < oldest_wal_lsn (and standby_tag == committed_tag), * ha_dispatch_repl_action_for_standby must call ha_send_snapshot_to_standby * (via hook in test mode). * * Steps: * 1. Set ha->standby_reported_tag[0] = committed_tag (tag is current) * 2. Set ha->standby_reported_lsn[0] = 10 (behind oldest_wal_lsn=50) * 3. Call ha_dispatch_repl_action_for_standby(..., oldest_wal_lsn=50) * 4. Assert hook called once with peer_idx=0 * * Requirements: 4.1 (snapshot when standby_lsn < oldest_wal_lsn) */ static void test_dispatch_repl_action_send_snapshot(void) { #ifdef XCE_TESTING ha_repl_send_snapshot_hook = stub_repl_send_snapshot; g_send_snapshot_count = 0; g_send_snapshot_idx = -1; ha_state_t *ha = build_ha_for_convergence_test(); /* standby[0] has current config tag but LSN predates WAL window */ ha->standby_reported_tag[0] = 5; ha->standby_reported_lsn[0] = 10; /* < oldest_wal_lsn=50 */ int rc = ha_dispatch_repl_action_for_standby( ha, /*c=*/NULL, /*ep=*/NULL, /*peer_idx=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); (void)rc; TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_send_snapshot_count, "SEND_SNAPSHOT: hook must fire once when standby_lsn < oldest_wal_lsn"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_send_snapshot_idx, "SEND_SNAPSHOT: hook must receive correct peer_idx=0"); ha_fini(ha); ha_repl_send_snapshot_hook = NULL; printf("test_dispatch_repl_action_send_snapshot: PASS\n"); #else printf("test_dispatch_repl_action_send_snapshot: SKIP (non-test build)\n"); #endif } /* ── Task 3.2: ha_evaluate_repl_action unified decision tree ─────────────── */ /* * Declare the function under test. It is defined in ha.c. * Pure function — no side effects — so no XCE_TESTING guard needed. */ ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag, uint64_t standby_lsn, uint64_t committed_tag, uint64_t committed_lsn, uint64_t oldest_wal_lsn); /* * Test: (tag=0, lsn=0) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → PUSH_CONFIG (tag behind). * Requirements: 2.2, 2.3 (ha-repl-hardening) */ static void test_evaluate_repl_action_push_config(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/0, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_PUSH_CONFIG, action); printf("test_evaluate_repl_action_push_config: PASS\n"); } /* * Test: (tag=5, lsn=0) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn=0 is below WAL retention window). * Requirements: 2.2, 2.3 */ static void test_evaluate_repl_action_send_snapshot_lsn_zero(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); /* lsn=0 < oldest_wal_lsn=50 → SEND_SNAPSHOT */ TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action); printf("test_evaluate_repl_action_send_snapshot_lsn_zero: PASS\n"); } /* * Test: (tag=5, lsn=100) with committed_tag=5, committed_lsn=100 → CONVERGED. * Requirements: 2.2, 2.4 */ static void test_evaluate_repl_action_converged(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/100, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, action); printf("test_evaluate_repl_action_converged: PASS\n"); } /* * Test: (tag=5, lsn=10) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → SEND_SNAPSHOT (lsn < oldest_wal). * Requirements: 2.2, 2.3 */ static void test_evaluate_repl_action_send_snapshot(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/10, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_SEND_SNAPSHOT, action); printf("test_evaluate_repl_action_send_snapshot: PASS\n"); } /* * Test: (tag=5, lsn=50) with committed_tag=5, committed_lsn=100, * oldest_wal_lsn=50 → STREAM_WAL (lsn >= oldest_wal but < committed). * Requirements: 2.2, 2.3 */ static void test_evaluate_repl_action_stream_wal(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/50, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/50); TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, action); printf("test_evaluate_repl_action_stream_wal: PASS\n"); } /* * Test: (tag=T, lsn=0) with oldest_wal_lsn=0 (fresh system, no WAL entries yet) * → STREAM_WAL (lsn=0 >= oldest_wal_lsn=0, so not in snapshot territory; * lsn=0 < committed_lsn → stream WAL from the beginning). * * This is the canonical Task 6.1 case: a standby that has caught up on config * (tag matches) but has no WAL applied yet on a fresh system. When there is no * WAL retention window to worry about (oldest_wal_lsn == 0), the correct action * is STREAM_WAL, not SEND_SNAPSHOT. * * Requirements: 1.1, 1.2, 2.2, 2.3 (ha-repl-hardening) */ static void test_evaluate_repl_action_stream_wal_fresh_system(void) { ha_repl_action_t action = ha_evaluate_repl_action( /*standby_tag=*/5, /*standby_lsn=*/0, /*committed_tag=*/5, /*committed_lsn=*/100, /*oldest_wal_lsn=*/0); /* lsn=0 >= oldest_wal_lsn=0 → NOT snapshot; lsn=0 < committed → STREAM_WAL */ TEST_ASSERT_EQUAL_INT_MESSAGE(HA_REPL_STREAM_WAL, action, "(tag=T, lsn=0) with oldest_wal_lsn=0 must route to STREAM_WAL (Req 1.1, 1.2)"); printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n"); } /* ── Task 5.3: committing timeout default value and degraded-mode WARN ─────── * * Requirements: 6.4, 7.5 (ha-repl-hardening) * * Verifies: * - HA_DEFAULT_COMMITTING_TIMEOUT_MS == 60000 (safety-net window is 60 s) * - committing_timeout_fn_for_test still promotes to CTRL_PRIMARY in degraded * mode even when only a subset of standbys have converged * - The ha->standby_converged[] state is readable so the WARN log can include * the converged/pending counts */ /* * Test: HA_DEFAULT_COMMITTING_TIMEOUT_MS is 60000 (not 10000). * Requirements: 6.4 (ha-repl-hardening) — "Change committing_timeout_ms default from 10000 to 60000" */ static void test_committing_timeout_default_is_60s(void) { TEST_ASSERT_EQUAL_UINT32_MESSAGE(60000, HA_DEFAULT_COMMITTING_TIMEOUT_MS, "HA_DEFAULT_COMMITTING_TIMEOUT_MS must be 60000 (Req 6.4, Task 5.3)"); printf("test_committing_timeout_default_is_60s: PASS\n"); } /* * Test: committing_timeout_fn promotes to PRIMARY even when only some standbys * have converged (degraded mode). The WARN log path fires when timeout expires * before majority convergence. After promotion, role must be CTRL_PRIMARY. * * Requirements: 6.4, 7.5 (ha-repl-hardening) */ #ifdef XCE_TESTING extern int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c, int cancelled_val); #endif static void test_committing_timeout_promotes_degraded_with_partial_convergence(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 3; /* 3-node cluster; 2 standbys */ atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* Only 1 of 2 standbys has converged — majority not yet reached */ c.ha.standby_converged[0] = true; c.ha.standby_converged[1] = false; /* * Count converged/pending so the test mirrors what the WARN log reads. * n_converged=1, n_pending=1 (Req 7.5). */ int n_converged = 0; int n_pending = 0; for (int i = 0; i < c.ha.n_peers - 1; i++) { if (c.ha.standby_converged[i]) n_converged++; else n_pending++; } TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_converged, "degraded-mode: 1 standby converged before timeout"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, n_pending, "degraded-mode: 1 standby still pending before timeout"); /* Fire the timeout — must promote despite partial convergence */ committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "committing timeout: must promote to CTRL_PRIMARY in degraded mode (Req 6.4)"); ha_fini(&c.ha); printf("test_committing_timeout_promotes_degraded_with_partial_convergence: PASS " "(n_converged=%d, n_pending=%d)\n", n_converged, n_pending); #else printf("test_committing_timeout_promotes_degraded_with_partial_convergence: SKIP\n"); #endif } /* * Test: committing_timeout_fn promotes when NO standbys have converged * (worst-case degraded mode — all standbys unreachable). * Requirements: 6.4 (ha-repl-hardening) */ static void test_committing_timeout_promotes_degraded_zero_converged(void) { #ifdef XCE_TESTING cluster_t c; memset(&c, 0, sizeof(c)); ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; ha_init(&c.ha, &cfg, 0); c.ha.n_peers = 3; /* 3-node cluster; all standbys unreachable */ atomic_store_explicit(&c.ha.role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); /* No standbys converged — n_converged=0, n_pending=2 (Req 7.5) */ c.ha.standby_converged[0] = false; c.ha.standby_converged[1] = false; committing_timeout_fn_for_test(&c.ha, &c, 0 /* not cancelled */); int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, role, "committing timeout: must promote even with 0 converged standbys (degraded, Req 6.4)"); ha_fini(&c.ha); printf("test_committing_timeout_promotes_degraded_zero_converged: PASS\n"); #else printf("test_committing_timeout_promotes_degraded_zero_converged: SKIP\n"); #endif } /* ── Task 3.3 (ha-wal-replication): dispatch gate allows OPCode_walCatchupEntry * in CTRL_STANDBY, CTRL_ELECTING, and CTRL_INIT. * * Requirements: ha-wal-replication Req 3.3 (standby role guard). * Verifies that ha_dispatch_gate(ha, OPCode_walCatchupEntry) returns 0 * (allow) for the three roles where a standby legitimately receives WAL * catch-up entries from the primary. CTRL_PRIMARY and * CTRL_PRIMARY_COMMITTING must NOT allow it (they never receive WAL pushes). */ static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void) { ha_state_t ha; ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, .committing_timeout_ms = HA_DEFAULT_COMMITTING_TIMEOUT_MS, }; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* CTRL_STANDBY — must allow */ atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_walCatchupEntry), "CTRL_STANDBY must allow OPCode_walCatchupEntry (Req 3.3)"); /* CTRL_ELECTING — must allow */ atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_walCatchupEntry), "CTRL_ELECTING must allow OPCode_walCatchupEntry (Req 3.3)"); /* CTRL_INIT — must allow */ atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_dispatch_gate(&ha, OPCode_walCatchupEntry), "CTRL_INIT must allow OPCode_walCatchupEntry (Req 3.3)"); ha_fini(&ha); printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n"); } void test_ha_role(void) { RUN_TEST(test_ha_init_default_role_is_standby); RUN_TEST(test_ha_init_rejects_fast_heartbeat); RUN_TEST(test_ha_init_rejects_short_election_timeout); RUN_TEST(test_ha_init_rejects_timeout_less_than_interval); RUN_TEST(test_ha_promote_sets_primary_role); /* P1-3a */ RUN_TEST(test_promote_increments_term); RUN_TEST(test_demote_sets_standby_role); /* Watchdog / SIGUSR1 tests deleted */ /* Task 2.1: election-iomgr-refactor new fields */ RUN_TEST(test_ha_init_zeroes_new_fields); /* Task 1.4: ha-repl-hardening per-standby convergence fields */ RUN_TEST(test_ha_init_zeroes_standby_convergence_fields); /* Task 3.1: ha_process_standby_state convergence tracking */ RUN_TEST(test_process_standby_state_converged); RUN_TEST(test_process_standby_state_lsn_behind); RUN_TEST(test_process_standby_state_tag_behind); RUN_TEST(test_process_standby_state_regression); RUN_TEST(test_process_standby_state_multiple_peers); /* Task 3.2: ha_evaluate_repl_action unified decision tree */ RUN_TEST(test_evaluate_repl_action_push_config); RUN_TEST(test_evaluate_repl_action_send_snapshot_lsn_zero); RUN_TEST(test_evaluate_repl_action_converged); RUN_TEST(test_evaluate_repl_action_send_snapshot); RUN_TEST(test_evaluate_repl_action_stream_wal); /* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */ RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system); /* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */ RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind); RUN_TEST(test_dispatch_repl_action_no_push_when_converged); /* Task 3.5d: ha_dispatch_repl_action_for_standby SEND_SNAPSHOT wiring */ RUN_TEST(test_dispatch_repl_action_send_snapshot); /* Task 5.2: ha_check_majority_convergence → PRIMARY transition */ RUN_TEST(test_majority_convergence_3node_none_stays_committing); RUN_TEST(test_majority_convergence_3node_one_standby_promotes); RUN_TEST(test_majority_convergence_5node_threshold); RUN_TEST(test_majority_convergence_3node_all_standbys_promotes); RUN_TEST(test_majority_convergence_single_node_immediate); /* Task 5.3: committing timeout default value and degraded-mode WARN */ RUN_TEST(test_committing_timeout_default_is_60s); RUN_TEST(test_committing_timeout_promotes_degraded_with_partial_convergence); RUN_TEST(test_committing_timeout_promotes_degraded_zero_converged); /* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */ RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles); } /* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */