/* * test_ha_failover.c — Unit tests for task 9.4: EPOLLRDHUP failover and * heartbeat timeout vtask. * * Tests: * 1. ha_on_primary_ctrl_fd_rdhup closes fd, sets to -1, transitions to CTRL_ELECTING * 2. ha_on_primary_ctrl_fd_rdhup calls persist_truncate_wal_above * 3. ha_arm_heartbeat_timeout_vtask fires failover after silence * 4. ha_on_heartbeat_received resets the timeout (no failover on heartbeat) * 5. FSM: CTRL_STANDBY after election → primary_ctrl_fd closed * 6. INFO log on FSM transition (Req 10.5) * * Requirements: 6.2, 6.3, 6.4, 6.8, 10.5 */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include "ha.h" #include "xce/cluster_def.h" /* full cluster_t layout for task-5.2 tests */ #include "myconf.h" /* myconf_t for task-5.2 tests */ /* ── Forward declarations — function-under-test (not yet implemented) ────── */ /* * ha_on_primary_ctrl_fd_rdhup: EPOLLRDHUP event handler for primary_ctrl_fd. * * Steps (Req 6.2, 6.3): * 1. Close and null ha->primary_ctrl_fd * 2. Transition ha role to CTRL_ELECTING * 3. Call persist_truncate_wal_above(ha->last_known_committed_lsn) via cluster * 4. Log INFO: FSM transition (Req 10.5) * * Pass c == NULL to skip persist_truncate and am_i_primary_quorum (unit test mode). */ extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c); /* * ha_arm_heartbeat_timeout_vtask: start background thread monitoring heartbeat * silence on primary_ctrl_fd (Req 6.3, 6.4). * * The thread sleeps for election_timeout_ms + rand_jitter. If no heartbeat * is received (ha_on_heartbeat_received not called) within that window, * it fires ha_on_primary_ctrl_fd_rdhup. * * Subsequent calls re-arm the timeout (same thread, reset deadline). * Calling ha_on_heartbeat_received() resets the deadline. */ extern void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c); /* * ha_on_heartbeat_received: reset the heartbeat timeout deadline. * Called whenever a heartbeat arrives on primary_ctrl_fd (Req 6.3). */ extern void ha_on_heartbeat_received(ha_state_t *ha); /* * ha_disarm_heartbeat_timeout_vtask: stop the heartbeat timeout thread. * Called when transitioning away from CTRL_STANDBY. */ extern void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha); /* ── Helpers ─────────────────────────────────────────────────────────────── */ static ha_config_t g_cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 500, /* minimum allowed by ha_init validation */ .probe_connect_timeout_ms = 50, .max_promotion_retries = 3, .election_jitter_max_ms = 0, /* no jitter: deterministic timeout */ .election_priority_step_ms = 500, }; /* ── Test 1: RDHUP closes fd and sets primary_ctrl_fd = -1 ─────────────── */ static void test_rdhup_closes_and_nulls_fd(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1)); /* Set up a socketpair as the "primary_ctrl_fd" */ int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; /* Fire RDHUP with NULL cluster (unit test mode — skips persist/election) */ ha_on_primary_ctrl_fd_rdhup(&ha, NULL); /* fd must be closed and set to -1 */ TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd); /* Verify sv[0] is really closed by trying to write to it */ ssize_t n = write(sv[0], "x", 1); TEST_ASSERT_LESS_THAN_INT(0, (int)n); TEST_ASSERT_EQUAL_INT(EBADF, errno); close(sv[1]); ha_fini(&ha); printf("test_rdhup_closes_and_nulls_fd: PASS\n"); } /* ── Test 2: RDHUP transitions role to CTRL_ELECTING ───────────────────── */ static void test_rdhup_transitions_to_ctrl_electing(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1)); /* Start as CTRL_STANDBY */ ha_do_demote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role)); /* Simulate having a primary fd */ int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&ha, NULL); /* Must transition to CTRL_ELECTING */ TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role)); close(sv[1]); ha_fini(&ha); printf("test_rdhup_transitions_to_ctrl_electing: PASS\n"); } /* ── Test 3: heartbeat received before timeout → no failover ───────────── */ static void test_heartbeat_received_resets_timeout(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1)); ha_do_demote(&ha); int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; /* Arm the timeout (200ms + jitter=0) */ ha_arm_heartbeat_timeout_vtask(&ha, NULL); /* Immediately receive heartbeat to reset the deadline */ ha_on_heartbeat_received(&ha); /* Wait briefly — less than one full election_timeout_ms */ struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000000L }; nanosleep(&ts, NULL); /* Role must still be CTRL_STANDBY (no failover yet) */ TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role)); TEST_ASSERT_NOT_EQUAL(-1, ha.primary_ctrl_fd); ha_disarm_heartbeat_timeout_vtask(&ha); close(sv[0]); close(sv[1]); ha.primary_ctrl_fd = -1; ha_fini(&ha); printf("test_heartbeat_received_resets_timeout: PASS\n"); } /* ── Test 4: heartbeat timeout fires failover ───────────────────────────── */ static void test_heartbeat_timeout_fires_failover(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1)); ha_do_demote(&ha); int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; /* Arm with 200ms timeout, NO heartbeat received */ ha_arm_heartbeat_timeout_vtask(&ha, NULL); /* Wait 800ms — well past the 500ms election_timeout_ms */ struct timespec ts = { .tv_sec = 0, .tv_nsec = 800 * 1000000L }; nanosleep(&ts, NULL); /* Failover must have fired: role = CTRL_ELECTING, fd = -1 */ TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role)); TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd); ha_disarm_heartbeat_timeout_vtask(&ha); close(sv[1]); /* sv[0] was closed by failover handler */ ha_fini(&ha); printf("test_heartbeat_timeout_fires_failover: PASS\n"); } /* ── Test 5: ha_on_primary_ctrl_fd_rdhup on already-closed fd is safe ───── */ static void test_rdhup_on_minus_one_fd_is_safe(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1)); ha.primary_ctrl_fd = -1; /* Should not crash or abort */ ha_on_primary_ctrl_fd_rdhup(&ha, NULL); TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd); ha_fini(&ha); printf("test_rdhup_on_minus_one_fd_is_safe: PASS\n"); } /* ── Task 5.2: RDHUP → CTRL_ELECTING → ha_start_election_async ─────────── */ /* * ha_rdhup_election_hook: XCE_TESTING seam — set in ha.c under XCE_TESTING. * When non-NULL, ha_on_primary_ctrl_fd_rdhup calls this instead of the real * ha_start_election_async. We use it to verify the RDHUP path triggers * async re-election (Req 6.3, 6.5, 9.3). */ extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name); /* Counters set by the hook so the test can inspect the call */ static int g_rdhup_hook_called = 0; static int g_rdhup_hook_n_peers = -1; static ha_state_t *g_rdhup_hook_ha = NULL; static void rdhup_election_spy(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) { (void)c; (void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name; g_rdhup_hook_called++; g_rdhup_hook_n_peers = n_peers; g_rdhup_hook_ha = ha; } /* * test_rdhup_calls_start_election_async: * Req 6.3: heartbeat timeout → ELECTING → async election fires. * Req 9.3: RDHUP path uses ha_start_election_async (not am_i_primary_quorum). * * We supply a minimal cluster_t with 2 candidates so the RDHUP handler * builds a peer array and calls ha_start_election_async via the hook. * The test also verifies role is CTRL_ELECTING after the handler. * * myconf_t.name is char* — use a string literal. * myhost == NULL → self-name compare falls back to "" → both candidates are * included in peer_ptrs (np == 2). That is fine for verifying the hook fires. */ static void test_rdhup_calls_start_election_async(void) { /* Minimal myconf with 2 candidates (self + 1 peer). * myconf_t.name is char* — point at a static string. */ myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.name = "ctrl-a"; myconf.peers_cfg.candidates_count = 2; snprintf(myconf.peers_cfg.candidates[0].name, sizeof(myconf.peers_cfg.candidates[0].name), "ctrl-a"); myconf.peers_cfg.candidates[0].port = 7001; snprintf(myconf.peers_cfg.candidates[1].name, sizeof(myconf.peers_cfg.candidates[1].name), "ctrl-b"); myconf.peers_cfg.candidates[1].port = 7002; /* Minimal cluster_t — ha embedded, myhost NULL (unit test mode) */ cluster_t c; memset(&c, 0, sizeof(c)); c.myconf = &myconf; /* Initialise the embedded ha_state_t directly */ TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0 /* priority=0 */)); ha_do_demote(&c.ha); /* Mark phase2 done — simulates a surviving standby after first election */ c.ha.phase2_done = 1; /* Set up a socketpair as the "primary_ctrl_fd" */ int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; /* Install spy hook */ g_rdhup_hook_called = 0; g_rdhup_hook_n_peers = -1; g_rdhup_hook_ha = NULL; ha_rdhup_election_hook = rdhup_election_spy; /* Fire RDHUP with the stub cluster */ ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); /* Restore hook */ ha_rdhup_election_hook = NULL; /* Verify: role must be CTRL_ELECTING */ TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&c.ha.role)); /* Verify: fd must be closed and set to -1 */ TEST_ASSERT_EQUAL_INT(-1, c.ha.primary_ctrl_fd); /* Verify: ha_start_election_async (hook) was called exactly once */ TEST_ASSERT_EQUAL_INT(1, g_rdhup_hook_called); /* Verify: n_peers > 0 (loop ran; myhost==NULL so both candidates included) */ TEST_ASSERT_GREATER_THAN_INT(0, g_rdhup_hook_n_peers); close(sv[1]); /* sv[0] was closed by the RDHUP handler */ ha_fini(&c.ha); printf("test_rdhup_calls_start_election_async: PASS\n"); } /* * test_rdhup_phase2_done_preserved: * Req 9.3: phase2_done flag must remain 1 after RDHUP so ha_elect_evaluate_and_signal * skips cluster_init_phase2 on the failover election. */ static void test_rdhup_phase2_done_preserved(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); ha.phase2_done = 1; /* set by first-boot election */ /* Fire RDHUP in NULL-cluster mode (unit-test shortcircuit) */ ha_on_primary_ctrl_fd_rdhup(&ha, NULL); /* phase2_done must still be 1 — RDHUP does NOT reset it */ TEST_ASSERT_EQUAL_INT(1, ha.phase2_done); ha_fini(&ha); printf("test_rdhup_phase2_done_preserved: PASS\n"); } /* ── Test 6: symbol existence check ─────────────────────────────────────── */ static void test_ha_failover_symbols_exist(void) { typedef void (*rdhup_fn)(ha_state_t *, cluster_t *); typedef void (*arm_fn)(ha_state_t *, cluster_t *); typedef void (*hb_fn)(ha_state_t *); typedef void (*disarm_fn)(ha_state_t *); rdhup_fn f1 = ha_on_primary_ctrl_fd_rdhup; arm_fn f2 = ha_arm_heartbeat_timeout_vtask; hb_fn f3 = ha_on_heartbeat_received; disarm_fn f4 = ha_disarm_heartbeat_timeout_vtask; TEST_ASSERT_NOT_NULL((void*)f1); TEST_ASSERT_NOT_NULL((void*)f2); TEST_ASSERT_NOT_NULL((void*)f3); TEST_ASSERT_NOT_NULL((void*)f4); printf("test_ha_failover_symbols_exist: PASS\n"); } /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_failover(void) { UNITY_BEGIN(); RUN_TEST(test_rdhup_closes_and_nulls_fd); RUN_TEST(test_rdhup_transitions_to_ctrl_electing); RUN_TEST(test_heartbeat_received_resets_timeout); RUN_TEST(test_heartbeat_timeout_fires_failover); RUN_TEST(test_rdhup_on_minus_one_fd_is_safe); RUN_TEST(test_ha_failover_symbols_exist); RUN_TEST(test_rdhup_calls_start_election_async); RUN_TEST(test_rdhup_phase2_done_preserved); UNITY_END(); }