/* * test_ha_failover_prevote.c — ha-prevote task 4.2: * route the failover triggers (RDHUP + liveness timeout) through PreVote and * defer the destructive step (truncate + CTRL_ELECTING) until PreVote WINS; * cancel an in-progress round when a primary heartbeat arrives mid-round. * * This is the rewire that activates PreVote on the LIVE failover path. The * NEW behaviour these tests pin down (which FAILS before the rewire, when the * triggers elect directly): * * (a) Firing the RDHUP failover trigger does NOT transition to CTRL_ELECTING * or increment the term — it starts a side-effect-free PreVote round and * the node stays CTRL_STANDBY with pre_vote_in_progress = 1. * (b) Firing the liveness-timeout trigger goes through the SAME PreVote-first * path (it inherits the rewired ha_on_primary_ctrl_fd_rdhup). * (c) A primary heartbeat arriving mid-round CANCELS the round and re-attaches * as standby with the persistent term unchanged (Req 4.2). * (d) When the PreVote round then WINS (a peer grants), the trigger path still * reaches the real election (CTRL_ELECTING) — failover is not blocked. * (e) The config-less voter-only guard (tag==0, peers>1) still trips and the * node does NOT even start a PreVote round (it can never win one). * * Boundary: PreVote orchestration (trigger path) — ha.c only. * Requirements: 1.1, 4.2, 4.3, 4.4. * * TDD: written FIRST (RED) before the trigger rewire. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include "ha.h" #include "myconf.h" #include "raft/raft.h" #include "xce/cluster_def.h" /* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */ extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c); extern void ha_on_heartbeat_received(ha_state_t *ha); extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c); /* * liveness_poll_ctx_t: defined privately in ha.c; the layout is mirrored here * (exactly as test_ha_liveness.c does) so the test can build one for the * test entry-point without a live delaymgr. */ typedef struct { ha_state_t *ha; cluster_t *c; _Atomic int cancelled; } liveness_poll_ctx_t; extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx); /* ── Test config ─────────────────────────────────────────────────────────── */ static ha_config_t g_cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 6000, .election_timeout_ms = 500, .probe_connect_timeout_ms = 50, .max_promotion_retries = 3, .election_jitter_max_ms = 0, .election_priority_step_ms = 500, }; static int g_rearm_calls = 0; static int stub_rearm(ha_state_t *ha, cluster_t *c) { (void)ha; (void)c; g_rearm_calls++; return 0; } /* * Widen the raft topology to a 3-node cluster (self=0) so a PreVote round * does NOT trivially self-win (quorum = 2; self-grant = 1). This lets the * round STAY in progress so we can observe the deferred / cancel behaviour. */ static void make_three_node(ha_state_t *ha) { ha->raft.n_peers = 3; ha->raft.self_idx = 0; ha->raft.role = RAFT_FOLLOWER; ha->n_peers = 3; } /* ── Test 1: RDHUP trigger defers the destructive step (Req 1.1, 4.4) ─────── */ /* * Firing ha_on_primary_ctrl_fd_rdhup must NO LONGER transition straight to * CTRL_ELECTING / truncate / increment the term. Instead it begins a PreVote * round and stays CTRL_STANDBY with the round in progress. (Unit-test mode, * c == NULL: ha_enter_real_election — which contains truncate + CTRL_ELECTING — * must NOT have run, so the role stays CTRL_STANDBY.) */ static void test_rdhup_trigger_starts_prevote_not_election(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY */ make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); /* Give the handler an fd to close (teardown bookkeeping still happens). */ int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&ha, NULL); /* unit-test mode */ close(sv[1]); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 1.1/4.4: RDHUP trigger must start a PreVote round and STAY " "CTRL_STANDBY, not transition to CTRL_ELECTING"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.4: RDHUP trigger must not change the persistent term before " "PreVote wins"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "Req 1.1: RDHUP trigger must begin a PreVote round (in-progress)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "PreVote round must record the self-grant (1)"); /* The primary fd must still be closed/cleared — teardown bookkeeping stays * at the trigger regardless of PreVote outcome. */ TEST_ASSERT_EQUAL_INT_MESSAGE(-1, ha.primary_ctrl_fd, "RDHUP teardown must still close + clear primary_ctrl_fd"); ha_fini(&ha); printf("test_rdhup_trigger_starts_prevote_not_election: PASS\n"); } /* ── Test 2: liveness-timeout trigger goes through the same PreVote path ──── */ /* * The liveness poll fn, on timeout, must reach the SAME PreVote-first path * (it calls ha_on_primary_ctrl_fd_rdhup, now rewired). Observable: after the * timeout, the node is in a PreVote round and STAYS CTRL_STANDBY — it does NOT * jump to CTRL_ELECTING. */ static void test_liveness_timeout_trigger_starts_prevote(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); /* last_activity far enough in the past to exceed heartbeat_timeout_ms. */ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); ha.last_activity = ts.tv_sec - (time_t)(g_cfg.heartbeat_timeout_ms / 1000) - 10; liveness_poll_ctx_t ctx; memset(&ctx, 0, sizeof(ctx)); ctx.ha = &ha; ctx.c = NULL; atomic_init(&ctx.cancelled, 0); liveness_poll_fn_test_entry(&ctx); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 1.1: liveness-timeout failover must start a PreVote round and stay " "CTRL_STANDBY, not transition straight to CTRL_ELECTING"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.4: liveness-timeout failover must not change the term before " "PreVote wins"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "Req 1.1: liveness-timeout failover must begin a PreVote round"); ha_fini(&ha); printf("test_liveness_timeout_trigger_starts_prevote: PASS\n"); } /* ── Test 3: heartbeat mid-round cancels and re-attaches (Req 4.2) ────────── */ /* * After the failover trigger starts a PreVote round, an incoming primary * heartbeat means the leader is alive — the campaign must be abandoned: the * round is cancelled (pre_vote_in_progress cleared), the node re-attaches as * CTRL_STANDBY, and the persistent term is UNCHANGED (Req 4.2). */ static void test_heartbeat_mid_round_cancels_prevote(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; /* Trigger fires → PreVote round in progress. */ ha_on_primary_ctrl_fd_rdhup(&ha, NULL); close(sv[1]); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: a PreVote round must be in progress before the heartbeat"); /* A fresh primary heartbeat arrives mid-round → cancel + re-attach. */ ha_on_heartbeat_received(&ha); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "Req 4.2: a heartbeat mid-round must CANCEL the PreVote round"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.2: a mid-round heartbeat must not change the persistent term"); ha_fini(&ha); printf("test_heartbeat_mid_round_cancels_prevote: PASS\n"); } /* ── Test 4: heartbeat with NO round in progress is a plain no-op ─────────── */ /* * The cancel logic must only fire when a round is actually in progress: a * heartbeat received while no PreVote round is running must just update * last_activity (the existing behaviour) and never touch role/term. */ static void test_heartbeat_without_round_is_plain_noop(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress); ha_on_heartbeat_received(&ha); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "no round in progress: heartbeat must leave pre_vote_in_progress = 0"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "no round: heartbeat must keep the node CTRL_STANDBY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "no round: heartbeat must not change the term"); TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity, "heartbeat must still update last_activity (existing behaviour preserved)"); ha_fini(&ha); printf("test_heartbeat_without_round_is_plain_noop: PASS\n"); } /* ── Test 5: a genuine failover (PreVote wins) still reaches real election ── */ /* * The whole point of PreVote is to defer, not block. After the RDHUP trigger * starts the round, a peer GRANT reaching quorum must drive the trigger path * into the real election (CTRL_ELECTING) — failover still proceeds when the * primary genuinely died. */ extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, int peer_idx, int granted); static void test_failover_prevote_win_reaches_real_election(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&ha, NULL); close(sv[1]); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress); /* Peer index 1 grants → quorum (2) reached → WON → real election. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 4.3/5.x: when PreVote WINS the trigger path must reach the real " "election (CTRL_ELECTING) — a genuine failover is not blocked"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "a WON round must clear the in-progress flag"); ha_fini(&ha); printf("test_failover_prevote_win_reaches_real_election: PASS (role=%d)\n", role); } /* ── Test 6: config-less voter-only guard still trips before PreVote ──────── */ /* * A config-less node (config_version_tag == 0, peers > 1) must NOT even start * a PreVote round on RDHUP — it can never win one (the freshness gate denies * it), and it must stay a passive voter (CTRL_STANDBY). This preserves the * ha-config-bootstrap Req 2.8 election-safety guard through the rewire. * * Requires a real cluster_t with a tag-0 config so the guard path is reached. */ static void test_config_less_node_does_not_start_prevote(void) { cluster_t c; myconf_t myconf; memset(&c, 0, sizeof(c)); memset(&myconf, 0, sizeof(myconf)); myconf.name = (char *)"ctrl-a"; myconf.config_version_tag = 0; /* config-less */ myconf.peers_cfg.candidates_count = 2; /* >1 node */ snprintf(myconf.peers_cfg.candidates[0].name, sizeof(myconf.peers_cfg.candidates[0].name), "ctrl-a"); snprintf(myconf.peers_cfg.candidates[1].name, sizeof(myconf.peers_cfg.candidates[1].name), "ctrl-b"); c.myconf = &myconf; TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); ha_do_demote(&c.ha); c.ha.phase2_done = 1; c.ha.n_peers = 1; make_three_node(&c.ha); /* widen raft so a round would NOT self-win */ int sv[2]; TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); c.ha.primary_ctrl_fd = sv[0]; ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); close(sv[1]); TEST_ASSERT_EQUAL_INT_MESSAGE(0, c.ha.raft.pre_vote_in_progress, "config-less node (tag=0, peers>1) must NOT start a PreVote round on RDHUP"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)atomic_load_explicit(&c.ha.role, memory_order_acquire), "config-less RDHUP node must stay CTRL_STANDBY (voter only)"); ha_fini(&c.ha); printf("test_config_less_node_does_not_start_prevote: PASS\n"); } /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_failover_prevote(void) { UnitySetTestFile(__FILE__); /* Ensure the round LOST/timeout branch never touches a live delaymgr. */ g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; RUN_TEST(test_rdhup_trigger_starts_prevote_not_election); RUN_TEST(test_liveness_timeout_trigger_starts_prevote); RUN_TEST(test_heartbeat_mid_round_cancels_prevote); RUN_TEST(test_heartbeat_without_round_is_plain_noop); RUN_TEST(test_failover_prevote_win_reaches_real_election); RUN_TEST(test_config_less_node_does_not_start_prevote); ha_prevote_rearm_liveness_hook = NULL; }