myxce/src/controller/testsuite/test_ha_prevote.c

/*
 * test_ha_prevote.c — Unit tests for ha-prevote task 4.1:
 * controller candidate-side PreVote orchestration in ha.c.
 *
 * Scope (task 4.1 only):
 *   - ha_start_prevote_async: starts a side-effect-free round (no persist, no
 *     role change, stays CTRL_STANDBY), arms a round timeout.
 *   - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches —
 *       WON  → real-election sequence (truncate + CTRL_ELECTING + start election)
 *       LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment
 *       PENDING → no-op
 *   - round-timeout firing = LOST branch.
 *
 * These tests drive the reply path through the XCE_TESTING test seam
 * (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the
 * existing vote-reply hook pattern.  ha.c is unit-test-linked, so the real
 * orchestration logic is exercised directly.
 *
 * Requirements: 1.1, 1.4, 1.5, 5.4, 7.3
 * Boundary: PreVote orchestration (ha.c)
 *
 * TDD: written first (RED) to specify the exact behaviour before implementation.
 */
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include <unity/unity.h>

#include "ha.h"
#include "netio.h"
#include "raft/raft.h"
#include "xce/cluster_def.h"

/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */

/*
 * ha_start_prevote_async: start a PreVote round.  No persist, no role change.
 * c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op,
 * raft_begin_pre_vote's send callback returns early in test mode).
 */
extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c);

/*
 * pre_vote_reply_dispatch_for_test: test entry point that calls the real
 * pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without
 * a live iomgr.  Mirrors the production reply path.
 */
extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
                                             int peer_idx, int granted);

/*
 * ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task
 * firing before a decision (drives the LOST branch).
 */
extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c);

/* Liveness arm-count hook so tests can prove the watchdog was re-armed on a
 * LOST outcome without a live delaymgr. */
extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);

/* ── Task 5.1 transport-route seams ──────────────────────────────────────── */

/*
 * pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL
 * peer_ctrl_msg_handler decode/route path while CTRL_STANDBY + round in
 * progress (the production condition).  Proves the reply is NOT dropped and
 * reaches pre_vote_reply_dispatch — end-to-end, not via the test hook.
 */
extern int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt);

/* vote_reply_build_pkt_for_test: encode a RequestVoteReply capnp packet (the
 * preVoteReply reuses the identical payload / union arm). */
extern io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
                                                  uint64_t peer_tag,
                                                  uint64_t peer_lsn);

/* vote_ep_ctx_build_for_test: allocate a vote_ep_ctx_t and wire ep->data with
 * the ABSOLUTE peer_idx (the index pre_vote_reply_dispatch expects). */
extern endpoint_t *vote_ep_ctx_build_for_test(endpoint_t    *ep,
                                              ha_state_t     *ha,
                                              ha_elect_ctx_t *ctx,
                                              int             peer_idx,
                                              uint64_t        my_term,
                                              uint64_t        my_tag,
                                              uint64_t        my_lsn,
                                              const char     *my_name);

/* ── Test config ─────────────────────────────────────────────────────────── */

static ha_config_t g_cfg = {
    .heartbeat_interval_ms     = 3000,
    .heartbeat_timeout_ms      = 6000,
    .election_timeout_ms       = 500,
    .probe_connect_timeout_ms  = 50,
    .max_promotion_retries     = 3,
    .election_jitter_max_ms    = 0,
    .election_priority_step_ms = 500,
};

/* Re-arm tracking */
static int g_rearm_calls = 0;
static int stub_rearm(ha_state_t *ha, cluster_t *c)
{
    (void)ha; (void)c;
    g_rearm_calls++;
    return 0;
}

/*
 * Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer
 * grant to win (quorum = 2; self-grant = 1).  raft_init in ha_init only sets a
 * 1-node placeholder, so we widen the topology directly on the public struct.
 */
static void make_three_node(ha_state_t *ha)
{
    ha->raft.n_peers  = 3;
    ha->raft.self_idx = ha->my_priority;
    ha->raft.role     = RAFT_FOLLOWER;
    ha->n_peers       = 3;
}

/* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */

/*
 * ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment
 * the persistent term, and must not flip the raft role to CANDIDATE.  It sets
 * the in-progress flag and self-grant in raft_state_t only.
 */
static void test_prevote_start_no_mutation(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);                 /* role = CTRL_STANDBY */
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
    uint64_t raft_term_before = ha.raft.term;
    int      raft_voted_before = ha.raft.voted_for;

    ha_start_prevote_async(&ha, NULL);

    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "Req 1.1: role must stay CTRL_STANDBY during the PreVote round");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "Req 1.2: persistent term must not change during the round");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
        "Req 1.2: raft term must not change during the round");
    TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
        "Req 1.2: raft voted_for must not change during the round");
    TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role,
        "Req 1.2: raft role must stay FOLLOWER during the round");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "round must be marked in-progress");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
        "self-grant must be recorded (1)");

    ha_fini(&ha);
    printf("test_prevote_start_no_mutation: PASS\n");
}

/* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */

/*
 * 3-node: after starting the round, feeding one peer GRANT reaches quorum (2).
 * The WON branch must run the real-election sequence: role becomes
 * CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election).
 */
static void test_prevote_majority_grant_starts_real_election(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    ha_start_prevote_async(&ha, NULL);

    /* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */
    pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);

    int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
    /* WON branch enters the real-election sequence: with c==NULL the election
     * cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY —
     * the candidate transitioned to CTRL_ELECTING (real-election path). */
    TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
        "Req 1.5: a PreVote WIN must enter the real-election sequence "
        "(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY");
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
        "WON must clear the in-progress flag (round over)");

    ha_fini(&ha);
    printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role);
}

/* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */

/*
 * 3-node: both peers DENY → no quorum → LOST.  The node must stay CTRL_STANDBY,
 * leave the persistent term unchanged (Req 1.4), and re-arm the liveness
 * watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still
 * trigger a new round).
 */
static void test_prevote_no_majority_stays_standby_and_rearms(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);

    g_rearm_calls = 0;
    ha_prevote_rearm_liveness_hook = stub_rearm;

    ha_start_prevote_async(&ha, NULL);

    /* Both non-self peers deny → all replied, no quorum → LOST. */
    pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
    pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);

    ha_prevote_rearm_liveness_hook = NULL;

    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "Req 1.4: no PreVote majority must keep the node CTRL_STANDBY");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "Req 1.4: no PreVote majority must leave the persistent term unchanged");
    TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
        "Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a "
        "later genuine failure still triggers a new round");

    ha_fini(&ha);
    printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n");
}

/* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */

/*
 * If the round times out before a decision, it is treated as a loss: stay
 * CTRL_STANDBY, term unchanged, re-arm liveness.
 */
static void test_prevote_round_timeout_is_loss(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);

    g_rearm_calls = 0;
    ha_prevote_rearm_liveness_hook = stub_rearm;

    ha_start_prevote_async(&ha, NULL);

    /* No replies arrive; the round timeout fires. */
    ha_prevote_round_timeout_fire_for_test(&ha, NULL);

    ha_prevote_rearm_liveness_hook = NULL;

    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "Req 5.4: a round timeout must leave the persistent term unchanged");
    TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
        "Req 5.4: a timed-out round must re-arm the liveness watchdog");
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
        "round timeout must clear the in-progress flag");

    ha_fini(&ha);
    printf("test_prevote_round_timeout_is_loss: PASS\n");
}

/* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */

/*
 * Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back
 * in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round
 * (self-grant + in-progress), which a peer grant can again win — proving the
 * candidate machinery is reusable, not one-shot.
 */
static void test_prevote_reusable_after_loss(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    ha_prevote_rearm_liveness_hook = stub_rearm;

    /* Round 1: lost. */
    ha_start_prevote_async(&ha, NULL);
    pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0);
    pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0);
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));

    /* Round 2: a fresh round arms cleanly (self-grant + in-progress). */
    ha_start_prevote_async(&ha, NULL);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "a new round after a loss must re-arm (in-progress)");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
        "a new round must reset the tally to the self-grant");

    /* Round 2 wins with a peer grant. */
    pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1);
    int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
    TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
        "a later genuine failure (new round) must still be able to win and "
        "start a real election");

    ha_prevote_rearm_liveness_hook = NULL;
    ha_fini(&ha);
    printf("test_prevote_reusable_after_loss: PASS\n");
}

/* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */

static void test_prevote_pending_is_noop(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    /* 5-node so a single grant is below quorum (3) → PENDING. */
    ha.raft.n_peers  = 5;
    ha.raft.self_idx = 0;
    ha.raft.role     = RAFT_FOLLOWER;
    ha.n_peers       = 5;

    ha_start_prevote_async(&ha, NULL);

    pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/);

    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "a PENDING (sub-quorum) reply must not change the role");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "a PENDING reply must keep the round in progress");

    ha_fini(&ha);
    printf("test_prevote_pending_is_noop: PASS\n");
}

/* ── Test 7: a preVoteReply over the REAL transport reaches the tally (5.1) ─ */

/*
 * Task 5.1 (Req 7.1): a preVoteReply arriving on the candidate's outbound
 * PreVote endpoint — while the candidate is still CTRL_STANDBY with a round in
 * progress — must be ROUTED to the PreVote tally by peer_ctrl_msg_handler (NOT
 * dropped because the role is STANDBY), and drive the round to a decision
 * end-to-end.  This exercises the production decode/route path, not the
 * pre_vote_reply_dispatch_for_test hook.
 */
static void test_prevote_reply_over_real_transport_reaches_tally(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);                 /* role = CTRL_STANDBY */
    make_three_node(&ha);

    /* Start a real round: self-grant + in-progress, role stays CTRL_STANDBY. */
    ha_start_prevote_async(&ha, NULL);
    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "precondition: role must be CTRL_STANDBY during the round");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "precondition: round must be in progress");

    /* Build an endpoint carrying ABSOLUTE peer_idx = 1 (the index the tally
     * dedups by), exactly as ha_send_pre_vote_request sets it. */
    endpoint_t ep;
    memset(&ep, 0, sizeof(ep));
    vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1,
                               /*my_term=*/1, /*my_tag=*/0, /*my_lsn=*/0, "test");

    /* Encode a GRANT preVoteReply (same wire payload as a real-vote reply). */
    io_packet_t *pkt = vote_reply_build_pkt_for_test(/*term=*/1, /*granted=*/1,
                                                     /*tag=*/0, /*lsn=*/0);
    TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "vote_reply_build_pkt_for_test must succeed");

    /* Route it through the REAL handler (peer_ctrl_msg_handler) while STANDBY. */
    int rc = pre_vote_reply_msg_handler_for_test(&ep, pkt);
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handler must return 0");

    /* Peer grant → quorum (2 of 3) → WON: the reply was NOT dropped; it reached
     * the tally and drove the round to a decision (real-election sequence). */
    int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
    TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
        "Req 7.1: a preVoteReply over the real transport must reach the tally "
        "and drive the round to a decision (left CTRL_STANDBY)");
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
        "the decided round must clear the in-progress flag");

    free(ep.data);   /* vctx allocated by vote_ep_ctx_build_for_test */
    ha_fini(&ha);
    printf("test_prevote_reply_over_real_transport_reaches_tally: PASS (role=%d)\n", role);
}

/* ── Suite entry point ──────────────────────────────────────────────────── */

void test_ha_prevote(void)
{
    UNITY_BEGIN();
    RUN_TEST(test_prevote_start_no_mutation);
    RUN_TEST(test_prevote_majority_grant_starts_real_election);
    RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms);
    RUN_TEST(test_prevote_round_timeout_is_loss);
    RUN_TEST(test_prevote_reusable_after_loss);
    RUN_TEST(test_prevote_pending_is_noop);
    RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally);
    UNITY_END();
}