myxce/src/controller/testsuite/test_ha_failover_prevote.c

/*
 * test_ha_failover_prevote.c — ha-prevote task 4.2:
 * route the failover triggers (RDHUP + liveness timeout) through PreVote and
 * defer the destructive step (truncate + CTRL_ELECTING) until PreVote WINS;
 * cancel an in-progress round when a primary heartbeat arrives mid-round.
 *
 * This is the rewire that activates PreVote on the LIVE failover path.  The
 * NEW behaviour these tests pin down (which FAILS before the rewire, when the
 * triggers elect directly):
 *
 *   (a) Firing the RDHUP failover trigger does NOT transition to CTRL_ELECTING
 *       or increment the term — it starts a side-effect-free PreVote round and
 *       the node stays CTRL_STANDBY with pre_vote_in_progress = 1.
 *   (b) Firing the liveness-timeout trigger goes through the SAME PreVote-first
 *       path (it inherits the rewired ha_on_primary_ctrl_fd_rdhup).
 *   (c) A primary heartbeat arriving mid-round CANCELS the round and re-attaches
 *       as standby with the persistent term unchanged (Req 4.2).
 *   (d) When the PreVote round then WINS (a peer grants), the trigger path still
 *       reaches the real election (CTRL_ELECTING) — failover is not blocked.
 *   (e) The config-less voter-only guard (tag==0, peers>1) still trips and the
 *       node does NOT even start a PreVote round (it can never win one).
 *
 * Boundary: PreVote orchestration (trigger path) — ha.c only.
 * Requirements: 1.1, 4.2, 4.3, 4.4.
 *
 * TDD: written FIRST (RED) before the trigger rewire.
 */
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include <sys/socket.h>
#include <unistd.h>

#include <unity/unity.h>

#include "ha.h"
#include "myconf.h"
#include "raft/raft.h"
#include "xce/cluster_def.h"

/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */

extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c);
extern void ha_on_heartbeat_received(ha_state_t *ha);
extern int  (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);

/*
 * liveness_poll_ctx_t: defined privately in ha.c; the layout is mirrored here
 * (exactly as test_ha_liveness.c does) so the test can build one for the
 * test entry-point without a live delaymgr.
 */
typedef struct {
    ha_state_t  *ha;
    cluster_t   *c;
    _Atomic int  cancelled;
} liveness_poll_ctx_t;

extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx);

/* ── Test config ─────────────────────────────────────────────────────────── */

static ha_config_t g_cfg = {
    .heartbeat_interval_ms     = 3000,
    .heartbeat_timeout_ms      = 6000,
    .election_timeout_ms       = 500,
    .probe_connect_timeout_ms  = 50,
    .max_promotion_retries     = 3,
    .election_jitter_max_ms    = 0,
    .election_priority_step_ms = 500,
};

static int g_rearm_calls = 0;
static int stub_rearm(ha_state_t *ha, cluster_t *c)
{
    (void)ha; (void)c;
    g_rearm_calls++;
    return 0;
}

/*
 * Widen the raft topology to a 3-node cluster (self=0) so a PreVote round
 * does NOT trivially self-win (quorum = 2; self-grant = 1).  This lets the
 * round STAY in progress so we can observe the deferred / cancel behaviour.
 */
static void make_three_node(ha_state_t *ha)
{
    ha->raft.n_peers  = 3;
    ha->raft.self_idx = 0;
    ha->raft.role     = RAFT_FOLLOWER;
    ha->n_peers       = 3;
}

/* ── Test 1: RDHUP trigger defers the destructive step (Req 1.1, 4.4) ─────── */

/*
 * Firing ha_on_primary_ctrl_fd_rdhup must NO LONGER transition straight to
 * CTRL_ELECTING / truncate / increment the term.  Instead it begins a PreVote
 * round and stays CTRL_STANDBY with the round in progress.  (Unit-test mode,
 * c == NULL: ha_enter_real_election — which contains truncate + CTRL_ELECTING —
 * must NOT have run, so the role stays CTRL_STANDBY.)
 */
static void test_rdhup_trigger_starts_prevote_not_election(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);          /* role = CTRL_STANDBY */
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);

    /* Give the handler an fd to close (teardown bookkeeping still happens). */
    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ha.primary_ctrl_fd = sv[0];

    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);   /* unit-test mode */
    close(sv[1]);

    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "Req 1.1/4.4: RDHUP trigger must start a PreVote round and STAY "
        "CTRL_STANDBY, not transition to CTRL_ELECTING");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "Req 4.4: RDHUP trigger must not change the persistent term before "
        "PreVote wins");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "Req 1.1: RDHUP trigger must begin a PreVote round (in-progress)");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
        "PreVote round must record the self-grant (1)");
    /* The primary fd must still be closed/cleared — teardown bookkeeping stays
     * at the trigger regardless of PreVote outcome. */
    TEST_ASSERT_EQUAL_INT_MESSAGE(-1, ha.primary_ctrl_fd,
        "RDHUP teardown must still close + clear primary_ctrl_fd");

    ha_fini(&ha);
    printf("test_rdhup_trigger_starts_prevote_not_election: PASS\n");
}

/* ── Test 2: liveness-timeout trigger goes through the same PreVote path ──── */

/*
 * The liveness poll fn, on timeout, must reach the SAME PreVote-first path
 * (it calls ha_on_primary_ctrl_fd_rdhup, now rewired).  Observable: after the
 * timeout, the node is in a PreVote round and STAYS CTRL_STANDBY — it does NOT
 * jump to CTRL_ELECTING.
 */
static void test_liveness_timeout_trigger_starts_prevote(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);

    /* last_activity far enough in the past to exceed heartbeat_timeout_ms. */
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    ha.last_activity = ts.tv_sec - (time_t)(g_cfg.heartbeat_timeout_ms / 1000) - 10;

    liveness_poll_ctx_t ctx;
    memset(&ctx, 0, sizeof(ctx));
    ctx.ha = &ha;
    ctx.c  = NULL;
    atomic_init(&ctx.cancelled, 0);

    liveness_poll_fn_test_entry(&ctx);

    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "Req 1.1: liveness-timeout failover must start a PreVote round and stay "
        "CTRL_STANDBY, not transition straight to CTRL_ELECTING");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "Req 4.4: liveness-timeout failover must not change the term before "
        "PreVote wins");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "Req 1.1: liveness-timeout failover must begin a PreVote round");

    ha_fini(&ha);
    printf("test_liveness_timeout_trigger_starts_prevote: PASS\n");
}

/* ── Test 3: heartbeat mid-round cancels and re-attaches (Req 4.2) ────────── */

/*
 * After the failover trigger starts a PreVote round, an incoming primary
 * heartbeat means the leader is alive — the campaign must be abandoned: the
 * round is cancelled (pre_vote_in_progress cleared), the node re-attaches as
 * CTRL_STANDBY, and the persistent term is UNCHANGED (Req 4.2).
 */
static void test_heartbeat_mid_round_cancels_prevote(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);

    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ha.primary_ctrl_fd = sv[0];

    /* Trigger fires → PreVote round in progress. */
    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
    close(sv[1]);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
        "precondition: a PreVote round must be in progress before the heartbeat");

    /* A fresh primary heartbeat arrives mid-round → cancel + re-attach. */
    ha_on_heartbeat_received(&ha);

    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
        "Req 4.2: a heartbeat mid-round must CANCEL the PreVote round");
    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "Req 4.2: a mid-round heartbeat must not change the persistent term");

    ha_fini(&ha);
    printf("test_heartbeat_mid_round_cancels_prevote: PASS\n");
}

/* ── Test 4: heartbeat with NO round in progress is a plain no-op ─────────── */

/*
 * The cancel logic must only fire when a round is actually in progress: a
 * heartbeat received while no PreVote round is running must just update
 * last_activity (the existing behaviour) and never touch role/term.
 */
static void test_heartbeat_without_round_is_plain_noop(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
    TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);

    ha_on_heartbeat_received(&ha);

    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
        "no round in progress: heartbeat must leave pre_vote_in_progress = 0");
    TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire),
        "no round: heartbeat must keep the node CTRL_STANDBY");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
        atomic_load_explicit(&ha.term, memory_order_acquire),
        "no round: heartbeat must not change the term");
    TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity,
        "heartbeat must still update last_activity (existing behaviour preserved)");

    ha_fini(&ha);
    printf("test_heartbeat_without_round_is_plain_noop: PASS\n");
}

/* ── Test 5: a genuine failover (PreVote wins) still reaches real election ── */

/*
 * The whole point of PreVote is to defer, not block.  After the RDHUP trigger
 * starts the round, a peer GRANT reaching quorum must drive the trigger path
 * into the real election (CTRL_ELECTING) — failover still proceeds when the
 * primary genuinely died.
 */
extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
                                             int peer_idx, int granted);

static void test_failover_prevote_win_reaches_real_election(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    make_three_node(&ha);

    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ha.primary_ctrl_fd = sv[0];

    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
    close(sv[1]);
    TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress);

    /* Peer index 1 grants → quorum (2) reached → WON → real election. */
    pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);

    int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
    TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
        "Req 4.3/5.x: when PreVote WINS the trigger path must reach the real "
        "election (CTRL_ELECTING) — a genuine failover is not blocked");
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
        "a WON round must clear the in-progress flag");

    ha_fini(&ha);
    printf("test_failover_prevote_win_reaches_real_election: PASS (role=%d)\n", role);
}

/* ── Test 6: config-less voter-only guard still trips before PreVote ──────── */

/*
 * A config-less node (config_version_tag == 0, peers > 1) must NOT even start
 * a PreVote round on RDHUP — it can never win one (the freshness gate denies
 * it), and it must stay a passive voter (CTRL_STANDBY).  This preserves the
 * ha-config-bootstrap Req 2.8 election-safety guard through the rewire.
 *
 * Requires a real cluster_t with a tag-0 config so the guard path is reached.
 */
static void test_config_less_node_does_not_start_prevote(void)
{
    cluster_t c;
    myconf_t  myconf;
    memset(&c, 0, sizeof(c));
    memset(&myconf, 0, sizeof(myconf));
    myconf.name = (char *)"ctrl-a";
    myconf.config_version_tag = 0;             /* config-less */
    myconf.peers_cfg.candidates_count = 2;     /* >1 node */
    snprintf(myconf.peers_cfg.candidates[0].name,
             sizeof(myconf.peers_cfg.candidates[0].name), "ctrl-a");
    snprintf(myconf.peers_cfg.candidates[1].name,
             sizeof(myconf.peers_cfg.candidates[1].name), "ctrl-b");
    c.myconf = &myconf;

    TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
    ha_do_demote(&c.ha);
    c.ha.phase2_done = 1;
    c.ha.n_peers     = 1;
    make_three_node(&c.ha);   /* widen raft so a round would NOT self-win */

    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    c.ha.primary_ctrl_fd = sv[0];

    ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
    close(sv[1]);

    TEST_ASSERT_EQUAL_INT_MESSAGE(0, c.ha.raft.pre_vote_in_progress,
        "config-less node (tag=0, peers>1) must NOT start a PreVote round on RDHUP");
    TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
        (int)atomic_load_explicit(&c.ha.role, memory_order_acquire),
        "config-less RDHUP node must stay CTRL_STANDBY (voter only)");

    ha_fini(&c.ha);
    printf("test_config_less_node_does_not_start_prevote: PASS\n");
}

/* ── Suite entry point ──────────────────────────────────────────────────── */

void test_ha_failover_prevote(void)
{
    UnitySetTestFile(__FILE__);
    /* Ensure the round LOST/timeout branch never touches a live delaymgr. */
    g_rearm_calls = 0;
    ha_prevote_rearm_liveness_hook = stub_rearm;

    RUN_TEST(test_rdhup_trigger_starts_prevote_not_election);
    RUN_TEST(test_liveness_timeout_trigger_starts_prevote);
    RUN_TEST(test_heartbeat_mid_round_cancels_prevote);
    RUN_TEST(test_heartbeat_without_round_is_plain_noop);
    RUN_TEST(test_failover_prevote_win_reaches_real_election);
    RUN_TEST(test_config_less_node_does_not_start_prevote);

    ha_prevote_rearm_liveness_hook = NULL;
}