myxce/src/controller/testsuite/test_ha_failover.c

/*
 * test_ha_failover.c — Unit tests for task 9.4: EPOLLRDHUP failover and
 * heartbeat timeout vtask.
 *
 * Tests:
 *  1. ha_on_primary_ctrl_fd_rdhup closes fd, sets to -1, transitions to CTRL_ELECTING
 *  2. ha_on_primary_ctrl_fd_rdhup calls persist_truncate_wal_above
 *  3. ha_arm_heartbeat_timeout_vtask fires failover after silence
 *  4. ha_on_heartbeat_received resets the timeout (no failover on heartbeat)
 *  5. FSM: CTRL_STANDBY after election → primary_ctrl_fd closed
 *  6. INFO log on FSM transition (Req 10.5)
 *
 * Requirements: 6.2, 6.3, 6.4, 6.8, 10.5
 */
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>

#include <unity/unity.h>
#include "ha.h"
#include "netio.h"
#include "xce/cluster_def.h"  /* full cluster_t layout for task-5.2 tests */
#include "myconf.h"           /* myconf_t for task-5.2 tests */

/* ── Forward declarations — function-under-test (not yet implemented) ────── */

/*
 * ha_on_primary_ctrl_fd_rdhup: EPOLLRDHUP event handler for primary_ctrl_fd.
 *
 * Steps (Req 6.2, 6.3):
 *  1. Close and null ha->primary_ctrl_fd
 *  2. Transition ha role to CTRL_ELECTING
 *  3. Call persist_truncate_wal_above(ha->last_known_committed_lsn) via cluster
 *  4. Log INFO: FSM transition (Req 10.5)
 *
 * Pass c == NULL to skip persist_truncate and am_i_primary_quorum (unit test mode).
 */
extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c);

/*
 * ha_arm_heartbeat_timeout_vtask: start background thread monitoring heartbeat
 * silence on primary_ctrl_fd (Req 6.3, 6.4).
 *
 * The thread sleeps for election_timeout_ms + rand_jitter. If no heartbeat
 * is received (ha_on_heartbeat_received not called) within that window,
 * it fires ha_on_primary_ctrl_fd_rdhup.
 *
 * Subsequent calls re-arm the timeout (same thread, reset deadline).
 * Calling ha_on_heartbeat_received() resets the deadline.
 */
extern void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c);

/*
 * ha_on_heartbeat_received: reset the heartbeat timeout deadline.
 * Called whenever a heartbeat arrives on primary_ctrl_fd (Req 6.3).
 */
extern void ha_on_heartbeat_received(ha_state_t *ha);

/*
 * ha_disarm_heartbeat_timeout_vtask: stop the heartbeat timeout thread.
 * Called when transitioning away from CTRL_STANDBY.
 */
extern void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha);

/* ── Helpers ─────────────────────────────────────────────────────────────── */

static ha_config_t g_cfg = {
    .heartbeat_interval_ms     = 3000,
    .heartbeat_timeout_ms      = 10000,
    .election_timeout_ms       = 500,   /* minimum allowed by ha_init validation */
    .probe_connect_timeout_ms  = 50,
    .max_promotion_retries     = 3,
    .election_jitter_max_ms    = 0,     /* no jitter: deterministic timeout */
    .election_priority_step_ms = 500,
};

/* Config with short heartbeat_timeout_ms for watchdog-fires tests */
static ha_config_t g_fast_cfg = {
    .heartbeat_interval_ms     = 2000,
    .heartbeat_timeout_ms      = 2000,  /* minimum valid: == heartbeat_interval_ms */
    .election_timeout_ms       = 500,
    .probe_connect_timeout_ms  = 50,
    .max_promotion_retries     = 3,
    .election_jitter_max_ms    = 0,
    .election_priority_step_ms = 500,
};

/* ── Test 1: RDHUP drops the primary channel and sets primary_ctrl_fd = -1 ──
 * bug-1865: the channel is a netio endpoint HA holds a reference on; RDHUP
 * shuts its socket down (the peer sees EOF) and releases the reference --
 * it never close()s the raw fd, which netio owns. */

static void test_rdhup_closes_and_nulls_fd(void)
{
    ha_state_t  ha;
    endpoint_t* ep;
    char        b;

    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));

    /* Set up a socketpair as the primary channel, held like handle_ctrlhello */
    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ep     = endpoint_get();
    TEST_ASSERT_NOT_NULL(ep);
    ep->fd = sv[0];
    endpoint_ref(ep);
    ha.primary_ctrl_fd = sv[0];
    ha.primary_ctrl_ep = ep;

    /* Fire RDHUP with NULL cluster (unit test mode — skips persist/election) */
    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);

    /* channel dropped: fields cleared, peer sees the hang-up */
    TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd);
    TEST_ASSERT_NULL(ha.primary_ctrl_ep);
    TEST_ASSERT_EQUAL_INT(0, (int)read(sv[1], &b, 1));

    endpoint_unref(ep);   /* netio's own reference: closes sv[0] */
    close(sv[1]);
    ha_fini(&ha);
    printf("test_rdhup_closes_and_nulls_fd: PASS\n");
}

/* ── Test 2: RDHUP transitions role to CTRL_ELECTING ───────────────────── */

static void test_rdhup_transitions_to_ctrl_electing(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));

    /* Start as CTRL_STANDBY */
    ha_do_demote(&ha);
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));

    /* Simulate having a primary fd */
    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ha.primary_ctrl_fd = sv[0];

    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);

    /* Must transition to CTRL_ELECTING */
    TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));

    close(sv[1]);
    ha_fini(&ha);
    printf("test_rdhup_transitions_to_ctrl_electing: PASS\n");
}

/* ── Test 3: heartbeat received before timeout → no failover ───────────── */

static void test_heartbeat_received_resets_timeout(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
    ha_do_demote(&ha);

    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ha.primary_ctrl_fd = sv[0];

    /* Arm the timeout (200ms + jitter=0) */
    ha_arm_heartbeat_timeout_vtask(&ha, NULL);

    /* Immediately receive heartbeat to reset the deadline */
    ha_on_heartbeat_received(&ha);

    /* Wait briefly — less than one full election_timeout_ms */
    struct timespec ts = { .tv_sec = 0, .tv_nsec = 50 * 1000000L };
    nanosleep(&ts, NULL);

    /* Role must still be CTRL_STANDBY (no failover yet) */
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load(&ha.role));
    TEST_ASSERT_NOT_EQUAL(-1, ha.primary_ctrl_fd);

    ha_disarm_heartbeat_timeout_vtask(&ha);
    close(sv[0]);
    close(sv[1]);
    ha.primary_ctrl_fd = -1;
    ha_fini(&ha);
    printf("test_heartbeat_received_resets_timeout: PASS\n");
}

/* ── Test 4: heartbeat timeout fires failover ───────────────────────────── */

/*
 * liveness_poll_ctx_t and liveness_poll_fn_test_entry: XCE_TESTING seam from ha.c.
 * Replicate the typedef here to construct a ctx directly.
 */
typedef struct {
    ha_state_t  *ha;
    cluster_t   *c;
    _Atomic int  cancelled;
} liveness_poll_ctx_t_failover;

extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t_failover *ctx);

static void test_heartbeat_timeout_fires_failover(void)
{
    ha_state_t ha;
    /* Use fast config: heartbeat_timeout_ms=2000ms */
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_fast_cfg, 1));
    ha_do_demote(&ha);

    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    ha.primary_ctrl_fd = sv[0];

    /* Set last_activity to beyond the timeout (simulate silence) */
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    ha.last_activity = ts.tv_sec - (time_t)(g_fast_cfg.heartbeat_timeout_ms / 1000) - 5;

    /* Invoke the liveness poll fn directly (no delaymgr needed) */
    liveness_poll_ctx_t_failover ctx;
    memset(&ctx, 0, sizeof(ctx));
    ctx.ha = &ha;
    ctx.c  = NULL;  /* NULL cluster — unit test mode */
    atomic_init(&ctx.cancelled, 0);

    liveness_poll_fn_test_entry(&ctx);

    /* Failover must have fired: role = CTRL_ELECTING, fd = -1 */
    TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&ha.role));
    TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd);

    close(sv[1]);  /* sv[0] was closed by failover handler */
    ha_fini(&ha);
    printf("test_heartbeat_timeout_fires_failover: PASS\n");
}

/* ── Test 5: ha_on_primary_ctrl_fd_rdhup on already-closed fd is safe ───── */

static void test_rdhup_on_minus_one_fd_is_safe(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 1));
    ha.primary_ctrl_fd = -1;

    /* Should not crash or abort */
    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
    TEST_ASSERT_EQUAL_INT(-1, ha.primary_ctrl_fd);

    ha_fini(&ha);
    printf("test_rdhup_on_minus_one_fd_is_safe: PASS\n");
}

/* ── Task 5.2: RDHUP → CTRL_ELECTING → ha_start_election_async ─────────── */

/*
 * ha_rdhup_election_hook: XCE_TESTING seam — set in ha.c under XCE_TESTING.
 * When non-NULL, ha_on_primary_ctrl_fd_rdhup calls this instead of the real
 * ha_start_election_async.  We use it to verify the RDHUP path triggers
 * async re-election (Req 6.3, 6.5, 9.3).
 */
extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
                                      int n_peers, xce_candidate_t **peers,
                                      uint64_t my_config_tag, uint64_t my_lsn,
                                      const char *my_name);

/* Counters set by the hook so the test can inspect the call */
static int     g_rdhup_hook_called   = 0;
static int     g_rdhup_hook_n_peers  = -1;
static ha_state_t *g_rdhup_hook_ha   = NULL;

static void rdhup_election_spy(ha_state_t *ha, cluster_t *c,
                               int n_peers, xce_candidate_t **peers,
                               uint64_t my_config_tag, uint64_t my_lsn,
                               const char *my_name)
{
    (void)c; (void)peers; (void)my_config_tag; (void)my_lsn; (void)my_name;
    g_rdhup_hook_called++;
    g_rdhup_hook_n_peers = n_peers;
    g_rdhup_hook_ha      = ha;
}

/*
 * test_rdhup_calls_start_election_async:
 *   Req 6.3: heartbeat timeout → ELECTING → async election fires.
 *   Req 9.3: RDHUP path uses ha_start_election_async (not am_i_primary_quorum).
 *
 * We supply a minimal cluster_t with 2 candidates so the RDHUP handler
 * builds a peer array and calls ha_start_election_async via the hook.
 * The test also verifies role is CTRL_ELECTING after the handler.
 *
 * myconf_t.name is char* — use a string literal.
 * myhost == NULL → self-name compare falls back to "" → both candidates are
 * included in peer_ptrs (np == 2).  That is fine for verifying the hook fires.
 */
static void test_rdhup_calls_start_election_async(void)
{
    /* Minimal myconf with 2 candidates (self + 1 peer).
     * myconf_t.name is char* — point at a static string. */
    myconf_t myconf;
    memset(&myconf, 0, sizeof(myconf));
    myconf.name = "ctrl-a";
    /* Configured node: a real (nonzero) config tag so the no-config voter-only
     * RDHUP guard does NOT trip — this test exercises the legitimate "a
     * CONFIGURED node re-elects after RDHUP" path (ha-config-bootstrap Req 2.8).
     * Without a tag, config_version_tag==0 with candidates_count>1 would (post
     * election-safety fix) keep the node a passive voter (CTRL_STANDBY) and the
     * election hook would never fire — that was the buggy condition this test
     * incidentally relied on. */
    myconf.config_version_tag = 0x1234;
    myconf.peers_cfg.candidates_count = 2;
    snprintf(myconf.peers_cfg.candidates[0].name,
             sizeof(myconf.peers_cfg.candidates[0].name), "ctrl-a");
    myconf.peers_cfg.candidates[0].port = 7001;
    snprintf(myconf.peers_cfg.candidates[1].name,
             sizeof(myconf.peers_cfg.candidates[1].name), "ctrl-b");
    myconf.peers_cfg.candidates[1].port = 7002;

    /* Minimal cluster_t — ha embedded, myhost NULL (unit test mode) */
    cluster_t c;
    memset(&c, 0, sizeof(c));
    c.myconf = &myconf;
    /* Initialise the embedded ha_state_t directly */
    TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0 /* priority=0 */));
    ha_do_demote(&c.ha);

    /* Mark phase2 done — simulates a surviving standby after first election */
    c.ha.phase2_done = 1;

    /* Set up a socketpair as the "primary_ctrl_fd" */
    int sv[2];
    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
    c.ha.primary_ctrl_fd = sv[0];

    /* Install spy hook */
    g_rdhup_hook_called  = 0;
    g_rdhup_hook_n_peers = -1;
    g_rdhup_hook_ha      = NULL;
    ha_rdhup_election_hook = rdhup_election_spy;

    /* Fire RDHUP with the stub cluster */
    ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);

    /* Restore hook */
    ha_rdhup_election_hook = NULL;

    /* Verify: role must be CTRL_ELECTING */
    TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load(&c.ha.role));

    /* Verify: fd must be closed and set to -1 */
    TEST_ASSERT_EQUAL_INT(-1, c.ha.primary_ctrl_fd);

    /* Verify: ha_start_election_async (hook) was called exactly once */
    TEST_ASSERT_EQUAL_INT(1, g_rdhup_hook_called);

    /* Verify: n_peers > 0 (loop ran; myhost==NULL so both candidates included) */
    TEST_ASSERT_GREATER_THAN_INT(0, g_rdhup_hook_n_peers);

    close(sv[1]); /* sv[0] was closed by the RDHUP handler */
    ha_fini(&c.ha);
    printf("test_rdhup_calls_start_election_async: PASS\n");
}

/*
 * test_rdhup_phase2_done_preserved:
 *   Req 9.3: phase2_done flag must remain 1 after RDHUP so ha_elect_evaluate_and_signal
 *            skips cluster_init_phase2 on the failover election.
 */
static void test_rdhup_phase2_done_preserved(void)
{
    ha_state_t ha;
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
    ha_do_demote(&ha);
    ha.phase2_done = 1; /* set by first-boot election */

    /* Fire RDHUP in NULL-cluster mode (unit-test shortcircuit) */
    ha_on_primary_ctrl_fd_rdhup(&ha, NULL);

    /* phase2_done must still be 1 — RDHUP does NOT reset it */
    TEST_ASSERT_EQUAL_INT(1, ha.phase2_done);

    ha_fini(&ha);
    printf("test_rdhup_phase2_done_preserved: PASS\n");
}

/* ── Test 6: symbol existence check ─────────────────────────────────────── */

static void test_ha_failover_symbols_exist(void)
{
    typedef void (*rdhup_fn)(ha_state_t *, cluster_t *);
    typedef void (*arm_fn)(ha_state_t *, cluster_t *);
    typedef void (*hb_fn)(ha_state_t *);
    typedef void (*disarm_fn)(ha_state_t *);

    rdhup_fn  f1 = ha_on_primary_ctrl_fd_rdhup;
    arm_fn    f2 = ha_arm_heartbeat_timeout_vtask;
    hb_fn     f3 = ha_on_heartbeat_received;
    disarm_fn f4 = ha_disarm_heartbeat_timeout_vtask;

    TEST_ASSERT_NOT_NULL((void*)f1);
    TEST_ASSERT_NOT_NULL((void*)f2);
    TEST_ASSERT_NOT_NULL((void*)f3);
    TEST_ASSERT_NOT_NULL((void*)f4);
    printf("test_ha_failover_symbols_exist: PASS\n");
}

/* ── Suite entry point ──────────────────────────────────────────────────── */

void test_ha_failover(void)
{
    UnitySetTestFile(__FILE__);
    RUN_TEST(test_rdhup_closes_and_nulls_fd);
    RUN_TEST(test_rdhup_transitions_to_ctrl_electing);
    RUN_TEST(test_heartbeat_received_resets_timeout);
    RUN_TEST(test_heartbeat_timeout_fires_failover);
    RUN_TEST(test_rdhup_on_minus_one_fd_is_safe);
    RUN_TEST(test_ha_failover_symbols_exist);
    RUN_TEST(test_rdhup_calls_start_election_async);
    RUN_TEST(test_rdhup_phase2_done_preserved);
}