myxce/src/controller/testsuite/test_ha_startup.c

/* test_ha_startup.c — Unity RED-phase tests for startup validation and
 *                     dispatch gate / FSM init state.
 *
 * Tests verify:
 *   Req 4.1-4.4: even-N peer count rejection, cert pre-flight -ENOENT with
 *                path in error message, --vcpus 0 rejection, and vcpus key in
 *                config logs WARN (via caller-supplied warn_buf) without failing.
 *   Req 1.3-1.4: dispatch gate rejects non-vote/non-probe ops in CTRL_INIT
 *                and CTRL_ELECTING states with XCE_ERR_NOT_READY; returns
 *                XCE_ERR_STANDBY for primary-only ops in CTRL_STANDBY state;
 *   Req 1.x:     heartbeat_seen semaphore is initialized in ha_init().
 *
 * Functions under test: validate_peer_count(), validate_cert_files(),
 *                       validate_vcpus(), validate_startup_config(),
 *                       ha_dispatch_gate()
 * These functions do NOT all exist yet — this file establishes contracts for
 * tasks 2.2/2.3 (validation) and 3.3 (dispatch gate).
 */
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <semaphore.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>

#include <unity/unity.h>

#include "controller.capnp.h"
#include "ha.h"

/* Functions under test — declared here; implementations come in tasks 2.2/2.3 */
int validate_peer_count(int n_peers);
int validate_cert_files(const char *cert_path, const char *key_path,
                        char *err_buf, size_t err_len);
int validate_vcpus(int vcpus);
int validate_startup_config(const char *config_path,
                            char *warn_buf, size_t warn_len);

/* Task 5.1 — ha_start_election_async (implemented in ha.c, task 2.4) */
void ha_start_election_async(ha_state_t *ha, cluster_t *c,
                             int n_peers, xce_candidate_t **peers,
                             uint64_t my_config_tag, uint64_t my_lsn,
                             const char *my_name);

/* ── Dispatch gate (task 3.3) — does NOT exist yet; linker failure expected ──
 *
 * ha_dispatch_gate: role-based operation filter for the vtaskmgr dispatch loop.
 *
 * Parameters:
 *   ha     - pointer to the controller's ha_state_t (role is read atomically)
 *   opcode - the OPCode integer value from the incoming request
 *
 * Returns:
 *   0                  — operation is permitted for the current role
 *   -XCE_ERR_NOT_READY — role is CTRL_INIT or CTRL_ELECTING and opcode is not
 *                        OP_requestVote or OP_probeConfigTag
 *   -XCE_ERR_STANDBY   — role is CTRL_STANDBY and opcode is a primary-only op
 *
 * Design ref: design.md "Dispatch gate" section (req 1.3, 1.4).
 * This declaration exists to make the RED-phase tests compile and fail to link.
 */
int ha_dispatch_gate(ha_state_t *ha, int opcode);

/* ── Error codes (task 3.3) — do NOT exist in errnox.h yet ──────────────────
 *
 * XCE_ERR_NOT_READY: returned when controller is in CTRL_INIT or CTRL_ELECTING
 *                    state and receives an opcode that is not allowed during
 *                    startup (req 1.3).
 * XCE_ERR_STANDBY:   returned when controller is in CTRL_STANDBY state and
 *                    receives a primary-only operation (req 1.4).
 *
 * These are declared as extern int to force a linker error when the
 * corresponding symbols are not yet defined in any translation unit.
 * Tasks 3.3 implementation will add them to errnox.h and provide definitions.
 */
extern int XCE_ERR_NOT_READY;
extern int XCE_ERR_STANDBY;

/* ── helpers ──────────────────────────────────────────────────────────── */

static char g_tmpdir[256];

static void startup_setup(void)
{
    snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/test_ha_startup_%d_%ld",
             (int)getpid(), (long)time(NULL));
    mkdir(g_tmpdir, 0700);
}

static void startup_teardown(void)
{
    char cmd[512];
    snprintf(cmd, sizeof(cmd), "rm -rf %s", g_tmpdir);
    system(cmd);
    g_tmpdir[0] = '\0';
}

/* write a tiny file inside g_tmpdir and return its path in out_path */
static void write_tmp_file(const char *name, const char *content,
                           char *out_path, size_t out_len)
{
    snprintf(out_path, out_len, "%s/%s", g_tmpdir, name);
    FILE *f = fopen(out_path, "w");
    if (f) {
        fputs(content, f);
        fclose(f);
    }
}

/* ── Req 4.1: even-N peer count rejection ─────────────────────────────── */

static void test_peer_count_2_is_invalid(void)
{
    /* 2 peers is even (> 1) — must reject with -EINVAL.
     * Raft quorum: a 2-node cluster cannot form a majority partition safely
     * (1-of-2 means any single failure loses quorum), so n_peers=2 is invalid. */
    TEST_ASSERT_EQUAL_INT(-EINVAL, validate_peer_count(2));
    printf("test_peer_count_2_is_invalid: PASS\n");
}

static void test_peer_count_4_is_invalid(void)
{
    /* 4 peers is even — must reject with -EINVAL */
    TEST_ASSERT_EQUAL_INT(-EINVAL, validate_peer_count(4));
    printf("test_peer_count_4_is_invalid: PASS\n");
}

static void test_peer_count_1_is_valid(void)
{
    /* 1 peer (single-node mode) must be accepted */
    TEST_ASSERT_EQUAL_INT(0, validate_peer_count(1));
    printf("test_peer_count_1_is_valid: PASS\n");
}

static void test_peer_count_3_is_valid(void)
{
    /* 3 peers is odd — majority quorum possible; must be accepted */
    TEST_ASSERT_EQUAL_INT(0, validate_peer_count(3));
    printf("test_peer_count_3_is_valid: PASS\n");
}

/* ── Req 4.2: cert pre-flight — missing cert returns -ENOENT with path ── */

static void test_cert_missing_returns_enoent_with_path(void)
{
    startup_setup();

    const char *cert_path = "/tmp/test_ha_startup_nonexistent_cert.pem";
    const char *key_path  = "/tmp/test_ha_startup_nonexistent_key.pem";

    char err_buf[256] = {0};

    /* Neither file exists — must return -ENOENT and include cert_path in msg */
    int rc = validate_cert_files(cert_path, key_path, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(-ENOENT, rc);
    TEST_ASSERT_NOT_NULL(strstr(err_buf, cert_path));

    startup_teardown();
    printf("test_cert_missing_returns_enoent_with_path: PASS\n");
}

static void test_cert_present_key_missing_returns_enoent_with_path(void)
{
    startup_setup();

    /* Write a real cert file but leave key absent */
    char cert_path[512];
    write_tmp_file("ctrl.pem", "-----BEGIN CERTIFICATE-----\n", cert_path,
                   sizeof(cert_path));

    const char *key_path = "/tmp/test_ha_startup_absent_key.pem";

    char err_buf[256] = {0};

    int rc = validate_cert_files(cert_path, key_path, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(-ENOENT, rc);
    /* Key is missing — error message must contain the key path */
    TEST_ASSERT_NOT_NULL(strstr(err_buf, key_path));

    startup_teardown();
    printf("test_cert_present_key_missing_returns_enoent_with_path: PASS\n");
}

static void test_cert_and_key_present_returns_zero(void)
{
    startup_setup();

    char cert_path[512], key_path[512];
    write_tmp_file("ctrl.pem", "-----BEGIN CERTIFICATE-----\n", cert_path,
                   sizeof(cert_path));
    write_tmp_file("ctrl.key", "-----BEGIN PRIVATE KEY-----\n", key_path,
                   sizeof(key_path));

    char err_buf[256] = {0};

    int rc = validate_cert_files(cert_path, key_path, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(0, rc);

    startup_teardown();
    printf("test_cert_and_key_present_returns_zero: PASS\n");
}

/* ── Req 4.3: --vcpus validation ─────────────────────────────────────── */

static void test_vcpus_zero_is_invalid(void)
{
    /* --vcpus 0 must be rejected with -EINVAL */
    TEST_ASSERT_EQUAL_INT(-EINVAL, validate_vcpus(0));
    printf("test_vcpus_zero_is_invalid: PASS\n");
}

static void test_vcpus_negative_is_invalid(void)
{
    TEST_ASSERT_EQUAL_INT(-EINVAL, validate_vcpus(-1));
    printf("test_vcpus_negative_is_invalid: PASS\n");
}

static void test_vcpus_positive_is_valid(void)
{
    /* --vcpus 4 is a normal value — must return 0 */
    TEST_ASSERT_EQUAL_INT(0, validate_vcpus(4));
    printf("test_vcpus_positive_is_valid: PASS\n");
}

/* ── Req 4.4: vcpus key in config file — WARN only, no failure ────────── */

static void test_vcpus_key_in_config_warns_but_does_not_fail(void)
{
    /* Create a temp config file with vcpus key using mkstemp */
    char tmppath[] = "/tmp/test_config_XXXXXX";
    int fd = mkstemp(tmppath);
    const char *content = "vcpus = 4\nqueue_depth = 100\n";
    write(fd, content, strlen(content));
    close(fd);

    char warn_buf[256] = {0};
    int result = validate_startup_config(tmppath, warn_buf, sizeof(warn_buf));

    /* vcpus key found — must NOT fail (return 0) */
    TEST_ASSERT_EQUAL_INT(0, result);
    /* warn_buf must contain a deprecation message mentioning "vcpus" */
    TEST_ASSERT_NOT_NULL(strstr(warn_buf, "vcpus"));

    unlink(tmppath);
    printf("test_vcpus_key_in_config_warns_but_does_not_fail: PASS\n");
}

static void test_config_without_vcpus_key_has_no_warning(void)
{
    char tmppath[] = "/tmp/test_config2_XXXXXX";
    int fd = mkstemp(tmppath);
    const char *content = "heartbeat_interval_ms = 3000\nqueue_depth = 100\n";
    write(fd, content, strlen(content));
    close(fd);

    char warn_buf[256] = {0};
    int result = validate_startup_config(tmppath, warn_buf, sizeof(warn_buf));

    /* no vcpus key — must succeed with no warning */
    TEST_ASSERT_EQUAL_INT(0, result);
    TEST_ASSERT_EQUAL_INT(0, warn_buf[0]); /* warn_buf empty — no warning */

    unlink(tmppath);
    printf("test_config_without_vcpus_key_has_no_warning: PASS\n");
}

/* ── Req 1.3: dispatch gate — CTRL_INIT blocks non-election opcodes ──────── */

/* Shared ha_config_t for dispatch gate tests — minimal valid config */
static ha_config_t g_dispatch_cfg = {
    .heartbeat_interval_ms    = 3000,
    .heartbeat_timeout_ms     = 10000,
    .election_timeout_ms      = 2000,
    .probe_connect_timeout_ms = 500,
    .max_promotion_retries    = 5,
};

static void test_dispatch_gate_init_blocks_non_election_ops(void)
{
    /* Req 1.3: When role is CTRL_INIT, any opcode that is not OP_requestVote
     * or OP_probeConfigTag must receive XCE_ERR_NOT_READY.
     *
     * OPCode_jobsub (11) is a representative primary-only opcode. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_jobsub);
    TEST_ASSERT_EQUAL_INT(-XCE_ERR_NOT_READY, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_init_blocks_non_election_ops: PASS\n");
}

static void test_dispatch_gate_electing_blocks_non_election_ops(void)
{
    /* Req 1.3: Same gate applies when role is CTRL_ELECTING.
     * OPCode_get (5) is another representative non-election opcode. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_get);
    TEST_ASSERT_EQUAL_INT(-XCE_ERR_NOT_READY, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_electing_blocks_non_election_ops: PASS\n");
}

static void test_dispatch_gate_init_allows_request_vote(void)
{
    /* Req 1.3: OP_requestVote must pass through in CTRL_INIT state.
     * Returns 0 (permitted). */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_requestVote);
    TEST_ASSERT_EQUAL_INT(0, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_init_allows_request_vote: PASS\n");
}

static void test_dispatch_gate_init_allows_probe_config_tag(void)
{
    /* Req 1.3: OP_probeConfigTag must pass through in CTRL_INIT state.
     * Returns 0 (permitted). */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_probeConfigTag);
    TEST_ASSERT_EQUAL_INT(0, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_init_allows_probe_config_tag: PASS\n");
}

static void test_dispatch_gate_electing_allows_request_vote(void)
{
    /* Req 1.3: OP_requestVote must pass through in CTRL_ELECTING state. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_requestVote);
    TEST_ASSERT_EQUAL_INT(0, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_electing_allows_request_vote: PASS\n");
}

static void test_dispatch_gate_electing_allows_probe_config_tag(void)
{
    /* Req 1.3: OP_probeConfigTag must pass through in CTRL_ELECTING state. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_probeConfigTag);
    TEST_ASSERT_EQUAL_INT(0, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_electing_allows_probe_config_tag: PASS\n");
}

/* ── Req 1.4: dispatch gate — CTRL_STANDBY blocks primary-only ops ───────── */

static void test_dispatch_gate_standby_blocks_primary_only_ops(void)
{
    /* Req 1.4: When role is CTRL_STANDBY, primary-only operations must
     * receive XCE_ERR_STANDBY.
     *
     * OPCode_jobsub (11) is a representative primary-only opcode.
     * The implementation populates redirect_host/port when a primary is known
     * (tested separately in join tests); here we only verify the error code. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_jobsub);
    TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_standby_blocks_primary_only_ops: PASS\n");
}

static void test_dispatch_gate_primary_allows_all_ops(void)
{
    /* Sanity: when role is CTRL_PRIMARY, all ops including primary-only ones
     * must be permitted (return 0). */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);

    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_jobsub));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_get));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_requestVote));

    ha_fini(&ha);
    printf("test_dispatch_gate_primary_allows_all_ops: PASS\n");
}

/* ── Req 1.x: ha_init() initializes heartbeat_seen semaphore ────────────── */

static void test_ha_init_initializes_heartbeat_semaphore(void)
{
    /* Req 1: ha_init() must initialize ha->heartbeat_seen so that
     * ha_wait_for_heartbeat_or_timeout() can safely call sem_timedwait on it.
     *
     * Verify the semaphore is initialized at value 0 (no heartbeat yet seen):
     *   sem_trywait() on a zero-value semaphore must return -1 with errno=EAGAIN.
     * Also verify sem_post() + sem_wait() round-trip works (semaphore is live). */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);

    /* sem_trywait on a freshly initialized semaphore (value=0) must fail with
     * EAGAIN — confirming it was initialized at 0, not garbage. */
    int rc = sem_trywait(&ha.heartbeat_seen);
    TEST_ASSERT_EQUAL_INT(-1, rc);
    TEST_ASSERT_EQUAL_INT(EAGAIN, errno);

    /* Round-trip: post then wait must succeed immediately */
    TEST_ASSERT_EQUAL_INT(0, sem_post(&ha.heartbeat_seen));
    TEST_ASSERT_EQUAL_INT(0, sem_wait(&ha.heartbeat_seen));

    ha_fini(&ha);
    printf("test_ha_init_initializes_heartbeat_semaphore: PASS\n");
}

/* ── Req 1.1/1.2: ha_wait_for_heartbeat_or_timeout() ────────────────────── */

/*
 * ha_wait_for_heartbeat_or_timeout: waits on ha->heartbeat_seen semaphore.
 * Deadline = now + election_timeout_ms + xorshift64_jitter(election_jitter_max_ms).
 *
 * Returns CTRL_STANDBY  if a heartbeat arrives before the deadline
 *         (calls ha_do_demote() which sets role = CTRL_STANDBY).
 * Returns CTRL_ELECTING if sem_timedwait times out (ETIMEDOUT).
 *
 * Takes ha_state_t* for standalone testability; production call-site wraps
 * with cluster_t* and passes &c->ha.
 *
 * Req 1.1: heartbeat before timeout → CTRL_STANDBY transition.
 * Req 1.2: timeout fires           → returns CTRL_ELECTING.
 */
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha);

static void test_wait_returns_standby_when_heartbeat_arrives_before_timeout(void)
{
    /* Req 1.1: When a heartbeat is posted before the deadline,
     * ha_wait_for_heartbeat_or_timeout() must:
     *   - return CTRL_STANDBY
     *   - set ha->role to CTRL_STANDBY (via ha_do_demote)
     *
     * We simulate a heartbeat by posting heartbeat_seen directly before calling
     * the function, so sem_timedwait returns immediately with value 0. */
    ha_state_t ha;
    /* Use a short timeout but with jitter disabled so the test is deterministic */
    ha_config_t cfg = {
        .heartbeat_interval_ms    = 3000,
        .heartbeat_timeout_ms     = 10000,
        .election_timeout_ms      = 500,
        .probe_connect_timeout_ms = 500,
        .max_promotion_retries    = 5,
        .election_jitter_max_ms   = 0,  /* no jitter — deterministic deadline */
    };
    ha_init(&ha, &cfg, 1);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);

    /* Post the semaphore to simulate a heartbeat arriving before the deadline */
    sem_post(&ha.heartbeat_seen);

    ctrl_role_t result = ha_wait_for_heartbeat_or_timeout(&ha);

    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)result);
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));

    ha_fini(&ha);
    printf("test_wait_returns_standby_when_heartbeat_arrives_before_timeout: PASS\n");
}

static void test_wait_returns_electing_when_timeout_fires(void)
{
    /* Req 1.2: When no heartbeat arrives before the deadline,
     * ha_wait_for_heartbeat_or_timeout() must:
     *   - return CTRL_ELECTING
     *   - NOT change ha->role (caller is responsible for the transition)
     *
     * Use election_timeout_ms=1 and election_jitter_max_ms=0 so the semaphore
     * times out almost immediately.  The test must complete in < 500ms. */
    ha_state_t ha;
    ha_config_t cfg = {
        .heartbeat_interval_ms    = 3000,
        .heartbeat_timeout_ms     = 10000,
        .election_timeout_ms      = 1,   /* 1 ms — times out almost instantly */
        .probe_connect_timeout_ms = 500,
        .max_promotion_retries    = 5,
        .election_jitter_max_ms   = 0,   /* no jitter */
    };
    /* Note: ha_init rejects election_timeout_ms < 500.
     * Bypass via direct struct assignment after ha_init with a valid config,
     * then overwrite the field for the test. */
    ha_config_t valid_cfg = cfg;
    valid_cfg.election_timeout_ms = 500; /* satisfy ha_init's >= 500 check */
    ha_init(&ha, &valid_cfg, 1);
    /* Now override to 1 ms for a near-instant timeout */
    ha.config.election_timeout_ms = 1;
    ha.config.election_jitter_max_ms = 0;
    atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);

    /* Do NOT post heartbeat_seen — let it time out */
    ctrl_role_t result = ha_wait_for_heartbeat_or_timeout(&ha);

    TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)result);
    /* Role must still be CTRL_INIT — caller drives the transition */
    TEST_ASSERT_EQUAL_INT(CTRL_INIT,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));

    ha_fini(&ha);
    printf("test_wait_returns_electing_when_timeout_fires: PASS\n");
}

/* ── Req 1.1/1.2: dispatch gate allows OPCode_heartbeat in CTRL_INIT ──────── */

static void test_dispatch_gate_init_allows_heartbeat(void)
{
    /* Req 1.1: OPCode_heartbeat must pass through in CTRL_INIT so the
     * heartbeat handler can post heartbeat_seen and trigger the
     * CTRL_INIT → CTRL_STANDBY transition. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_INIT, memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_heartbeat);
    TEST_ASSERT_EQUAL_INT(0, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_init_allows_heartbeat: PASS\n");
}

/* ── Task 3.1: dispatch gate CTRL_PRIMARY_COMMITTING ─────────────────────── */

static void test_dispatch_gate_committing_denies_job_ops(void)
{
    /* Req 5.1, 8.1: When role is CTRL_PRIMARY_COMMITTING, primary-only
     * job operations must be denied with XCE_ERR_NOT_READY.
     * OPCode_jobsub is a representative job-submission opcode. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
                          memory_order_release);

    int rc = ha_dispatch_gate(&ha, OPCode_jobsub);
    TEST_ASSERT_EQUAL_INT(-XCE_ERR_NOT_READY, rc);

    ha_fini(&ha);
    printf("test_dispatch_gate_committing_denies_job_ops: PASS\n");
}

static void test_dispatch_gate_committing_allows_repl_ops(void)
{
    /* Req 5.2, 8.1: When role is CTRL_PRIMARY_COMMITTING, replication
     * operations must be permitted (return 0).
     * OPCode_replCatchUpDone is the key signal that ends the COMMITTING phase. */
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);
    atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY_COMMITTING,
                          memory_order_release);

    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_replCatchUpDone));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_heartbeat));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_replConfig));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_replConfigPush));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_upgradeConfigAck));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_hello));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_walAck));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_replEntry));
    TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&ha, OPCode_replAck));

    ha_fini(&ha);
    printf("test_dispatch_gate_committing_allows_repl_ops: PASS\n");
}

/* ── Task 1.1 (ha-heartbeat-liveness): ha_init() zero-initializes last_activity ─
 *
 * Req 1.1, 1.3, 3.1: ha_state_t must carry a `time_t last_activity` field that
 * is zero-initialized by ha_init().  Later tasks (2.1) write to this field via
 * ha_on_heartbeat_received(); task 3.1 reads it for liveness polling.
 */
static void test_ha_init_last_activity_is_zero(void)
{
    ha_state_t ha;
    ha_init(&ha, &g_dispatch_cfg, 0);

    TEST_ASSERT_EQUAL_INT_MESSAGE(0, (int)ha.last_activity,
        "ha_init must zero-initialize last_activity (liveness field)");

    ha_fini(&ha);
    printf("test_ha_init_last_activity_is_zero: PASS\n");
}

/* ── Task 5.1: myapp_run startup sequence ────────────────────────────────
 *
 * Req 1.1, 9.3, 9.4: myapp_run must call ha_start_election_async (non-blocking)
 * and then cluster_run without any intervening blocking calls.
 *
 * These tests verify the call-sequence contract by checking that:
 *   1. ha_start_election_async with n_peers=1 sets ha->n_peers = 1 (single-node
 *      path — the same path myapp_run takes for a single-node cluster).
 *   2. The role after ha_start_election_async(n_peers=1) is CTRL_PRIMARY_COMMITTING
 *      (not CTRL_PRIMARY) — matching what myapp_run would produce in the new design.
 *   3. ha_start_election_async completes in < 10ms — confirming myapp_run does not
 *      block before entering cluster_run.
 *
 * These tests are intentionally thin: myapp_run itself is an integration entry
 * point that requires a live network socket.  The unit tests here verify that
 * the building blocks behave correctly; the integration test is test 13 in the
 * design testing strategy (regression: single-node self-elects).
 */

static ha_config_t g_startup_cfg = {
    .heartbeat_interval_ms     = 3000,
    .heartbeat_timeout_ms      = 10000,
    .election_timeout_ms       = 2000,
    .probe_connect_timeout_ms  = 500,
    .max_promotion_retries     = 5,
    .election_jitter_max_ms    = 0,
    .election_priority_step_ms = 500,
    .committing_timeout_ms     = 10000,
};

static void test_nonblocking_single_node_election_sets_n_peers(void)
{
    /* Req 1.1, 9.3: the new myapp_run path calls ha_start_election_async with
     * n_peers=1 for a single-node cluster.  This test verifies the call stores
     * n_peers in ha->n_peers, matching the new myapp_run contract. */
    ha_state_t ha;
    ha_init(&ha, &g_startup_cfg, 0);
    ha.n_peers   = 0;
    ha.phase2_done = 0;

    ha_start_election_async(&ha, NULL, 1, NULL, 0ULL, 0ULL, "ctrl-a");

    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.n_peers,
        "myapp_run path: ha_start_election_async must store n_peers=1 for single-node");

    ha_fini(&ha);
    printf("test_nonblocking_single_node_election_sets_n_peers: PASS\n");
}

static void test_nonblocking_single_node_election_enters_committing(void)
{
    /* Req 9.3, 8.1: After ha_start_election_async(n_peers=1) the role must be
     * CTRL_PRIMARY_COMMITTING, NOT CTRL_PRIMARY.
     * This validates that the new myapp_run path (without blocking for(;;) loop)
     * correctly transitions through the COMMITTING state. */
    ha_state_t ha;
    ha_init(&ha, &g_startup_cfg, 0);
    ha.phase2_done = 0;
    atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);

    ha_start_election_async(&ha, NULL, 1, NULL, 100ULL, 50ULL, "ctrl-a");

    ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(
        &ha.role, memory_order_acquire);
    TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY_COMMITTING, (int)role,
        "myapp_run path: single-node must enter CTRL_PRIMARY_COMMITTING after election");

    ha_fini(&ha);
    printf("test_nonblocking_single_node_election_enters_committing: PASS (role=%d)\n",
           (int)role);
}

static void test_nonblocking_election_returns_in_under_10ms(void)
{
    /* Req 9.4, 1.1: ha_start_election_async must return in < 5ms (design spec).
     * Observable: myapp_run exits into cluster_run within 5ms of startup;
     * no blocking calls visible between ha_start_election_async and cluster_run. */
    ha_state_t ha;
    ha_init(&ha, &g_startup_cfg, 0);
    ha.phase2_done = 0;

    struct timespec t0;
    clock_gettime(CLOCK_MONOTONIC, &t0);

    ha_start_election_async(&ha, NULL, 1, NULL, 0ULL, 0ULL, "ctrl-a");

    struct timespec t1;
    clock_gettime(CLOCK_MONOTONIC, &t1);
    long elapsed_ms_val = (t1.tv_sec - t0.tv_sec) * 1000L
                        + (t1.tv_nsec - t0.tv_nsec) / 1000000L;

    TEST_ASSERT_LESS_OR_EQUAL_INT_MESSAGE(5, (int)elapsed_ms_val,
        "myapp_run path: ha_start_election_async must return in < 5ms (design spec)");

    ha_fini(&ha);
    printf("test_nonblocking_election_returns_in_under_10ms: PASS (%ldms)\n",
           elapsed_ms_val);
}

/* ── Req 2.4: heartbeat timing validation at startup ─────────────────────── */

/*
 * validate_heartbeat_config: Req 2.4 — startup validation of heartbeat
 * timing relationship.
 *
 * Returns 0 when heartbeat_timeout_ms >= 2 × heartbeat_interval_ms.
 * Returns -EINVAL when heartbeat_timeout_ms < 2 × heartbeat_interval_ms;
 * writes human-readable error into err_buf.
 */
int validate_heartbeat_config(uint32_t heartbeat_timeout_ms,
                              uint32_t heartbeat_interval_ms,
                              char *err_buf, size_t err_len);

static void test_heartbeat_config_valid_exact_double(void)
{
    /* timeout == 2 × interval — exactly at the boundary; must return 0. */
    char err_buf[256] = {0};
    int rc = validate_heartbeat_config(6000, 3000, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* No error message on success */
    TEST_ASSERT_EQUAL_INT(0, err_buf[0]);
    printf("test_heartbeat_config_valid_exact_double: PASS\n");
}

static void test_heartbeat_config_valid_above_double(void)
{
    /* timeout > 2 × interval — clearly valid (3× ratio) */
    char err_buf[256] = {0};
    int rc = validate_heartbeat_config(9000, 3000, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(0, rc);
    printf("test_heartbeat_config_valid_above_double: PASS\n");
}

static void test_heartbeat_config_invalid_equal_interval(void)
{
    /* timeout == interval — below the 2× floor; must reject with -EINVAL.
     * Req 2.4: heartbeat_timeout_ms >= 2 × heartbeat_interval_ms */
    char err_buf[256] = {0};
    int rc = validate_heartbeat_config(3000, 3000, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
    /* err_buf must contain a human-readable message */
    TEST_ASSERT_GREATER_THAN(0, (int)strlen(err_buf));
    printf("test_heartbeat_config_invalid_equal_interval: PASS\n");
}

static void test_heartbeat_config_invalid_less_than_double(void)
{
    /* timeout = 1.5 × interval — also below the floor */
    char err_buf[256] = {0};
    int rc = validate_heartbeat_config(4500, 3000, err_buf, sizeof(err_buf));
    TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
    TEST_ASSERT_GREATER_THAN(0, (int)strlen(err_buf));
    printf("test_heartbeat_config_invalid_less_than_double: PASS\n");
}

/* ── Test suite entry point ───────────────────────────────────────────── */

void test_ha_startup(void)
{
    /* Req 4.1 — even-N peer count */
    RUN_TEST(test_peer_count_2_is_invalid);
    RUN_TEST(test_peer_count_4_is_invalid);
    RUN_TEST(test_peer_count_1_is_valid);
    RUN_TEST(test_peer_count_3_is_valid);

    /* Req 4.2 — cert pre-flight with full path in error message */
    RUN_TEST(test_cert_missing_returns_enoent_with_path);
    RUN_TEST(test_cert_present_key_missing_returns_enoent_with_path);
    RUN_TEST(test_cert_and_key_present_returns_zero);

    /* Req 4.3 — vcpus validation */
    RUN_TEST(test_vcpus_zero_is_invalid);
    RUN_TEST(test_vcpus_negative_is_invalid);
    RUN_TEST(test_vcpus_positive_is_valid);

    /* Req 4.4 — vcpus key WARN via warn_buf; no failure */
    RUN_TEST(test_vcpus_key_in_config_warns_but_does_not_fail);
    RUN_TEST(test_config_without_vcpus_key_has_no_warning);

    /* Req 1.3 — dispatch gate: CTRL_INIT and CTRL_ELECTING block non-election ops */
    RUN_TEST(test_dispatch_gate_init_blocks_non_election_ops);
    RUN_TEST(test_dispatch_gate_electing_blocks_non_election_ops);
    RUN_TEST(test_dispatch_gate_init_allows_request_vote);
    RUN_TEST(test_dispatch_gate_init_allows_probe_config_tag);
    RUN_TEST(test_dispatch_gate_electing_allows_request_vote);
    RUN_TEST(test_dispatch_gate_electing_allows_probe_config_tag);

    /* Req 1.4 — dispatch gate: CTRL_STANDBY blocks primary-only ops */
    RUN_TEST(test_dispatch_gate_standby_blocks_primary_only_ops);
    RUN_TEST(test_dispatch_gate_primary_allows_all_ops);

    /* Req 1.x — ha_init() initializes heartbeat_seen semaphore */
    RUN_TEST(test_ha_init_initializes_heartbeat_semaphore);

    /* Req 1.1 — ha_wait_for_heartbeat_or_timeout: heartbeat → CTRL_STANDBY */
    RUN_TEST(test_wait_returns_standby_when_heartbeat_arrives_before_timeout);

    /* Req 1.2 — ha_wait_for_heartbeat_or_timeout: timeout → CTRL_ELECTING */
    RUN_TEST(test_wait_returns_electing_when_timeout_fires);

    /* Req 1.1 — dispatch gate allows heartbeat in CTRL_INIT */
    RUN_TEST(test_dispatch_gate_init_allows_heartbeat);

    /* Task 3.1 — dispatch gate: CTRL_PRIMARY_COMMITTING */
    RUN_TEST(test_dispatch_gate_committing_denies_job_ops);
    RUN_TEST(test_dispatch_gate_committing_allows_repl_ops);

    /* Task 5.1 — myapp_run non-blocking startup sequence (Req 1.1, 9.3, 9.4) */
    RUN_TEST(test_nonblocking_single_node_election_sets_n_peers);
    RUN_TEST(test_nonblocking_single_node_election_enters_committing);
    RUN_TEST(test_nonblocking_election_returns_in_under_10ms);

    /* Task 1.1 (ha-heartbeat-liveness) — ha_init zero-initializes last_activity */
    RUN_TEST(test_ha_init_last_activity_is_zero);

    /* Req 2.4 — heartbeat timing validation at startup */
    RUN_TEST(test_heartbeat_config_valid_exact_double);
    RUN_TEST(test_heartbeat_config_valid_above_double);
    RUN_TEST(test_heartbeat_config_invalid_equal_interval);
    RUN_TEST(test_heartbeat_config_invalid_less_than_double);
}