myxce/src/controller/testsuite/test_p3_hardening.c

/* P3-1 through P3-5: Integration & Hardening test suite
 *
 * P3-1: Failover state-machine — promote/demote cycle role + term consistency
 * P3-2: Redirect corner cases — primary never redirects; standby redirects iff
 *        primary is known; ha_no_primary_known() complement.
 * P3-3: WAL corruption handling — torn write after checkpoint; CRC corruption
 *        in mixed-type WAL sequence.
 * P3-4: WAL write throughput baseline — 1 000 async appends < 5 s.
 * P3-5: Checkpoint GC retention predicates — running, expired-finished,
 *        transient-timeout, expired-worker.
 *
 * No new implementation code is required — all called functions exist.
 */
#define _GNU_SOURCE
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>

#include <unity/unity.h>

#include "config.h"
#include "ha.h"
#include "persist.h"
#include "xce/types.h"

/* ================================================================
 * Shared helpers
 * ================================================================ */

static ha_config_t make_p3_ha_cfg(void)
{
    ha_config_t c;
    memset(&c, 0, sizeof(c));
    c.heartbeat_interval_ms    = 3000;
    c.heartbeat_timeout_ms     = 10000;
    c.election_timeout_ms      = 2000;
    c.probe_connect_timeout_ms = 500;
    c.max_promotion_retries    = 5;
    return c;
}

/* Create and start a fresh persist_t in a fresh mkdtemp directory.
 * Caller owns the returned pointer and must persist_stop/fini/free it. */
static persist_t *make_p3_persist(const char *suffix, char *tmpdir_out,
                                  size_t tmpdir_outsz)
{
    char buf[256];
    snprintf(buf, sizeof(buf), "/tmp/xce_p3_%s_XXXXXX", suffix);
    if (!mkdtemp(buf)) return NULL;
    if (tmpdir_out)
        snprintf(tmpdir_out, tmpdir_outsz, "%s", buf);

    persist_t *p = calloc(1, sizeof(persist_t));
    if (!p) return NULL;

    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000; /* suppress background fsync races in tests */

    persist_set_workdir(p, buf, "ctrl");
    if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
    if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; }
    return p;
}

static void teardown_p3_persist(persist_t *p)
{
    persist_stop(p);
    persist_fini(p);
    free(p);
}

/* Find the first .wal segment file in wal_path; fills out[] and returns 0. */
static int p3_find_wal_segment(const char *wal_path, char *out, size_t outsz)
{
    DIR *d = opendir(wal_path);
    if (!d) return -1;

    struct dirent *e;
    while ((e = readdir(d)) != NULL) {
        size_t len = strlen(e->d_name);
        if (len > 4 && strcmp(e->d_name + len - 4, ".wal") == 0) {
            snprintf(out, outsz, "%s/%s", wal_path, e->d_name);
            closedir(d);
            return 0;
        }
    }
    closedir(d);
    return -1;
}

/* ================================================================
 * P3-1: Failover state-machine
 * ================================================================ */

/* Full promote → demote → re-promote cycle:
 *   - role transitions must be consistent
 *   - term must be monotonically increasing across promotions
 *   - hooks_registered must reset to 0 on demotion
 */
static void test_p3_promote_demote_cycle_consistent_role(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_p3_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));

    /* Initial state: STANDBY */
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));

    /* Promote once */
    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));
    uint64_t term1 = atomic_load_explicit(&ha.term, memory_order_acquire);
    TEST_ASSERT_TRUE(term1 > 0);

    /* Demote */
    ha_do_demote(&ha);
    TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));
    /* hooks_registered idempotency guard must be cleared on demotion */
    TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered);

    /* Re-promote: term must be strictly greater than term1 */
    ha_do_promote(&ha);
    uint64_t term2 = atomic_load_explicit(&ha.term, memory_order_acquire);
    TEST_ASSERT_TRUE(term2 > term1);
    TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));

    ha_fini(&ha);
    printf("test_p3_promote_demote_cycle_consistent_role: PASS\n");
}

/* Heartbeat timeout with a known primary must set failover_requested
 * (replicates the critical path exercised by the watchdog). */
static void test_p3_heartbeat_timeout_sets_failover_flag(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_p3_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1 /* not highest priority */));

    /* Seed a known primary so the timeout check fires */
    strncpy(ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1);
    ha.primary_port = 7788;

    /* Force an expired heartbeat timestamp */
    clock_gettime(CLOCK_MONOTONIC, &ha.last_heartbeat);
    ha.last_heartbeat.tv_sec -= 30; /* 30 s ago — far past 10 s timeout */

    ha_check_heartbeat_nocluster(&ha);

    TEST_ASSERT_TRUE(
        atomic_load_explicit(&ha.failover_requested, memory_order_acquire) != 0);

    ha_fini(&ha);
    printf("test_p3_heartbeat_timeout_sets_failover_flag: PASS\n");
}

/* ================================================================
 * P3-2: Redirect state-machine corner cases
 * ================================================================ */

/* A PRIMARY controller must never redirect requests to itself. */
static void test_p3_redirect_primary_never_redirects(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_p3_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
    ha_do_promote(&ha);
    TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
        (int)atomic_load_explicit(&ha.role, memory_order_acquire));
    TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha));
    ha_fini(&ha);
    printf("test_p3_redirect_primary_never_redirects: PASS\n");
}

/* STANDBY with no primary_host: must return 0 from ha_should_redirect()
 * and 1 from ha_no_primary_known() so the client knows to retry. */
static void test_p3_redirect_standby_no_primary_returns_nomaster(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_p3_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));
    /* primary_host is empty after ha_init */
    TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha));
    TEST_ASSERT_EQUAL_INT(1, ha_no_primary_known(&ha));
    ha_fini(&ha);
    printf("test_p3_redirect_standby_no_primary_returns_nomaster: PASS\n");
}

/* STANDBY with a populated primary_host + port: must redirect AND
 * ha_no_primary_known() must return 0. */
static void test_p3_redirect_standby_with_primary_redirects(void)
{
    ha_state_t ha;
    ha_config_t cfg = make_p3_ha_cfg();
    TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));
    strncpy(ha.primary_host, "10.0.1.1", sizeof(ha.primary_host) - 1);
    ha.primary_port = 7000;
    TEST_ASSERT_EQUAL_INT(1, ha_should_redirect(&ha));
    TEST_ASSERT_EQUAL_INT(0, ha_no_primary_known(&ha));
    ha_fini(&ha);
    printf("test_p3_redirect_standby_with_primary_redirects: PASS\n");
}

/* ================================================================
 * P3-3: WAL corruption handling
 *
 * These complement test_e2e_wal.c by covering a mixed-type WAL sequence
 * (submit + update + worker events) rather than a uniform payload.
 * ================================================================ */

/* Torn write after a checkpoint: recovery must succeed and the recovered
 * LSN must be strictly less than the pre-truncation LSN. */
static void test_p3_wal_torn_write_after_checkpoint(void)
{
    char tmpdir[256];
    persist_t *p = make_p3_persist("torn_ckpt", tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* Write entries of different types then checkpoint */
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT,  "job-a",  5));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE,  "job-b",  5));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH,  "job-c",  5));
    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));

    /* Write more entries after checkpoint */
    for (int i = 0; i < 5; i++)
        TEST_ASSERT_EQUAL_INT(0,
            persist_wal_async(p, WAL_JOB_UPDATE, "post-ckpt", 9));

    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
    uint64_t lsn_before = p->current_lsn;

    /* Locate segment and truncate the last WAL entry */
    char seg_path[512];
    TEST_ASSERT_EQUAL_INT(0,
        p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));

    teardown_p3_persist(p);

    /* Truncate to exactly 4 complete post-checkpoint entries (keep lsns 4..7,
     * drop lsn 8).  The segment also holds pre-checkpoint entries 0..2 and
     * the checkpoint marker at lsn 3 — that is 4 entries × entry_size bytes
     * of header+payload before the post-checkpoint block.
     *
     * Pre-checkpoint entries: 3 × (hdr + 5) + 1 × (hdr + 0) checkpoint marker
     * Post-checkpoint entries: 5 × (hdr + 9); keep 4, drop 1.
     *
     * Use the same technique as test_e2e_wal.c: compute the byte offset that
     * includes exactly N-1 post-checkpoint entries and truncate there.
     * Entry sizes: pre-ckpt payload=5, ckpt payload=0, post-ckpt payload=9.
     */
    off_t pre_ckpt_entry_sz  = (off_t)(sizeof(wal_entry_hdr_t) + 5);
    off_t ckpt_marker_sz     = (off_t)(sizeof(wal_entry_hdr_t) + 0);
    off_t post_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 9);
    /* Keep 3 pre-ckpt + 1 ckpt marker + 4 post-ckpt = drop the 5th post-ckpt */
    off_t trunc_offset = 3 * pre_ckpt_entry_sz
                       + ckpt_marker_sz
                       + 4 * post_ckpt_entry_sz;

    int fd = open(seg_path, O_RDWR);
    if (fd >= 0) {
        ftruncate(fd, trunc_offset);
        close(fd);
    }

    /* Re-init with same workdir and recover */
    persist_t *p2 = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p2);
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    persist_set_workdir(p2, tmpdir, "ctrl");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p2));

    int rc = persist_recover(p2);
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* Torn write: recovered LSN must be strictly less than pre-truncation LSN */
    TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);

    teardown_p3_persist(p2);
    printf("test_p3_wal_torn_write_after_checkpoint: PASS\n");
}

/* CRC corruption in a mixed-type WAL: recovery must stop at the first
 * corrupted entry and return success (not an error code). */
static void test_p3_wal_crc_corruption_mixed_types_stops_replay(void)
{
    char tmpdir[256];
    persist_t *p = make_p3_persist("crc_mixed", tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    /* 4 entries of differing types */
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT,  "submit",  6));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE,  "update",  6));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH,  "finish",  6));
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE,  "update2", 7));
    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));

    char seg_path[512];
    TEST_ASSERT_EQUAL_INT(0,
        p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));

    uint64_t lsn_before = p->current_lsn;
    teardown_p3_persist(p);

    /* Corrupt the payload area of the 2nd entry (index 1) */
    off_t entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 6);
    off_t corrupt_offset = entry_sz + (off_t)sizeof(wal_entry_hdr_t);
    int fd = open(seg_path, O_RDWR);
    if (fd >= 0) {
        const uint8_t garbage[4] = {0xDE, 0xAD, 0xBE, 0xEF};
        pwrite(fd, garbage, sizeof(garbage), corrupt_offset);
        close(fd);
    }

    /* Recover with fresh persist_t on the same workdir */
    persist_t *p2 = calloc(1, sizeof(persist_t));
    TEST_ASSERT_NOT_NULL(p2);
    persist_config_t cfg = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    persist_set_workdir(p2, tmpdir, "ctrl");
    TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(p2));

    int rc = persist_recover(p2);
    /* Recovery must succeed (0) even when it stops early */
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* Fewer than 4 entries replayed — stopped at the corrupted entry */
    TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);

    teardown_p3_persist(p2);
    printf("test_p3_wal_crc_corruption_mixed_types_stops_replay: PASS\n");
}

/* ================================================================
 * P3-4: WAL write throughput baseline
 *
 * 1 000 async WAL appends must complete in under 5 seconds.
 * This is a floor test — any regression that serialises all writes
 * on a single lock will blow this budget.
 * ================================================================ */

static void test_p3_wal_write_throughput_baseline(void)
{
    char tmpdir[256];
    persist_t *p = make_p3_persist("perf", tmpdir, sizeof(tmpdir));
    TEST_ASSERT_NOT_NULL(p);

    struct timespec start, end;
    clock_gettime(CLOCK_MONOTONIC, &start);

    for (int i = 0; i < 1000; i++) {
        int rc = persist_wal_async(p, WAL_JOB_UPDATE,
                                   "perf_payload_data", 17);
        TEST_ASSERT_EQUAL_INT(0, rc);
    }

    clock_gettime(CLOCK_MONOTONIC, &end);

    long elapsed_ms = (end.tv_sec  - start.tv_sec)  * 1000L
                    + (end.tv_nsec - start.tv_nsec) / 1000000L;

    /* All 1 000 writes must complete in under 5 s */
    TEST_ASSERT_TRUE(elapsed_ms < 5000);
    /* LSN must have advanced by at least 1 000 */
    TEST_ASSERT_TRUE(p->current_lsn >= 1000);

    teardown_p3_persist(p);
    printf("test_p3_wal_write_throughput_baseline: %ldms for 1000 writes, PASS\n",
           elapsed_ms);
}

/* ================================================================
 * P3-5: Checkpoint GC retention predicates
 *
 * Exercises job_should_retain() and worker_should_retain() using
 * the job_t wrapper type (consistent with persist.c / test_e2e_gc.c).
 * These tests are complementary to test_gc_retention.c — they focus
 * on the interaction with non-default config values.
 * ================================================================ */

/* A running job must always be retained regardless of config. */
static void test_p3_gc_running_job_retained(void)
{
    job_t j;
    memset(&j, 0, sizeof(j));
    j.job.state = JOB_STAT_RUN;

    persist_config_t cfg = persist_config_defaults();
    TEST_ASSERT_TRUE(job_should_retain(&j, &cfg));
    printf("test_p3_gc_running_job_retained: PASS\n");
}

/* An EXIT job whose finish_time is 2 h before the 1 h retention window
 * must be cleaned up. */
static void test_p3_gc_expired_finished_job_cleaned(void)
{
    job_t j;
    memset(&j, 0, sizeof(j));
    j.job.state       = JOB_STAT_EXIT;
    j.job.finish_time = time(NULL) - 7200; /* 2 hours ago */

    persist_config_t cfg = persist_config_defaults();
    cfg.finished_retain_s = 3600; /* 1-hour retention window */

    TEST_ASSERT_FALSE(job_should_retain(&j, &cfg));
    printf("test_p3_gc_expired_finished_job_cleaned: PASS\n");
}

/* A TRANSIENT job whose transient_since is past the configured timeout
 * must NOT be retained. */
static void test_p3_gc_transient_job_times_out(void)
{
    job_t j;
    memset(&j, 0, sizeof(j));
    j.job.state          = JOB_STAT_TRANSIENT;
    j.job.transient_since = (uint64_t)(time(NULL) - 400); /* 400 s ago */

    persist_config_t cfg = persist_config_defaults();
    cfg.transient_job_timeout_s = 300; /* 5-minute window */

    TEST_ASSERT_FALSE(job_should_retain(&j, &cfg));
    printf("test_p3_gc_transient_job_times_out: PASS\n");
}

/* An UNAVAIL worker whose dead_since is far past the retention window
 * must be cleaned up. */
static void test_p3_gc_expired_worker_cleaned(void)
{
    xce_worker_t w;
    memset(&w, 0, sizeof(w));
    w.status     = XCE_WORKER_UNAVAIL;
    w.dead_since = time(NULL) - 100000; /* ~28 h ago */

    persist_config_t cfg = persist_config_defaults();
    cfg.worker_retain_s = 86400; /* 24-hour retention */

    TEST_ASSERT_FALSE(worker_should_retain(&w, &cfg));
    printf("test_p3_gc_expired_worker_cleaned: PASS\n");
}

/* A DONE job still within the retention window must be retained. */
static void test_p3_gc_recent_done_job_retained(void)
{
    job_t j;
    memset(&j, 0, sizeof(j));
    j.job.state       = JOB_STAT_DONE;
    j.job.finish_time = time(NULL) - 1800; /* 30 min ago */

    persist_config_t cfg = persist_config_defaults();
    cfg.finished_retain_s = 3600;

    TEST_ASSERT_TRUE(job_should_retain(&j, &cfg));
    printf("test_p3_gc_recent_done_job_retained: PASS\n");
}

/* ================================================================
 * Suite entry point
 * ================================================================ */

void test_p3_hardening(void)
{
    printf("\n>> P3-1..P3-5: Integration & Hardening\n\n");

    /* P3-1: Failover state machine */
    printf("-- P3-1: Failover state machine --\n");
    RUN_TEST(test_p3_promote_demote_cycle_consistent_role);
    RUN_TEST(test_p3_heartbeat_timeout_sets_failover_flag);

    /* P3-2: Redirect corner cases */
    printf("-- P3-2: Redirect --\n");
    RUN_TEST(test_p3_redirect_primary_never_redirects);
    RUN_TEST(test_p3_redirect_standby_no_primary_returns_nomaster);
    RUN_TEST(test_p3_redirect_standby_with_primary_redirects);

    /* P3-3: WAL corruption */
    printf("-- P3-3: WAL corruption --\n");
    RUN_TEST(test_p3_wal_torn_write_after_checkpoint);
    RUN_TEST(test_p3_wal_crc_corruption_mixed_types_stops_replay);

    /* P3-4: Throughput baseline */
    printf("-- P3-4: Throughput baseline --\n");
    RUN_TEST(test_p3_wal_write_throughput_baseline);

    /* P3-5: GC retention */
    printf("-- P3-5: GC retention --\n");
    RUN_TEST(test_p3_gc_running_job_retained);
    RUN_TEST(test_p3_gc_expired_finished_job_cleaned);
    RUN_TEST(test_p3_gc_transient_job_times_out);
    RUN_TEST(test_p3_gc_expired_worker_cleaned);
    RUN_TEST(test_p3_gc_recent_done_job_retained);
}