diff --git a/src/controller/meson.build b/src/controller/meson.build index 00515e68..86809704 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -93,6 +93,7 @@ controller_unit_test_src = [ 'testsuite/test_persist_repl.c', 'testsuite/test_repl_catchup.c', 'testsuite/test_p2_integration.c', + 'testsuite/test_p3_hardening.c', 'persist.c', 'ha.c', 'repl.c', diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index 736e2b53..d1b7f33e 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -28,6 +28,7 @@ extern void test_repl_apply(void); extern void test_persist_repl(void); extern void test_repl_catchup(void); extern void test_p2_integration(void); +extern void test_p3_hardening(void); void setUp(void) { return; } @@ -125,6 +126,7 @@ int main(int argc, char* argv[]) TEST(test_persist_repl); TEST(test_repl_catchup); TEST(test_p2_integration); + TEST(test_p3_hardening); return UNITY_END(); } diff --git a/src/controller/testsuite/test_p3_hardening.c b/src/controller/testsuite/test_p3_hardening.c new file mode 100644 index 00000000..4284e8e9 --- /dev/null +++ b/src/controller/testsuite/test_p3_hardening.c @@ -0,0 +1,505 @@ +/* P3-1 through P3-5: Integration & Hardening test suite + * + * P3-1: Failover state-machine — promote/demote cycle role + term consistency + * P3-2: Redirect corner cases — primary never redirects; standby redirects iff + * primary is known; ha_no_primary_known() complement. + * P3-3: WAL corruption handling — torn write after checkpoint; CRC corruption + * in mixed-type WAL sequence. + * P3-4: WAL write throughput baseline — 1 000 async appends < 5 s. + * P3-5: Checkpoint GC retention predicates — running, expired-finished, + * transient-timeout, expired-worker. + * + * No new implementation code is required — all called functions exist. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "config.h" +#include "ha.h" +#include "persist.h" +#include "xce/types.h" + +/* ================================================================ + * Shared helpers + * ================================================================ */ + +static ha_config_t make_p3_ha_cfg(void) +{ + ha_config_t c; + memset(&c, 0, sizeof(c)); + c.heartbeat_interval_ms = 3000; + c.heartbeat_timeout_ms = 10000; + c.election_timeout_ms = 2000; + c.probe_connect_timeout_ms = 500; + c.max_promotion_retries = 5; + return c; +} + +/* Create and start a fresh persist_t in a fresh mkdtemp directory. + * Caller owns the returned pointer and must persist_stop/fini/free it. */ +static persist_t *make_p3_persist(const char *suffix, char *tmpdir_out, + size_t tmpdir_outsz) +{ + char buf[256]; + snprintf(buf, sizeof(buf), "/tmp/xce_p3_%s_XXXXXX", suffix); + if (!mkdtemp(buf)) return NULL; + if (tmpdir_out) + snprintf(tmpdir_out, tmpdir_outsz, "%s", buf); + + persist_t *p = calloc(1, sizeof(persist_t)); + if (!p) return NULL; + + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; /* suppress background fsync races in tests */ + + persist_set_workdir(p, buf, "ctrl"); + if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } + if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } + return p; +} + +static void teardown_p3_persist(persist_t *p) +{ + persist_stop(p); + persist_fini(p); + free(p); +} + +/* Find the first .wal segment file in wal_path; fills out[] and returns 0. */ +static int p3_find_wal_segment(const char *wal_path, char *out, size_t outsz) +{ + DIR *d = opendir(wal_path); + if (!d) return -1; + + struct dirent *e; + while ((e = readdir(d)) != NULL) { + size_t len = strlen(e->d_name); + if (len > 4 && strcmp(e->d_name + len - 4, ".wal") == 0) { + snprintf(out, outsz, "%s/%s", wal_path, e->d_name); + closedir(d); + return 0; + } + } + closedir(d); + return -1; +} + +/* ================================================================ + * P3-1: Failover state-machine + * ================================================================ */ + +/* Full promote → demote → re-promote cycle: + * - role transitions must be consistent + * - term must be monotonically increasing across promotions + * - hooks_registered must reset to 0 on demotion + */ +static void test_p3_promote_demote_cycle_consistent_role(void) +{ + ha_state_t ha; + ha_config_t cfg = make_p3_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + + /* Initial state: STANDBY */ + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + + /* Promote once */ + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + uint64_t term1 = atomic_load_explicit(&ha.term, memory_order_acquire); + TEST_ASSERT_TRUE(term1 > 0); + + /* Demote */ + ha_do_demote(&ha); + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + /* hooks_registered idempotency guard must be cleared on demotion */ + TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); + + /* Re-promote: term must be strictly greater than term1 */ + ha_do_promote(&ha); + uint64_t term2 = atomic_load_explicit(&ha.term, memory_order_acquire); + TEST_ASSERT_TRUE(term2 > term1); + TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + + ha_fini(&ha); + printf("test_p3_promote_demote_cycle_consistent_role: PASS\n"); +} + +/* Heartbeat timeout with a known primary must set failover_requested + * (replicates the critical path exercised by the watchdog). */ +static void test_p3_heartbeat_timeout_sets_failover_flag(void) +{ + ha_state_t ha; + ha_config_t cfg = make_p3_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1 /* not highest priority */)); + + /* Seed a known primary so the timeout check fires */ + strncpy(ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1); + ha.primary_port = 7788; + + /* Force an expired heartbeat timestamp */ + clock_gettime(CLOCK_MONOTONIC, &ha.last_heartbeat); + ha.last_heartbeat.tv_sec -= 30; /* 30 s ago — far past 10 s timeout */ + + ha_check_heartbeat_nocluster(&ha); + + TEST_ASSERT_TRUE( + atomic_load_explicit(&ha.failover_requested, memory_order_acquire) != 0); + + ha_fini(&ha); + printf("test_p3_heartbeat_timeout_sets_failover_flag: PASS\n"); +} + +/* ================================================================ + * P3-2: Redirect state-machine corner cases + * ================================================================ */ + +/* A PRIMARY controller must never redirect requests to itself. */ +static void test_p3_redirect_primary_never_redirects(void) +{ + ha_state_t ha; + ha_config_t cfg = make_p3_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); + ha_do_promote(&ha); + TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha)); + ha_fini(&ha); + printf("test_p3_redirect_primary_never_redirects: PASS\n"); +} + +/* STANDBY with no primary_host: must return 0 from ha_should_redirect() + * and 1 from ha_no_primary_known() so the client knows to retry. */ +static void test_p3_redirect_standby_no_primary_returns_nomaster(void) +{ + ha_state_t ha; + ha_config_t cfg = make_p3_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); + /* primary_host is empty after ha_init */ + TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha)); + TEST_ASSERT_EQUAL_INT(1, ha_no_primary_known(&ha)); + ha_fini(&ha); + printf("test_p3_redirect_standby_no_primary_returns_nomaster: PASS\n"); +} + +/* STANDBY with a populated primary_host + port: must redirect AND + * ha_no_primary_known() must return 0. */ +static void test_p3_redirect_standby_with_primary_redirects(void) +{ + ha_state_t ha; + ha_config_t cfg = make_p3_ha_cfg(); + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); + strncpy(ha.primary_host, "10.0.1.1", sizeof(ha.primary_host) - 1); + ha.primary_port = 7000; + TEST_ASSERT_EQUAL_INT(1, ha_should_redirect(&ha)); + TEST_ASSERT_EQUAL_INT(0, ha_no_primary_known(&ha)); + ha_fini(&ha); + printf("test_p3_redirect_standby_with_primary_redirects: PASS\n"); +} + +/* ================================================================ + * P3-3: WAL corruption handling + * + * These complement test_e2e_wal.c by covering a mixed-type WAL sequence + * (submit + update + worker events) rather than a uniform payload. + * ================================================================ */ + +/* Torn write after a checkpoint: recovery must succeed and the recovered + * LSN must be strictly less than the pre-truncation LSN. */ +static void test_p3_wal_torn_write_after_checkpoint(void) +{ + char tmpdir[256]; + persist_t *p = make_p3_persist("torn_ckpt", tmpdir, sizeof(tmpdir)); + TEST_ASSERT_NOT_NULL(p); + + /* Write entries of different types then checkpoint */ + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, "job-a", 5)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "job-b", 5)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH, "job-c", 5)); + TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p)); + + /* Write more entries after checkpoint */ + for (int i = 0; i < 5; i++) + TEST_ASSERT_EQUAL_INT(0, + persist_wal_async(p, WAL_JOB_UPDATE, "post-ckpt", 9)); + + TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); + uint64_t lsn_before = p->current_lsn; + + /* Locate segment and truncate the last WAL entry */ + char seg_path[512]; + TEST_ASSERT_EQUAL_INT(0, + p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path))); + + teardown_p3_persist(p); + + /* Truncate to exactly 4 complete post-checkpoint entries (keep lsns 4..7, + * drop lsn 8). The segment also holds pre-checkpoint entries 0..2 and + * the checkpoint marker at lsn 3 — that is 4 entries × entry_size bytes + * of header+payload before the post-checkpoint block. + * + * Pre-checkpoint entries: 3 × (hdr + 5) + 1 × (hdr + 0) checkpoint marker + * Post-checkpoint entries: 5 × (hdr + 9); keep 4, drop 1. + * + * Use the same technique as test_e2e_wal.c: compute the byte offset that + * includes exactly N-1 post-checkpoint entries and truncate there. + * Entry sizes: pre-ckpt payload=5, ckpt payload=0, post-ckpt payload=9. + */ + off_t pre_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 5); + off_t ckpt_marker_sz = (off_t)(sizeof(wal_entry_hdr_t) + 0); + off_t post_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 9); + /* Keep 3 pre-ckpt + 1 ckpt marker + 4 post-ckpt = drop the 5th post-ckpt */ + off_t trunc_offset = 3 * pre_ckpt_entry_sz + + ckpt_marker_sz + + 4 * post_ckpt_entry_sz; + + int fd = open(seg_path, O_RDWR); + if (fd >= 0) { + ftruncate(fd, trunc_offset); + close(fd); + } + + /* Re-init with same workdir and recover */ + persist_t *p2 = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(p2); + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; + persist_set_workdir(p2, tmpdir, "ctrl"); + TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); + + int rc = persist_recover(p2); + TEST_ASSERT_EQUAL_INT(0, rc); + /* Torn write: recovered LSN must be strictly less than pre-truncation LSN */ + TEST_ASSERT_TRUE(p2->current_lsn < lsn_before); + + teardown_p3_persist(p2); + printf("test_p3_wal_torn_write_after_checkpoint: PASS\n"); +} + +/* CRC corruption in a mixed-type WAL: recovery must stop at the first + * corrupted entry and return success (not an error code). */ +static void test_p3_wal_crc_corruption_mixed_types_stops_replay(void) +{ + char tmpdir[256]; + persist_t *p = make_p3_persist("crc_mixed", tmpdir, sizeof(tmpdir)); + TEST_ASSERT_NOT_NULL(p); + + /* 4 entries of differing types */ + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, "submit", 6)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "update", 6)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH, "finish", 6)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "update2", 7)); + TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); + + char seg_path[512]; + TEST_ASSERT_EQUAL_INT(0, + p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path))); + + uint64_t lsn_before = p->current_lsn; + teardown_p3_persist(p); + + /* Corrupt the payload area of the 2nd entry (index 1) */ + off_t entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 6); + off_t corrupt_offset = entry_sz + (off_t)sizeof(wal_entry_hdr_t); + int fd = open(seg_path, O_RDWR); + if (fd >= 0) { + const uint8_t garbage[4] = {0xDE, 0xAD, 0xBE, 0xEF}; + pwrite(fd, garbage, sizeof(garbage), corrupt_offset); + close(fd); + } + + /* Recover with fresh persist_t on the same workdir */ + persist_t *p2 = calloc(1, sizeof(persist_t)); + TEST_ASSERT_NOT_NULL(p2); + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; + persist_set_workdir(p2, tmpdir, "ctrl"); + TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); + TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); + + int rc = persist_recover(p2); + /* Recovery must succeed (0) even when it stops early */ + TEST_ASSERT_EQUAL_INT(0, rc); + /* Fewer than 4 entries replayed — stopped at the corrupted entry */ + TEST_ASSERT_TRUE(p2->current_lsn < lsn_before); + + teardown_p3_persist(p2); + printf("test_p3_wal_crc_corruption_mixed_types_stops_replay: PASS\n"); +} + +/* ================================================================ + * P3-4: WAL write throughput baseline + * + * 1 000 async WAL appends must complete in under 5 seconds. + * This is a floor test — any regression that serialises all writes + * on a single lock will blow this budget. + * ================================================================ */ + +static void test_p3_wal_write_throughput_baseline(void) +{ + char tmpdir[256]; + persist_t *p = make_p3_persist("perf", tmpdir, sizeof(tmpdir)); + TEST_ASSERT_NOT_NULL(p); + + struct timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); + + for (int i = 0; i < 1000; i++) { + int rc = persist_wal_async(p, WAL_JOB_UPDATE, + "perf_payload_data", 17); + TEST_ASSERT_EQUAL_INT(0, rc); + } + + clock_gettime(CLOCK_MONOTONIC, &end); + + long elapsed_ms = (end.tv_sec - start.tv_sec) * 1000L + + (end.tv_nsec - start.tv_nsec) / 1000000L; + + /* All 1 000 writes must complete in under 5 s */ + TEST_ASSERT_TRUE(elapsed_ms < 5000); + /* LSN must have advanced by at least 1 000 */ + TEST_ASSERT_TRUE(p->current_lsn >= 1000); + + teardown_p3_persist(p); + printf("test_p3_wal_write_throughput_baseline: %ldms for 1000 writes, PASS\n", + elapsed_ms); +} + +/* ================================================================ + * P3-5: Checkpoint GC retention predicates + * + * Exercises job_should_retain() and worker_should_retain() using + * the job_t wrapper type (consistent with persist.c / test_e2e_gc.c). + * These tests are complementary to test_gc_retention.c — they focus + * on the interaction with non-default config values. + * ================================================================ */ + +/* A running job must always be retained regardless of config. */ +static void test_p3_gc_running_job_retained(void) +{ + job_t j; + memset(&j, 0, sizeof(j)); + j.job.state = JOB_STAT_RUN; + + persist_config_t cfg = persist_config_defaults(); + TEST_ASSERT_TRUE(job_should_retain(&j, &cfg)); + printf("test_p3_gc_running_job_retained: PASS\n"); +} + +/* An EXIT job whose finish_time is 2 h before the 1 h retention window + * must be cleaned up. */ +static void test_p3_gc_expired_finished_job_cleaned(void) +{ + job_t j; + memset(&j, 0, sizeof(j)); + j.job.state = JOB_STAT_EXIT; + j.job.finish_time = time(NULL) - 7200; /* 2 hours ago */ + + persist_config_t cfg = persist_config_defaults(); + cfg.finished_retain_s = 3600; /* 1-hour retention window */ + + TEST_ASSERT_FALSE(job_should_retain(&j, &cfg)); + printf("test_p3_gc_expired_finished_job_cleaned: PASS\n"); +} + +/* A TRANSIENT job whose transient_since is past the configured timeout + * must NOT be retained. */ +static void test_p3_gc_transient_job_times_out(void) +{ + job_t j; + memset(&j, 0, sizeof(j)); + j.job.state = JOB_STAT_TRANSIENT; + j.job.transient_since = (uint64_t)(time(NULL) - 400); /* 400 s ago */ + + persist_config_t cfg = persist_config_defaults(); + cfg.transient_job_timeout_s = 300; /* 5-minute window */ + + TEST_ASSERT_FALSE(job_should_retain(&j, &cfg)); + printf("test_p3_gc_transient_job_times_out: PASS\n"); +} + +/* An UNAVAIL worker whose dead_since is far past the retention window + * must be cleaned up. */ +static void test_p3_gc_expired_worker_cleaned(void) +{ + xce_worker_t w; + memset(&w, 0, sizeof(w)); + w.status = XCE_WORKER_UNAVAIL; + w.dead_since = time(NULL) - 100000; /* ~28 h ago */ + + persist_config_t cfg = persist_config_defaults(); + cfg.worker_retain_s = 86400; /* 24-hour retention */ + + TEST_ASSERT_FALSE(worker_should_retain(&w, &cfg)); + printf("test_p3_gc_expired_worker_cleaned: PASS\n"); +} + +/* A DONE job still within the retention window must be retained. */ +static void test_p3_gc_recent_done_job_retained(void) +{ + job_t j; + memset(&j, 0, sizeof(j)); + j.job.state = JOB_STAT_DONE; + j.job.finish_time = time(NULL) - 1800; /* 30 min ago */ + + persist_config_t cfg = persist_config_defaults(); + cfg.finished_retain_s = 3600; + + TEST_ASSERT_TRUE(job_should_retain(&j, &cfg)); + printf("test_p3_gc_recent_done_job_retained: PASS\n"); +} + +/* ================================================================ + * Suite entry point + * ================================================================ */ + +void test_p3_hardening(void) +{ + printf("\n>> P3-1..P3-5: Integration & Hardening\n\n"); + + /* P3-1: Failover state machine */ + printf("-- P3-1: Failover state machine --\n"); + RUN_TEST(test_p3_promote_demote_cycle_consistent_role); + RUN_TEST(test_p3_heartbeat_timeout_sets_failover_flag); + + /* P3-2: Redirect corner cases */ + printf("-- P3-2: Redirect --\n"); + RUN_TEST(test_p3_redirect_primary_never_redirects); + RUN_TEST(test_p3_redirect_standby_no_primary_returns_nomaster); + RUN_TEST(test_p3_redirect_standby_with_primary_redirects); + + /* P3-3: WAL corruption */ + printf("-- P3-3: WAL corruption --\n"); + RUN_TEST(test_p3_wal_torn_write_after_checkpoint); + RUN_TEST(test_p3_wal_crc_corruption_mixed_types_stops_replay); + + /* P3-4: Throughput baseline */ + printf("-- P3-4: Throughput baseline --\n"); + RUN_TEST(test_p3_wal_write_throughput_baseline); + + /* P3-5: GC retention */ + printf("-- P3-5: GC retention --\n"); + RUN_TEST(test_p3_gc_running_job_retained); + RUN_TEST(test_p3_gc_expired_finished_job_cleaned); + RUN_TEST(test_p3_gc_transient_job_times_out); + RUN_TEST(test_p3_gc_expired_worker_cleaned); + RUN_TEST(test_p3_gc_recent_done_job_retained); +}