/* P3-1 through P3-5: Integration & Hardening test suite * * P3-1: Failover state-machine — promote/demote cycle role + term consistency * P3-2: Redirect corner cases — primary never redirects; standby redirects iff * primary is known; ha_no_primary_known() complement. * P3-3: WAL corruption handling — torn write after checkpoint; CRC corruption * in mixed-type WAL sequence. * P3-4: WAL write throughput baseline — 1 000 async appends < 5 s. * P3-5: Checkpoint GC retention predicates — running, expired-finished, * transient-timeout, expired-worker. * * No new implementation code is required — all called functions exist. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "ha.h" #include "persist.h" #include "xce/types.h" /* ================================================================ * Shared helpers * ================================================================ */ static ha_config_t make_p3_ha_cfg(void) { ha_config_t c; memset(&c, 0, sizeof(c)); c.heartbeat_interval_ms = 3000; c.heartbeat_timeout_ms = 10000; c.election_timeout_ms = 2000; c.probe_connect_timeout_ms = 500; c.max_promotion_retries = 5; return c; } /* Create and start a fresh persist_t in a fresh mkdtemp directory. * Caller owns the returned pointer and must persist_stop/fini/free it. */ static persist_t *make_p3_persist(const char *suffix, char *tmpdir_out, size_t tmpdir_outsz) { char buf[256]; snprintf(buf, sizeof(buf), "/tmp/xce_p3_%s_XXXXXX", suffix); if (!mkdtemp(buf)) return NULL; if (tmpdir_out) snprintf(tmpdir_out, tmpdir_outsz, "%s", buf); persist_t *p = calloc(1, sizeof(persist_t)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* suppress background fsync races in tests */ persist_set_workdir(p, buf, "ctrl"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } return p; } static void teardown_p3_persist(persist_t *p) { persist_stop(p); persist_fini(p); free(p); } /* Find the first .wal segment file in wal_path; fills out[] and returns 0. */ static int p3_find_wal_segment(const char *wal_path, char *out, size_t outsz) { DIR *d = opendir(wal_path); if (!d) return -1; struct dirent *e; while ((e = readdir(d)) != NULL) { size_t len = strlen(e->d_name); if (len > 4 && strcmp(e->d_name + len - 4, ".wal") == 0) { snprintf(out, outsz, "%s/%s", wal_path, e->d_name); closedir(d); return 0; } } closedir(d); return -1; } /* ================================================================ * P3-1: Failover state-machine * ================================================================ */ /* Full promote → demote → re-promote cycle: * - role transitions must be consistent * - term must be monotonically increasing across promotions * - hooks_registered must reset to 0 on demotion */ static void test_p3_promote_demote_cycle_consistent_role(void) { ha_state_t ha; ha_config_t cfg = make_p3_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* Initial state: STANDBY */ TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); /* Promote once */ ha_do_promote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); uint64_t term1 = atomic_load_explicit(&ha.term, memory_order_acquire); TEST_ASSERT_TRUE(term1 > 0); /* Demote */ ha_do_demote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); /* hooks_registered idempotency guard must be cleared on demotion */ TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered); /* Re-promote: term must be strictly greater than term1 */ ha_do_promote(&ha); uint64_t term2 = atomic_load_explicit(&ha.term, memory_order_acquire); TEST_ASSERT_TRUE(term2 > term1); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); printf("test_p3_promote_demote_cycle_consistent_role: PASS\n"); } /* Heartbeat timeout with a known primary must set failover_requested * (replicates the critical path exercised by the watchdog). */ static void test_p3_heartbeat_timeout_sets_failover_flag(void) { ha_state_t ha; ha_config_t cfg = make_p3_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1 /* not highest priority */)); /* Seed a known primary so the timeout check fires */ strncpy(ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1); ha.primary_port = 7788; /* Force an expired heartbeat timestamp */ clock_gettime(CLOCK_MONOTONIC, &ha.last_heartbeat); ha.last_heartbeat.tv_sec -= 30; /* 30 s ago — far past 10 s timeout */ ha_check_heartbeat_nocluster(&ha); TEST_ASSERT_TRUE( atomic_load_explicit(&ha.failover_requested, memory_order_acquire) != 0); ha_fini(&ha); printf("test_p3_heartbeat_timeout_sets_failover_flag: PASS\n"); } /* ================================================================ * P3-2: Redirect state-machine corner cases * ================================================================ */ /* A PRIMARY controller must never redirect requests to itself. */ static void test_p3_redirect_primary_never_redirects(void) { ha_state_t ha; ha_config_t cfg = make_p3_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); ha_do_promote(&ha); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha)); ha_fini(&ha); printf("test_p3_redirect_primary_never_redirects: PASS\n"); } /* STANDBY with no primary_host: must return 0 from ha_should_redirect() * and 1 from ha_no_primary_known() so the client knows to retry. */ static void test_p3_redirect_standby_no_primary_returns_nomaster(void) { ha_state_t ha; ha_config_t cfg = make_p3_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); /* primary_host is empty after ha_init */ TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha)); TEST_ASSERT_EQUAL_INT(1, ha_no_primary_known(&ha)); ha_fini(&ha); printf("test_p3_redirect_standby_no_primary_returns_nomaster: PASS\n"); } /* STANDBY with a populated primary_host + port: must redirect AND * ha_no_primary_known() must return 0. */ static void test_p3_redirect_standby_with_primary_redirects(void) { ha_state_t ha; ha_config_t cfg = make_p3_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1)); strncpy(ha.primary_host, "10.0.1.1", sizeof(ha.primary_host) - 1); ha.primary_port = 7000; TEST_ASSERT_EQUAL_INT(1, ha_should_redirect(&ha)); TEST_ASSERT_EQUAL_INT(0, ha_no_primary_known(&ha)); ha_fini(&ha); printf("test_p3_redirect_standby_with_primary_redirects: PASS\n"); } /* ================================================================ * P3-3: WAL corruption handling * * These complement test_e2e_wal.c by covering a mixed-type WAL sequence * (submit + update + worker events) rather than a uniform payload. * ================================================================ */ /* Torn write after a checkpoint: recovery must succeed and the recovered * LSN must be strictly less than the pre-truncation LSN. */ static void test_p3_wal_torn_write_after_checkpoint(void) { char tmpdir[256]; persist_t *p = make_p3_persist("torn_ckpt", tmpdir, sizeof(tmpdir)); TEST_ASSERT_NOT_NULL(p); /* Write entries of different types then checkpoint */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, "job-a", 5)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "job-b", 5)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH, "job-c", 5)); TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p)); /* Write more entries after checkpoint */ for (int i = 0; i < 5; i++) TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "post-ckpt", 9)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); uint64_t lsn_before = p->current_lsn; /* Locate segment and truncate the last WAL entry */ char seg_path[512]; TEST_ASSERT_EQUAL_INT(0, p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path))); teardown_p3_persist(p); /* Truncate to exactly 4 complete post-checkpoint entries (keep lsns 4..7, * drop lsn 8). The segment also holds pre-checkpoint entries 0..2 and * the checkpoint marker at lsn 3 — that is 4 entries × entry_size bytes * of header+payload before the post-checkpoint block. * * Pre-checkpoint entries: 3 × (hdr + 5) + 1 × (hdr + 0) checkpoint marker * Post-checkpoint entries: 5 × (hdr + 9); keep 4, drop 1. * * Use the same technique as test_e2e_wal.c: compute the byte offset that * includes exactly N-1 post-checkpoint entries and truncate there. * Entry sizes: pre-ckpt payload=5, ckpt payload=0, post-ckpt payload=9. */ off_t pre_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 5); off_t ckpt_marker_sz = (off_t)(sizeof(wal_entry_hdr_t) + 0); off_t post_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 9); /* Keep 3 pre-ckpt + 1 ckpt marker + 4 post-ckpt = drop the 5th post-ckpt */ off_t trunc_offset = 3 * pre_ckpt_entry_sz + ckpt_marker_sz + 4 * post_ckpt_entry_sz; int fd = open(seg_path, O_RDWR); if (fd >= 0) { ftruncate(fd, trunc_offset); close(fd); } /* Re-init with same workdir and recover */ persist_t *p2 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p2); persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(p2, tmpdir, "ctrl"); TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); /* Torn write: recovered LSN must be strictly less than pre-truncation LSN */ TEST_ASSERT_TRUE(p2->current_lsn < lsn_before); teardown_p3_persist(p2); printf("test_p3_wal_torn_write_after_checkpoint: PASS\n"); } /* CRC corruption in a mixed-type WAL: recovery must stop at the first * corrupted entry and return success (not an error code). */ static void test_p3_wal_crc_corruption_mixed_types_stops_replay(void) { char tmpdir[256]; persist_t *p = make_p3_persist("crc_mixed", tmpdir, sizeof(tmpdir)); TEST_ASSERT_NOT_NULL(p); /* 4 entries of differing types */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, "submit", 6)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "update", 6)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH, "finish", 6)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "update2", 7)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); char seg_path[512]; TEST_ASSERT_EQUAL_INT(0, p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path))); uint64_t lsn_before = p->current_lsn; teardown_p3_persist(p); /* Corrupt the payload area of the 2nd entry (index 1) */ off_t entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 6); off_t corrupt_offset = entry_sz + (off_t)sizeof(wal_entry_hdr_t); int fd = open(seg_path, O_RDWR); if (fd >= 0) { const uint8_t garbage[4] = {0xDE, 0xAD, 0xBE, 0xEF}; pwrite(fd, garbage, sizeof(garbage), corrupt_offset); close(fd); } /* Recover with fresh persist_t on the same workdir */ persist_t *p2 = calloc(1, sizeof(persist_t)); TEST_ASSERT_NOT_NULL(p2); persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(p2, tmpdir, "ctrl"); TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL)); TEST_ASSERT_EQUAL_INT(0, persist_start(p2)); int rc = persist_recover(p2); /* Recovery must succeed (0) even when it stops early */ TEST_ASSERT_EQUAL_INT(0, rc); /* Fewer than 4 entries replayed — stopped at the corrupted entry */ TEST_ASSERT_TRUE(p2->current_lsn < lsn_before); teardown_p3_persist(p2); printf("test_p3_wal_crc_corruption_mixed_types_stops_replay: PASS\n"); } /* ================================================================ * P3-4: WAL write throughput baseline * * 1 000 async WAL appends must complete in under 5 seconds. * This is a floor test — any regression that serialises all writes * on a single lock will blow this budget. * ================================================================ */ static void test_p3_wal_write_throughput_baseline(void) { char tmpdir[256]; persist_t *p = make_p3_persist("perf", tmpdir, sizeof(tmpdir)); TEST_ASSERT_NOT_NULL(p); struct timespec start, end; clock_gettime(CLOCK_MONOTONIC, &start); for (int i = 0; i < 1000; i++) { int rc = persist_wal_async(p, WAL_JOB_UPDATE, "perf_payload_data", 17); TEST_ASSERT_EQUAL_INT(0, rc); } clock_gettime(CLOCK_MONOTONIC, &end); long elapsed_ms = (end.tv_sec - start.tv_sec) * 1000L + (end.tv_nsec - start.tv_nsec) / 1000000L; /* All 1 000 writes must complete in under 5 s */ TEST_ASSERT_TRUE(elapsed_ms < 5000); /* LSN must have advanced by at least 1 000 */ TEST_ASSERT_TRUE(p->current_lsn >= 1000); teardown_p3_persist(p); printf("test_p3_wal_write_throughput_baseline: %ldms for 1000 writes, PASS\n", elapsed_ms); } /* ================================================================ * P3-5: Checkpoint GC retention predicates * * Exercises job_should_retain() and worker_should_retain() using * the job_t wrapper type (consistent with persist.c / test_e2e_gc.c). * These tests are complementary to test_gc_retention.c — they focus * on the interaction with non-default config values. * ================================================================ */ /* A running job must always be retained regardless of config. */ static void test_p3_gc_running_job_retained(void) { job_t j; memset(&j, 0, sizeof(j)); j.job.state = JOB_STAT_RUN; persist_config_t cfg = persist_config_defaults(); TEST_ASSERT_TRUE(job_should_retain(&j, &cfg)); printf("test_p3_gc_running_job_retained: PASS\n"); } /* An EXIT job whose finish_time is 2 h before the 1 h retention window * must be cleaned up. */ static void test_p3_gc_expired_finished_job_cleaned(void) { job_t j; memset(&j, 0, sizeof(j)); j.job.state = JOB_STAT_EXIT; j.job.finish_time = time(NULL) - 7200; /* 2 hours ago */ persist_config_t cfg = persist_config_defaults(); cfg.finished_retain_s = 3600; /* 1-hour retention window */ TEST_ASSERT_FALSE(job_should_retain(&j, &cfg)); printf("test_p3_gc_expired_finished_job_cleaned: PASS\n"); } /* A TRANSIENT job whose transient_since is past the configured timeout * must NOT be retained. */ static void test_p3_gc_transient_job_times_out(void) { job_t j; memset(&j, 0, sizeof(j)); j.job.state = JOB_STAT_TRANSIENT; j.job.transient_since = (uint64_t)(time(NULL) - 400); /* 400 s ago */ persist_config_t cfg = persist_config_defaults(); cfg.transient_job_timeout_s = 300; /* 5-minute window */ TEST_ASSERT_FALSE(job_should_retain(&j, &cfg)); printf("test_p3_gc_transient_job_times_out: PASS\n"); } /* An UNAVAIL worker whose dead_since is far past the retention window * must be cleaned up. */ static void test_p3_gc_expired_worker_cleaned(void) { xce_worker_t w; memset(&w, 0, sizeof(w)); w.status = XCE_WORKER_UNAVAIL; w.dead_since = time(NULL) - 100000; /* ~28 h ago */ persist_config_t cfg = persist_config_defaults(); cfg.worker_retain_s = 86400; /* 24-hour retention */ TEST_ASSERT_FALSE(worker_should_retain(&w, &cfg)); printf("test_p3_gc_expired_worker_cleaned: PASS\n"); } /* A DONE job still within the retention window must be retained. */ static void test_p3_gc_recent_done_job_retained(void) { job_t j; memset(&j, 0, sizeof(j)); j.job.state = JOB_STAT_DONE; j.job.finish_time = time(NULL) - 1800; /* 30 min ago */ persist_config_t cfg = persist_config_defaults(); cfg.finished_retain_s = 3600; TEST_ASSERT_TRUE(job_should_retain(&j, &cfg)); printf("test_p3_gc_recent_done_job_retained: PASS\n"); } /* ================================================================ * Suite entry point * ================================================================ */ void test_p3_hardening(void) { printf("\n>> P3-1..P3-5: Integration & Hardening\n\n"); /* P3-1: Failover state machine */ printf("-- P3-1: Failover state machine --\n"); RUN_TEST(test_p3_promote_demote_cycle_consistent_role); RUN_TEST(test_p3_heartbeat_timeout_sets_failover_flag); /* P3-2: Redirect corner cases */ printf("-- P3-2: Redirect --\n"); RUN_TEST(test_p3_redirect_primary_never_redirects); RUN_TEST(test_p3_redirect_standby_no_primary_returns_nomaster); RUN_TEST(test_p3_redirect_standby_with_primary_redirects); /* P3-3: WAL corruption */ printf("-- P3-3: WAL corruption --\n"); RUN_TEST(test_p3_wal_torn_write_after_checkpoint); RUN_TEST(test_p3_wal_crc_corruption_mixed_types_stops_replay); /* P3-4: Throughput baseline */ printf("-- P3-4: Throughput baseline --\n"); RUN_TEST(test_p3_wal_write_throughput_baseline); /* P3-5: GC retention */ printf("-- P3-5: GC retention --\n"); RUN_TEST(test_p3_gc_running_job_retained); RUN_TEST(test_p3_gc_expired_finished_job_cleaned); RUN_TEST(test_p3_gc_transient_job_times_out); RUN_TEST(test_p3_gc_expired_worker_cleaned); RUN_TEST(test_p3_gc_recent_done_job_retained); }