lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test: P3-1..P3-5 hardening tests — failover, redirect, WAL, GC, throughput baseline

Browse source 
13 new Unity tests in test_p3_hardening.c covering:
- P3-1: promote/demote cycle verifies role+term monotonicity and
        hooks_registered reset; heartbeat timeout sets failover_requested
- P3-2: ha_should_redirect/ha_no_primary_known corner cases for
        PRIMARY, STANDBY-no-primary, STANDBY-with-primary
- P3-3: torn-write recovery after checkpoint (mixed WAL types, exact
        offset truncation); CRC corruption stops replay in mixed-type WAL
- P3-4: 1000 async WAL writes complete in < 5 s (throughput floor)
- P3-5: GC retention predicates — running retained, expired EXIT
        cleaned, TRANSIENT timeout, expired UNAVAIL worker cleaned,
        recent DONE retained

No new implementation code. All called functions existed from P0-P2.
This commit is contained in:
Rongsong Shen 2026-05-07 23:39:11 +00:00
parent 6bae7b382c
commit 1f97aee1a3
3 changed files with 508 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/controller/meson.build
View file

@ -93,6 +93,7 @@ controller_unit_test_src = [
'testsuite/test_persist_repl.c',
'testsuite/test_repl_catchup.c',
'testsuite/test_p2_integration.c',
'testsuite/test_p3_hardening.c',
'persist.c',
'ha.c',
'repl.c',

2
src/controller/testsuite/controller_test.c
View file

@ -28,6 +28,7 @@ extern void test_repl_apply(void);
extern void test_persist_repl(void);
extern void test_repl_catchup(void);
extern void test_p2_integration(void);
extern void test_p3_hardening(void);
void setUp(void) { return; }
@ -125,6 +126,7 @@ int main(int argc, char* argv[])
TEST(test_persist_repl);
TEST(test_repl_catchup);
TEST(test_p2_integration);
TEST(test_p3_hardening);
return UNITY_END();
}

505
src/controller/testsuite/test_p3_hardening.c Normal file
View file

@ -0,0 +1,505 @@
/* P3-1 through P3-5: Integration & Hardening test suite
*
* P3-1: Failover state-machine — promote/demote cycle role + term consistency
* P3-2: Redirect corner cases — primary never redirects; standby redirects iff
* primary is known; ha_no_primary_known() complement.
* P3-3: WAL corruption handling — torn write after checkpoint; CRC corruption
* in mixed-type WAL sequence.
* P3-4: WAL write throughput baseline — 1 000 async appends < 5 s.
* P3-5: Checkpoint GC retention predicates — running, expired-finished,
* transient-timeout, expired-worker.
*
* No new implementation code is required — all called functions exist.
*/
#define _GNU_SOURCE
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "config.h"
#include "ha.h"
#include "persist.h"
#include "xce/types.h"
/* ================================================================
* Shared helpers
* ================================================================ */
static ha_config_t make_p3_ha_cfg(void)
{
ha_config_t c;
memset(&c, 0, sizeof(c));
c.heartbeat_interval_ms = 3000;
c.heartbeat_timeout_ms = 10000;
c.election_timeout_ms = 2000;
c.probe_connect_timeout_ms = 500;
c.max_promotion_retries = 5;
return c;
}
/* Create and start a fresh persist_t in a fresh mkdtemp directory.
* Caller owns the returned pointer and must persist_stop/fini/free it. */
static persist_t *make_p3_persist(const char *suffix, char *tmpdir_out,
size_t tmpdir_outsz)
{
char buf[256];
snprintf(buf, sizeof(buf), "/tmp/xce_p3_%s_XXXXXX", suffix);
if (!mkdtemp(buf)) return NULL;
if (tmpdir_out)
snprintf(tmpdir_out, tmpdir_outsz, "%s", buf);
persist_t *p = calloc(1, sizeof(persist_t));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* suppress background fsync races in tests */
persist_set_workdir(p, buf, "ctrl");
if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; }
return p;
}
static void teardown_p3_persist(persist_t *p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* Find the first .wal segment file in wal_path; fills out[] and returns 0. */
static int p3_find_wal_segment(const char *wal_path, char *out, size_t outsz)
{
DIR *d = opendir(wal_path);
if (!d) return -1;
struct dirent *e;
while ((e = readdir(d)) != NULL) {
size_t len = strlen(e->d_name);
if (len > 4 && strcmp(e->d_name + len - 4, ".wal") == 0) {
snprintf(out, outsz, "%s/%s", wal_path, e->d_name);
closedir(d);
return 0;
}
}
closedir(d);
return -1;
}
/* ================================================================
* P3-1: Failover state-machine
* ================================================================ */
/* Full promote → demote → re-promote cycle:
* - role transitions must be consistent
* - term must be monotonically increasing across promotions
* - hooks_registered must reset to 0 on demotion
*/
static void test_p3_promote_demote_cycle_consistent_role(void)
{
ha_state_t ha;
ha_config_t cfg = make_p3_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Initial state: STANDBY */
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
/* Promote once */
ha_do_promote(&ha);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
uint64_t term1 = atomic_load_explicit(&ha.term, memory_order_acquire);
TEST_ASSERT_TRUE(term1 > 0);
/* Demote */
ha_do_demote(&ha);
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
/* hooks_registered idempotency guard must be cleared on demotion */
TEST_ASSERT_EQUAL_INT(0, ha.hooks_registered);
/* Re-promote: term must be strictly greater than term1 */
ha_do_promote(&ha);
uint64_t term2 = atomic_load_explicit(&ha.term, memory_order_acquire);
TEST_ASSERT_TRUE(term2 > term1);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
printf("test_p3_promote_demote_cycle_consistent_role: PASS\n");
}
/* Heartbeat timeout with a known primary must set failover_requested
* (replicates the critical path exercised by the watchdog). */
static void test_p3_heartbeat_timeout_sets_failover_flag(void)
{
ha_state_t ha;
ha_config_t cfg = make_p3_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1 /* not highest priority */));
/* Seed a known primary so the timeout check fires */
strncpy(ha.primary_host, "dead-primary", XCE_MAXNAME_LEN - 1);
ha.primary_port = 7788;
/* Force an expired heartbeat timestamp */
clock_gettime(CLOCK_MONOTONIC, &ha.last_heartbeat);
ha.last_heartbeat.tv_sec -= 30; /* 30 s ago — far past 10 s timeout */
ha_check_heartbeat_nocluster(&ha);
TEST_ASSERT_TRUE(
atomic_load_explicit(&ha.failover_requested, memory_order_acquire) != 0);
ha_fini(&ha);
printf("test_p3_heartbeat_timeout_sets_failover_flag: PASS\n");
}
/* ================================================================
* P3-2: Redirect state-machine corner cases
* ================================================================ */
/* A PRIMARY controller must never redirect requests to itself. */
static void test_p3_redirect_primary_never_redirects(void)
{
ha_state_t ha;
ha_config_t cfg = make_p3_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
ha_do_promote(&ha);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha));
ha_fini(&ha);
printf("test_p3_redirect_primary_never_redirects: PASS\n");
}
/* STANDBY with no primary_host: must return 0 from ha_should_redirect()
* and 1 from ha_no_primary_known() so the client knows to retry. */
static void test_p3_redirect_standby_no_primary_returns_nomaster(void)
{
ha_state_t ha;
ha_config_t cfg = make_p3_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));
/* primary_host is empty after ha_init */
TEST_ASSERT_EQUAL_INT(0, ha_should_redirect(&ha));
TEST_ASSERT_EQUAL_INT(1, ha_no_primary_known(&ha));
ha_fini(&ha);
printf("test_p3_redirect_standby_no_primary_returns_nomaster: PASS\n");
}
/* STANDBY with a populated primary_host + port: must redirect AND
* ha_no_primary_known() must return 0. */
static void test_p3_redirect_standby_with_primary_redirects(void)
{
ha_state_t ha;
ha_config_t cfg = make_p3_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 1));
strncpy(ha.primary_host, "10.0.1.1", sizeof(ha.primary_host) - 1);
ha.primary_port = 7000;
TEST_ASSERT_EQUAL_INT(1, ha_should_redirect(&ha));
TEST_ASSERT_EQUAL_INT(0, ha_no_primary_known(&ha));
ha_fini(&ha);
printf("test_p3_redirect_standby_with_primary_redirects: PASS\n");
}
/* ================================================================
* P3-3: WAL corruption handling
*
* These complement test_e2e_wal.c by covering a mixed-type WAL sequence
* (submit + update + worker events) rather than a uniform payload.
* ================================================================ */
/* Torn write after a checkpoint: recovery must succeed and the recovered
* LSN must be strictly less than the pre-truncation LSN. */
static void test_p3_wal_torn_write_after_checkpoint(void)
{
char tmpdir[256];
persist_t *p = make_p3_persist("torn_ckpt", tmpdir, sizeof(tmpdir));
TEST_ASSERT_NOT_NULL(p);
/* Write entries of different types then checkpoint */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, "job-a", 5));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "job-b", 5));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH, "job-c", 5));
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
/* Write more entries after checkpoint */
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "post-ckpt", 9));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
uint64_t lsn_before = p->current_lsn;
/* Locate segment and truncate the last WAL entry */
char seg_path[512];
TEST_ASSERT_EQUAL_INT(0,
p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));
teardown_p3_persist(p);
/* Truncate to exactly 4 complete post-checkpoint entries (keep lsns 4..7,
* drop lsn 8). The segment also holds pre-checkpoint entries 0..2 and
* the checkpoint marker at lsn 3 — that is 4 entries × entry_size bytes
* of header+payload before the post-checkpoint block.
*
* Pre-checkpoint entries: 3 × (hdr + 5) + 1 × (hdr + 0) checkpoint marker
* Post-checkpoint entries: 5 × (hdr + 9); keep 4, drop 1.
*
* Use the same technique as test_e2e_wal.c: compute the byte offset that
* includes exactly N-1 post-checkpoint entries and truncate there.
* Entry sizes: pre-ckpt payload=5, ckpt payload=0, post-ckpt payload=9.
*/
off_t pre_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 5);
off_t ckpt_marker_sz = (off_t)(sizeof(wal_entry_hdr_t) + 0);
off_t post_ckpt_entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 9);
/* Keep 3 pre-ckpt + 1 ckpt marker + 4 post-ckpt = drop the 5th post-ckpt */
off_t trunc_offset = 3 * pre_ckpt_entry_sz
+ ckpt_marker_sz
+ 4 * post_ckpt_entry_sz;
int fd = open(seg_path, O_RDWR);
if (fd >= 0) {
ftruncate(fd, trunc_offset);
close(fd);
}
/* Re-init with same workdir and recover */
persist_t *p2 = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(p2);
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(p2, tmpdir, "ctrl");
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Torn write: recovered LSN must be strictly less than pre-truncation LSN */
TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);
teardown_p3_persist(p2);
printf("test_p3_wal_torn_write_after_checkpoint: PASS\n");
}
/* CRC corruption in a mixed-type WAL: recovery must stop at the first
* corrupted entry and return success (not an error code). */
static void test_p3_wal_crc_corruption_mixed_types_stops_replay(void)
{
char tmpdir[256];
persist_t *p = make_p3_persist("crc_mixed", tmpdir, sizeof(tmpdir));
TEST_ASSERT_NOT_NULL(p);
/* 4 entries of differing types */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_SUBMIT, "submit", 6));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "update", 6));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_FINISH, "finish", 6));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "update2", 7));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
char seg_path[512];
TEST_ASSERT_EQUAL_INT(0,
p3_find_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));
uint64_t lsn_before = p->current_lsn;
teardown_p3_persist(p);
/* Corrupt the payload area of the 2nd entry (index 1) */
off_t entry_sz = (off_t)(sizeof(wal_entry_hdr_t) + 6);
off_t corrupt_offset = entry_sz + (off_t)sizeof(wal_entry_hdr_t);
int fd = open(seg_path, O_RDWR);
if (fd >= 0) {
const uint8_t garbage[4] = {0xDE, 0xAD, 0xBE, 0xEF};
pwrite(fd, garbage, sizeof(garbage), corrupt_offset);
close(fd);
}
/* Recover with fresh persist_t on the same workdir */
persist_t *p2 = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(p2);
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(p2, tmpdir, "ctrl");
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
int rc = persist_recover(p2);
/* Recovery must succeed (0) even when it stops early */
TEST_ASSERT_EQUAL_INT(0, rc);
/* Fewer than 4 entries replayed — stopped at the corrupted entry */
TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);
teardown_p3_persist(p2);
printf("test_p3_wal_crc_corruption_mixed_types_stops_replay: PASS\n");
}
/* ================================================================
* P3-4: WAL write throughput baseline
*
* 1 000 async WAL appends must complete in under 5 seconds.
* This is a floor test — any regression that serialises all writes
* on a single lock will blow this budget.
* ================================================================ */
static void test_p3_wal_write_throughput_baseline(void)
{
char tmpdir[256];
persist_t *p = make_p3_persist("perf", tmpdir, sizeof(tmpdir));
TEST_ASSERT_NOT_NULL(p);
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < 1000; i++) {
int rc = persist_wal_async(p, WAL_JOB_UPDATE,
"perf_payload_data", 17);
TEST_ASSERT_EQUAL_INT(0, rc);
}
clock_gettime(CLOCK_MONOTONIC, &end);
long elapsed_ms = (end.tv_sec - start.tv_sec) * 1000L
+ (end.tv_nsec - start.tv_nsec) / 1000000L;
/* All 1 000 writes must complete in under 5 s */
TEST_ASSERT_TRUE(elapsed_ms < 5000);
/* LSN must have advanced by at least 1 000 */
TEST_ASSERT_TRUE(p->current_lsn >= 1000);
teardown_p3_persist(p);
printf("test_p3_wal_write_throughput_baseline: %ldms for 1000 writes, PASS\n",
elapsed_ms);
}
/* ================================================================
* P3-5: Checkpoint GC retention predicates
*
* Exercises job_should_retain() and worker_should_retain() using
* the job_t wrapper type (consistent with persist.c / test_e2e_gc.c).
* These tests are complementary to test_gc_retention.c — they focus
* on the interaction with non-default config values.
* ================================================================ */
/* A running job must always be retained regardless of config. */
static void test_p3_gc_running_job_retained(void)
{
job_t j;
memset(&j, 0, sizeof(j));
j.job.state = JOB_STAT_RUN;
persist_config_t cfg = persist_config_defaults();
TEST_ASSERT_TRUE(job_should_retain(&j, &cfg));
printf("test_p3_gc_running_job_retained: PASS\n");
}
/* An EXIT job whose finish_time is 2 h before the 1 h retention window
* must be cleaned up. */
static void test_p3_gc_expired_finished_job_cleaned(void)
{
job_t j;
memset(&j, 0, sizeof(j));
j.job.state = JOB_STAT_EXIT;
j.job.finish_time = time(NULL) - 7200; /* 2 hours ago */
persist_config_t cfg = persist_config_defaults();
cfg.finished_retain_s = 3600; /* 1-hour retention window */
TEST_ASSERT_FALSE(job_should_retain(&j, &cfg));
printf("test_p3_gc_expired_finished_job_cleaned: PASS\n");
}
/* A TRANSIENT job whose transient_since is past the configured timeout
* must NOT be retained. */
static void test_p3_gc_transient_job_times_out(void)
{
job_t j;
memset(&j, 0, sizeof(j));
j.job.state = JOB_STAT_TRANSIENT;
j.job.transient_since = (uint64_t)(time(NULL) - 400); /* 400 s ago */
persist_config_t cfg = persist_config_defaults();
cfg.transient_job_timeout_s = 300; /* 5-minute window */
TEST_ASSERT_FALSE(job_should_retain(&j, &cfg));
printf("test_p3_gc_transient_job_times_out: PASS\n");
}
/* An UNAVAIL worker whose dead_since is far past the retention window
* must be cleaned up. */
static void test_p3_gc_expired_worker_cleaned(void)
{
xce_worker_t w;
memset(&w, 0, sizeof(w));
w.status = XCE_WORKER_UNAVAIL;
w.dead_since = time(NULL) - 100000; /* ~28 h ago */
persist_config_t cfg = persist_config_defaults();
cfg.worker_retain_s = 86400; /* 24-hour retention */
TEST_ASSERT_FALSE(worker_should_retain(&w, &cfg));
printf("test_p3_gc_expired_worker_cleaned: PASS\n");
}
/* A DONE job still within the retention window must be retained. */
static void test_p3_gc_recent_done_job_retained(void)
{
job_t j;
memset(&j, 0, sizeof(j));
j.job.state = JOB_STAT_DONE;
j.job.finish_time = time(NULL) - 1800; /* 30 min ago */
persist_config_t cfg = persist_config_defaults();
cfg.finished_retain_s = 3600;
TEST_ASSERT_TRUE(job_should_retain(&j, &cfg));
printf("test_p3_gc_recent_done_job_retained: PASS\n");
}
/* ================================================================
* Suite entry point
* ================================================================ */
void test_p3_hardening(void)
{
printf("\n>> P3-1..P3-5: Integration & Hardening\n\n");
/* P3-1: Failover state machine */
printf("-- P3-1: Failover state machine --\n");
RUN_TEST(test_p3_promote_demote_cycle_consistent_role);
RUN_TEST(test_p3_heartbeat_timeout_sets_failover_flag);
/* P3-2: Redirect corner cases */
printf("-- P3-2: Redirect --\n");
RUN_TEST(test_p3_redirect_primary_never_redirects);
RUN_TEST(test_p3_redirect_standby_no_primary_returns_nomaster);
RUN_TEST(test_p3_redirect_standby_with_primary_redirects);
/* P3-3: WAL corruption */
printf("-- P3-3: WAL corruption --\n");
RUN_TEST(test_p3_wal_torn_write_after_checkpoint);
RUN_TEST(test_p3_wal_crc_corruption_mixed_types_stops_replay);
/* P3-4: Throughput baseline */
printf("-- P3-4: Throughput baseline --\n");
RUN_TEST(test_p3_wal_write_throughput_baseline);
/* P3-5: GC retention */
printf("-- P3-5: GC retention --\n");
RUN_TEST(test_p3_gc_running_job_retained);
RUN_TEST(test_p3_gc_expired_finished_job_cleaned);
RUN_TEST(test_p3_gc_transient_job_times_out);
RUN_TEST(test_p3_gc_expired_worker_cleaned);
RUN_TEST(test_p3_gc_recent_done_job_retained);
}