test: P0-G end-to-end smoke tests — WAL, checkpoint, hooks, GC, Cap'n Proto (P0-15a-e)
Add 5 E2E integration test files that cross sub-phase boundaries: - test_e2e_wal.c (P0-15a): WAL write+read-back (entry count), torn-write recovery via ftruncate + persist_recover(), chained-CRC corruption stops replay at last valid entry. - test_e2e_checkpoint.c (P0-15b): 20-entry WAL + checkpoint + 10 post- checkpoint entries + recover asserts current_lsn >= 30 and checkpoint_lsn > 0; WAL trim after two checkpoints; snap_keep_n rotation with 4 rounds. - test_e2e_hooks.c (P0-15c): persist_job_hook_fun(HOOK_DETAIL_JOB_NEW) writes WAL_JOB_SUBMIT entry; persist_worker_hook_fun(HOOK_DETAIL_WORKER_JOIN) writes WAL_WORKER_CTRL; 3-call sequence preserves monotone LSN order and verifies header LSN fields on disk. - test_e2e_gc.c (P0-15d): running job retained; expired finished job not retained; JOB_STAT_TRANSIENT times out; expired unavailable worker not retained; checkpoint-after-WAL integration smoke. - test_e2e_capnp.c (P0-15e): persist_config_t → capnp_persist_cfg_t 8-field round-trip; capnp_snap_hdr_t field values and struct size (48 bytes); capnp_repl_envelope_t fields including WAL_CRC_NONE sentinel. Total: 89 tests, 0 failures (was 72 before this commit).
This commit is contained in:
parent
b52fbef36d
commit
a7094fc47f
7 changed files with 1002 additions and 0 deletions
269
src/controller/testsuite/test_e2e_wal.c
Normal file
269
src/controller/testsuite/test_e2e_wal.c
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
/* P0-15a: WAL integrity end-to-end smoke tests
|
||||
*
|
||||
* Exercises cross-phase interactions:
|
||||
* - WAL writer (P0-B) produces segment files that can be read back
|
||||
* - Torn-write simulation (ftruncate) + persist_recover() (P0-C)
|
||||
* - Chained-CRC corruption detection during recovery replay
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
#include <zlib.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "persist.h"
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static char g_e2e_wal_tmpdir[256];
|
||||
|
||||
static persist_t* make_e2e_wal_persist(void)
|
||||
{
|
||||
strcpy(g_e2e_wal_tmpdir, "/tmp/xce_e2e_wal_XXXXXX");
|
||||
if (!mkdtemp(g_e2e_wal_tmpdir)) return NULL;
|
||||
|
||||
persist_t* p = calloc(1, sizeof(persist_t));
|
||||
if (!p) return NULL;
|
||||
|
||||
persist_config_t cfg = persist_config_defaults();
|
||||
cfg.wal_sync_ms = 5000; /* suppress background fsync races */
|
||||
|
||||
persist_set_workdir(p, g_e2e_wal_tmpdir, "e2e-wal");
|
||||
|
||||
if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
|
||||
if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; }
|
||||
return p;
|
||||
}
|
||||
|
||||
static void teardown_e2e_wal(persist_t* p)
|
||||
{
|
||||
persist_stop(p);
|
||||
persist_fini(p);
|
||||
free(p);
|
||||
}
|
||||
|
||||
/* Return the path to the first .wal segment in the WAL directory. */
|
||||
static int find_first_wal_segment(const char* wal_dir, char* out, size_t outsz)
|
||||
{
|
||||
DIR* d = opendir(wal_dir);
|
||||
if (!d) return -1;
|
||||
|
||||
struct dirent* e;
|
||||
while ((e = readdir(d)) != NULL) {
|
||||
size_t len = strlen(e->d_name);
|
||||
if (len > 4 && strcmp(e->d_name + len - 4, ".wal") == 0) {
|
||||
snprintf(out, outsz, "%s/%s", wal_dir, e->d_name);
|
||||
closedir(d);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
closedir(d);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Count valid wal_entry_hdr_t records in a segment file by reading
|
||||
* sequentially and checking header CRC (first 32 bytes of header). */
|
||||
static int count_valid_wal_entries(const char* seg_path)
|
||||
{
|
||||
int fd = open(seg_path, O_RDONLY);
|
||||
if (fd < 0) return -1;
|
||||
|
||||
int count = 0;
|
||||
off_t offset = 0;
|
||||
|
||||
for (;;) {
|
||||
wal_entry_hdr_t hdr;
|
||||
ssize_t n = pread(fd, &hdr, sizeof(hdr), offset);
|
||||
if (n != (ssize_t)sizeof(hdr)) break;
|
||||
|
||||
/* Verify header CRC (first WAL_HDR_CRC_SIZE bytes) */
|
||||
uint32_t expected_crc = (uint32_t)crc32(0L,
|
||||
(const unsigned char*)&hdr, WAL_HDR_CRC_SIZE);
|
||||
if (hdr.payload_len > 0) {
|
||||
unsigned char* payload = malloc(hdr.payload_len);
|
||||
if (!payload) break;
|
||||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||||
offset + (off_t)sizeof(hdr));
|
||||
if (pn == (ssize_t)hdr.payload_len)
|
||||
expected_crc = (uint32_t)crc32(expected_crc,
|
||||
payload, hdr.payload_len);
|
||||
free(payload);
|
||||
}
|
||||
|
||||
if (expected_crc != hdr.crc) break;
|
||||
|
||||
count++;
|
||||
offset += (off_t)(sizeof(hdr) + hdr.payload_len);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
return count;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Test 1: write 10 WAL entries and verify they appear in segment file */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static void test_wal_write_and_verify_entry_count(void)
|
||||
{
|
||||
persist_t* p = make_e2e_wal_persist();
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
|
||||
const char payload[] = "e2e-entry";
|
||||
for (int i = 0; i < 10; i++) {
|
||||
int rc = persist_wal_async(p, WAL_JOB_UPDATE, payload,
|
||||
(uint32_t)strlen(payload));
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
}
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
|
||||
char seg_path[512];
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));
|
||||
|
||||
int count = count_valid_wal_entries(seg_path);
|
||||
TEST_ASSERT_EQUAL_INT(10, count);
|
||||
|
||||
teardown_e2e_wal(p);
|
||||
printf("test_wal_write_and_verify_entry_count: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Test 2: torn-write recovery — truncate last entry, recover, check */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static void test_wal_torn_write_recovery(void)
|
||||
{
|
||||
/* Phase 1: write 5 entries and flush */
|
||||
persist_t* p = make_e2e_wal_persist();
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
|
||||
const char payload[] = "torn";
|
||||
for (int i = 0; i < 5; i++) {
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
persist_wal_async(p, WAL_JOB_UPDATE, payload,
|
||||
(uint32_t)strlen(payload)));
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
|
||||
/* Locate segment file and determine size of 4 complete entries */
|
||||
char seg_path[512];
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));
|
||||
|
||||
/* One entry = sizeof(wal_entry_hdr_t) + payload_len */
|
||||
off_t entry_size = (off_t)(sizeof(wal_entry_hdr_t) + strlen(payload));
|
||||
off_t trunc_offset = entry_size * 4; /* keep only 4 complete entries */
|
||||
|
||||
/* Truncate: simulate torn write on entry 5 */
|
||||
int fd = open(seg_path, O_RDWR);
|
||||
TEST_ASSERT_TRUE(fd >= 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, ftruncate(fd, trunc_offset));
|
||||
close(fd);
|
||||
|
||||
uint64_t lsn_before = p->current_lsn;
|
||||
|
||||
teardown_e2e_wal(p); /* stop / destroy the first instance */
|
||||
|
||||
/* Phase 2: fresh persist_t pointing at the same directory */
|
||||
persist_t* p2 = calloc(1, sizeof(persist_t));
|
||||
TEST_ASSERT_NOT_NULL(p2);
|
||||
|
||||
persist_config_t cfg = persist_config_defaults();
|
||||
cfg.wal_sync_ms = 5000;
|
||||
persist_set_workdir(p2, g_e2e_wal_tmpdir, "e2e-wal");
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
|
||||
|
||||
int rc = persist_recover(p2);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
/* Recovery must have replayed at most 4 entries (not 5 — torn write) */
|
||||
TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);
|
||||
|
||||
teardown_e2e_wal(p2);
|
||||
printf("test_wal_torn_write_recovery: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Test 3: chained CRC corruption stops replay at last valid entry */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static void test_wal_chained_crc_detects_corruption(void)
|
||||
{
|
||||
/* Phase 1: write 5 entries and flush */
|
||||
persist_t* p = make_e2e_wal_persist();
|
||||
TEST_ASSERT_NOT_NULL(p);
|
||||
|
||||
const char payload[] = "crc-chain";
|
||||
for (int i = 0; i < 5; i++) {
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
persist_wal_async(p, WAL_JOB_UPDATE, payload,
|
||||
(uint32_t)strlen(payload)));
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
|
||||
|
||||
char seg_path[512];
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
find_first_wal_segment(p->wal_path, seg_path, sizeof(seg_path)));
|
||||
|
||||
/* Corrupt payload of entry 3 (zero-indexed: entries 0,1 valid; entry 2
|
||||
* corrupted; entries 3,4 unreachable via CRC chain). */
|
||||
off_t entry_size = (off_t)(sizeof(wal_entry_hdr_t) + strlen(payload));
|
||||
off_t corrupt_offset = entry_size * 2 + (off_t)sizeof(wal_entry_hdr_t);
|
||||
/* Write garbage over the payload of entry 2 */
|
||||
int fd = open(seg_path, O_RDWR);
|
||||
TEST_ASSERT_TRUE(fd >= 0);
|
||||
const char garbage[] = "\xFF\xFF\xFF\xFF";
|
||||
pwrite(fd, garbage, sizeof(garbage) - 1, corrupt_offset);
|
||||
close(fd);
|
||||
|
||||
uint64_t lsn_before = p->current_lsn;
|
||||
teardown_e2e_wal(p);
|
||||
|
||||
/* Phase 2: fresh persist_t — recover must stop before the corrupt entry */
|
||||
persist_t* p2 = calloc(1, sizeof(persist_t));
|
||||
TEST_ASSERT_NOT_NULL(p2);
|
||||
|
||||
persist_config_t cfg = persist_config_defaults();
|
||||
cfg.wal_sync_ms = 5000;
|
||||
persist_set_workdir(p2, g_e2e_wal_tmpdir, "e2e-wal");
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
|
||||
|
||||
int rc = persist_recover(p2);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
/* Must have replayed fewer than 5 entries — stopped at corruption */
|
||||
TEST_ASSERT_TRUE(p2->current_lsn < lsn_before);
|
||||
|
||||
teardown_e2e_wal(p2);
|
||||
printf("test_wal_chained_crc_detects_corruption: PASS\n");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Suite entry point */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
void test_e2e_wal(void)
|
||||
{
|
||||
printf("\n>> P0-15a: WAL E2E — torn write + CRC corruption recovery\n\n");
|
||||
|
||||
RUN_TEST(test_wal_write_and_verify_entry_count);
|
||||
RUN_TEST(test_wal_torn_write_recovery);
|
||||
RUN_TEST(test_wal_chained_crc_detects_corruption);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue