wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/controller/testsuite/test_e2e_checkpoint.c
Rongsong Shen f28ce57993 fix(raft-apply-decoupling): bug-1419 layer 3 — close cross-primary client-retry dedup gap 
jobid_range_exists() (c->jobmgr) and the in-flight registry (job.c,
per-process, only populated by this node's own handle_jobsubmit calls)
both miss a real cross-node case: a client's XCECLI_FAILOVER_CALL retry
resends the same client-generated jobid after a connection drop, and the
resend lands on a newly elected primary that never processed the original
request. If the original was already committed before the old primary
died, the new primary's own local WAL already holds it (Raft Leader
Completeness) but jobmgr hasn't applied it yet — both existing guards see
nothing, so the resend appends a second, genuinely committed WAL_JOB_SUBMIT
entry for the same jobid. A live chaotic soak reproduced this exactly
(waldump: two committed entries, same term, matching apiserverId/timestamp/
sequence, one second apart) — apply's own idempotent dedup keeps jobmgr
correct, but replay's raw WAL_JOB_SUBMIT count no longer matches jobmgr's
growth, firing a false APPLY_DIVERGENT fence and deadlocking promotion.

Fix: apply_committed_entry()'s WAL_JOB_SUBMIT case now claims the jobid in
job.c's in-flight registry (new job_wal_commit_claim/release, wrapping the
existing inflight_range_try_add/remove) for the duration of dispatch,
regardless of whether this node originated the submission or is applying a
replicated/replayed entry. A resend landing mid-dispatch on any node now
sees the same -EEXIST rejection self-originated in-flight submits already
get. Release only fires when the claim call added the entry, so a
self-originated submission's own claim/release lifecycle (unchanged) is
never double-released or stolen.

Also fixes 4 pre-existing tests that broke under the earlier checkpoint_lsn
fix (0eb1054) and were missed in that commit's regression sweep: cluster-
less fixtures where last_applied_lsn never advances now correctly get
checkpoint_lsn=0 unless the test simulates apply catch-up, matching the
new (correct) semantics. One assertion in
test_checkpoint_tail_with_no_snapshot_is_apply_clean compared Phase 1's
checkpoint_lsn against Phase 2's independently-reconstructed physical WAL
tip — that equality only held under the pre-fix current_lsn-tracking
behavior and is removed, not patched around.

Also hardens run-ha-soak.sh's start_node() with setsid+nohup: a forked/
background test-harness invocation whose session tears down mid-run was
silently killing every controller process via SIGHUP forwarding (observed
twice — deploy completes, ~1s of activity, then all 3 nodes die together
with no error logged).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VgT3n4vTv4ZjCuCTmzed1F
2026-07-08 08:45:22 +00:00

347 lines
13 KiB
C
Raw Blame History

/* P0-15b: Checkpoint E2E — round trip
*
* Exercises cross-phase interactions:
* - WAL writer (P0-B) + checkpoint (P0-C) + recovery (P0-C)
* - LSN continuity across checkpoint + post-checkpoint WAL entries
* - WAL segment trimming after checkpoint
* - Snapshot rotation keeping snap_keep_n files
*/
#define _GNU_SOURCE
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <unity/unity.h>
#include "config.h"
#include "persist.h"
#include "repl.h" /* repl_add_standby / repl_on_ack for trim-guard tests
* (raft-drive-async task 3.2, bug-1147) */
#include "typemap.h" /* wal_catchup_entry_t */
/* Drop hook for repl_broadcast: fixtures with dummy (int) endpoints installed
* via repl_add_standby must install this so persist_wal_async→repl_broadcast does
* not crash on the fake endpoint. Pattern from test_commit_engine.c, test_p2_integration.c. */
static int ckpt_drop_broadcast(void *ep, wal_catchup_entry_t **entries, int count)
{
(void)ep;
(void)entries;
(void)count;
return 0;
}
extern int (*repl_broadcast_send_hook)(void *ep,
wal_catchup_entry_t **entries,
int count);
/* ------------------------------------------------------------------ */
/* Helpers */
/* ------------------------------------------------------------------ */
static int count_files_suffix(const char* dir, const char* suffix)
{
DIR* d = opendir(dir);
if (!d) return 0;
int n = 0;
size_t slen = strlen(suffix);
struct dirent* e;
while ((e = readdir(d)) != NULL) {
size_t len = strlen(e->d_name);
if (len > slen && strcmp(e->d_name + len - slen, suffix) == 0)
n++;
}
closedir(d);
return n;
}
/* Make a persist_t with a fresh tmpdir. Caller must persist_stop/fini/free. */
static persist_t* make_ckpt_e2e(char* tmpdir, size_t sz,
uint32_t snap_keep_n)
{
snprintf(tmpdir, sz, "/tmp/xce_e2e_ckpt_XXXXXX");
if (!mkdtemp(tmpdir)) return NULL;
persist_t* p = calloc(1, sizeof(persist_t));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = snap_keep_n;
persist_set_workdir(p, tmpdir, "e2e-ckpt");
if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; }
return p;
}
static void teardown_ckpt_e2e(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* ------------------------------------------------------------------ */
/* Test 1: LSN continuity — 20 WAL + checkpoint + 10 WAL + recovery */
/* ------------------------------------------------------------------ */
static void test_checkpoint_recovery_lsn_continuity(void)
{
char tmpdir[256];
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* Phase 1: 20 WAL entries */
for (int i = 0; i < 20; i++) {
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
}
/* No live apply pump runs in this cluster-less fixture, so
* last_applied_lsn stays 0 unless simulated -- persist_checkpoint now
* stamps checkpoint_lsn from last_applied_lsn, not current_lsn
* (checkpoint_lsn-vs-last_applied_lsn fix); simulate apply having
* caught up to what was appended before the checkpoint runs. */
atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
memory_order_relaxed);
/* Checkpoint */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
uint64_t ckpt_lsn = p->checkpoint_lsn;
TEST_ASSERT_TRUE(ckpt_lsn > 0);
/* Phase 2: 10 more WAL entries after checkpoint */
for (int i = 0; i < 10; i++) {
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
}
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_ckpt_e2e(p);
/* Phase 3: fresh persist_t, recover */
persist_t* p2 = calloc(1, sizeof(persist_t));
TEST_ASSERT_NOT_NULL(p2);
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = 2;
persist_set_workdir(p2, tmpdir, "e2e-ckpt");
TEST_ASSERT_EQUAL_INT(0, persist_init(p2, &cfg, NULL));
TEST_ASSERT_EQUAL_INT(0, persist_start(p2));
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
/* All 30+ entries must be accounted for */
TEST_ASSERT_TRUE(p2->current_lsn >= 30);
/* The snapshot was used */
TEST_ASSERT_TRUE(p2->checkpoint_lsn > 0);
teardown_ckpt_e2e(p2);
printf("test_checkpoint_recovery_lsn_continuity: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 2: WAL trim — old segments removed after two checkpoints */
/* ------------------------------------------------------------------ */
static void test_checkpoint_wal_trim_after_checkpoint(void)
{
char tmpdir[256];
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* Write enough entries and checkpoint once */
for (int i = 0; i < 5; i++)
persist_wal_async(p, WAL_JOB_UPDATE, "trim", 4);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
/* Write a few more entries then checkpoint again */
for (int i = 0; i < 3; i++)
persist_wal_async(p, WAL_JOB_UPDATE, "trim2", 5);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
/* After two checkpoints the wal directory must still be functional
* (at least one segment exists — the currently-active one). */
char wal_path[512];
snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);
int n = count_files_suffix(wal_path, ".wal");
TEST_ASSERT_TRUE(n >= 1);
teardown_ckpt_e2e(p);
printf("test_checkpoint_wal_trim_after_checkpoint: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 3: snapshot rotation keeps at most snap_keep_n files */
/* ------------------------------------------------------------------ */
static void test_snapshot_rotation_keeps_snap_keep_n(void)
{
char tmpdir[256];
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* Run 4 checkpoints with snap_keep_n = 2 */
for (int i = 0; i < 4; i++) {
persist_wal_async(p, WAL_JOB_UPDATE, "r", 1);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
}
char snap_path[512];
snprintf(snap_path, sizeof(snap_path), "%s/e2e-ckpt/data/snap", tmpdir);
int n = count_files_suffix(snap_path, ".capnp");
TEST_ASSERT_TRUE(n <= (int)p->config.snap_keep_n);
teardown_ckpt_e2e(p);
printf("test_snapshot_rotation_keeps_snap_keep_n: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Helper: check if a WAL segment file at a given start LSN exists. */
/* Builds the name "seg-<lsn16>.wal" inside dir. */
/* ------------------------------------------------------------------ */
static int seg_file_exists(const char* wal_dir, uint64_t seg_start_lsn)
{
char path[4096];
snprintf(path, sizeof(path), "%s/seg-%016" PRIx64 ".wal",
wal_dir, seg_start_lsn);
return access(path, F_OK) == 0 ? 1 : 0;
}
/* ------------------------------------------------------------------ */
/* Test 4: standby cursor-bound trim guard — Req 1.1 (task 3.2, */
/* bug-1147). With a standby whose acked_lsn is below the */
/* active segment, segments the standby still needs for */
/* catch-up are NOT deleted. */
/* */
/* RED proof: revert the repl_trim_lsn floor call in trim_wal_segments */
/* → the segment at the standby's acked LSN is deleted because it's */
/* below the active segment → re-creates the bug-795/828 gap where */
/* catch-up hits a missing segment → permanent committed-job loss. */
/* ------------------------------------------------------------------ */
static void test_trim_guard_preserves_standby_segments(void)
{
char tmpdir[256];
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* Install drop hook so repl_broadcast doesn't crash on the dummy ep */
repl_broadcast_send_hook = ckpt_drop_broadcast;
/* Add a standby with acked_lsn = 2 (simulating a standby that has
* caught up through LSN 2 but not beyond) */
int dummy_ep = 0;
int idx = repl_add_standby(&p->repl, (void*)&dummy_ep, "standby1", 7001);
TEST_ASSERT_TRUE(idx >= 0);
repl_on_ack(&p->repl, (void*)&dummy_ep, 2);
/* Write entries to create segments.
* init: current_lsn = 1 (LSN 0 reserved, fresh store)
* write "x" @ LSN 1 → current_lsn = 2
* force → close seg-at-1, open seg-at-2
* write "y" @ LSN 2 → current_lsn = 3
* force → close seg-at-2, open seg-at-3
*
* Now we have 3 segments: seg-...001.wal, seg-...002.wal, seg-...003.wal.
* Active segment = seg-at-3. Standby acked_lsn = 2. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
/* Trigger checkpoint → trim_wal_segments runs */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
/* floor = repl_trim_lsn(active_seg_lsn=3, ...) = min(3, acked=2) = 2.
* So seg_start_lsn < 2 means only seg-at-1 is deleted (1 < 2).
* seg-at-2 (the standby still needs it) is NOT deleted (2 >= 2). */
char wal_path[512];
snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);
/* The old segment below the standby's acked LSN IS trimmed */
TEST_ASSERT_FALSE_MESSAGE(
seg_file_exists(wal_path, 1),
"seg-at-1: below standby acked_lsn=2, MUST be trimmed");
/* The segment at the standby's acked LSN is PRESERVED (bug-1147 guard) */
TEST_ASSERT_TRUE_MESSAGE(
seg_file_exists(wal_path, 2),
"seg-at-2: at standby acked_lsn=2, MUST be preserved (bug-1147 guard)");
/* The active segment is never trimmed */
TEST_ASSERT_TRUE_MESSAGE(
seg_file_exists(wal_path, 3),
"seg-at-3: active segment, MUST always be preserved");
repl_broadcast_send_hook = NULL;
teardown_ckpt_e2e(p);
printf("test_trim_guard_preserves_standby_segments: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Test 5: trim with no standbys proceeds as before — Req 1.1 */
/* (task 3.2). With no standbys registered, the floor */
/* defaults to active_seg_lsn, so all old segments are */
/* trimmed. This confirms the guard does not prevent normal */
/* cleanup when no standbys need protection. */
/* ------------------------------------------------------------------ */
static void test_trim_without_standbys_proceeds_normally(void)
{
char tmpdir[256];
persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
TEST_ASSERT_NOT_NULL(p);
/* No standbys — just create segments and checkpoint */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
char wal_path[512];
snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);
/* Without standbys, floor = active_seg_lsn = 3.
* seg_at_1 < 3 → deleted. seg_at_2 < 3 → deleted too. */
TEST_ASSERT_FALSE_MESSAGE(
seg_file_exists(wal_path, 1),
"seg-at-1: no standbys, MUST be trimmed");
TEST_ASSERT_FALSE_MESSAGE(
seg_file_exists(wal_path, 2),
"seg-at-2: no standbys, MUST be trimmed");
/* Active segment still there */
TEST_ASSERT_TRUE_MESSAGE(
seg_file_exists(wal_path, 3),
"seg-at-3: active segment, MUST always be preserved");
teardown_ckpt_e2e(p);
printf("test_trim_without_standbys_proceeds_normally: PASS\n");
}
/* ------------------------------------------------------------------ */
/* Suite entry point */
/* ------------------------------------------------------------------ */
void test_e2e_checkpoint(void)
{
printf("\n>> P0-15b: Checkpoint E2E — round trip\n\n");
RUN_TEST(test_checkpoint_recovery_lsn_continuity);
RUN_TEST(test_checkpoint_wal_trim_after_checkpoint);
RUN_TEST(test_snapshot_rotation_keeps_snap_keep_n);
RUN_TEST(test_trim_guard_preserves_standby_segments);
RUN_TEST(test_trim_without_standbys_proceeds_normally);
}
View git blame Copy permalink