lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(walcompare): add per-member checkpoint-floor awareness (raft-ha-core-safety 5.10a / ha-entity-snapshots 7.1)

Browse source 
walcompare previously had no concept of a per-member compaction floor:
committed_lsn only capped the HIGH end (skip LSNs a member hasn't
caught up to yet), but the LOW end unconditionally expected every
member to hold raw WAL from LSN 1. A member that legitimately caught
up via a full-checkpoint snapshot holds no raw WAL at or below its own
checkpoint LSN by design (that range was entity-restored, not
WAL-replayed) -- so it was always flagged MISSING/divergent there,
regardless of whether the restore was correct. This is the root cause
ha-entity-snapshots task 7.1 diagnosed (committed_divergence=11 on an
otherwise-healthy soak) and raft-ha-core-safety task 5.10a's residual.

Adds member_snapshot_floor(): each member's own snap/ sibling
directory (matching the deployed <data>/wal + <data>/snap layout) is
scanned for the highest installed snap-<lsn>.{capnp,bin} file. Any LSN
at or below that member's own floor is excluded from its comparison
the same way an LSN above its own committed_lsn already is -- never
uniform/shared across members, since compaction is a per-member fact.
wc_status() gains a new "compacted" token distinct from "MISSING" so
the machine-readable report can tell the two apart.

Two new regression tests prove: (1) the floor correctly suppresses the
compacted range from comparison, and (2) it does NOT mask a genuine
divergence within the range a member still claims to hold. Caught and
fixed one bug myself before landing: the floor comparison was
off-by-one (< instead of <=), which the first test run surfaced
immediately (Expected 0 Was 1).

test_walcompare 17/17, test_ha_core_safety 28/0, test_config_repl
66/0, test_waldump 7/0 (all no-regression). Soak --selftest 17/17.
This commit is contained in:
Rongsong Shen 2026-07-02 02:59:08 +00:00
parent 2806a96c37
commit f3c86a683a
3 changed files with 246 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

121
src/controller/testsuite/test_walcompare.c
View file

@ -2,6 +2,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zlib.h>
@ -217,6 +218,124 @@ static void test_divergence_above_committed_boundary_not_flagged(void)
TEST_ASSERT_EQUAL_UINT64(3, div.lsn);
}
/* ------------------------------------------------------------------ *
* Harness for the per-member checkpoint-floor tests (raft-ha-core-safety
* task 5.10a / ha-entity-snapshots task 7.1): builds `<dir_out>/wal/` (real
* segs, LSNs starting at `start_lsn`, contiguous CRC chain) and, when
* `floor_lsn > 0`, an empty `<dir_out>/snap/snap-<floor_lsn:016x>.capnp`
* marker (walcompare only parses the FILENAME's embedded LSN, never the
* content) — the sibling `wal/`+`snap/` layout `member_snapshot_floor`'s
* "Candidate 1" path expects, matching the real deployed layout
* (`<data>/wal/`, `<data>/snap/`).
* ------------------------------------------------------------------ */
static void build_member_floored(char* dir_out, size_t dirsz,
const wal_spec_t* specs, int n,
uint64_t start_lsn, uint64_t floor_lsn)
{
snprintf(dir_out, dirsz, "/tmp/xce_walcmp_fl_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(dir_out));
char waldir[288];
snprintf(waldir, sizeof(waldir), "%s/wal", dir_out);
TEST_ASSERT_EQUAL_INT(0, mkdir(waldir, 0755));
char path[512];
snprintf(path, sizeof(path), "%s/seg-%016lx.wal", waldir,
(unsigned long)start_lsn);
FILE* fp = fopen(path, "wb");
TEST_ASSERT_NOT_NULL(fp);
uint32_t last = WAL_CRC_NONE;
for (int i = 0; i < n; i++) {
uint32_t plen =
specs[i].payload ? (uint32_t)strlen(specs[i].payload) : 0;
put_entry(fp, specs[i].term, start_lsn + (uint64_t)i, specs[i].type,
specs[i].payload, plen, &last, i == 0);
}
fclose(fp);
if (floor_lsn > 0) {
char snapdir[288];
snprintf(snapdir, sizeof(snapdir), "%s/snap", dir_out);
TEST_ASSERT_EQUAL_INT(0, mkdir(snapdir, 0755));
char snapfile[512];
snprintf(snapfile, sizeof(snapfile), "%s/snap-%016lx.capnp", snapdir,
(unsigned long)floor_lsn);
FILE* sf = fopen(snapfile, "wb");
TEST_ASSERT_NOT_NULL(sf);
fclose(sf); /* content is irrelevant — only the filename is parsed */
}
}
/* ------------------------------------------------------------------ */
/* A member that caught up via a full-checkpoint snapshot legitimately */
/* holds NO raw WAL below its own installed checkpoint LSN — that range */
/* must be excluded from comparison, never flagged MISSING/divergent */
/* (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1). */
/* ------------------------------------------------------------------ */
static void test_checkpoint_floor_excludes_compacted_range(void)
{
/* Member A: full history, LSNs 1..5, uniform term 1. */
wal_spec_t a_specs[] = {
{ 1, WAL_NOOP, NULL },
{ 1, WAL_JOB_SUBMIT, "jobA" },
{ 1, WAL_JOB_SUBMIT, "jobB" },
{ 1, WAL_JOB_SUBMIT, "jobC" },
{ 1, WAL_JOB_SUBMIT, "jobD" },
};
/* Member B: caught up via a checkpoint at LSN 3 — its raw WAL only
* physically holds LSNs 4..5 (matching A's LSN 4..5 exactly), and it
* carries a snap-...0003.capnp marker declaring that floor. */
wal_spec_t b_specs[] = {
{ 1, WAL_JOB_SUBMIT, "jobC" }, /* LSN 4 */
{ 1, WAL_JOB_SUBMIT, "jobD" }, /* LSN 5 */
};
char a[256], b[256];
build_member(a, sizeof(a), a_specs, 5);
build_member_floored(b, sizeof(b), b_specs, 2, /*start_lsn=*/4,
/*floor_lsn=*/3);
walcompare_member_t m[2] = { { a, 5 }, { b, 5 } };
walcompare_divergence_t div;
int rc = walcompare(m, 2, stdout, &div);
TEST_ASSERT_EQUAL_INT_MESSAGE(WALCMP_OK, rc,
"a member's own checkpoint floor must suppress the compacted range "
"from comparison, not flag it MISSING");
}
/* ------------------------------------------------------------------ */
/* The checkpoint floor must NOT mask a genuine divergence within the */
/* range the compacted member DOES claim to hold (past its own floor). */
/* ------------------------------------------------------------------ */
static void test_checkpoint_floor_does_not_mask_real_divergence(void)
{
wal_spec_t a_specs[] = {
{ 1, WAL_NOOP, NULL },
{ 1, WAL_JOB_SUBMIT, "jobA" },
{ 1, WAL_JOB_SUBMIT, "jobB" },
{ 1, WAL_JOB_SUBMIT, "jobC" },
{ 1, WAL_JOB_SUBMIT, "jobD" },
};
/* Member B's post-floor LSN 4 entry DISAGREES with A's (different
* term+payload) — this is a real divergence and must still be caught. */
wal_spec_t b_specs[] = {
{ 9, WAL_JOB_SUBMIT, "DIFFERENT" }, /* LSN 4 — diverges from A */
{ 1, WAL_JOB_SUBMIT, "jobD" }, /* LSN 5 */
};
char a[256], b[256];
build_member(a, sizeof(a), a_specs, 5);
build_member_floored(b, sizeof(b), b_specs, 2, /*start_lsn=*/4,
/*floor_lsn=*/3);
walcompare_member_t m[2] = { { a, 5 }, { b, 5 } };
walcompare_divergence_t div;
int rc = walcompare(m, 2, stdout, &div);
TEST_ASSERT_EQUAL_INT_MESSAGE(WALCMP_DIVERGENT, rc,
"a checkpoint floor excludes the compacted range only -- a real "
"divergence within the member's post-floor range must still be caught");
TEST_ASSERT_EQUAL_UINT64(4, div.lsn);
}
/* ------------------------------------------------------------------ */
/* A member MISSING a committed entry (a hole inside the committed */
/* prefix the boundary covers) is a divergence (R4.2/R6.4). */
@ -515,6 +634,8 @@ void test_walcompare(void)
RUN_TEST(test_same_lsn_diff_payload_divergent);
RUN_TEST(test_uncommitted_tail_not_flagged);
RUN_TEST(test_divergence_above_committed_boundary_not_flagged);
RUN_TEST(test_checkpoint_floor_excludes_compacted_range);
RUN_TEST(test_checkpoint_floor_does_not_mask_real_divergence);
RUN_TEST(test_missing_committed_entry_divergent);
RUN_TEST(test_legacy_zero_term_not_divergent);
RUN_TEST(test_three_members_identical_ok);