diff --git a/src/controller/testsuite/test_walcompare.c b/src/controller/testsuite/test_walcompare.c index 7daa4438..6f81afd5 100644 --- a/src/controller/testsuite/test_walcompare.c +++ b/src/controller/testsuite/test_walcompare.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -217,6 +218,124 @@ static void test_divergence_above_committed_boundary_not_flagged(void) TEST_ASSERT_EQUAL_UINT64(3, div.lsn); } +/* ------------------------------------------------------------------ * + * Harness for the per-member checkpoint-floor tests (raft-ha-core-safety + * task 5.10a / ha-entity-snapshots task 7.1): builds `/wal/` (real + * segs, LSNs starting at `start_lsn`, contiguous CRC chain) and, when + * `floor_lsn > 0`, an empty `/snap/snap-.capnp` + * marker (walcompare only parses the FILENAME's embedded LSN, never the + * content) — the sibling `wal/`+`snap/` layout `member_snapshot_floor`'s + * "Candidate 1" path expects, matching the real deployed layout + * (`/wal/`, `/snap/`). + * ------------------------------------------------------------------ */ +static void build_member_floored(char* dir_out, size_t dirsz, + const wal_spec_t* specs, int n, + uint64_t start_lsn, uint64_t floor_lsn) +{ + snprintf(dir_out, dirsz, "/tmp/xce_walcmp_fl_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(dir_out)); + + char waldir[288]; + snprintf(waldir, sizeof(waldir), "%s/wal", dir_out); + TEST_ASSERT_EQUAL_INT(0, mkdir(waldir, 0755)); + + char path[512]; + snprintf(path, sizeof(path), "%s/seg-%016lx.wal", waldir, + (unsigned long)start_lsn); + FILE* fp = fopen(path, "wb"); + TEST_ASSERT_NOT_NULL(fp); + + uint32_t last = WAL_CRC_NONE; + for (int i = 0; i < n; i++) { + uint32_t plen = + specs[i].payload ? (uint32_t)strlen(specs[i].payload) : 0; + put_entry(fp, specs[i].term, start_lsn + (uint64_t)i, specs[i].type, + specs[i].payload, plen, &last, i == 0); + } + fclose(fp); + + if (floor_lsn > 0) { + char snapdir[288]; + snprintf(snapdir, sizeof(snapdir), "%s/snap", dir_out); + TEST_ASSERT_EQUAL_INT(0, mkdir(snapdir, 0755)); + char snapfile[512]; + snprintf(snapfile, sizeof(snapfile), "%s/snap-%016lx.capnp", snapdir, + (unsigned long)floor_lsn); + FILE* sf = fopen(snapfile, "wb"); + TEST_ASSERT_NOT_NULL(sf); + fclose(sf); /* content is irrelevant — only the filename is parsed */ + } +} + +/* ------------------------------------------------------------------ */ +/* A member that caught up via a full-checkpoint snapshot legitimately */ +/* holds NO raw WAL below its own installed checkpoint LSN — that range */ +/* must be excluded from comparison, never flagged MISSING/divergent */ +/* (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1). */ +/* ------------------------------------------------------------------ */ +static void test_checkpoint_floor_excludes_compacted_range(void) +{ + /* Member A: full history, LSNs 1..5, uniform term 1. */ + wal_spec_t a_specs[] = { + { 1, WAL_NOOP, NULL }, + { 1, WAL_JOB_SUBMIT, "jobA" }, + { 1, WAL_JOB_SUBMIT, "jobB" }, + { 1, WAL_JOB_SUBMIT, "jobC" }, + { 1, WAL_JOB_SUBMIT, "jobD" }, + }; + /* Member B: caught up via a checkpoint at LSN 3 — its raw WAL only + * physically holds LSNs 4..5 (matching A's LSN 4..5 exactly), and it + * carries a snap-...0003.capnp marker declaring that floor. */ + wal_spec_t b_specs[] = { + { 1, WAL_JOB_SUBMIT, "jobC" }, /* LSN 4 */ + { 1, WAL_JOB_SUBMIT, "jobD" }, /* LSN 5 */ + }; + char a[256], b[256]; + build_member(a, sizeof(a), a_specs, 5); + build_member_floored(b, sizeof(b), b_specs, 2, /*start_lsn=*/4, + /*floor_lsn=*/3); + + walcompare_member_t m[2] = { { a, 5 }, { b, 5 } }; + walcompare_divergence_t div; + int rc = walcompare(m, 2, stdout, &div); + TEST_ASSERT_EQUAL_INT_MESSAGE(WALCMP_OK, rc, + "a member's own checkpoint floor must suppress the compacted range " + "from comparison, not flag it MISSING"); +} + +/* ------------------------------------------------------------------ */ +/* The checkpoint floor must NOT mask a genuine divergence within the */ +/* range the compacted member DOES claim to hold (past its own floor). */ +/* ------------------------------------------------------------------ */ +static void test_checkpoint_floor_does_not_mask_real_divergence(void) +{ + wal_spec_t a_specs[] = { + { 1, WAL_NOOP, NULL }, + { 1, WAL_JOB_SUBMIT, "jobA" }, + { 1, WAL_JOB_SUBMIT, "jobB" }, + { 1, WAL_JOB_SUBMIT, "jobC" }, + { 1, WAL_JOB_SUBMIT, "jobD" }, + }; + /* Member B's post-floor LSN 4 entry DISAGREES with A's (different + * term+payload) — this is a real divergence and must still be caught. */ + wal_spec_t b_specs[] = { + { 9, WAL_JOB_SUBMIT, "DIFFERENT" }, /* LSN 4 — diverges from A */ + { 1, WAL_JOB_SUBMIT, "jobD" }, /* LSN 5 */ + }; + char a[256], b[256]; + build_member(a, sizeof(a), a_specs, 5); + build_member_floored(b, sizeof(b), b_specs, 2, /*start_lsn=*/4, + /*floor_lsn=*/3); + + walcompare_member_t m[2] = { { a, 5 }, { b, 5 } }; + walcompare_divergence_t div; + int rc = walcompare(m, 2, stdout, &div); + TEST_ASSERT_EQUAL_INT_MESSAGE(WALCMP_DIVERGENT, rc, + "a checkpoint floor excludes the compacted range only -- a real " + "divergence within the member's post-floor range must still be caught"); + TEST_ASSERT_EQUAL_UINT64(4, div.lsn); +} + /* ------------------------------------------------------------------ */ /* A member MISSING a committed entry (a hole inside the committed */ /* prefix the boundary covers) is a divergence (R4.2/R6.4). */ @@ -515,6 +634,8 @@ void test_walcompare(void) RUN_TEST(test_same_lsn_diff_payload_divergent); RUN_TEST(test_uncommitted_tail_not_flagged); RUN_TEST(test_divergence_above_committed_boundary_not_flagged); + RUN_TEST(test_checkpoint_floor_excludes_compacted_range); + RUN_TEST(test_checkpoint_floor_does_not_mask_real_divergence); RUN_TEST(test_missing_committed_entry_divergent); RUN_TEST(test_legacy_zero_term_not_divergent); RUN_TEST(test_three_members_identical_ok); diff --git a/src/controller/testsuite/walcompare.c b/src/controller/testsuite/walcompare.c index fdebd8dc..ff12924f 100644 --- a/src/controller/testsuite/walcompare.c +++ b/src/controller/testsuite/walcompare.c @@ -1,8 +1,11 @@ #define _GNU_SOURCE #include "walcompare.h" +#include +#include #include #include +#include #include #include @@ -16,6 +19,16 @@ typedef struct { uint64_t recovered_tip; /* highest CRC-intact LSN held (waldump tip) */ uint64_t committed_lsn; /* effective committed boundary for this member */ + /* raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: the + * highest checkpoint LSN this member has installed (0 = never installed + * a snapshot -- expect raw WAL from LSN 1, the pre-existing behavior, + * unchanged). A member that caught up via a full-checkpoint snapshot + * legitimately holds NO raw WAL entries at or below its own floor_lsn -- + * that range was restored via entity content, not WAL replay -- so it + * must never be flagged MISSING there. Read once per member from its own + * snap/ directory (scan_snapshot_floor); never uniform across members + * (unlike committed_lsn, which the caller may force to one shared value). */ + uint64_t floor_lsn; uint64_t cap; /* allocated index size (LSN range [0..cap-1]) */ uint64_t* term; /* raw on-disk term per LSN (0 = legacy / absent) */ uint32_t* pcrc; /* payload CRC per LSN */ @@ -96,6 +109,86 @@ static void member_log_free(member_log_t* m) m->cap = 0; } +/* Scan one directory for `snap-<16-hex-digit-lsn>.{capnp,bin}` files (the + * on-disk naming write_snapshot()/legacy checkpoints use — persist.c) and + * return the highest embedded LSN found, or 0 if the directory doesn't exist + * or holds no matching file. A bare 16-hex-digit match is required (skips + * `.tmp` in-progress files and anything else) so this never mis-parses an + * unrelated entry as a checkpoint LSN. */ +static uint64_t scan_snap_dir_floor(const char* dir) +{ + DIR* d = opendir(dir); + if (!d) + return 0; /* no such dir (no snapshot ever installed here) — floor 0 */ + + uint64_t max_lsn = 0; + struct dirent* de; + while ((de = readdir(d)) != NULL) { + unsigned long long lsn = 0; + char ext[8]; + /* "snap-XXXXXXXXXXXXXXXX.capnp" / ".bin" — %16llx would also accept + * fewer than 16 digits, so additionally check the literal width via + * strlen on the numeric run; simplest robust check: require the + * filename to be exactly "snap-" + 16 hex chars + "." + ext. */ + size_t len = strlen(de->d_name); + if (len < 5 + 16 + 1 + 3 || strncmp(de->d_name, "snap-", 5) != 0) + continue; + int matched = sscanf(de->d_name + 5, "%16llx.%7s", &lsn, ext); + if (matched != 2) + continue; + if (strcmp(ext, "capnp") != 0 && strcmp(ext, "bin") != 0) + continue; + /* Confirm the hex run is EXACTLY 16 chars (reject a short-digit name + * like "snap-1.capnp" that sscanf's %16llx would still happily take). */ + int hexlen = 0; + while (hexlen < 16 && isxdigit((unsigned char)de->d_name[5 + hexlen])) + hexlen++; + if (hexlen != 16 || de->d_name[5 + 16] != '.') + continue; + if ((uint64_t)lsn > max_lsn) + max_lsn = (uint64_t)lsn; + } + closedir(d); + return max_lsn; +} + +/* Find this member's own checkpoint floor (raft-ha-core-safety task 5.10a / + * ha-entity-snapshots task 7.1). `src->path` may be either a "data" dir + * containing sibling `wal/`+`snap/` subdirs (the soak harness's convention: + * `$TOP/controller/ctrl-N/data`), or the WAL dir itself passed directly (seen + * in unit-test call sites, e.g. `persist_t.wal_path`) — in the latter case the + * sibling snap dir sits at the same parent level with `wal` replaced by + * `snap`. Try both; a nonexistent candidate just yields floor 0 (harmless). */ +static uint64_t member_snapshot_floor(const char* path) +{ + if (!path || !*path) + return 0; + + char candidate[4160]; + uint64_t floor1 = 0, floor2 = 0; + + /* Candidate 1: /snap (path is a "data" parent dir). */ + snprintf(candidate, sizeof(candidate), "%s/snap", path); + floor1 = scan_snap_dir_floor(candidate); + + /* Candidate 2: sibling of with a trailing "/wal" (or "wal") + * component replaced by "/snap" (path is the WAL dir itself). */ + size_t plen = strlen(path); + const char* base = strrchr(path, '/'); + const char* leaf = base ? base + 1 : path; + if (strcmp(leaf, "wal") == 0) { + size_t prefix_len = (size_t)(leaf - path); + if (prefix_len + 4 < sizeof(candidate)) { + memcpy(candidate, path, prefix_len); + snprintf(candidate + prefix_len, sizeof(candidate) - prefix_len, "snap"); + floor2 = scan_snap_dir_floor(candidate); + } + } + (void)plen; + + return floor1 > floor2 ? floor1 : floor2; +} + /* Load one member's WAL into `*m`. Returns 0 or a negative errno. */ static int load_member(const walcompare_member_t* src, member_log_t* m, FILE* out) { @@ -129,6 +222,10 @@ static int load_member(const walcompare_member_t* src, member_log_t* m, FILE* ou /* An explicit committed_lsn wins; 0 means "derive from the shared * boundary" (filled by the caller after all members are loaded). */ m->committed_lsn = src->committed_lsn; + /* Always PER-MEMBER, never uniform/shared (unlike committed_lsn) — a + * member's own checkpoint floor has nothing to do with any other + * member's state. */ + m->floor_lsn = member_snapshot_floor(src->path); return 0; } @@ -146,6 +243,10 @@ static const char* wc_status(const member_log_t* m, uint64_t lsn) { if ((lsn < m->cap) && m->present[lsn]) return "present"; + if (lsn <= m->floor_lsn) + return "compacted"; /* below this member's OWN checkpoint floor — + * legitimately absent (entity-restored, not + * WAL-held), never a divergence signal */ if (lsn <= m->committed_lsn) return "MISSING"; /* a committed entry this member SHOULD hold (R4.2) */ return "beyond"; /* legitimately not held (above its committed prefix) */ @@ -247,6 +348,9 @@ static int wc_compare_committed(const wc_cluster_t* cl, for (int i = 0; i < cl->n; i++) { if (lsn > cl->logs[i].committed_lsn) continue; /* beyond member i's committed prefix */ + if (lsn <= cl->logs[i].floor_lsn) + continue; /* below member i's OWN checkpoint floor — legitimately + * compacted (entity-restored), never a divergence */ have++; const member_log_t* m = &cl->logs[i]; int holds = (lsn < m->cap) && m->present[lsn]; @@ -276,8 +380,8 @@ static int wc_compare_committed(const wc_cluster_t* cl, div->committed = 1; for (int i = 0; i < cl->n && i < 16; i++) { const member_log_t* m = &cl->logs[i]; - int holds = (lsn <= m->committed_lsn) && (lsn < m->cap) && - m->present[lsn]; + int holds = (lsn <= m->committed_lsn) && (lsn > m->floor_lsn) && + (lsn < m->cap) && m->present[lsn]; div->member_present[i] = holds ? 1 : 0; div->member_term[i] = holds ? match_term(m->term[lsn]) : 0; div->member_pcrc[i] = holds ? m->pcrc[lsn] : 0; @@ -296,6 +400,13 @@ static int wc_compare_committed(const wc_cluster_t* cl, i, members[i].path, cl->logs[i].committed_lsn); continue; } + if (lsn <= cl->logs[i].floor_lsn) { + fprintf(out, + " member %d ('%s'): LSN below its own checkpoint floor" + " (floor=%" PRIu64 ") — legitimately compacted\n", + i, members[i].path, cl->logs[i].floor_lsn); + continue; + } int holds = (lsn < cl->logs[i].cap) && cl->logs[i].present[lsn]; if (holds) fprintf(out, " member %d ('%s'): term=%" PRIu64 " pcrc=%08x\n", diff --git a/src/controller/testsuite/walcompare.h b/src/controller/testsuite/walcompare.h index 3a98ea06..420fe608 100644 --- a/src/controller/testsuite/walcompare.h +++ b/src/controller/testsuite/walcompare.h @@ -55,6 +55,18 @@ enum { * flagged for the uncommitted tail it doesn't share (R6.1 is the committed * prefix, not the full log). * + * Per-member checkpoint floor (raft-ha-core-safety task 5.10a / ha-entity- + * snapshots task 7.1): a member that caught up via a full-checkpoint snapshot + * legitimately holds no raw WAL entries at or below its own installed + * checkpoint LSN — that range was restored via entity content, not WAL + * replay. Each member's floor is read independently from its own `snap/` + * sibling directory (0 if none installed, the unchanged pre-existing + * behavior); an LSN below a member's own floor is excluded from that + * member's comparison the same way an LSN above its committed_lsn already + * is — never flagged MISSING/divergent. This is a PER-MEMBER floor, never a + * shared/uniform value (unlike committed_lsn, which the caller may force to + * one shared value across all members). + * * Legacy-zero term (R1.8) is mapped to WAL_LEGACY_TERM for matching, so two * pre-feature entries are not spuriously divergent. *