test(ha-term-safety): cross-member walcompare divergence tool (task 5.3)
Diffs member WALs by (lsn -> term, pcrc) over the committed prefix, exits non-zero on any committed-prefix divergence with the diverging LSN + per-member entry identities. Reuses the canonical waldump decoder via a new per-entry visitor seam (no second WAL parser). Detects same-LSN/diff-term and same-LSN/diff-content divergences (the bug-851 class) that an LSN-range check would mask.
This commit is contained in:
parent
d39500ba48
commit
a166ecb45b
9 changed files with 842 additions and 5 deletions
272
src/controller/testsuite/walcompare.c
Normal file
272
src/controller/testsuite/walcompare.c
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
#define _GNU_SOURCE
|
||||
#include "walcompare.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "persist.h" /* WAL_LEGACY_TERM */
|
||||
#include "waldump.h"
|
||||
|
||||
#define WALCMP_MAX_MEMBERS 16
|
||||
|
||||
/* One member's collected log, indexed by LSN. `slot[lsn]` is valid when
|
||||
* `present[lsn]` is set. LSNs are 1-based; index 0 is unused. */
|
||||
typedef struct {
|
||||
uint64_t recovered_tip; /* highest CRC-intact LSN held (waldump tip) */
|
||||
uint64_t committed_lsn; /* effective committed boundary for this member */
|
||||
uint64_t cap; /* allocated index size (LSN range [0..cap-1]) */
|
||||
uint64_t* term; /* raw on-disk term per LSN (0 = legacy / absent) */
|
||||
uint32_t* pcrc; /* payload CRC per LSN */
|
||||
uint8_t* present; /* 1 if this LSN is held */
|
||||
} member_log_t;
|
||||
|
||||
/* Visitor context: collect entries into a member_log_t. */
|
||||
typedef struct {
|
||||
member_log_t* log;
|
||||
int oom; /* set on an allocation failure mid-scan */
|
||||
} collect_ctx_t;
|
||||
|
||||
/* Map a raw on-disk term to its matching term: legacy 0 -> WAL_LEGACY_TERM
|
||||
* (R1.8) so two pre-feature entries are not spuriously divergent. */
|
||||
static uint64_t match_term(uint64_t raw)
|
||||
{
|
||||
return raw == 0 ? WAL_LEGACY_TERM : raw;
|
||||
}
|
||||
|
||||
/* Grow the per-LSN index so `lsn` is addressable. Returns 0 or -ENOMEM. */
|
||||
static int member_log_ensure(member_log_t* m, uint64_t lsn)
|
||||
{
|
||||
if (lsn < m->cap)
|
||||
return 0;
|
||||
uint64_t newcap = m->cap ? m->cap : 64;
|
||||
while (newcap <= lsn)
|
||||
newcap *= 2;
|
||||
|
||||
uint64_t* nt = realloc(m->term, newcap * sizeof(*nt));
|
||||
uint32_t* np = realloc(m->pcrc, newcap * sizeof(*np));
|
||||
uint8_t* ne = realloc(m->present, newcap * sizeof(*ne));
|
||||
if (!nt || !np || !ne) {
|
||||
/* realloc may have partially succeeded; keep whatever grew so the
|
||||
* eventual free() is safe, but report OOM. */
|
||||
if (nt)
|
||||
m->term = nt;
|
||||
if (np)
|
||||
m->pcrc = np;
|
||||
if (ne)
|
||||
m->present = ne;
|
||||
return -ENOMEM;
|
||||
}
|
||||
/* Zero the newly added tail. */
|
||||
memset(nt + m->cap, 0, (newcap - m->cap) * sizeof(*nt));
|
||||
memset(np + m->cap, 0, (newcap - m->cap) * sizeof(*np));
|
||||
memset(ne + m->cap, 0, (newcap - m->cap) * sizeof(*ne));
|
||||
m->term = nt;
|
||||
m->pcrc = np;
|
||||
m->present = ne;
|
||||
m->cap = newcap;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int collect_visit(void* ctx, const waldump_entry_t* e)
|
||||
{
|
||||
collect_ctx_t* cc = ctx;
|
||||
member_log_t* m = cc->log;
|
||||
if (e->lsn == 0)
|
||||
return 0; /* LSN 0 is never a real committed entry */
|
||||
if (member_log_ensure(m, e->lsn) != 0) {
|
||||
cc->oom = 1;
|
||||
return 1; /* stop the scan */
|
||||
}
|
||||
m->term[e->lsn] = e->term;
|
||||
m->pcrc[e->lsn] = e->pcrc;
|
||||
m->present[e->lsn] = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void member_log_free(member_log_t* m)
|
||||
{
|
||||
free(m->term);
|
||||
free(m->pcrc);
|
||||
free(m->present);
|
||||
m->term = NULL;
|
||||
m->pcrc = NULL;
|
||||
m->present = NULL;
|
||||
m->cap = 0;
|
||||
}
|
||||
|
||||
/* Load one member's WAL into `*m`. Returns 0 or a negative errno. */
|
||||
static int load_member(const walcompare_member_t* src, member_log_t* m, FILE* out)
|
||||
{
|
||||
memset(m, 0, sizeof(*m));
|
||||
|
||||
waldump_stats_t st;
|
||||
memset(&st, 0, sizeof(st));
|
||||
collect_ctx_t cc = { .log = m, .oom = 0 };
|
||||
|
||||
/* Send the human dump to the caller's `out` (or a discard sink). The
|
||||
* visitor harvests the (lsn -> term, pcrc) index from the same decode. */
|
||||
FILE* dump_sink = out ? out : fopen("/dev/null", "w");
|
||||
int own_sink = (out == NULL);
|
||||
if (!dump_sink)
|
||||
return -errno;
|
||||
|
||||
int rc = waldump_path_visit(src->path, dump_sink, &st, collect_visit, &cc);
|
||||
if (own_sink)
|
||||
fclose(dump_sink);
|
||||
|
||||
if (rc != 0) {
|
||||
member_log_free(m);
|
||||
return rc; /* hard load error (missing path / no segments) */
|
||||
}
|
||||
if (cc.oom) {
|
||||
member_log_free(m);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
m->recovered_tip = st.recovered_tip;
|
||||
/* An explicit committed_lsn wins; 0 means "derive from the shared
|
||||
* boundary" (filled by the caller after all members are loaded). */
|
||||
m->committed_lsn = src->committed_lsn;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int walcompare(const walcompare_member_t* members, int n, FILE* out,
|
||||
walcompare_divergence_t* div)
|
||||
{
|
||||
if (!members || n < 2)
|
||||
return -EINVAL;
|
||||
if (n > WALCMP_MAX_MEMBERS)
|
||||
return -E2BIG;
|
||||
|
||||
member_log_t logs[WALCMP_MAX_MEMBERS];
|
||||
int loaded = 0;
|
||||
int rc = 0;
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
rc = load_member(&members[i], &logs[i], out);
|
||||
if (rc != 0) {
|
||||
if (out)
|
||||
fprintf(out, "walcompare: cannot load member %d ('%s'): %s\n", i,
|
||||
members[i].path ? members[i].path : "(null)",
|
||||
strerror(-rc));
|
||||
goto done;
|
||||
}
|
||||
loaded = i + 1;
|
||||
}
|
||||
|
||||
/* Shared committed boundary = the minimum recovered tip across members:
|
||||
* the largest committed prefix every member can be expected to hold. A
|
||||
* member with an explicit committed_lsn keeps its own. */
|
||||
uint64_t shared = logs[0].recovered_tip;
|
||||
for (int i = 1; i < n; i++)
|
||||
if (logs[i].recovered_tip < shared)
|
||||
shared = logs[i].recovered_tip;
|
||||
|
||||
uint64_t boundary = 0; /* highest committed LSN compared across the cluster */
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (logs[i].committed_lsn == 0)
|
||||
logs[i].committed_lsn = shared;
|
||||
if (logs[i].committed_lsn > boundary)
|
||||
boundary = logs[i].committed_lsn;
|
||||
}
|
||||
|
||||
if (out)
|
||||
fprintf(out,
|
||||
"== walcompare: %d members, committed boundary up to LSN %" PRIu64
|
||||
" ==\n",
|
||||
n, boundary);
|
||||
|
||||
/* Walk every committed LSN. At each LSN, compare the (term, pcrc) of the
|
||||
* members whose own committed boundary covers it and that hold the entry.
|
||||
* A member that does not hold a committed LSN it SHOULD (boundary covers it
|
||||
* but the entry is absent) is itself a committed-prefix divergence: a
|
||||
* committed entry is missing (R4.2 / R6.4). */
|
||||
for (uint64_t lsn = 1; lsn <= boundary; lsn++) {
|
||||
int have = 0; /* members whose boundary covers this LSN */
|
||||
int present = 0; /* of those, how many actually hold it */
|
||||
uint64_t ref_term = 0;
|
||||
uint32_t ref_pcrc = 0;
|
||||
int have_ref = 0;
|
||||
int divergent = 0;
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (lsn > logs[i].committed_lsn)
|
||||
continue; /* this LSN is beyond member i's committed prefix */
|
||||
have++;
|
||||
|
||||
int holds = (lsn < logs[i].cap) && logs[i].present[lsn];
|
||||
if (!holds) {
|
||||
/* member i should hold this committed LSN but does not */
|
||||
divergent = 1;
|
||||
continue;
|
||||
}
|
||||
present++;
|
||||
uint64_t mt = match_term(logs[i].term[lsn]);
|
||||
uint32_t mp = logs[i].pcrc[lsn];
|
||||
if (!have_ref) {
|
||||
ref_term = mt;
|
||||
ref_pcrc = mp;
|
||||
have_ref = 1;
|
||||
} else if (mt != ref_term || mp != ref_pcrc) {
|
||||
divergent = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (have < 2)
|
||||
continue; /* nothing to compare at this LSN */
|
||||
if (!divergent)
|
||||
continue;
|
||||
|
||||
/* First divergent committed LSN — report and fail. */
|
||||
if (div) {
|
||||
memset(div, 0, sizeof(*div));
|
||||
div->lsn = lsn;
|
||||
div->n_members = n;
|
||||
for (int i = 0; i < n && i < 16; i++) {
|
||||
int holds = (lsn <= logs[i].committed_lsn) &&
|
||||
(lsn < logs[i].cap) && logs[i].present[lsn];
|
||||
div->member_present[i] = holds ? 1 : 0;
|
||||
div->member_term[i] = holds ? match_term(logs[i].term[lsn]) : 0;
|
||||
div->member_pcrc[i] = holds ? logs[i].pcrc[lsn] : 0;
|
||||
}
|
||||
}
|
||||
if (out) {
|
||||
fprintf(out,
|
||||
"*** DIVERGENCE at committed LSN %" PRIu64
|
||||
" (compared %d, present %d) ***\n",
|
||||
lsn, have, present);
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (lsn > logs[i].committed_lsn) {
|
||||
fprintf(out,
|
||||
" member %d ('%s'): LSN beyond its committed prefix"
|
||||
" (committed=%" PRIu64 ")\n",
|
||||
i, members[i].path, logs[i].committed_lsn);
|
||||
continue;
|
||||
}
|
||||
int holds = (lsn < logs[i].cap) && logs[i].present[lsn];
|
||||
if (holds)
|
||||
fprintf(out,
|
||||
" member %d ('%s'): term=%" PRIu64 " pcrc=%08x\n", i,
|
||||
members[i].path, match_term(logs[i].term[lsn]),
|
||||
logs[i].pcrc[lsn]);
|
||||
else
|
||||
fprintf(out,
|
||||
" member %d ('%s'): MISSING committed entry\n", i,
|
||||
members[i].path);
|
||||
}
|
||||
}
|
||||
rc = WALCMP_DIVERGENT;
|
||||
goto done;
|
||||
}
|
||||
|
||||
if (out)
|
||||
fprintf(out, "walcompare: OK — committed prefixes identical\n");
|
||||
rc = WALCMP_OK;
|
||||
|
||||
done:
|
||||
for (int i = 0; i < loaded; i++)
|
||||
member_log_free(&logs[i]);
|
||||
return rc;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue