lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(waldump): decode WAL_CONFIG_MARKER, distinguish trimmed gaps from real loss

Browse source 
wal_type_name() never had a case for WAL_CONFIG_MARKER (type 50) -- every
config-change marker entry printed as UNKNOWN instead of a real, defined
type. Added the case plus payload decode (tag/sha256) matching config_repl.c's
config_marker_encode/_decode byte layout, hand-decoded here rather than
linking config_repl.c into xce_test's link graph.

Gap detection also had no way to tell "committed prefix missing!" (real
loss) apart from a normal checkpoint-triggered WAL trim (persist.c's
trim_wal_segments, floor-guarded so it never removes a segment a standby
still needs) -- every long-running node's WAL, once checkpointed, would
false-alarm on every inspection. Cross-checks the sibling data/snap/
directory (snapshot LSN is literally in the filename, snap-<hex>.capnp)
and downgrades the message when a snapshot covers the gap.

Adds waldump_stats_t.leading_gap_explained and 3 new tests plus one
assertion on the existing gap test.
This commit is contained in:
Rongsong Shen 2026-07-31 09:05:22 +00:00
parent 0ca96359f5
commit a86de77f91
3 changed files with 228 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

92
src/controller/testsuite/waldump.c
View file

@ -46,6 +46,7 @@ static const char* wal_type_name(uint32_t t)
case WAL_QUEUE_CTRL: return "QUEUE_CTRL";
case WAL_WORKER_CTRL: return "WORKER_CTRL";
case WAL_NOOP: return "NOOP";
case WAL_CONFIG_MARKER: return "CONFIG_MARKER";
case WAL_CHECKPOINT: return "CHECKPOINT";
default: return "UNKNOWN";
}
@ -127,6 +128,63 @@ static int decode_jobstate_info(const uint8_t* payload, uint32_t len,
return 0;
}
/* Decode a WAL_CONFIG_MARKER payload -- fixed 40-byte form (8-byte
* big-endian version_tag + 32-byte raw sha256), never capnp-encoded
* (config_repl.c's config_marker_encode/config_marker_decode). Hand-decoded
* here rather than linking config_repl.c, which would pull controller-
* internal dependencies into xce_test's link graph (meson.build's
* xce_test_src is deliberately limited to read-only dump/compare tools).
* Mirrors config_marker_decode's byte layout exactly. */
static int decode_config_marker_info(const uint8_t* payload, uint32_t len,
uint64_t* out_tag, uint8_t sha256_out[32])
{
if (!payload || len < WAL_CONFIG_MARKER_PAYLOAD_LEN)
return -1;
uint64_t tag = 0;
for (int i = 0; i < 8; i++)
tag = (tag << 8) | payload[i];
*out_tag = tag;
memcpy(sha256_out, payload + 8, 32);
return 0;
}
/* Highest LSN among sibling data/snap/snap-<hex>.capnp files next to
* `wal_dir` (persist.c hardcodes both as siblings under a node's data/,
* e.g. .../data/wal and .../data/snap). A leading WAL gap up to this LSN is
* explained by normal checkpoint-triggered trim (persist.c's
* trim_wal_segments only removes segments below a replication-safe floor,
* never a segment a standby still needs to catch up) rather than genuine
* loss -- without this check, EVERY long-running node's WAL (which will
* have been checkpointed at least once) prints the same alarming
* "(committed prefix missing!)" wording a real, unexplained loss would,
* making the message a standing false alarm. Returns 0 if `wal_dir` isn't
* named "wal", or no sibling snap dir/files exist (unknown -- callers keep
* today's un-downgraded wording, the conservative default). */
static uint64_t highest_snapshot_lsn(const char* wal_dir)
{
char snap_dir[4096];
snprintf(snap_dir, sizeof(snap_dir), "%s", wal_dir);
char* slash = strrchr(snap_dir, '/');
if (!slash || strcmp(slash + 1, "wal") != 0)
return 0;
snprintf(slash + 1, sizeof(snap_dir) - (size_t)(slash + 1 - snap_dir),
"snap");
DIR* d = opendir(snap_dir);
if (!d)
return 0;
uint64_t max_lsn = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL) {
uint64_t lsn;
if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) == 1 &&
lsn > max_lsn)
max_lsn = lsn;
}
closedir(d);
return max_lsn;
}
/* True if `dir` directly contains at least one seg-*.wal file. */
static int dir_has_segments(const char* dir)
{
@ -241,7 +299,7 @@ static int collect_segments(const char* wal_dir, uint64_t* seg_lsns, int max)
static void scan_segment(const char* seg_path, FILE* out,
uint32_t* last_crc, uint64_t* prev_lsn,
uint64_t* prev_submit_jobid, uint32_t* prev_submit_pcrc,
waldump_stats_t* st,
waldump_stats_t* st, uint64_t snap_lsn,
waldump_visit_fn visit, void* visit_ctx,
int* visit_stop)
{
@ -309,10 +367,15 @@ static void scan_segment(const char* seg_path, FILE* out,
/* Gap detection against the running LSN sequence. */
if (*prev_lsn == 0) {
if (hdr.lsn > 1) {
int explained = snap_lsn >= hdr.lsn - 1;
fprintf(out, " *** GAP: missing prefix LSN 1..%" PRIu64
" (%" PRIu64 " entries absent) ***\n",
hdr.lsn - 1, hdr.lsn - 1);
" (%" PRIu64 " entries absent)%s ***\n",
hdr.lsn - 1, hdr.lsn - 1,
explained ? " -- covered by snapshot, expected "
"post-checkpoint trim"
: "");
st->n_gaps++;
st->leading_gap_explained = explained;
}
} else if (hdr.lsn > *prev_lsn + 1) {
fprintf(out, " *** GAP: missing LSN %" PRIu64 "..%" PRIu64 " ***\n",
@ -404,6 +467,20 @@ static void scan_segment(const char* seg_path, FILE* out,
}
/* a non-submit entry breaks the adjacency for dup detection */
*prev_submit_jobid = 0;
} else if (hdr.type == WAL_CONFIG_MARKER) {
uint64_t tag = 0;
uint8_t sha[32];
if (decode_config_marker_info(payload, hdr.payload_len, &tag,
sha) == 0) {
char hex[65];
for (int i = 0; i < 32; i++)
snprintf(hex + i * 2, 3, "%02x", sha[i]);
snprintf(info, sizeof(info), "tag=%016" PRIx64 " sha256=%s",
tag, hex);
} else {
snprintf(info, sizeof(info), "tag=?");
}
*prev_submit_jobid = 0;
} else {
/* a non-submit entry breaks the adjacency for dup detection */
*prev_submit_jobid = 0;
@ -505,6 +582,7 @@ int waldump_path_visit(const char* path, FILE* out, waldump_stats_t* stats,
}
st.n_segments = (uint32_t)nseg;
uint64_t snap_lsn = highest_snapshot_lsn(wal_dir);
fprintf(out, "== WAL: %s (%d segment%s) ==\n", wal_dir, nseg,
nseg == 1 ? "" : "s");
@ -521,7 +599,7 @@ int waldump_path_visit(const char* path, FILE* out, waldump_stats_t* stats,
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, wal_dir,
seg_lsns[i]);
scan_segment(seg_path, out, &last_crc, &prev_lsn,
&prev_submit_jobid, &prev_submit_pcrc, &st,
&prev_submit_jobid, &prev_submit_pcrc, &st, snap_lsn,
visit, ctx, &visit_stop);
}
@ -532,7 +610,11 @@ int waldump_path_visit(const char* path, FILE* out, waldump_stats_t* stats,
" recovered tip: %" PRIu64 "\n",
st.first_lsn, st.last_lsn, st.recovered_tip);
fprintf(out, " gaps: %u%s crc chain: %s\n", st.n_gaps,
(st.n_gaps && st.first_lsn > 1) ? " (committed prefix missing!)" : "",
(st.n_gaps && st.first_lsn > 1)
? (st.leading_gap_explained
? " (explained by checkpoint/snapshot -- not a loss)"
: " (committed prefix missing!)")
: "",
st.crc_broken ? "BROKEN" : "intact");
if (st.n_dup_submits > 0)
fprintf(out, " duplicate submits: %u (byte-identical: %u) — bug-845\n",