checkpoint: bug-851 root cause + raft TLA+ conformance spec family
Investigation & fixes (failover committed-job loss): - root-cause bug-851 on an owned 3-node cluster: chaotic failover causes silent committed-prefix divergence (WAL hdr.term always 0; two leaders per term; no (term,lsn) reconcile). Evidence in repro-851/ + INVESTIGATION. - live-validated fixes 848/849/850 (recovery-on-promotion, advertise real applied_lsn, persist-layer contiguity guard); add waldump WAL inspector. Specs (strict raft.tla conformance — library owns the log): - TLA-gap-analysis.md: impl vs raft/raft.tla, gaps G1-G6. - raft-log-replication, raft-commit-safety, raft-sim-conformance: extend src/lib/raft to own log/commit/verification via callbacks (specs generated, cross-spec review passed). - ha-term-safety: controller-side adapter + config marker (R1-R7); design done. - config-upgrade-snapshot: R8 descoped here (deps entity-bearing-snapshots, bug-857). - roadmap + buglog (851/854/857) + .wolf knowledge updated.
This commit is contained in:
parent
7d70820aac
commit
20e59d0b42
85 changed files with 10525 additions and 1993 deletions
444
src/controller/testsuite/waldump.c
Normal file
444
src/controller/testsuite/waldump.c
Normal file
|
|
@ -0,0 +1,444 @@
|
|||
#define _GNU_SOURCE
|
||||
#include "waldump.h"
|
||||
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <zlib.h>
|
||||
|
||||
#include "config.h" /* WAL_HDR_CRC_SIZE, WAL_CRC_NONE, MAX_WAL_PAYLOAD_BYTES */
|
||||
#include "controller.capnp.h" /* Job decode for WAL_JOB_SUBMIT */
|
||||
#include "wal_types.h"
|
||||
|
||||
#define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
|
||||
#define MAX_SEGS 4096
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Small helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static const char* wal_type_name(uint32_t t)
|
||||
{
|
||||
switch (t) {
|
||||
case WAL_JOB_SUBMIT: return "JOB_SUBMIT";
|
||||
case WAL_JOB_START: return "JOB_START";
|
||||
case WAL_JOB_FINISH: return "JOB_FINISH";
|
||||
case WAL_JOB_FAIL: return "JOB_FAIL";
|
||||
case WAL_JOB_STOP: return "JOB_STOP";
|
||||
case WAL_JOB_RESUME: return "JOB_RESUME";
|
||||
case WAL_JOB_UPDATE: return "JOB_UPDATE";
|
||||
case WAL_JOB_RUSAGE: return "JOB_RUSAGE";
|
||||
case WAL_JOB_CKPT: return "JOB_CKPT";
|
||||
case WAL_JOB_PREASON: return "JOB_PREASON";
|
||||
case WAL_QUEUE_CTRL: return "QUEUE_CTRL";
|
||||
case WAL_WORKER_CTRL: return "WORKER_CTRL";
|
||||
case WAL_NOOP: return "NOOP";
|
||||
case WAL_CHECKPOINT: return "CHECKPOINT";
|
||||
default: return "UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
/* Decoded fields of a WAL_JOB_SUBMIT Cap'n Proto payload (debug view). */
|
||||
typedef struct {
|
||||
uint64_t jobid;
|
||||
uint64_t subtime; /* job_submit stamps this with time(NULL) per submit */
|
||||
uint32_t state;
|
||||
} submit_info_t;
|
||||
|
||||
/* Decode a WAL_JOB_SUBMIT payload. Mirrors restore_job() (job.c) but reads a
|
||||
* few fields only, zero-alloc. Returns 0 on success, -1 if undecodable.
|
||||
* subtime is the decisive bug-845 signal: two entries of the SAME job written
|
||||
* in one submit share a subtime (a literal double-append); two separate
|
||||
* submits (e.g. a failover retry) would differ. */
|
||||
static int decode_submit_info(const uint8_t* payload, uint32_t len,
|
||||
submit_info_t* out)
|
||||
{
|
||||
if (!payload || len == 0)
|
||||
return -1;
|
||||
|
||||
struct capn cctx;
|
||||
if (capn_init_mem(&cctx, payload, len, 0) != 0)
|
||||
return -1;
|
||||
|
||||
Job_ptr jp;
|
||||
jp.p = capn_getp(capn_root(&cctx), 0, 1);
|
||||
if (jp.p.type == CAPN_NULL) {
|
||||
capn_free(&cctx);
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct Job cj;
|
||||
memset(&cj, 0, sizeof(cj));
|
||||
read_Job(&cj, jp);
|
||||
out->jobid = cj.jobid;
|
||||
out->subtime = cj.subtime;
|
||||
out->state = cj.state;
|
||||
capn_free(&cctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* True if `dir` directly contains at least one seg-*.wal file. */
|
||||
static int dir_has_segments(const char* dir)
|
||||
{
|
||||
DIR* d = opendir(dir);
|
||||
if (!d)
|
||||
return 0;
|
||||
int found = 0;
|
||||
struct dirent* e;
|
||||
while ((e = readdir(d)) != NULL) {
|
||||
size_t n = strlen(e->d_name);
|
||||
if (n > 4 && strncmp(e->d_name, "seg-", 4) == 0 &&
|
||||
strcmp(e->d_name + n - 4, ".wal") == 0) {
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
closedir(d);
|
||||
return found;
|
||||
}
|
||||
|
||||
/* Resolve `path` (seg file / wal dir / node-or-data dir) to the wal dir that
|
||||
* holds the seg-*.wal files. Writes it to `out`. Returns 0, or -errno. When
|
||||
* `path` is itself a single seg file, sets *single_seg to that file path. */
|
||||
static int resolve_wal_dir(const char* path, char* out, size_t outsz,
|
||||
char* single_seg, size_t segsz)
|
||||
{
|
||||
struct stat st;
|
||||
if (stat(path, &st) != 0)
|
||||
return -errno;
|
||||
|
||||
single_seg[0] = '\0';
|
||||
|
||||
if (S_ISREG(st.st_mode)) {
|
||||
/* Only a seg-*.wal file is a WAL segment. Reject anything else (e.g. a
|
||||
* config blob like conf/cluster.bin) instead of parsing it as a WAL and
|
||||
* silently reporting 0 entries. */
|
||||
const char* base = strrchr(path, '/');
|
||||
base = base ? base + 1 : path;
|
||||
size_t bl = strlen(base);
|
||||
if (strncmp(base, "seg-", 4) != 0 || bl < 5 ||
|
||||
strcmp(base + bl - 4, ".wal") != 0)
|
||||
return -EINVAL;
|
||||
snprintf(single_seg, segsz, "%s", path);
|
||||
/* wal dir = dirname(path); only used for display */
|
||||
snprintf(out, outsz, "%s", path);
|
||||
char* slash = strrchr(out, '/');
|
||||
if (slash)
|
||||
*slash = '\0';
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (!S_ISDIR(st.st_mode))
|
||||
return -ENOTDIR;
|
||||
|
||||
/* A directory: try it directly, then data/wal, then wal/. */
|
||||
if (dir_has_segments(path)) {
|
||||
snprintf(out, outsz, "%s", path);
|
||||
return 0;
|
||||
}
|
||||
char cand[4096];
|
||||
snprintf(cand, sizeof(cand), "%s/data/wal", path);
|
||||
if (dir_has_segments(cand)) {
|
||||
snprintf(out, outsz, "%s", cand);
|
||||
return 0;
|
||||
}
|
||||
snprintf(cand, sizeof(cand), "%s/wal", path);
|
||||
if (dir_has_segments(cand)) {
|
||||
snprintf(out, outsz, "%s", cand);
|
||||
return 0;
|
||||
}
|
||||
return -ENOENT; /* no segments anywhere under path */
|
||||
}
|
||||
|
||||
/* Parse the start LSN out of a seg file basename ("seg-<hex16>.wal"). */
|
||||
static int parse_seg_start(const char* name, uint64_t* lsn)
|
||||
{
|
||||
return sscanf(name, "seg-%" SCNx64 ".wal", lsn) == 1 ? 0 : -1;
|
||||
}
|
||||
|
||||
/* Collect + sort segment start LSNs under wal_dir. Returns count, or -errno. */
|
||||
static int collect_segments(const char* wal_dir, uint64_t* seg_lsns, int max)
|
||||
{
|
||||
DIR* d = opendir(wal_dir);
|
||||
if (!d)
|
||||
return -errno;
|
||||
|
||||
int n = 0;
|
||||
struct dirent* e;
|
||||
while ((e = readdir(d)) != NULL && n < max) {
|
||||
uint64_t s;
|
||||
if (parse_seg_start(e->d_name, &s) == 0)
|
||||
seg_lsns[n++] = s;
|
||||
}
|
||||
closedir(d);
|
||||
|
||||
for (int i = 1; i < n; i++) { /* insertion sort — segment count is tiny */
|
||||
uint64_t key = seg_lsns[i];
|
||||
int j = i - 1;
|
||||
while (j >= 0 && seg_lsns[j] > key) {
|
||||
seg_lsns[j + 1] = seg_lsns[j];
|
||||
j--;
|
||||
}
|
||||
seg_lsns[j + 1] = key;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Per-segment scan: prints entries, validates the CRC chain, detects */
|
||||
/* LSN gaps. Carries the chain CRC and prev LSN across segment calls. */
|
||||
/* ------------------------------------------------------------------ */
|
||||
static void scan_segment(const char* seg_path, FILE* out,
|
||||
uint32_t* last_crc, uint64_t* prev_lsn,
|
||||
uint64_t* prev_submit_jobid, uint32_t* prev_submit_pcrc,
|
||||
waldump_stats_t* st)
|
||||
{
|
||||
int fd = open(seg_path, O_RDONLY);
|
||||
if (fd < 0) {
|
||||
fprintf(out, " !! cannot open %s: %s\n", seg_path, strerror(errno));
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t off = 0;
|
||||
for (;;) {
|
||||
wal_entry_hdr_t hdr;
|
||||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||||
if (hn == 0)
|
||||
break; /* clean EOF */
|
||||
if (hn < (ssize_t)sizeof(hdr))
|
||||
break; /* truncated / pre-allocated zeros */
|
||||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||||
break; /* pre-allocated zero region = clean end */
|
||||
|
||||
/* Bound payload before trusting it (corruption guard). */
|
||||
if (hdr.payload_len > MAX_WAL_PAYLOAD_BYTES) {
|
||||
fprintf(out, " !! LSN %" PRIu64 " payload_len %u exceeds max — stop\n",
|
||||
hdr.lsn, hdr.payload_len);
|
||||
st->crc_broken = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
/* CRC chain check (identical rule to persist recovery). */
|
||||
int chain_ok;
|
||||
if (off == 0)
|
||||
chain_ok = (hdr.prev_crc == WAL_CRC_NONE || *last_crc == WAL_CRC_NONE ||
|
||||
hdr.prev_crc == *last_crc);
|
||||
else
|
||||
chain_ok = (hdr.prev_crc == *last_crc);
|
||||
if (!chain_ok) {
|
||||
fprintf(out, " !! LSN %" PRIu64 " prev_crc chain break — stop\n",
|
||||
hdr.lsn);
|
||||
st->crc_broken = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
/* Verify entry CRC over header + payload. */
|
||||
uint8_t* payload = NULL;
|
||||
uint32_t want_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||||
if (hdr.payload_len > 0) {
|
||||
payload = malloc(hdr.payload_len);
|
||||
if (!payload)
|
||||
break;
|
||||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||||
(off_t)(off + sizeof(hdr)));
|
||||
if (pn != (ssize_t)hdr.payload_len) {
|
||||
free(payload);
|
||||
break; /* short read — clean end */
|
||||
}
|
||||
want_crc = crc32(want_crc, (const Bytef*)payload, hdr.payload_len);
|
||||
}
|
||||
if (hdr.crc != want_crc) {
|
||||
fprintf(out, " !! LSN %" PRIu64 " CRC mismatch — stop\n", hdr.lsn);
|
||||
st->crc_broken = 1;
|
||||
free(payload);
|
||||
break;
|
||||
}
|
||||
|
||||
/* Gap detection against the running LSN sequence. */
|
||||
if (*prev_lsn == 0) {
|
||||
if (hdr.lsn > 1) {
|
||||
fprintf(out, " *** GAP: missing prefix LSN 1..%" PRIu64
|
||||
" (%" PRIu64 " entries absent) ***\n",
|
||||
hdr.lsn - 1, hdr.lsn - 1);
|
||||
st->n_gaps++;
|
||||
}
|
||||
} else if (hdr.lsn > *prev_lsn + 1) {
|
||||
fprintf(out, " *** GAP: missing LSN %" PRIu64 "..%" PRIu64 " ***\n",
|
||||
*prev_lsn + 1, hdr.lsn - 1);
|
||||
st->n_gaps++;
|
||||
}
|
||||
|
||||
/* payload-only CRC — isolates the blob from the header (LSN/term/ts),
|
||||
* so two entries carrying the SAME job blob share a pcrc regardless of
|
||||
* their differing LSNs. Decisive for bug-845: equal pcrc on an adjacent
|
||||
* same-jobid pair ⇒ a literal byte-for-byte double-append. */
|
||||
uint32_t pcrc = (hdr.payload_len > 0)
|
||||
? crc32(0L, (const Bytef*)payload, hdr.payload_len)
|
||||
: 0;
|
||||
|
||||
/* Per-entry line, with jobid/subtime for submits. */
|
||||
char info[96] = "";
|
||||
int is_dup = 0;
|
||||
int dup_identical = 0;
|
||||
if (hdr.type == WAL_JOB_SUBMIT) {
|
||||
submit_info_t si;
|
||||
uint64_t jobid = 0;
|
||||
if (decode_submit_info(payload, hdr.payload_len, &si) == 0) {
|
||||
jobid = si.jobid;
|
||||
snprintf(info, sizeof(info),
|
||||
"jobid=%" PRIu64 " subtime=%" PRIu64 " state=%u",
|
||||
si.jobid, si.subtime, si.state);
|
||||
} else {
|
||||
snprintf(info, sizeof(info), "jobid=?");
|
||||
}
|
||||
st->n_submits++;
|
||||
|
||||
/* bug-845: adjacent duplicate submit — same jobid as the previous
|
||||
* entry at lsn = prev_lsn + 1. Capture identity BEFORE updating the
|
||||
* carried pcrc. */
|
||||
if (jobid != 0 && jobid == *prev_submit_jobid &&
|
||||
hdr.lsn == *prev_lsn + 1) {
|
||||
is_dup = 1;
|
||||
dup_identical = (pcrc == *prev_submit_pcrc);
|
||||
st->n_dup_submits++;
|
||||
if (dup_identical)
|
||||
st->n_dup_identical++;
|
||||
}
|
||||
*prev_submit_jobid = jobid;
|
||||
*prev_submit_pcrc = pcrc;
|
||||
} else {
|
||||
/* a non-submit entry breaks the adjacency for dup detection */
|
||||
*prev_submit_jobid = 0;
|
||||
}
|
||||
|
||||
fprintf(out, " %8" PRIu64 " | %5" PRIu64 " | %-11s | %5u | %08x | %s\n",
|
||||
hdr.lsn, hdr.term, wal_type_name(hdr.type), hdr.payload_len,
|
||||
pcrc, info);
|
||||
if (is_dup)
|
||||
fprintf(out, " ^^ DUP submit: payload %s the prior entry\n",
|
||||
dup_identical ? "BYTE-IDENTICAL to" : "DIFFERS from");
|
||||
|
||||
free(payload);
|
||||
|
||||
/* Advance watermarks. */
|
||||
if (st->first_lsn == 0 || hdr.lsn < st->first_lsn)
|
||||
st->first_lsn = hdr.lsn;
|
||||
if (hdr.lsn > st->last_lsn)
|
||||
st->last_lsn = hdr.lsn;
|
||||
if (hdr.lsn > st->recovered_tip)
|
||||
st->recovered_tip = hdr.lsn;
|
||||
st->n_entries++;
|
||||
*prev_lsn = hdr.lsn;
|
||||
*last_crc = hdr.crc;
|
||||
|
||||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Public entry point */
|
||||
/* ------------------------------------------------------------------ */
|
||||
int waldump_path(const char* path, FILE* out, waldump_stats_t* stats)
|
||||
{
|
||||
if (!path || !out)
|
||||
return -EINVAL;
|
||||
|
||||
char wal_dir[4096];
|
||||
char single_seg[4096];
|
||||
int rc = resolve_wal_dir(path, wal_dir, sizeof(wal_dir),
|
||||
single_seg, sizeof(single_seg));
|
||||
if (rc != 0) {
|
||||
/* Clear, actionable error instead of silently scanning a non-WAL path
|
||||
* (e.g. conf/cluster.bin) and reporting "0 entries". */
|
||||
fprintf(out,
|
||||
"waldump: no WAL segments at '%s' (%s)\n"
|
||||
" point at a node's WAL dir, e.g. "
|
||||
"<deploy>/controller/ctrl-N/data/wal\n"
|
||||
" (or a specific seg-*.wal file inside it)\n",
|
||||
path, strerror(-rc));
|
||||
return rc;
|
||||
}
|
||||
|
||||
waldump_stats_t st;
|
||||
memset(&st, 0, sizeof(st));
|
||||
|
||||
uint64_t seg_lsns[MAX_SEGS];
|
||||
int nseg;
|
||||
/* wider than wal_dir[4096] so the "/seg-<hex16>.wal" suffix never truncates */
|
||||
char seg_path[4096 + 64];
|
||||
|
||||
if (single_seg[0] != '\0') {
|
||||
/* A single seg file: parse its start for ordering display. */
|
||||
const char* base = strrchr(single_seg, '/');
|
||||
base = base ? base + 1 : single_seg;
|
||||
uint64_t start = 0;
|
||||
parse_seg_start(base, &start);
|
||||
seg_lsns[0] = start;
|
||||
nseg = 1;
|
||||
snprintf(seg_path, sizeof(seg_path), "%s", single_seg);
|
||||
|
||||
/* If the dir holds OTHER segments, this single-file dump shows only a
|
||||
* slice of the log. A node that recovered then caught up keeps its
|
||||
* prefix in seg-...001.wal and the suffix in seg-...<current_lsn>.wal,
|
||||
* so a single-file dump looks (wrongly) like missing data. Steer the
|
||||
* user to the whole dir. */
|
||||
uint64_t others[MAX_SEGS];
|
||||
int total = collect_segments(wal_dir, others, MAX_SEGS);
|
||||
if (total > 1)
|
||||
fprintf(out,
|
||||
"NOTE: %s holds %d segments — this dumps only %s.\n"
|
||||
" Run 'waldump %s' (the directory) for the full "
|
||||
"stitched 1..N log.\n",
|
||||
wal_dir, total, base, wal_dir);
|
||||
} else {
|
||||
nseg = collect_segments(wal_dir, seg_lsns, MAX_SEGS);
|
||||
if (nseg < 0)
|
||||
return nseg;
|
||||
if (nseg == 0)
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
st.n_segments = (uint32_t)nseg;
|
||||
|
||||
fprintf(out, "== WAL: %s (%d segment%s) ==\n", wal_dir, nseg,
|
||||
nseg == 1 ? "" : "s");
|
||||
fprintf(out, " %8s | %5s | %-11s | %5s | %8s | %s\n",
|
||||
"LSN", "TERM", "TYPE", "PLEN", "PCRC", "INFO");
|
||||
|
||||
uint32_t last_crc = WAL_CRC_NONE;
|
||||
uint64_t prev_lsn = 0;
|
||||
uint64_t prev_submit_jobid = 0;
|
||||
uint32_t prev_submit_pcrc = 0;
|
||||
for (int i = 0; i < nseg; i++) {
|
||||
if (single_seg[0] == '\0')
|
||||
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, wal_dir,
|
||||
seg_lsns[i]);
|
||||
scan_segment(seg_path, out, &last_crc, &prev_lsn,
|
||||
&prev_submit_jobid, &prev_submit_pcrc, &st);
|
||||
}
|
||||
|
||||
fprintf(out, "-- summary --\n");
|
||||
fprintf(out, " segments: %u entries: %u submits: %u\n",
|
||||
st.n_segments, st.n_entries, st.n_submits);
|
||||
fprintf(out, " lsn range: %" PRIu64 "..%" PRIu64
|
||||
" recovered tip: %" PRIu64 "\n",
|
||||
st.first_lsn, st.last_lsn, st.recovered_tip);
|
||||
fprintf(out, " gaps: %u%s crc chain: %s\n", st.n_gaps,
|
||||
(st.n_gaps && st.first_lsn > 1) ? " (committed prefix missing!)" : "",
|
||||
st.crc_broken ? "BROKEN" : "intact");
|
||||
if (st.n_dup_submits > 0)
|
||||
fprintf(out, " duplicate submits: %u (byte-identical: %u) — bug-845\n",
|
||||
st.n_dup_submits, st.n_dup_identical);
|
||||
|
||||
if (stats)
|
||||
*stats = st;
|
||||
return 0;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue