checkpoint: bug-851 root cause + raft TLA+ conformance spec family

Investigation & fixes (failover committed-job loss):
- root-cause bug-851 on an owned 3-node cluster: chaotic failover causes
  silent committed-prefix divergence (WAL hdr.term always 0; two leaders
  per term; no (term,lsn) reconcile). Evidence in repro-851/ + INVESTIGATION.
- live-validated fixes 848/849/850 (recovery-on-promotion, advertise real
  applied_lsn, persist-layer contiguity guard); add waldump WAL inspector.

Specs (strict raft.tla conformance — library owns the log):
- TLA-gap-analysis.md: impl vs raft/raft.tla, gaps G1-G6.
- raft-log-replication, raft-commit-safety, raft-sim-conformance: extend
  src/lib/raft to own log/commit/verification via callbacks (specs generated,
  cross-spec review passed).
- ha-term-safety: controller-side adapter + config marker (R1-R7); design done.
- config-upgrade-snapshot: R8 descoped here (deps entity-bearing-snapshots, bug-857).
- roadmap + buglog (851/854/857) + .wolf knowledge updated.
This commit is contained in:
Rongsong Shen 2026-06-21 14:02:15 +00:00
parent 7d70820aac
commit 20e59d0b42
85 changed files with 10525 additions and 1993 deletions

View file

@ -22,6 +22,7 @@
#include "delay.h"
#include "idgen.h"
#include "preason.h"
#include "waldump.h" /* offline WAL inspector (cmd_waldump) */
#include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */
#include "xce/types.h"
@ -122,6 +123,7 @@ extern int cmd_expect_jobs(int, char*[]);
extern int cmd_expect_job(int, char*[]);
extern int cmd_wait_primary(int, char*[]);
extern int cmd_ctrlstatus(int, char*[]);
extern int cmd_waldump(int, char*[]);
static int worker_house_keeping(void*);
/* Session verdict / loop state (xce-test-e2e-primitives task 1.1).
@ -179,6 +181,7 @@ static command_t top_commands[] = {
{ "client", cmd_client, "In client context" },
{ "set", cmd_set, "Show/Set settings" },
{ "echo", cmd_echo, "Print a message" },
{ "waldump", cmd_waldump, "Inspect WAL segments" },
{ "quit", cmd_quit, "Quit xce test" },
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
};
@ -216,6 +219,7 @@ static command_t client_commands[] = {
{ "wait-primary", cmd_wait_primary,
"wait until a primary serves [timeout_ms]" },
{ "ctrlstatus", cmd_ctrlstatus, "show controller/cluster status" },
{ "waldump", cmd_waldump, "Inspect WAL segments" },
{ "shutdown", cmd_shutdown, "shutdown xce controller" },
{ "jobrun", cmd_jobrun, "run a job" },
{ "jobsub", cmd_jobsub, "submit a job" },
@ -683,6 +687,35 @@ int cmd_echo(int argc, char* argv[])
return 0;
}
/* waldump <path> [<path> ...] — offline WAL inspector for failover/durability
* triage. Each path may be a seg-*.wal file, a wal/ dir, or a node/data dir.
* Read-only: lists every entry, decodes JOB_SUBMIT jobids, and flags LSN gaps
* (a missing committed prefix is THE failover job-loss signal) plus the CRC
* recovered tip. Returns nonzero if any path failed or any gap was found, so
* it doubles as an assertion in scripted E2E runs. */
int cmd_waldump(int argc, char* argv[])
{
if (argc < 2) {
fprintf(stderr,
"usage: waldump <wal_dir|data_dir|seg_file> [more...]\n");
return -1;
}
int rc = 0;
for (int i = 1; i < argc; i++) {
waldump_stats_t st;
int r = waldump_path(argv[i], stdout, &st);
if (r != 0) {
fprintf(stderr, "waldump: %s: %s\n", argv[i], strerror(-r));
rc = -1;
continue;
}
if (st.n_gaps > 0 || st.crc_broken)
rc = -1; /* surface the defect as a nonzero command verdict */
}
return rc;
}
int cmd_up(int argc, char* argv[])
{
if (active_ctx->parent != NULL) {