forked from lei.chen/myxce
checkpoint: bug-851 root cause + raft TLA+ conformance spec family
Investigation & fixes (failover committed-job loss): - root-cause bug-851 on an owned 3-node cluster: chaotic failover causes silent committed-prefix divergence (WAL hdr.term always 0; two leaders per term; no (term,lsn) reconcile). Evidence in repro-851/ + INVESTIGATION. - live-validated fixes 848/849/850 (recovery-on-promotion, advertise real applied_lsn, persist-layer contiguity guard); add waldump WAL inspector. Specs (strict raft.tla conformance — library owns the log): - TLA-gap-analysis.md: impl vs raft/raft.tla, gaps G1-G6. - raft-log-replication, raft-commit-safety, raft-sim-conformance: extend src/lib/raft to own log/commit/verification via callbacks (specs generated, cross-spec review passed). - ha-term-safety: controller-side adapter + config marker (R1-R7); design done. - config-upgrade-snapshot: R8 descoped here (deps entity-bearing-snapshots, bug-857). - roadmap + buglog (851/854/857) + .wolf knowledge updated.
This commit is contained in:
parent
7d70820aac
commit
20e59d0b42
85 changed files with 10525 additions and 1993 deletions
|
|
@ -22,6 +22,7 @@
|
|||
#include "delay.h"
|
||||
#include "idgen.h"
|
||||
#include "preason.h"
|
||||
#include "waldump.h" /* offline WAL inspector (cmd_waldump) */
|
||||
#include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */
|
||||
#include "xce/types.h"
|
||||
|
||||
|
|
@ -122,6 +123,7 @@ extern int cmd_expect_jobs(int, char*[]);
|
|||
extern int cmd_expect_job(int, char*[]);
|
||||
extern int cmd_wait_primary(int, char*[]);
|
||||
extern int cmd_ctrlstatus(int, char*[]);
|
||||
extern int cmd_waldump(int, char*[]);
|
||||
static int worker_house_keeping(void*);
|
||||
|
||||
/* Session verdict / loop state (xce-test-e2e-primitives task 1.1).
|
||||
|
|
@ -179,6 +181,7 @@ static command_t top_commands[] = {
|
|||
{ "client", cmd_client, "In client context" },
|
||||
{ "set", cmd_set, "Show/Set settings" },
|
||||
{ "echo", cmd_echo, "Print a message" },
|
||||
{ "waldump", cmd_waldump, "Inspect WAL segments" },
|
||||
{ "quit", cmd_quit, "Quit xce test" },
|
||||
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
||||
};
|
||||
|
|
@ -216,6 +219,7 @@ static command_t client_commands[] = {
|
|||
{ "wait-primary", cmd_wait_primary,
|
||||
"wait until a primary serves [timeout_ms]" },
|
||||
{ "ctrlstatus", cmd_ctrlstatus, "show controller/cluster status" },
|
||||
{ "waldump", cmd_waldump, "Inspect WAL segments" },
|
||||
{ "shutdown", cmd_shutdown, "shutdown xce controller" },
|
||||
{ "jobrun", cmd_jobrun, "run a job" },
|
||||
{ "jobsub", cmd_jobsub, "submit a job" },
|
||||
|
|
@ -683,6 +687,35 @@ int cmd_echo(int argc, char* argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* waldump <path> [<path> ...] — offline WAL inspector for failover/durability
|
||||
* triage. Each path may be a seg-*.wal file, a wal/ dir, or a node/data dir.
|
||||
* Read-only: lists every entry, decodes JOB_SUBMIT jobids, and flags LSN gaps
|
||||
* (a missing committed prefix is THE failover job-loss signal) plus the CRC
|
||||
* recovered tip. Returns nonzero if any path failed or any gap was found, so
|
||||
* it doubles as an assertion in scripted E2E runs. */
|
||||
int cmd_waldump(int argc, char* argv[])
|
||||
{
|
||||
if (argc < 2) {
|
||||
fprintf(stderr,
|
||||
"usage: waldump <wal_dir|data_dir|seg_file> [more...]\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
int rc = 0;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
waldump_stats_t st;
|
||||
int r = waldump_path(argv[i], stdout, &st);
|
||||
if (r != 0) {
|
||||
fprintf(stderr, "waldump: %s: %s\n", argv[i], strerror(-r));
|
||||
rc = -1;
|
||||
continue;
|
||||
}
|
||||
if (st.n_gaps > 0 || st.crc_broken)
|
||||
rc = -1; /* surface the defect as a nonzero command verdict */
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
int cmd_up(int argc, char* argv[])
|
||||
{
|
||||
if (active_ctx->parent != NULL) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue