fix(raft-apply-decoupling): kill-mid-fact-stream soak + close bug-1416 boot-time TOCTOU (task 7.2)
Adds --factstream HA soak mode driving real jstart/jfinish/jfail worker-fact traffic against a live 3-node cluster, killing the primary mid-stream and asserting zero committed-state divergence and zero phantom fact reactions. Building this scenario surfaced bug-1416: persist_recover()'s boot-time apply-correctness verification can race the live apply pump and the standby live-replication apply path, both of which mutate the same last_applied_lsn/ job-table state the verification re-reads live -- producing a false APPLY_DIVERGENT that permanently fences an otherwise-healthy node. Fixed with a new persist_t.recovery_complete gate that defers only the in-memory apply (never the durability write or replication ACK) while a recovery pass is in flight, closed from the earliest safe point in myapp_run() so an election- retry candidate can't apply live traffic before persist_recover() ever runs. Round-2 adversarial review independently verified the ACK/durability paths stay ungated, reproduced the myapp.c fix as load-bearing via revert-and- reproduce, and confirmed real-divergence detection still fires. Live 3-node factstream soak: 2/2 rounds PASS, zero APPLY_DIVERGENT across 5 recover calls. controller_unit_test: zero new failures across 23 groups.
This commit is contained in:
parent
5f98d1b0ef
commit
a5e29a6ffa
17 changed files with 74793 additions and 156 deletions
|
|
@ -107,6 +107,8 @@ extern int cmd_shutdown(int, char*[]);
|
|||
extern int cmd_set(int, char*[]);
|
||||
extern int cmd_reslist(int, char*[]);
|
||||
extern int cmd_join_worker(int, char*[]);
|
||||
extern int cmd_jobstart(int, char*[]);
|
||||
extern int cmd_jobfail(int, char*[]);
|
||||
extern int cmd_join_sched(int, char*[]);
|
||||
extern int cmd_workerlist(int, char*[]);
|
||||
extern int cmd_joblist(int, char*[]);
|
||||
|
|
@ -206,7 +208,9 @@ static command_t worker_commands[] = {
|
|||
{ "help", cmd_help, "Display this text" },
|
||||
{ "quit", cmd_quit, "Quit xce test" },
|
||||
{ "join", cmd_join_worker, "Join the cluster" },
|
||||
{ "jstart", cmd_jobstart, "tell controller jobstart" },
|
||||
{ "jfinish", cmd_jobfinish, "tell controller jobfinish" },
|
||||
{ "jfail", cmd_jobfail, "tell controller jobfail" },
|
||||
{ "resupdate", cmd_resupdate, "update res vals" },
|
||||
{ "echo", cmd_echo, "Print a message" },
|
||||
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
||||
|
|
@ -1534,6 +1538,16 @@ static void print_jobinfo(xce_job_t* job)
|
|||
fmt_jobid(idstring, sizeof(idstring) - 1, job->jobid);
|
||||
printf("JobID: %s\n", idstring);
|
||||
printf("state: %s\n", job_state_str(job->state));
|
||||
/* raft-apply-decoupling task 7.2: job_state_str's if/else-if chain checks
|
||||
* JOB_STAT_RUN before JOB_STAT_EXIT, so a job that is simultaneously RUN
|
||||
* and EXIT|PERR (e.g. a worker-reported FAIL received without a prior
|
||||
* FINISH — apply_job_fail_transition ORs in EXIT|PERR without clearing
|
||||
* RUN) prints as "RUN", silently hiding the EXIT/PERR bits from the
|
||||
* pretty string. Print the raw bitmask too so a scriptable caller (the
|
||||
* xce-test-ha factstream regression scenario's phantom-fact check) can
|
||||
* test for a specific bit's presence unambiguously instead of relying on
|
||||
* the display string's priority ordering. */
|
||||
printf("state_raw: 0x%x\n", job->state);
|
||||
printf("Queue: %s\n", job->submit->queue);
|
||||
printf("User: %s\n", job->submit->user);
|
||||
printf("Slots: %d,%d\n", job->submit->mintasks, job->submit->maxtasks);
|
||||
|
|
@ -2443,6 +2457,69 @@ int cmd_join_worker(int argc, char* argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* jstart -j jobid [-p pid] — raft-apply-decoupling task 7.2: synthetic
|
||||
* worker-fact-stream primitive. Mirrors cmd_jobfinish's existing pattern
|
||||
* exactly (same getopt/parse_jobid/SET_ERROR shape) but drives the
|
||||
* xcecli_notify_jobstart wire call that cmd_jobfinish's sibling
|
||||
* xcecli_notify_jobfinish already exercises — libxce (client.c) already
|
||||
* implements the full worker-fact API (jobstart/jobfinish/jobfail/...); this
|
||||
* command was the missing REPL-level primitive for jobstart specifically.
|
||||
* Drives the SAME production receipt path (on_job_start -> record_job_start_fact
|
||||
* -> persist_submit_job_fact -> durable append -> commit -> apply pump ->
|
||||
* apply_committed_entry()) that a real worker's report would, without
|
||||
* requiring a real scheduler dispatch (on_job_start only requires the job to
|
||||
* already exist in jobmgr, i.e. have been submitted). */
|
||||
int cmd_jobstart(int argc, char* argv[])
|
||||
{
|
||||
int rc;
|
||||
int opt;
|
||||
uint64_t jobid = 0;
|
||||
unsigned int pid = 0;
|
||||
unsigned int pids[1];
|
||||
job_pidinfo_t info;
|
||||
|
||||
optind = 1;
|
||||
memset(&info, 0x0, sizeof(info));
|
||||
|
||||
while ((opt = getopt(argc, argv, "j:p:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'j':
|
||||
parse_jobid(&jobid, optarg);
|
||||
break;
|
||||
case 'p':
|
||||
pid = (unsigned int)atoi(optarg);
|
||||
break;
|
||||
default:
|
||||
printf(" jstart -j jobid [-p pid]\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
if (jobid == 0) {
|
||||
printf(" you need specify a positive jobid\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
pids[0] = (pid != 0) ? pid : (unsigned int)getpid();
|
||||
info.jobid = jobid;
|
||||
info.npids = 1;
|
||||
info.pids = pids;
|
||||
info.npgids = 0;
|
||||
info.pgids = NULL;
|
||||
info.exdata = NULL;
|
||||
|
||||
rc = xcecli_notify_jobstart(active_ctx->ctx, &info);
|
||||
if (rc < 0) {
|
||||
SET_ERROR(-rc);
|
||||
perror("xcecli_jobstart");
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("ok\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmd_jobfinish(int argc, char* argv[])
|
||||
{
|
||||
int rc;
|
||||
|
|
@ -2451,6 +2528,17 @@ int cmd_jobfinish(int argc, char* argv[])
|
|||
int code = 0;
|
||||
xce_jobfinish_t finish;
|
||||
|
||||
/* raft-apply-decoupling task 7.2: reset optind so repeated jfinish
|
||||
* invocations within one xce_test process (the fact-stream soak scenario
|
||||
* calls this many times in a single worker session) each re-parse their
|
||||
* own argv from the start. Without this, getopt's optind carries over
|
||||
* from the PRIOR command's parse position, silently skipping this
|
||||
* command's own "-j jobid" on the second and later calls (jobid stays 0
|
||||
* and the command errors out) — matches the reset convention most other
|
||||
* cmd_ handlers in this file already follow (e.g. cmd_join_worker just
|
||||
* above, cmd_jobsub, cmd_resupdate). */
|
||||
optind = 1;
|
||||
|
||||
while ((opt = getopt(argc, argv, "j:c:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'j':
|
||||
|
|
@ -2485,6 +2573,64 @@ int cmd_jobfinish(int argc, char* argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* jfail -j jobid [-c code] [-s stage] — raft-apply-decoupling task 7.2:
|
||||
* synthetic worker-fact-stream primitive, sibling of jstart/jfinish above.
|
||||
* Drives xcecli_notify_jobfail -> on_job_fail -> record_job_fail_fact ->
|
||||
* persist_submit_job_fact_ex (the _pad-discriminated path, task 6.1 round 2)
|
||||
* -> apply_job_fail_transition() at apply time. Reuses the existing
|
||||
* job_failinfo_t shape already exercised internally by this file's own
|
||||
* notify_job_fail() helper (used by run_job() on a local exec failure). */
|
||||
int cmd_jobfail(int argc, char* argv[])
|
||||
{
|
||||
int rc;
|
||||
int opt;
|
||||
uint64_t jobid = 0;
|
||||
int code = 0;
|
||||
int stage = JOB_EXEC_STAGE_POST_EXEC;
|
||||
job_failinfo_t failinfo;
|
||||
|
||||
optind = 1;
|
||||
memset(&failinfo, 0x0, sizeof(failinfo));
|
||||
|
||||
while ((opt = getopt(argc, argv, "j:c:s:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'j':
|
||||
parse_jobid(&jobid, optarg);
|
||||
break;
|
||||
case 'c':
|
||||
code = atoi(optarg);
|
||||
break;
|
||||
case 's':
|
||||
stage = atoi(optarg);
|
||||
break;
|
||||
default:
|
||||
printf(" jfail -j jobid [-c code] [-s stage]\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
}
|
||||
|
||||
if (jobid == 0) {
|
||||
printf(" you need specify a positive jobid\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
failinfo.jobid = jobid;
|
||||
failinfo.stage = stage;
|
||||
failinfo.code = code;
|
||||
failinfo.detail = "synthetic fact-stream failure (xce_test jfail)";
|
||||
|
||||
rc = xcecli_notify_jobfail(active_ctx->ctx, &failinfo);
|
||||
if (rc < 0) {
|
||||
SET_ERROR(-rc);
|
||||
perror("xcecli_jobfail");
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("ok\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int set_resval(xce_resval_t* vals, int idx, char* name, char* val)
|
||||
{
|
||||
static struct {
|
||||
|
|
|
|||
|
|
@ -84,6 +84,43 @@ static int decode_submit_info(const uint8_t* payload, uint32_t len,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* True for the WAL types whose payload is the flat, non-capnp
|
||||
* wal_jobstate_t {jobid, state, _pad} (wal_types.h) — every worker-reported
|
||||
* job-state fact (START/FINISH/FAIL/STOP/RESUME/UPDATE/RUSAGE/CKPT/PREASON).
|
||||
* Added for raft-apply-decoupling task 7.2: the kill-mid-fact-stream live
|
||||
* regression needs to verify, per jobid, which fact WAL entries actually
|
||||
* committed on each node — before this, waldump only decoded jobid for
|
||||
* WAL_JOB_SUBMIT, leaving every fact entry printed as "jobid=?". */
|
||||
static int wal_type_is_jobstate(uint32_t type)
|
||||
{
|
||||
switch (type) {
|
||||
case WAL_JOB_START:
|
||||
case WAL_JOB_FINISH:
|
||||
case WAL_JOB_FAIL:
|
||||
case WAL_JOB_STOP:
|
||||
case WAL_JOB_RESUME:
|
||||
case WAL_JOB_UPDATE:
|
||||
case WAL_JOB_RUSAGE:
|
||||
case WAL_JOB_CKPT:
|
||||
case WAL_JOB_PREASON:
|
||||
return 1;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Decode a wal_jobstate_t payload. Native-endian flat struct (single-node WAL
|
||||
* — see wal_types.h), so this is a bounds-checked copy, not a parser. Returns
|
||||
* 0 on success, -1 if the payload is too short to hold the struct. */
|
||||
static int decode_jobstate_info(const uint8_t* payload, uint32_t len,
|
||||
wal_jobstate_t* out)
|
||||
{
|
||||
if (!payload || len < sizeof(*out))
|
||||
return -1;
|
||||
memcpy(out, payload, sizeof(*out));
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* True if `dir` directly contains at least one seg-*.wal file. */
|
||||
static int dir_has_segments(const char* dir)
|
||||
{
|
||||
|
|
@ -342,6 +379,25 @@ static void scan_segment(const char* seg_path, FILE* out,
|
|||
}
|
||||
*prev_submit_jobid = jobid;
|
||||
*prev_submit_pcrc = pcrc;
|
||||
} else if (wal_type_is_jobstate(hdr.type)) {
|
||||
/* Worker-reported job-state fact (task 7.2): decode the flat
|
||||
* {jobid, state, _pad} payload so a per-jobid committed-fact check
|
||||
* (e.g. "did WAL_JOB_FINISH for jobid X actually commit on this
|
||||
* node") can grep waldump output directly, the same way the
|
||||
* existing WAL_JOB_SUBMIT branch above already supports
|
||||
* expect-job/joblist cross-checks. */
|
||||
wal_jobstate_t js;
|
||||
if (decode_jobstate_info(payload, hdr.payload_len, &js) == 0) {
|
||||
char jidStr[128];
|
||||
if (fmt_jobid(jidStr, sizeof(jidStr), js.jobid) < 0)
|
||||
snprintf(jidStr, sizeof(jidStr), "%" PRIu64, js.jobid);
|
||||
snprintf(info, sizeof(info), "jobid=%s state=0x%x pad=0x%x",
|
||||
jidStr, js.state, js._pad);
|
||||
} else {
|
||||
snprintf(info, sizeof(info), "jobid=?");
|
||||
}
|
||||
/* a non-submit entry breaks the adjacency for dup detection */
|
||||
*prev_submit_jobid = 0;
|
||||
} else {
|
||||
/* a non-submit entry breaks the adjacency for dup detection */
|
||||
*prev_submit_jobid = 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue