diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index 756f60f..72947d2 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -23,16 +23,27 @@ # fdatasync) are isolated from the divergence/job-loss result. # --sequential default single-failover mode (kill → wait-new-primary → # restart → reconverge). Backward-compatible. +# --force-snapshot raft-ha-core-safety task 5.10a / ha-entity-snapshots +# task 7.1: deploys with a low --max-wal-records so a real +# checkpoint fires, then wipes a killed node's data dir +# before restarting it (a genuine empty rejoin) so the +# leader MUST send a full-checkpoint snapshot rather than +# stream incremental WAL. Confirms via the restarted node's +# own log that a snapshot install actually happened (not +# just a claim), then asserts zero walcompare divergence +# and zero job loss. Single dedicated round, not a loop. # --selftest run the verdict-parser self-test (soak-verdict-selftest.sh) # and exit — no cluster, no binaries needed. # # Env overrides: # XCE_HA_TOP deploy dir (default /home/claude/tmp/xce-ha-deploy) # XCE_HA_DATADIR config/certs source (default /ha/data) -# XCE_HA_MODE sequential | chaotic (default sequential; --chaotic wins) +# XCE_HA_MODE sequential | chaotic | force-snapshot (default sequential) # XCE_HA_ROUNDS kill/restart rounds (default 6, clamp 5..10) # XCE_HA_JOBS jobs per submit batch (default 3) # XCE_HA_WAITMS wait-primary budget (default 30000) +# MAX_WAL_RECORDS --max-wal-records passed to every node (default: unset +# for sequential/chaotic; 100 for --force-snapshot) set -u # ── Mode / flag parsing (flags may precede the positional [ROUNDS] [DEPLOY_DIR]) ─ @@ -47,8 +58,9 @@ ROUNDS_CLI="" # rounds from a bare-integer positional (wins over env/default) ARGS=() while [ $# -gt 0 ]; do case "$1" in - --chaotic) MODE="chaotic" ;; - --sequential) MODE="sequential" ;; + --chaotic) MODE="chaotic" ;; + --sequential) MODE="sequential" ;; + --force-snapshot) MODE="force-snapshot" ;; --selftest) exec bash "$SELF_DIR/soak-verdict-selftest.sh" ;; *[!0-9]*|'') ARGS+=("$1") ;; # non-integer ⇒ deploy dir *) ROUNDS_CLI="$1" ;; # bare integer ⇒ rounds @@ -94,6 +106,16 @@ fi [ "$ROUNDS" -gt 10 ] && ROUNDS=10 NJOBS="${XCE_HA_JOBS:-3}" WAITMS="${XCE_HA_WAITMS:-30000}" +# raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: force-snapshot +# mode needs a real checkpoint to fire on a job count the soak can actually +# reach, so it defaults --max-wal-records to persist's enforced floor (100) +# unless the caller already set MAX_WAL_RECORDS explicitly. Every other mode +# leaves it unset (empty ⇒ start_node passes no override, unchanged behavior). +if [ "$MODE" = "force-snapshot" ]; then + MAX_WAL_RECORDS="${MAX_WAL_RECORDS:-100}" +else + MAX_WAL_RECORDS="${MAX_WAL_RECORDS:-}" +fi NODES=(ctrl-1 ctrl-2 ctrl-3) declare -A PORT=( [ctrl-1]=9991 [ctrl-2]=9992 [ctrl-3]=9993 ) @@ -153,7 +175,25 @@ start_node() { # libc/ASAN crash message) leaves a forensic trace. Appended (>>) so a # restarted node's later crash does not clobber the earlier instance's output. mkdir -p "$TOP/logs" - bash "$TOP/scripts/start.sh" "$1" -vvv >>"$TOP/logs/$1.stderr" 2>&1 & + if [ -n "$MAX_WAL_RECORDS" ]; then + bash "$TOP/scripts/start.sh" "$1" -vvv --max-wal-records "$MAX_WAL_RECORDS" \ + >>"$TOP/logs/$1.stderr" 2>&1 & + else + bash "$TOP/scripts/start.sh" "$1" -vvv >>"$TOP/logs/$1.stderr" 2>&1 & + fi +} + +# Wipe a node's WAL + snapshot history (keeps conf/certs so it can still +# authenticate + rejoin the same cluster identity) — simulates a genuine +# "empty rejoin" (raft-ha-core-safety task 5.10a / ha-entity-snapshots task +# 7.1). Also drops the persisted term marker so no stale term survives the +# wipe. Node must already be stopped (kill_node first). +wipe_node_data() { + local node="$1" + rm -rf "$TOP/controller/$node/data/wal"/* "$TOP/controller/$node/data/snap"/* \ + 2>/dev/null + rm -f "$TOP/controller/$node/data/ha-term.bin" 2>/dev/null + return 0 } # ── Primary / convergence detection: CLIENT PROBE (release-build-log-immune) ── @@ -644,6 +684,72 @@ run_chaotic_rounds() { fi } +# raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: force a REAL +# full-checkpoint catch-up round (not incremental WAL streaming) and confirm +# the restored member's job/queue state matches the rest of the cluster. +# +# The cluster is deployed with --max-wal-records (set low by MAX_WAL_RECORDS, +# default 100) so a real async checkpoint fires once enough WAL records have +# accumulated (checkpoint_task_fn -> persist_checkpoint -> trim_wal_segments, +# persist.c). After that checkpoint, the leader's oldest retained WAL entry +# is above LSN 1. A node whose data dir is then wiped (wipe_node_data — a +# genuine empty rejoin, standby_lsn=0) MUST receive HA_REPL_SEND_SNAPSHOT +# (ha_evaluate_repl_action, ha.c: standby_lsn+1 < oldest_wal_lsn) rather than +# HA_REPL_STREAM_WAL — there is no incremental history left to stream. +# +# This does NOT loop or share state with run_chaotic_rounds/the sequential +# loop — it is one dedicated, deterministic scenario, run standalone. +run_force_snapshot_round() { + local target other1 other2 n before_lsn snap_log + + # Submit enough jobs to cross the checkpoint threshold. Each jobsub is one + # WAL_JOB_SUBMIT record; comfortably clear MAX_WAL_RECORDS so the async + # checkpoint (1s delaymgr coalescing delay, persist.c) has unambiguously + # fired by the time we act. + local need=$((MAX_WAL_RECORDS + 10)) + log "force-snapshot: submitting $need jobs to cross the max-wal-records=$MAX_WAL_RECORDS checkpoint threshold" + submit_jobs "$need" || fail "force-snapshot: could not submit $need jobs to trigger a checkpoint" + verify_no_loss || fail "force-snapshot: job loss before the forced round even started" + + log "force-snapshot: waiting for the async checkpoint to fire and trim WAL" + sleep 8 # >> the 1s delaymgr coalescing delay + trim_wal_segments' own work + + # Pick a non-primary target to wipe+rejoin (killing the primary is a + # different, already-covered scenario — this round isolates the snapshot- + # catchup path specifically). + target="" + for n in "${NODES[@]}"; do + [ "$n" = "$PRIMARY" ] || { target="$n"; break; } + done + [ -n "$target" ] || fail "force-snapshot: could not pick a non-primary target" + log "force-snapshot: target=$target (will be killed, wiped, and rejoined empty)" + + kill_node "$target" || fail "force-snapshot: could not kill $target" + sleep 1 + wipe_node_data "$target" + log "force-snapshot: $target data wiped (wal/ + snap/ + ha-term.bin) — rejoining as a genuinely empty member" + + start_node "$target" + wait_for_full_convergence \ + || fail "force-snapshot: cluster did not reconverge to 3-of-3 after $target rejoined empty" + log "force-snapshot: cluster reconverged (3-of-3)" + + # Direct proof this exercised a REAL snapshot install, not incremental + # catch-up: ha_handle_snapshot_chunk (ha.c) logs "HA: snapshot loaded" / + # "HA: snapshot ... legacy position-only" on success. If neither line + # appears, the forcing mechanism did not work and the round proves + # nothing — fail loud instead of reporting a false PASS. + snap_log="$TOP/logs/$target.log" + if ! grep -q "HA: snapshot" "$snap_log" 2>/dev/null; then + fail "force-snapshot: $target's log shows NO snapshot install (grep 'HA: snapshot' $snap_log) — the forced scenario did not actually exercise a checkpoint catch-up; this round proves nothing" + fi + log "force-snapshot: confirmed a real snapshot install in $target's log" + + verify_no_loss \ + || fail "force-snapshot: a committed job was LOST after the forced snapshot catch-up" + walcheck_count "${#ALL_JOBIDS[@]}" +} + # Drive a SPECIFIC node to primary — used to exercise leadership on the # lowest-priority node (ctrl-3), which the natural single-failover loop NEVER # elects: the election timeout is priority-staggered (my_priority * step_ms, p0 @@ -741,6 +847,29 @@ submit_jobs "$NJOBS" || fail "initial job submission did not get $NJOBS clean ac verify_no_loss || fail "initial joblist verification failed" walcheck "${#ALL_JOBIDS[@]}" +# ── Force-snapshot mode: one dedicated forced full-checkpoint catch-up round ── +# Opt-in (--force-snapshot / XCE_HA_MODE=force-snapshot). See +# run_force_snapshot_round's own header comment. Single round, not a loop — +# emits a verdict and exits, like chaotic mode. +if [ "$MODE" = "force-snapshot" ]; then + log "MODE=force-snapshot — forcing one full-checkpoint catch-up round (max-wal-records=$MAX_WAL_RECORDS)" + run_force_snapshot_round + scan_bug924 + verdict_emit "force-snapshot" 1 "${#ALL_JOBIDS[@]}" \ + "$V_DIVERGENCE" "$V_JOBLOSS" "0" "$V_BUG924" + VERDICT_RC=$? + if [ "$VERDICT_RC" -eq 0 ]; then + log "PASS (force-snapshot): ${#ALL_JOBIDS[@]} jobs, zero committed divergence, zero committed job loss, confirmed real snapshot install (bug924=$V_BUG924 excluded)" + cleanup + trap - EXIT INT TERM + exit 0 + fi + echo "[xce-test-ha] FAIL: force-snapshot verdict — committed_divergence=$V_DIVERGENCE committed_job_loss=$V_JOBLOSS (bug924=$V_BUG924 excluded)" >&2 + cleanup + trap - EXIT INT TERM + exit 1 +fi + # ── Chaotic mode: overlapping-kill rounds → isolated verdict ────────────────── # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land inside election/catch-up # windows; outcomes are COUNTED (committed divergence + committed job loss) with diff --git a/src/controller/cluster.c b/src/controller/cluster.c index c668b04..5525a75 100644 --- a/src/controller/cluster.c +++ b/src/controller/cluster.c @@ -1324,6 +1324,12 @@ int cluster_init_phase2(cluster_t* c) * the requested cadence (ha-commit-durability Req 3.1). */ persist_config_set_wal_sync_ms(&pcfg, c->myconf->wal_sync_ms); + /* --max-wal-records checkpoint-trigger override (0 = keep the + * persist default). raft-ha-core-safety task 5.10a / ha-entity- + * snapshots task 7.1: lets a soak force real checkpoints on a + * deterministic record count instead of the 10000-record default. */ + persist_config_set_max_wal_records(&pcfg, c->myconf->max_wal_records); + persist_set_workdir(&c->persist, workdir, name); rc = persist_init(&c->persist, &pcfg, c); if (rc != 0) { diff --git a/src/controller/main.c b/src/controller/main.c index fc1fc86..28c868c 100644 --- a/src/controller/main.c +++ b/src/controller/main.c @@ -31,6 +31,7 @@ const char* argp_program_bug_address = BUG_EMAIL; #define OPT_CERTDIR 260 #define OPT_SYNC_TIMEOUT 261 #define OPT_WAL_SYNC 262 +#define OPT_MAX_WAL_RECORDS 263 struct arguments { char* myname; @@ -44,6 +45,7 @@ struct arguments { char* certdir; uint32_t sync_timeout_ms; uint32_t wal_sync_ms; + uint32_t max_wal_records; }; static struct argp_option options[] = { @@ -66,6 +68,9 @@ static struct argp_option options[] = { "Synchronous majority-replication commit barrier timeout in ms (0 = default)" }, { "wal-sync-ms", OPT_WAL_SYNC, "MS", 0, "WAL flush (fsync) cadence in ms (0 = default)" }, + { "max-wal-records", OPT_MAX_WAL_RECORDS, "N", 0, + "WAL records since last checkpoint before an async checkpoint is " + "scheduled (0 = default; persist enforces a minimum of 100)" }, { 0 } }; @@ -122,6 +127,13 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state) arguments->wal_sync_ms = (uint32_t)v; break; } + case OPT_MAX_WAL_RECORDS: { + int v = atoi(arg); + if (v < 0) + argp_error(state, "--max-wal-records must be >= 0 (got: %s)", arg); + arguments->max_wal_records = (uint32_t)v; + break; + } default: return ARGP_ERR_UNKNOWN; } @@ -175,6 +187,7 @@ int main(int argc, char* argv[]) arguments.vcpus = 0; /* 0 = not set; use cluster config default */ arguments.sync_timeout_ms = 0; /* 0 = not set; use persist config default */ arguments.wal_sync_ms = 0; /* 0 = not set; use persist config default */ + arguments.max_wal_records = 0; /* 0 = not set; use persist config default */ argp_parse(&argp, argc, argv, 0, 0, &arguments); @@ -225,6 +238,7 @@ int main(int argc, char* argv[]) myconf.vcpus = arguments.vcpus; myconf.sync_timeout_ms = arguments.sync_timeout_ms; myconf.wal_sync_ms = arguments.wal_sync_ms; + myconf.max_wal_records = arguments.max_wal_records; memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg)); if (arguments.peers_str) { diff --git a/src/controller/myapp.c b/src/controller/myapp.c index 25001e6..d702a9b 100644 --- a/src/controller/myapp.c +++ b/src/controller/myapp.c @@ -171,6 +171,12 @@ int myapp_run(myconf_t *myconf) * the requested cadence (ha-commit-durability Req 3.1). */ persist_config_set_wal_sync_ms(&pcfg, myconf->wal_sync_ms); + /* --max-wal-records checkpoint-trigger override (0 = keep the + * persist default). raft-ha-core-safety task 5.10a / ha-entity- + * snapshots task 7.1: lets a soak force real checkpoints on a + * deterministic record count instead of the 10000-record default. */ + persist_config_set_max_wal_records(&pcfg, myconf->max_wal_records); + persist_set_workdir(&c.persist, workdir, name); rc = persist_init(&c.persist, &pcfg, &c); if (rc != 0) { diff --git a/src/include/myconf.h b/src/include/myconf.h index 99af8be..96edf9a 100644 --- a/src/include/myconf.h +++ b/src/include/myconf.h @@ -37,6 +37,7 @@ typedef struct { int vcpus; /* Req 4.3: --vcpus CLI value; 0 = use config default */ uint32_t sync_timeout_ms; /* --sync-timeout-ms; 0 = use persist default */ uint32_t wal_sync_ms; /* --wal-sync-ms; 0 = use persist default */ + uint32_t max_wal_records; /* --max-wal-records; 0 = use persist default */ } myconf_t; extern cluster_config_t* get_default_conf(void*); diff --git a/src/include/persist.h b/src/include/persist.h index ad9a75f..db7e025 100644 --- a/src/include/persist.h +++ b/src/include/persist.h @@ -135,6 +135,25 @@ static inline void persist_config_set_wal_sync_ms(persist_config_t *c, c->wal_sync_ms = override_ms; } +/* + * persist_config_set_max_wal_records — apply the --max-wal-records + * checkpoint-trigger override (raft-ha-core-safety task 5.10a / + * ha-entity-snapshots task 7.1: forcing a real full-checkpoint catch-up + * round in the chaotic soak requires a low, deterministic record-count + * threshold rather than waiting on the 10000-record production default). + * + * Same convention as persist_config_set_wal_sync_ms: override == 0 means + * "flag not passed", leaves the default threshold untouched. A positive + * value sets it exactly (persist_init's own minimum-100 floor still + * applies afterward — this setter does not duplicate that enforcement). + */ +static inline void persist_config_set_max_wal_records(persist_config_t *c, + uint32_t override_n) +{ + if (override_n > 0) + c->max_wal_records = override_n; +} + /* ============================================================ * Key→index mapping for checkpoint serialization (Part 3.8) * ============================================================ */