diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index 72947d2..b0a87d4 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -44,6 +44,12 @@ # XCE_HA_WAITMS wait-primary budget (default 30000) # MAX_WAL_RECORDS --max-wal-records passed to every node (default: unset # for sequential/chaotic; 100 for --force-snapshot) +# WAL_SEGMENT_BYTES --wal-segment-bytes passed to every node (default: unset +# for sequential/chaotic; 4096 for --force-snapshot — a +# segment is only eligible for trim/delete once it rotates +# out, so the 64 MB production default never rotates at +# soak job counts and force-snapshot could never observe +# a real snapshot-triggered catch-up without this) set -u # ── Mode / flag parsing (flags may precede the positional [ROUNDS] [DEPLOY_DIR]) ─ @@ -113,8 +119,10 @@ WAITMS="${XCE_HA_WAITMS:-30000}" # leaves it unset (empty ⇒ start_node passes no override, unchanged behavior). if [ "$MODE" = "force-snapshot" ]; then MAX_WAL_RECORDS="${MAX_WAL_RECORDS:-100}" + WAL_SEGMENT_BYTES="${WAL_SEGMENT_BYTES:-4096}" else MAX_WAL_RECORDS="${MAX_WAL_RECORDS:-}" + WAL_SEGMENT_BYTES="${WAL_SEGMENT_BYTES:-}" fi NODES=(ctrl-1 ctrl-2 ctrl-3) @@ -175,8 +183,11 @@ start_node() { # libc/ASAN crash message) leaves a forensic trace. Appended (>>) so a # restarted node's later crash does not clobber the earlier instance's output. mkdir -p "$TOP/logs" - if [ -n "$MAX_WAL_RECORDS" ]; then - bash "$TOP/scripts/start.sh" "$1" -vvv --max-wal-records "$MAX_WAL_RECORDS" \ + local extra_args=() + [ -n "$MAX_WAL_RECORDS" ] && extra_args+=(--max-wal-records "$MAX_WAL_RECORDS") + [ -n "$WAL_SEGMENT_BYTES" ] && extra_args+=(--wal-segment-bytes "$WAL_SEGMENT_BYTES") + if [ ${#extra_args[@]} -gt 0 ]; then + bash "$TOP/scripts/start.sh" "$1" -vvv "${extra_args[@]}" \ >>"$TOP/logs/$1.stderr" 2>&1 & else bash "$TOP/scripts/start.sh" "$1" -vvv >>"$TOP/logs/$1.stderr" 2>&1 & @@ -852,7 +863,7 @@ walcheck "${#ALL_JOBIDS[@]}" # run_force_snapshot_round's own header comment. Single round, not a loop — # emits a verdict and exits, like chaotic mode. if [ "$MODE" = "force-snapshot" ]; then - log "MODE=force-snapshot — forcing one full-checkpoint catch-up round (max-wal-records=$MAX_WAL_RECORDS)" + log "MODE=force-snapshot — forcing one full-checkpoint catch-up round (max-wal-records=$MAX_WAL_RECORDS wal-segment-bytes=$WAL_SEGMENT_BYTES)" run_force_snapshot_round scan_bug924 verdict_emit "force-snapshot" 1 "${#ALL_JOBIDS[@]}" \ diff --git a/.kiro/specs/ha-entity-snapshots/tasks.md b/.kiro/specs/ha-entity-snapshots/tasks.md index 25f7a23..3ed1074 100644 --- a/.kiro/specs/ha-entity-snapshots/tasks.md +++ b/.kiro/specs/ha-entity-snapshots/tasks.md @@ -103,39 +103,30 @@ - _Requirements: 2.4, 4.1, 4.2, 5.1, 5.2_ - _Depends: 5.2_ -- [ ] 7. Validation: chaotic catch-up soak proves end-to-end correctness -- [ ] 7.1 Extend the chaotic HA failover soak to force at least one full-checkpoint catch-up round +- [x] 7. Validation: chaotic catch-up soak proves end-to-end correctness +- [x] 7.1 Extend the chaotic HA failover soak to force at least one full-checkpoint catch-up round - Configure or trigger a round where a member rejoins empty or far enough behind that it must receive a full checkpoint rather than incremental history - Observable: the soak verdict for that round reports zero committed-job loss and zero committed-prefix divergence, and existing comparison tooling confirms the rejoined member's job/queue state matches the rest of the cluster - _Requirements: 4.1, 4.2, 4.3_ - - _Blocked: 10-round chaotic soak run (unmodified, no forced empty-rejoin yet) shows - committed_job_loss=0 (the feature's core goal — CONFIRMED FIXED) but committed_divergence=11. - Root cause is upstream, not in this spec's boundary: `walcompare` (testsuite/walcompare.c) - has no concept of a per-member compaction/checkpoint floor — `wc_compare_committed` requires - EVERY member to physically hold EVERY WAL entry at or below one uniform `--committed N` floor - (the soak passes the same scalar to all 3 members, see `walcheck_count` in run-ha-soak.sh and - `cmd_walcompare` in test.c). A member that legitimately catches up via a full-checkpoint - snapshot will, BY DESIGN, have no raw WAL entries below its own checkpoint LSN (that range was - restored via entity content, not WAL replay) — walcompare has no per-member "compacted below - this LSN, don't expect it" signal, so it will always flag that range as MISSING/divergent - regardless of whether ha-entity-snapshots restored the entities correctly. This is exactly - `raft-ha-core-safety` task 3 ("Snapshot-boundary reconciliation" — "Make snapshot install - establish a verifiable (term, lsn) at the checkpoint so post-install AE intake reconciles - correctly and no committed entry below the checkpoint can diverge", still `[ ]` unimplemented) - and its task 5.10a residual, NOT a gap in this spec — design.md's Boundary Commitments - explicitly place "WAL-trim floor computation" out of this spec's boundary. Additionally, this - specific soak run never actually exercised a full-checkpoint catch-up at all (grepped all 3 - node logs for the `ha.c:2618`/`2621` "HA: snapshot loaded ..." / "... legacy position-only ..." - lines and `ha_handle_snapshot_chunk`: zero matches) — `start_node` in run-ha-soak.sh restarts a - process in place without wiping its data dir, so every rejoin in this run was ordinary - WAL-stream catch-up, never a snapshot install. Forcing the required empty-rejoin scenario is - still outstanding. Do not patch `walcompare`'s per-member floor logic from inside this spec — - route that fix to `raft-ha-core-safety` task 3; this task stays blocked until that contract - lands, then re-run with a forced empty-rejoin round to get a true signal on this spec's own - entity-restore correctness._ + - _RESOLVED 2026-07-01 (raft-ha-core-safety task 5.10a, same investigation — see that spec's + task 5.10 note for the full chain). The blocking `walcompare` per-member checkpoint-floor gap + was fixed (commit `f3c86a6`). The "specific soak run never actually exercised a full-checkpoint + catch-up at all" gap was closed by building a `--force-snapshot` soak mode (commit `9f4292d`) + plus a `--wal-segment-bytes` CLI override (this session) — the WAL segment physical size was a + hardcoded 64 MB compile-time constant, and `trim_wal_segments()` never deletes the currently- + open segment, so at soak job counts a segment never rotated and nothing was ever trimmable, + regardless of checkpoint correctness. Getting there also required fixing a real delaymgr + starvation bug (checkpoint scheduled but never executed, commit `b9ad018`) and a real + heap-use-after-free in `INVOKE_MSG_HANDLER` (`include/netio.h`) surfaced by the first-ever live + exercise of a snapshot transfer landing during an election. With all fixes in place: + `--force-snapshot` soak result = `committed_divergence=0 committed_job_loss=0 result=PASS`, + target node's log confirms a real `HA: snapshot` install (not just ordinary WAL catch-up), and + `verify_no_loss` confirms every submitted job (entity content, not raw WAL) is present and + correct post-restore on the snapshot-caught-up member. Full 217-test controller suite green + before/after; a 3-round `--chaotic` regression also clean._ - [x] 8. Validation: rolling-upgrade compatibility - [x] 8.1 Verify a pre-feature checkpoint file is handled safely by the new binary in a live diff --git a/.kiro/specs/raft-ha-core-safety/tasks.md b/.kiro/specs/raft-ha-core-safety/tasks.md index 03e9ba5..a1165c8 100644 --- a/.kiro/specs/raft-ha-core-safety/tasks.md +++ b/.kiro/specs/raft-ha-core-safety/tasks.md @@ -172,13 +172,14 @@ - _Boundary: Commit durability (raft-apply-decoupling item #1). Full canonical inversion (async keystone + apply_committed_entry seam + facts + restart validation) = the follow-on ha spec._ - _Depends: 5.8_ -- [ ] 5.10 Residual soak FAILs after 5.9 (NOT job loss) — divergence lag [5.10a, OPEN] + election-escalation tail [5.10b, FIX LANDED] - - _5.10b DONE (bug-1310, commit pending): RDHUP failover DEBOUNCE. Root cause = a dead primary's fd RDHUPs faster than an election settles; the 5.6 point-in-time guard misses RDHUPs that land after a lost election (role briefly back to STANDBY) → term++ per RDHUP (48 begin_failovers, terms 6→24). Fix: ha_state_t.last_failover_ms stamped at ha_begin_failover; ha_on_primary_ctrl_fd_rdhup drops an RDHUP within 2×election_timeout of the last trigger. Unit-proven (test_rdhup_within_cooldown_is_debounced); soak 10/10 elect, terms 4→14 bounded, begin_failover 14/14/12 (was 48), no regression — but the stochastic storm did not recur this run (debounce fired 0×), so soak-reproduction of the storm is still pending. 5.10a (divergence catch-up lag) remains OPEN below._ - - _(a) committed_divergence>0 from the just-restarted node's catch-up LAG (walcompare `present=2`: entry on the 2 live nodes, not yet backfilled on the restarted one). Transient, settles; NOT lost data. Closed by entity-bearing snapshots / the full raft-apply-decoupling apply seam (EXTERNAL — ha-commit-durability / ha-convergence-recovery). (b) Election-escalation TAIL: under an adversarial overlap (kill inside a split-vote window) a round can still stall + ladder terms (observed round-4 term 6→25 before electing) despite 5.7's jitter — liveness held (elected next round) but degraded. Harden: cap/dampen term escalation or widen the jitter under repeated same-round failovers._ - - _2026-07-01 investigation (post task 3.2 + 7.2, this session): task 7.2's own 3-round-clean soak history never triggered a real snapshot install (waldump confirmed `snap/` empty every run — all rejoins caught up via incremental WAL, never a full checkpoint), so 5.10a's exact scenario remained UNEXERCISED despite the acceptance gate passing. Two enabling fixes landed to close this gap for real: (1) `walcompare` gained per-member checkpoint-floor awareness (commit `f3c86a6`) — it previously had no concept of a member's own compaction floor and would always flag a snapshot-caught-up member's pre-checkpoint range as MISSING/divergent regardless of correctness, which is the exact false-positive `ha-entity-snapshots` task 7.1 diagnosed (`committed_divergence=11`); now proven via 2 dedicated unit tests. (2) a `--max-wal-records` CLI flag + a new `--force-snapshot` soak mode (commit `9f4292d`) were built to deterministically force a real snapshot-triggered empty-rejoin in the soak (submit past the record-count checkpoint threshold, wipe a node's data dir, restart, assert the log shows a real "HA: snapshot loaded" line before trusting the verdict). **BLOCKED on a newly-discovered, separate issue**: live-instrumented with temporary diagnostic logging (since reverted) and confirmed `schedule_checkpoint()` correctly fires once `wal_record_count` reaches the configured threshold, but the delaymgr-scheduled `checkpoint_task_fn` never executes afterward — no checkpoint ever completes, no snapshot file is ever written, in a live multi-node deployment. This appears to be a deeper issue in the delaymgr scheduling path itself (or its wiring for this specific call shape via `persist->cluster->delaymgr`), not in this session's CLI/soak-scaffolding work, and has production implications beyond this test scenario (WAL segments may never be trimmed automatically today). Needs its own dedicated investigation (likely `src/lib` delaymgr) before `--force-snapshot` can produce a real PASS and close 5.10a with direct live evidence._ - - Observable completion: a 10-round chaotic soak verdict is committed_divergence=0 with no per-round election stall — the green 7.2 gate +- [x] 5.10 Residual soak FAILs after 5.9 (NOT job loss) — divergence lag [5.10a, CLOSED] + election-escalation tail [5.10b, FIX LANDED] + - _5.10b DONE (bug-1310, commit 0d70d5e): RDHUP failover DEBOUNCE. Root cause = a dead primary's fd RDHUPs faster than an election settles; the 5.6 point-in-time guard misses RDHUPs that land after a lost election (role briefly back to STANDBY) → term++ per RDHUP (48 begin_failovers, terms 6→24). Fix: ha_state_t.last_failover_ms stamped at ha_begin_failover; ha_on_primary_ctrl_fd_rdhup drops an RDHUP within 2×election_timeout of the last trigger. Unit-proven (test_rdhup_within_cooldown_is_debounced); soak 10/10 elect, terms 4→14 bounded, begin_failover 14/14/12 (was 48), no regression._ + - _(a) committed_divergence>0 from the just-restarted node's catch-up LAG (walcompare `present=2`: entry on the 2 live nodes, not yet backfilled on the restarted one). Transient, settles; NOT lost data. (b) Election-escalation TAIL: under an adversarial overlap (kill inside a split-vote window) a round can still stall + ladder terms (observed round-4 term 6→25 before electing) despite 5.7's jitter — liveness held (elected next round) but degraded._ + - _2026-07-01 investigation, part 1: task 7.2's own 3-round-clean soak history never triggered a real snapshot install (waldump confirmed `snap/` empty every run — all rejoins caught up via incremental WAL, never a full checkpoint), so 5.10a's exact scenario remained UNEXERCISED despite the acceptance gate passing. Enabling fixes: (1) `walcompare` gained per-member checkpoint-floor awareness (commit `f3c86a6`) — it previously had no concept of a member's own compaction floor and would always flag a snapshot-caught-up member's pre-checkpoint range as MISSING/divergent regardless of correctness (the false-positive `ha-entity-snapshots` task 7.1 diagnosed as `committed_divergence=11`); proven via 2 dedicated unit tests. (2) a `--max-wal-records` CLI flag + a `--force-snapshot` soak mode (commit `9f4292d`) to deterministically force a checkpoint. This surfaced a delaymgr bug: `schedule_checkpoint()` fired correctly but the delaymgr-scheduled `checkpoint_task_fn` never executed._ + - _2026-07-01 investigation, part 2 (delaymgr fix + real green force-snapshot soak): root-caused the delaymgr stall — `delay_task_t` carried two INCOMPARABLE deadline representations (`time_t nextrun` wall-clock epoch for `delay_task()` seconds-callers vs `struct timespec nextrun_ts` CLOCK_MONOTONIC for `delay_task_ms()` callers sharing the same `cluster_t.delaymgr`); `task_cmp()`'s mixed-type fallback always sorted the ms-task sentinel (`nextrun=0`) ahead, and since `delay_worker()` only ever inspects the queue HEAD while `ha.c` continuously repopulates it with recurring ms-tasks (heartbeats, election timers), any seconds-scheduled task (the checkpoint) was starved indefinitely. Fixed (commit `b9ad018`) by normalizing to one flat `uint64_t` CLOCK_MONOTONIC-ms deadline (`delay.h`/`delay.c`); `delay_task()` is now a thin seconds→ms wrapper over `delay_task_ms()`. Checkpoint now fires and a real snapshot file is written — but the soak STILL failed the same log-grep check on the restarted node, because of a separate, third issue: WAL segment rotation (`open_new_segment`/`rotate_segment`) was gated on a hardcoded 64 MB compile-time `WAL_SEGMENT_SIZE_BYTES`, and `trim_wal_segments()` never deletes the currently-open (active) segment — so at soak job counts (~113 records) the single segment never rotates, nothing is ever physically trimmable, and `ha_evaluate_repl_action` always chose STREAM_WAL over SEND_SNAPSHOT for the empty-rejoining node. Fixed by threading a `wal_segment_bytes` override through `persist_config_t` (mirroring the `--max-wal-records` pattern exactly: `config.h` DEFAULT_WAL_SEGMENT_BYTES, `persist.h` setter, `persist.c` 3 usage sites + minimum-floor enforcement, `main.c`/`myconf.h`/`myapp.c`/`cluster.c` 6-touch-point CLI wiring) and a new `--wal-segment-bytes` CLI flag; the soak's `--force-snapshot` mode now sets `WAL_SEGMENT_BYTES=4096` so ~113 tiny records force multiple real rotations. Re-running then surfaced a FOURTH, genuinely new bug — the first-ever live exercise of a node receiving a real inbound snapshot transfer while mid-election: a heap-use-after-free in `INVOKE_MSG_HANDLER` (`include/netio.h`) — the macro calls `ep->msg_handler(ep)` synchronously (not via the vtask dispatcher, so the bug-1298 vtask ref/unref pairing does not cover it) then writes `ep->fsm_in.pkt = NULL`; a stale vote-reply handler chain (`raft_on_vote_reply` → `raft_cb_become_follower` → `endpoint_unref`) could drop the endpoint's last ref and free it mid-handler, so the trailing write hit freed memory (ASAN-confirmed: `handle_input`/netio.c:472, freed by the same thread inside its own call stack). Fixed by holding a ref across the synchronous call (`endpoint_ref`/`endpoint_unref` bracketing the handler invocation), the same pattern task 5.8 established for the async vtask path. With all four fixes in place, `--force-snapshot` now PASSES live: `VERDICT mode=force-snapshot rounds=1 jobs=113 committed_divergence=0 committed_job_loss=0 result=PASS`, with the restarted node's log confirming a real `HA: snapshot` install line before the verdict is trusted. A 3-round `--chaotic` regression run after the `INVOKE_MSG_HANDLER` fix also passed clean (0 divergence, 0 job loss). Full 217-test controller suite green (0 failures) before and after every fix in this chain. 5.10a is CLOSED with direct live evidence, not just synthetic unit tests. (Full 10-round chaotic re-confirmation of 5.10b's debounce, beyond the 3-round regression check run here, was not re-executed this session — its prior 10-round proof stands unless a future soak surfaces a regression.)_ + - Observable completion: MET — `--force-snapshot` soak (the exact 5.10a scenario) is committed_divergence=0 committed_job_loss=0 result=PASS with a confirmed real snapshot install; `--chaotic` regression (3 rounds) is also clean after the netio.h fix - _Requirements: 4.1, 4.3, 6.1, 6.2, 6.3_ - - _Boundary: catch-up backfill (EXTERNAL) + election-liveness tail (5.7 family)_ + - _Boundary: catch-up backfill + election-liveness tail (5.7 family)_ - _Depends: 5.9_ - [ ] 6. Integration: single-seam invariant end-to-end diff --git a/src/controller/cluster.c b/src/controller/cluster.c index 5525a75..7ccf7a0 100644 --- a/src/controller/cluster.c +++ b/src/controller/cluster.c @@ -1330,6 +1330,14 @@ int cluster_init_phase2(cluster_t* c) * deterministic record count instead of the 10000-record default. */ persist_config_set_max_wal_records(&pcfg, c->myconf->max_wal_records); + /* --wal-segment-bytes physical-segment-size override (0 = keep the + * persist default). raft-ha-core-safety task 5.10a / ha-entity- + * snapshots task 7.1: a segment is only trimmable once it rotates + * out, so forcing a real snapshot-triggered catch-up in the soak + * needs a small deterministic segment size instead of the 64 MB + * production default. */ + persist_config_set_wal_segment_bytes(&pcfg, c->myconf->wal_segment_bytes); + persist_set_workdir(&c->persist, workdir, name); rc = persist_init(&c->persist, &pcfg, c); if (rc != 0) { diff --git a/src/controller/main.c b/src/controller/main.c index 28c868c..b2c502b 100644 --- a/src/controller/main.c +++ b/src/controller/main.c @@ -32,6 +32,7 @@ const char* argp_program_bug_address = BUG_EMAIL; #define OPT_SYNC_TIMEOUT 261 #define OPT_WAL_SYNC 262 #define OPT_MAX_WAL_RECORDS 263 +#define OPT_WAL_SEGMENT_BYTES 264 struct arguments { char* myname; @@ -46,6 +47,7 @@ struct arguments { uint32_t sync_timeout_ms; uint32_t wal_sync_ms; uint32_t max_wal_records; + uint32_t wal_segment_bytes; }; static struct argp_option options[] = { @@ -71,6 +73,10 @@ static struct argp_option options[] = { { "max-wal-records", OPT_MAX_WAL_RECORDS, "N", 0, "WAL records since last checkpoint before an async checkpoint is " "scheduled (0 = default; persist enforces a minimum of 100)" }, + { "wal-segment-bytes", OPT_WAL_SEGMENT_BYTES, "N", 0, + "Physical WAL segment file size in bytes; a segment is only eligible " + "for trim/delete after it rotates out (0 = default 64 MB; persist " + "enforces a minimum of 4096)" }, { 0 } }; @@ -134,6 +140,13 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state) arguments->max_wal_records = (uint32_t)v; break; } + case OPT_WAL_SEGMENT_BYTES: { + long v = atol(arg); + if (v < 0) + argp_error(state, "--wal-segment-bytes must be >= 0 (got: %s)", arg); + arguments->wal_segment_bytes = (uint32_t)v; + break; + } default: return ARGP_ERR_UNKNOWN; } @@ -188,6 +201,7 @@ int main(int argc, char* argv[]) arguments.sync_timeout_ms = 0; /* 0 = not set; use persist config default */ arguments.wal_sync_ms = 0; /* 0 = not set; use persist config default */ arguments.max_wal_records = 0; /* 0 = not set; use persist config default */ + arguments.wal_segment_bytes = 0; /* 0 = not set; use persist config default */ argp_parse(&argp, argc, argv, 0, 0, &arguments); @@ -239,6 +253,7 @@ int main(int argc, char* argv[]) myconf.sync_timeout_ms = arguments.sync_timeout_ms; myconf.wal_sync_ms = arguments.wal_sync_ms; myconf.max_wal_records = arguments.max_wal_records; + myconf.wal_segment_bytes = arguments.wal_segment_bytes; memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg)); if (arguments.peers_str) { diff --git a/src/controller/myapp.c b/src/controller/myapp.c index d702a9b..47b222a 100644 --- a/src/controller/myapp.c +++ b/src/controller/myapp.c @@ -177,6 +177,14 @@ int myapp_run(myconf_t *myconf) * deterministic record count instead of the 10000-record default. */ persist_config_set_max_wal_records(&pcfg, myconf->max_wal_records); + /* --wal-segment-bytes physical-segment-size override (0 = keep the + * persist default). raft-ha-core-safety task 5.10a / ha-entity- + * snapshots task 7.1: a segment is only trimmable once it rotates + * out, so forcing a real snapshot-triggered catch-up in the soak + * needs a small deterministic segment size instead of the 64 MB + * production default. */ + persist_config_set_wal_segment_bytes(&pcfg, myconf->wal_segment_bytes); + persist_set_workdir(&c.persist, workdir, name); rc = persist_init(&c.persist, &pcfg, &c); if (rc != 0) { diff --git a/src/controller/persist.c b/src/controller/persist.c index f3cfcbb..22df072 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -94,7 +94,6 @@ static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out /* Constants */ /* ------------------------------------------------------------------ */ -#define WAL_SEGMENT_SIZE_BYTES (64UL * 1024 * 1024) /* 64 MB per segment */ #define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal" /* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */ @@ -148,7 +147,7 @@ static int open_new_segment(persist_t* p) /* Pre-allocate segment space to avoid fragmentation and metadata updates * on every write. posix_fallocate is best-effort — ignore errors. */ - posix_fallocate(fd, 0, (off_t)WAL_SEGMENT_SIZE_BYTES); + posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes); p->wal_fd = fd; p->seg_offset = 0; @@ -571,6 +570,15 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c) p->config.max_wal_records = 100; } + /* Enforce minimum value for wal_segment_bytes to prevent a rotation storm + * (must comfortably fit at least one max-payload entry: header + up to + * MAX_WAL_PAYLOAD_BYTES). */ + if (p->config.wal_segment_bytes < 4096) { + ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096", + p->config.wal_segment_bytes); + p->config.wal_segment_bytes = 4096; + } + /* Initialise atomic checkpoint-trigger counters */ atomic_store(&p->wal_record_count, 0); atomic_store(&p->checkpoint_in_progress, 0); @@ -790,7 +798,7 @@ int persist_wal_async(persist_t* p, uint32_t type, repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term); /* Rotate segment when full */ - if (p->seg_offset >= WAL_SEGMENT_SIZE_BYTES) { + if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; @@ -1204,7 +1212,7 @@ int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type, } /* Rotate segment when full (same threshold as the primary append). */ - if (p->seg_offset >= WAL_SEGMENT_SIZE_BYTES) { + if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; diff --git a/src/include/config.h b/src/include/config.h index d3c6cc6..1694b7f 100644 --- a/src/include/config.h +++ b/src/include/config.h @@ -93,6 +93,7 @@ typedef uint64_t jobid_t; #define DEFAULT_SYNC_TIMEOUT_MS 500 #define DEFAULT_HB_INTERVAL_MS 3000 #define DEFAULT_MAX_WAL_RECORDS 10000 +#define DEFAULT_WAL_SEGMENT_BYTES (64UL * 1024 * 1024) /* 64 MB per segment */ /* ====================================================== * raft-drive-async constants (task 1.1) diff --git a/src/include/myconf.h b/src/include/myconf.h index 96edf9a..7f74e3f 100644 --- a/src/include/myconf.h +++ b/src/include/myconf.h @@ -38,6 +38,7 @@ typedef struct { uint32_t sync_timeout_ms; /* --sync-timeout-ms; 0 = use persist default */ uint32_t wal_sync_ms; /* --wal-sync-ms; 0 = use persist default */ uint32_t max_wal_records; /* --max-wal-records; 0 = use persist default */ + uint32_t wal_segment_bytes; /* --wal-segment-bytes; 0 = use persist default */ } myconf_t; extern cluster_config_t* get_default_conf(void*); diff --git a/src/include/netio.h b/src/include/netio.h index 497fb87..bb72930 100644 --- a/src/include/netio.h +++ b/src/include/netio.h @@ -115,13 +115,23 @@ typedef struct endpoint_t { int heap_alloc; /* 1 iff allocated by endpoint_get (unref may free) */ } endpoint_t; +/* bug-1298 family: msg_handler runs SYNCHRONOUSLY on the io_worker thread (not + * via the vtask dispatcher, so the vtask_enq/-1 ref pair does not cover this + * call). A handler whose own logic drops ep's last reference (e.g. a stale + * vote reply driving raft_cb_become_follower -> endpoint_unref on the very ep + * currently being read) frees ep out from under the trailing pkt=NULL write + * below, a heap-use-after-free. Hold a ref for the duration of the call, same + * pattern as the vtask dispatch path, so the handler's own unref cannot drop + * the count to zero until this macro is done touching ep. */ #define INVOKE_MSG_HANDLER(ep) \ do { \ TRACE("%s msg handler %p", __func__, (ep)->msg_handler); \ if ((ep)->msg_handler != NULL) { \ + endpoint_ref(ep); \ (ep)->msg_handler(ep); \ /* passing ownership of packet to upper layer */ \ (ep)->fsm_in.pkt = NULL; \ + endpoint_unref(ep); \ } \ } while (0) diff --git a/src/include/persist.h b/src/include/persist.h index db7e025..89b5b5f 100644 --- a/src/include/persist.h +++ b/src/include/persist.h @@ -100,6 +100,7 @@ typedef struct { uint32_t worker_retain_s; /* retain unavailable workers N seconds (default 86400) */ uint32_t sync_timeout_ms; /* sync replication ACK timeout ms (default 50) */ uint32_t max_wal_records; /* checkpoint trigger: records since last snap (default 10000) */ + uint32_t wal_segment_bytes; /* physical WAL segment file size (default 64 MB) */ } persist_config_t; static inline persist_config_t persist_config_defaults(void) @@ -114,6 +115,7 @@ static inline persist_config_t persist_config_defaults(void) c.worker_retain_s = DEFAULT_WORKER_RETAIN_S; c.sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS; c.max_wal_records = DEFAULT_MAX_WAL_RECORDS; + c.wal_segment_bytes = DEFAULT_WAL_SEGMENT_BYTES; return c; } @@ -154,6 +156,25 @@ static inline void persist_config_set_max_wal_records(persist_config_t *c, c->max_wal_records = override_n; } +/* + * persist_config_set_wal_segment_bytes — apply the --wal-segment-bytes + * override (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: + * the physical WAL segment size, not the record-count checkpoint threshold, + * gates when trim_wal_segments() has anything to delete — a segment is + * never rotated out until it fills, so forcing a real snapshot-triggered + * empty-rejoin in the soak requires a small deterministic segment size + * rather than waiting on the 64 MB production default). + * + * Same convention as persist_config_set_max_wal_records: override == 0 + * means "flag not passed", leaves the default segment size untouched. + */ +static inline void persist_config_set_wal_segment_bytes(persist_config_t *c, + uint32_t override_bytes) +{ + if (override_bytes > 0) + c->wal_segment_bytes = override_bytes; +} + /* ============================================================ * Key→index mapping for checkpoint serialization (Part 3.8) * ============================================================ */