From c205b5b25d08e49c7ba93cd7b618e36d31685fd7 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sat, 20 Jun 2026 01:28:26 +0000 Subject: [PATCH] =?UTF-8?q?spec(ha-commit-durability):=20sharpen=20bug-795?= =?UTF-8?q?=20=E2=80=94=20durability=20loss=20reproduces=20in=20rolling=20?= =?UTF-8?q?failover=20(trustworthy=20soak)=20+=20>60s=20election=20livenes?= =?UTF-8?q?s=20finding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .kiro/specs/ha-commit-durability/research.md | 9 ++ .wolf/buglog.json | 137 ++++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/.kiro/specs/ha-commit-durability/research.md b/.kiro/specs/ha-commit-durability/research.md index f335cf9..cd8c96d 100644 --- a/.kiro/specs/ha-commit-durability/research.md +++ b/.kiro/specs/ha-commit-durability/research.md @@ -195,3 +195,12 @@ No CRITICAL/HIGH residuals. No criterion is **Missing** or **Unknown**: 16/16 ve - The durable-gating (2.2/2.4) should make `committed_lsn` HOLD if `fdatasync` failed — so an un-fdatasync'd job should NOT have returned `jobsubmit ok`. Yet these jobs DID get `jobsubmit ok`. So either `fdatasync` returned success on overlay without real durability, or a non-durable-gated ack path exists under the total-outage timing, or recovery truncates committed entries. **Disposition:** the `xce-test-e2e-primitives` soak is the reproduction surface and is working correctly (it detects the loss via exit codes). The fix (if real) belongs HERE. See Phase 6. + +### UPDATE 2026-06-19b — trustworthy soak sharpens the finding (rolling-failover loss + liveness) + +After the `xce-test-e2e-primitives` soak was HARDENED (commit d82cc78: it no longer conflates liveness with durability — durability is asserted ONLY against a confirmed-serving cluster, with a catch-up-lag confirm re-check; a non-serving cluster is INCONCLUSIVE, never "DATA LOSS"), three 6–10-round live runs give a cleaner signal: + +1. **The committed-job loss reproduces in ROLLING single-node failover too — not only cold-restart-all.** One run lost a committed job at round 6 with the cluster CONFIRMED SERVING (`DATA LOSS — persistently absent from a SERVING cluster`); two other runs had zero loss. So the durability race is NON-DETERMINISTIC (~1 in 3 runs here) and broader than the cold-restart-all framing in the original reopen note — it is a general acked-but-not-durable-on-majority race that a kill can expose during normal rolling failover. **This is exactly the bug-661 invariant this spec claimed CLOSED for rolling failover (task 5.2) — it is not fully closed.** (Caveat still applies: sandbox `/tmp` is overlayfs; 6.1 must confirm on a real fsync-able fs.) +2. **Separate LIVENESS concern:** every run shows 2–3 rounds where, after killing the current primary (consistently `ctrl-1`), NO new primary served within 60s (the cluster always eventually recovered). A >60s failover for a single-node kill in a 3-node cluster is poor availability and may share a root cause with the durability race (a behind/just-rejoined node interacting with the promotion fence / freshness floor). Worth a dedicated liveness check in 6.2. + +**Phase 6 scope note:** 6.1's reproduction MUST include BOTH the cold-restart-all step AND the rolling-failover rounds (the hardened soak runs both and reports trustworthy per-round verdicts), and should capture the >60s-election cases for the liveness angle. diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 0e642b4..05dbd1b 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -13004,7 +13004,7 @@ "timestamp": "2026-06-19", "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", "file": "src/controller/persist.c (recovery/truncation) + commit/replication path \u2014 UPSTREAM (ha-commit-durability), NOT xce-test-e2e-primitives", - "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware.", + "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) \u2014 broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers.", "fix": "NOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", "tags": [ "ha-commit-durability", @@ -13014,15 +13014,146 @@ "wal-truncate", "upstream", "xce-test-e2e-primitives", - "needs-real-fs" + "needs-real-fs", + "rolling-failover", + "liveness", + "non-deterministic" ], "related_bugs": [ "bug-661", "bug-778", "bug-786" ], - "occurrences": 1, + "occurrences": 2, "last_seen": "2026-06-19" + }, + { + "id": "bug-796", + "timestamp": "2026-06-20T00:32:43.144Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/spec.json", + "root_cause": "Had \"2026-06-18T04:00:00Z\"", + "fix": "Changed to \"2026-06-19T00:00:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T00:32:43.144Z" + }, + { + "id": "bug-797", + "timestamp": "2026-06-20T00:42:36.967Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/xce-test-e2e-primitives/spec.json", + "root_cause": "2 lines replaced/restructured", + "fix": "Rewrote 3\u21928 lines (2 removed)", + "tags": [ + "auto-detected", + "refactor", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T00:42:36.967Z" + }, + { + "id": "bug-798", + "timestamp": "2026-06-20T01:05:04.547Z", + "error_message": "Missing error handling in unknown", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:05:04.547Z" + }, + { + "id": "bug-799", + "timestamp": "2026-06-20T01:05:25.091Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"$label: DATA LOSS \u2014 a committed job vanished (bug", + "fix": "Changed to \"$label: DATA LOSS \u2014 a committed job vanished from", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:05:25.091Z" + }, + { + "id": "bug-800", + "timestamp": "2026-06-20T01:05:51.010Z", + "error_message": "Significant refactor of ", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 8\u219212 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:05:51.010Z" + }, + { + "id": "bug-801", + "timestamp": "2026-06-20T01:14:42.334Z", + "error_message": "Significant refactor of ", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 8\u219210 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:14:42.334Z" + }, + { + "id": "bug-802", + "timestamp": "2026-06-20T01:21:21.789Z", + "error_message": "Missing error handling in unknown", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:21:21.789Z" + }, + { + "id": "bug-803", + "timestamp": "2026-06-20T01:27:31.000Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:27:31.000Z" } ] } \ No newline at end of file