wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

spec(ha-commit-durability): sharpen bug-795 — durability loss reproduces in rolling failover (trustworthy soak) + >60s election liveness finding

Browse source
This commit is contained in:
Rongsong Shen 2026-06-20 01:28:26 +00:00
parent d82cc78284
commit c205b5b25d
2 changed files with 143 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

137
.wolf/buglog.json
View file

@ -13004,7 +13004,7 @@
"timestamp": "2026-06-19",
"error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job <id> not found). Recovery logs show WAL truncate.",
"file": "src/controller/persist.c (recovery/truncation) + commit/replication path \u2014 UPSTREAM (ha-commit-durability), NOT xce-test-e2e-primitives",
"root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware.",
"root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) \u2014 broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers.",
"fix": "NOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.",
"tags": [
"ha-commit-durability",
@ -13014,15 +13014,146 @@
"wal-truncate",
"upstream",
"xce-test-e2e-primitives",
"needs-real-fs"
"needs-real-fs",
"rolling-failover",
"liveness",
"non-deterministic"
],
"related_bugs": [
"bug-661",
"bug-778",
"bug-786"
],
"occurrences": 1,
"occurrences": 2,
"last_seen": "2026-06-19"
},
{
"id": "bug-796",
"timestamp": "2026-06-20T00:32:43.144Z",
"error_message": "Incorrect value in code",
"file": ".kiro/specs/ha-commit-durability/spec.json",
"root_cause": "Had \"2026-06-18T04:00:00Z\"",
"fix": "Changed to \"2026-06-19T00:00:00Z\"",
"tags": [
"auto-detected",
"wrong-value",
"json"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T00:32:43.144Z"
},
{
"id": "bug-797",
"timestamp": "2026-06-20T00:42:36.967Z",
"error_message": "Significant refactor of ",
"file": ".kiro/specs/xce-test-e2e-primitives/spec.json",
"root_cause": "2 lines replaced/restructured",
"fix": "Rewrote 3\u21928 lines (2 removed)",
"tags": [
"auto-detected",
"refactor",
"json"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T00:42:36.967Z"
},
{
"id": "bug-798",
"timestamp": "2026-06-20T01:05:04.547Z",
"error_message": "Missing error handling in unknown",
"file": "src/tools/test-commit-durability.sh",
"root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught",
"fix": "Added try/catch block",
"tags": [
"auto-detected",
"error-handling",
"sh"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T01:05:04.547Z"
},
{
"id": "bug-799",
"timestamp": "2026-06-20T01:05:25.091Z",
"error_message": "Incorrect value in code",
"file": "src/tools/test-commit-durability.sh",
"root_cause": "Had \"$label: DATA LOSS \u2014 a committed job vanished (bug",
"fix": "Changed to \"$label: DATA LOSS \u2014 a committed job vanished from",
"tags": [
"auto-detected",
"wrong-value",
"sh"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T01:05:25.091Z"
},
{
"id": "bug-800",
"timestamp": "2026-06-20T01:05:51.010Z",
"error_message": "Significant refactor of ",
"file": "src/tools/test-commit-durability.sh",
"root_cause": "4 lines replaced/restructured",
"fix": "Rewrote 8\u219212 lines (4 removed)",
"tags": [
"auto-detected",
"refactor",
"sh"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T01:05:51.010Z"
},
{
"id": "bug-801",
"timestamp": "2026-06-20T01:14:42.334Z",
"error_message": "Significant refactor of ",
"file": "src/tools/test-commit-durability.sh",
"root_cause": "4 lines replaced/restructured",
"fix": "Rewrote 8\u219210 lines (4 removed)",
"tags": [
"auto-detected",
"refactor",
"sh"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T01:14:42.334Z"
},
{
"id": "bug-802",
"timestamp": "2026-06-20T01:21:21.789Z",
"error_message": "Missing error handling in unknown",
"file": "src/tools/test-commit-durability.sh",
"root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught",
"fix": "Added try/catch block",
"tags": [
"auto-detected",
"error-handling",
"sh"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T01:21:21.789Z"
},
{
"id": "bug-803",
"timestamp": "2026-06-20T01:27:31.000Z",
"error_message": "Missing error handling in unknown",
"file": ".kiro/specs/xce-test-e2e-primitives/tasks.md",
"root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught",
"fix": "Added try/catch block",
"tags": [
"auto-detected",
"error-handling",
"md"
],
"related_bugs": [],
"occurrences": 1,
"last_seen": "2026-06-20T01:27:31.000Z"
}
]
}