diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 702d5898..ad0f2f23 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -17149,6 +17149,33 @@ ], "fix_commits": [], "record": ".wolf/bugs/bug-1913.md" + }, + { + "id": "bug-1914", + "status": "open", + "status_updated": "2026-10-03", + "status_note": "newly logged", + "component": "controller", + "severity": "wrong-result", + "timestamp": "2026-10-03T15:07:26Z", + "last_seen": "2026-10-03T15:07:26Z", + "occurrences": 1, + "error_message": "config push fails -110 under heavy CPU load: commit lands just after the 500 ms barrier\n\n**Observed:**\n```\nError: controller returned status -110\n```\n`xceadmin config push` to a healthy, normally elected HA primary (standbys promoted from the vote connections, own-term no-op already committed) was rejected with -ETIMEDOUT: the config marker's majority commit landed ~75 ms after the `repl_wait_majority_ack(..., CONF_PUSH_TIMEOUT_MS)` barrier (`config_repl.c` ~1658; `CONF_PUSH_TIMEOUT_MS` = 500, `config_repl.h:33`) expired. Seen only with the CPU heavily oversubscribed (40 busy loops on 32 cores).\n\n**Expected:** a push to a healthy primary with a reachable majority succeeds under load, or the barrier is sized/structured so normal scheduling delay cannot turn a soon-to-commit marker into a rejection. (A commit that lands after the client was told -110 also leaves the outcome ambiguous to the client: rejected per the reply, yet committed later.)\n**Impact:** spurious push failures under heavy load (fail-loud, no data loss). If the marker then commits after the -110 reply, the client believes the push failed while the cluster activated it -- a retry is idempotent per tag, but the reply is misleading. Check before changing: whether the late commit is actually applied after the -110 reply, and the async-I/O rules for the commit-wait barrier (`COMMIT_WAIT_RESERVED`, `.kiro/steering/ha-async-io.md`).\n**Found by:** test (bug-1485 investigation, 2026-10-03)", + "file": "src/controller/config_repl.c, src/controller/config_repl.h", + "root_cause": "TODO: required for fixed/partial", + "fix": "TODO: required for fixed/partial\n\nVerification:\n- Test: TODO: name + file; RED before the fix, GREEN after\n- Suites: TODO: suite results and build profile\n- Live: TODO: soak/deploy run, or n/a and why", + "tags": [ + "ha", + "config-push", + "timeout", + "load" + ], + "related_bugs": [ + "bug-1485", + "bug-1913" + ], + "fix_commits": [], + "record": ".wolf/bugs/bug-1914.md" } ] } diff --git a/.wolf/bugs/bug-1914.md b/.wolf/bugs/bug-1914.md new file mode 100644 index 00000000..6e81d679 --- /dev/null +++ b/.wolf/bugs/bug-1914.md @@ -0,0 +1,56 @@ +--- +id: "bug-1914" +status: "open" +status_updated: "2026-10-03" +status_note: "newly logged" +component: "controller" +severity: "wrong-result" +timestamp: "2026-10-03T15:07:26Z" +last_seen: "2026-10-03T15:07:26Z" +occurrences: 1 +files: ["src/controller/config_repl.c", "src/controller/config_repl.h"] +tags: ["ha", "config-push", "timeout", "load"] +related: ["bug-1485", "bug-1913"] +--- + +# bug-1914: config push fails -110 under heavy CPU load: commit lands just after the 500 ms barrier + +## Description +**Observed:** +``` +Error: controller returned status -110 +``` +`xceadmin config push` to a healthy, normally elected HA primary (standbys promoted from the vote connections, own-term no-op already committed) was rejected with -ETIMEDOUT: the config marker's majority commit landed ~75 ms after the `repl_wait_majority_ack(..., CONF_PUSH_TIMEOUT_MS)` barrier (`config_repl.c` ~1658; `CONF_PUSH_TIMEOUT_MS` = 500, `config_repl.h:33`) expired. Seen only with the CPU heavily oversubscribed (40 busy loops on 32 cores). + +**Expected:** a push to a healthy primary with a reachable majority succeeds under load, or the barrier is sized/structured so normal scheduling delay cannot turn a soon-to-commit marker into a rejection. (A commit that lands after the client was told -110 also leaves the outcome ambiguous to the client: rejected per the reply, yet committed later.) +**Impact:** spurious push failures under heavy load (fail-loud, no data loss). If the marker then commits after the -110 reply, the client believes the push failed while the cluster activated it -- a retry is idempotent per tag, but the reply is misleading. Check before changing: whether the late commit is actually applied after the -110 reply, and the async-I/O rules for the commit-wait barrier (`COMMIT_WAIT_RESERVED`, `.kiro/steering/ha-async-io.md`). +**Found by:** test (bug-1485 investigation, 2026-10-03) + +## Environment +- OS: no /etc/os-release; Docker container; NixOS kernel; kernel Linux 6.18.33 x86_64 +- libc: glibc 2.42-47 (interpreter /nix/store/wb6rhpznjfczwlwx23zmdrrw74bayxw4-glibc-2.42-47/lib/ld-linux-x86-64.so.2) +- Toolchain: gcc (GCC) 15.2.0, meson 1.9.1 +- Build: asan (xce.build) +- Code: scheduler-framework @ 4cd5fb7d-dirty +- Deployment: local 3-node HA (xceadmin `live_ha_cluster` fixture), CPU oversubscribed with 40 busy loops on 32 cores + +## Reproduction +1. Start 40 CPU busy loops on a 32-core host. +2. Loop `src/xceadmin/tests/test_e2e_live_ha_push.py` (with the bug-1485 readiness wait in place). +3. Rarely, the push fails with `controller returned status -110` on a primary whose own-term no-op had already committed; the primary log shows the marker commit ~75 ms after the 500 ms barrier expired. +- Frequency: 1/70 HA push loops under heavy load, 0 idle (2026-10-03) +- Evidence: not kept (scratchpad loop logs from the bug-1485 investigation) + +## Root cause +TODO: required for fixed/partial + +## Fix +TODO: required for fixed/partial + +## Verification +- Test: TODO: name + file; RED before the fix, GREEN after +- Suites: TODO: suite results and build profile +- Live: TODO: soak/deploy run, or n/a and why + +## References +- Related: none