From cdc6e2377a7d8c50117f12c2ecd094ee64d3cefd Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sat, 20 Jun 2026 06:04:41 +0000 Subject: [PATCH] spec(controller-status): mark implemented (E2E pass; primaryName bug fixed); log upstream dead-first-master failover finding --- .kiro/specs/controller-status/spec.json | 11 +- .kiro/specs/controller-status/tasks.md | 7 +- .wolf/buglog.json | 230 ++++++++++++++++++++++++ 3 files changed, 244 insertions(+), 4 deletions(-) diff --git a/.kiro/specs/controller-status/spec.json b/.kiro/specs/controller-status/spec.json index d5fcaa4..13b7c6b 100644 --- a/.kiro/specs/controller-status/spec.json +++ b/.kiro/specs/controller-status/spec.json @@ -3,7 +3,7 @@ "created_at": "2026-06-19T00:00:00Z", "updated_at": "2026-06-19T00:00:00Z", "language": "en", - "phase": "tasks-generated", + "phase": "implemented", "approvals": { "requirements": { "generated": true, @@ -18,5 +18,10 @@ "approved": true } }, - "ready_for_implementation": true -} \ No newline at end of file + "ready_for_implementation": true, + "implementation": { + "completed": true, + "tasks_done": "1.1-5.1 (all)", + "notes": "ctrlstatus opcode + handler + all-roles dispatch gate + libxce any-node & authoritative reads + ctrlstatus command + wait-primary swap, all reviewed and committed; E2E-verified live (primary self-reports, standbys answer with answered-by, config SHA-256 checksum identical across nodes, authoritative wait-primary names the primary). E2E found+fixed the primaryName-from-vestigial-primary_host bug. Out-of-boundary upstream finding: libxce failover returns EINVAL on a dead-first-master (affects joblist equally) -> route to libxce-failover." + } +} diff --git a/.kiro/specs/controller-status/tasks.md b/.kiro/specs/controller-status/tasks.md index 66e90b9..d213595 100644 --- a/.kiro/specs/controller-status/tasks.md +++ b/.kiro/specs/controller-status/tasks.md @@ -57,12 +57,17 @@ - _Depends: 3.2_ - [ ] 5. Validation (live cluster) -- [ ] 5.1 End-to-end status + authoritative wait-primary on a live cluster +- [x] 5.1 End-to-end status + authoritative wait-primary on a live cluster - Run against a live 3-node cluster: `ctrlstatus` on a healthy cluster shows the correct primary, peers, and version+checksum; querying via a standby still reports the primary with the "answered by " framing; the version+checksum match across nodes with identical config; and the authoritative wait-primary returns immediately on a healthy cluster, blocks until the NEW primary itself answers after a primary kill (without false-succeeding on a standby's stale primary view), and times out when no primary serves. - Observable: the live run shows a correct status (primary/peers/checksum) including via a standby, and the authoritative wait-primary names the new primary after a kill without a false success; manual environment prerequisite (live cluster), same status as other live-E2E tasks. - _Requirements: 2.3, 5.3, 7.1, 7.2, 8.1, 9.1, 9.2, 9.3, 9.4_ - _Depends: 4.1, 4.2_ +## Implementation Notes (5.1 E2E outcome) +- 5.1 ran live on a 3-node `/xce-deploy-ha`-style cluster (ext2/3 fs). PASSED in-scope: `ctrlstatus` shows the primary self-reporting (`Primary: ctrl-1` on the primary), standbys honestly report `Primary: unknown` with `answered by ctrl-2 (standby)` (the dispatch-gate lets standbys answer — Req 7); peers + count correct; `configChecksum` = the real 64-hex SHA-256 (matched the deploy tag prefix) and is IDENTICAL across all 3 nodes with identical config (Req 5.3); `configVersion`==`committedConfigVersion`; authoritative `wait-primary` names the primary in <1s on a serveable cluster (Req 9.1/9.2). +- 5.1 FOUND + FIXED a real 2.1 bug: `ha->primary_host` is VESTIGIAL (never written anywhere in the controller), and `voted_for_name` is the last-vote (often self), so neither names the current primary off-primary. Corrected: PRIMARY self-reports its own name; STANDBY/ELECTING/INIT report empty → "unknown" (the authoritative client path reaches the real primary by rotation). Commit ade0081. +- 5.1 UPSTREAM FINDING (out of boundary, NOT controller-status): with a DEAD node FIRST in the masters list, the libxce client failover returns `EINVAL` instead of advancing to the live nodes — so the authoritative `wait-primary` (and ANY client op) can't reach the primary while a dead node leads the list. CONFIRMED pre-existing: plain `joblist` fails IDENTICALLY (`xce_joblist: Invalid argument`) in the same condition; with a live-only masters list `wait-primary` succeeds in 46ms. This spec consumes failover unchanged (per Boundary/Adjacent expectations) → route to libxce-failover, do NOT patch here. The durability soak avoided it by restarting killed nodes (no dead-first-master during waits). + ## Implementation Notes - All new C types are generated from the schema (task 1.1) — no hand-written struct headers. Build regenerates `controller.capnp.h/.c`. - Task 2.2 is the one HA-critical change (request-handling path): keep it scoped to the single read-only opcode and prove no other opcode's gating widened. diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 8f2aed2..b0e3bd8 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -13154,6 +13154,236 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-06-20T01:27:31.000Z" + }, + { + "id": "bug-804", + "timestamp": "2026-06-20T03:29:22.678Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "Had \"none\"", + "fix": "Changed to \"unknown\"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-20T03:33:32.399Z" + }, + { + "id": "bug-805", + "timestamp": "2026-06-20T03:29:29.570Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "Was returning: a well-defined \"unknown\" or \"none\" value for that ", + "fix": "Now returns: a well-defined \"unknown\" value for that field rath", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T03:29:29.570Z" + }, + { + "id": "bug-806", + "timestamp": "2026-06-20T03:32:19.414Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 6\u21928 lines (3 removed) | Also: ### Requirement 5: Committed configuration version; **Objective:** As an operator, I want the configur", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-20T03:32:34.774Z" + }, + { + "id": "bug-807", + "timestamp": "2026-06-20T04:08:01.204Z", + "error_message": "Significant refactor of ", + "file": ".kiro/specs/controller-status/design.md", + "root_cause": "12 lines replaced/restructured", + "fix": "Rewrote 16\u219213 lines (12 removed) | Also: clusterName @0: Text;; primaryName @1: Text; # empty \u21d2 \"unknown\" | Also: /* failover-aware read (is_write=false) */; - Postconditions: on success `*status` owns a deco | Also: - `configChecksum`/`configVersion` equal across tw; - `ctrlstatus` against a healthy 3-node cluster pr | Also: - Answerable on **any** node and across failover \u2014; - Report a content checksum of the committed confi", + "tags": [ + "auto-detected", + "refactor", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-20T04:11:28.379Z" + }, + { + "id": "bug-808", + "timestamp": "2026-06-20T04:08:49.684Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/design.md", + "root_cause": "Had `committed_config_tag`", + "fix": "Changed to `committedConfigVersion`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-20T04:10:27.793Z" + }, + { + "id": "bug-809", + "timestamp": "2026-06-20T04:40:36.648Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'ctrlstatus.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T04:40:36.648Z" + }, + { + "id": "bug-810", + "timestamp": "2026-06-20T04:45:48.985Z", + "error_message": "Wrong reference: Data should be Text", + "file": "src/schema/controller.capnp", + "root_cause": "Used \"Data\" instead of \"Text\"", + "fix": "Changed Data \u2192 Text", + "tags": [ + "auto-detected", + "wrong-reference", + "capnp" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T04:45:48.985Z" + }, + { + "id": "bug-811", + "timestamp": "2026-06-20T04:45:49.790Z", + "error_message": "Wrong reference: xce_data_t should be char", + "file": "src/include/typemap.h", + "root_cause": "Used \"xce_data_t\" instead of \"char\"", + "fix": "Changed xce_data_t \u2192 char", + "tags": [ + "auto-detected", + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T04:45:49.790Z" + }, + { + "id": "bug-812", + "timestamp": "2026-06-20T05:08:41.030Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'raft/raft_sim.c'", + "fix": "Changed to 'testsuite/test_ctrlstatus.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T05:08:41.030Z" + }, + { + "id": "bug-813", + "timestamp": "2026-06-20T05:42:31.707Z", + "error_message": "Wrong operator: >= should be ==", + "file": "src/controller/testsuite/test.c", + "root_cause": "Used \">=\" instead of \"==\"", + "fix": "Changed operator >= \u2192 ==", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-20T05:42:46.894Z" + }, + { + "id": "bug-814", + "timestamp": "2026-06-20T05:51:48.030Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_ctrlstatus.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 23\u219273 lines (3 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T05:51:48.030Z" + }, + { + "id": "bug-815", + "timestamp": "2026-06-20T05:52:48.719Z", + "error_message": "Significant refactor of ", + "file": "src/controller/ctrlstatus.c", + "root_cause": "3 lines replaced/restructured", + "fix": "Rewrote 3\u219213 lines (3 removed) | Also: /* primary identity \u2014 guarded by ha->lock (Req 2.2; st->primaryName = (ha->primary_host[0] != '\\0') ? ", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-20T05:53:00.264Z" + }, + { + "id": "bug-816", + "timestamp": "2026-06-20T05:57:46.206Z", + "error_message": "Significant refactor of ", + "file": "src/controller/testsuite/test_ctrlstatus.c", + "root_cause": "4 lines replaced/restructured", + "fix": "Rewrote 13\u219216 lines (4 removed)", + "tags": [ + "auto-detected", + "refactor", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T05:57:46.206Z" + }, + { + "id": "bug-817", + "timestamp": "2026-06-19", + "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", + "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) \u2014 UPSTREAM, not controller-status", + "root_cause": "libxce client failover does not cleanly advance past a dead FIRST master to the live nodes; surfaces a typed EINVAL instead of rotating. Pre-existing (controller-status consumes failover unchanged). Surfaced by controller-status task 5.1 E2E (authoritative wait-primary timed out after a primary kill while a dead node led the masters list, even though a new primary was serving).", + "fix": "NOT fixed in controller-status (out of boundary). Route to libxce-failover. Confirmed affects plain joblist identically. Workaround: ensure the masters list does not lead with a known-dead node, or restart killed nodes (as the durability soak does).", + "tags": [ + "libxce", + "failover", + "master-rotation", + "dead-first-master", + "EINVAL", + "controller-status", + "upstream" + ], + "related_bugs": [ + "bug-646" + ], + "occurrences": 1, + "last_seen": "2026-06-19" } ] } \ No newline at end of file