diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 4b3db276..dec06d69 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -19512,6 +19512,34 @@ ], "occurrences": 1, "last_seen": "2026-08-26T18:35:00Z" + }, + { + "id": "bug-1715", + "timestamp": "2026-08-26T18:45:00Z", + "error_message": "User-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901` failed once with `Error: scheduler disconnected`, then succeeded on the very next identical invocation moments later. Root-caused to a full cluster redeploy (user confirmed: 'This is a full rebuilt and re-deployed cluster') restarting all 5 processes (3 controllers + worker + scheduler) together -- PIDs across ctrl-1/ctrl-2/ctrl-3/scheduler/worker logs are all tightly clustered (4137667/4137684/4137696/4137745/4138046), confirming a coordinated restart, not an isolated scheduler crash. The request happened to land in the brief, normal window between the old scheduler disconnecting and the new one reconnecting.", + "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)", + "root_cause": "FULLY ROOT-CAUSED, same family as bug-1713 but NOT covered by that fix. 'scheduler disconnected' is delivered via a DIFFERENT signaling path than bug-1713's two numeric status codes: cluster.c's bug-1693 disconnect-detection mechanism (cluster.c:1052-1067) sends `reply.status = 0` (success at the RPC level) with `schedadminreplymsg{ok=false, error=\"scheduler disconnected\"}` -- a normal-status reply carrying a domain-level failure, not a nonzero `reply.status` the way E_XCE_NOT_READY/E_XCE_NOMASTER are. `_send_admin_request()` (scheduler.py:145-154) treats `admin_reply.ok == False` as an unconditional hard SchedAdminError for EVERY reason string, with no distinction between genuinely transient conditions ('scheduler disconnected', 'scheduler not connected', 'request timeout' -- all connection/timing related, expected to self-resolve within seconds, especially during a redeploy/restart cycle) and genuinely permanent domain errors ('not recording', 'already recording', 'invalid record file name' -- real error states that must NOT be retried, e.g. calling record-stop on an already-stopped recorder is a correct, meaningful rejection, not a transient hiccup).", + "fix": "PENDING -- deliberately NOT dispatched yet: bug-1713's fix agent is concurrently editing this exact same file (scheduler.py) for the sibling numeric-status-code retry gap. Dispatch this fix ONLY after bug-1713 lands and its diff is reviewed, to avoid two agents colliding on the same function. Scope: extend (or share the same retry-loop infrastructure bug-1713 adds) to also retry when `admin_reply.ok == False` and `admin_reply.error` matches one of the known-transient reason strings ('scheduler disconnected', 'scheduler not connected', 'request timeout') -- exact string match against the literal strings cluster.c/sched.c actually send (grep cluster.c/sched.c/handler.c for every `sched_admin_reply_t.error =`/`SchedAdminReply.error` assignment to get the authoritative list before hardcoding it, don't guess). Do NOT retry on any other ok=false reason (those are real domain errors). Needs a unit test mirroring bug-1713's mocked-server pattern: a mock returning ok=false/error='scheduler disconnected' N times then succeeding results in eventual success; ok=false/error='not recording' is NEVER retried (fails immediately, unchanged behavior). Live-verify against a real redeploy cycle (kill+restart the whole cluster, immediately hammer `record status` through the restart window) showing zero user-visible 'scheduler disconnected' failures.", + "tags": [ + "xceadmin", + "scheduler-admin", + "scheduler-disconnected", + "retry", + "transient-state", + "bug-1713-adjacent", + "bug-1693-adjacent", + "found-live", + "root-caused", + "fix-scoped", + "blocked-on-bug-1713", + "unfixed" + ], + "related_bugs": [ + "bug-1713", + "bug-1693" + ], + "occurrences": 1, + "last_seen": "2026-08-26T18:45:00Z" } ] } \ No newline at end of file