feat(xceadmin): accept a comma-separated --controller failover list (bug-1821, bug-1822)

xceadmin's --controller option (config push, and scheduler record
start/stop/status) accepted only a single host:port. Passing every
configured controller as a comma-separated list -- the natural way to
query an HA cluster without knowing which node is alive/primary --
mis-parsed the whole joined string as one bogus hostname and failed
with a DNS-resolution error.

Adds xceadmin.discovery.resolve_controllers() (comma-split candidate
parsing) and follow_redirects_any() (tries each candidate in order,
skipping ones that are unreachable at the network level -- connection
refused, DNS failure, or a discovery timeout -- mirroring xcecli/
xce_test's own client-side master-rotation failover). A candidate that
IS reachable but returns a genuine protocol-level failure is not
retried against a different node. push.py and scheduler.py's own
duplicated single-host _split_controller helpers are removed in favor
of this shared implementation.

Also fixes a related gap bug-1821's own live verification surfaced:
right after a primary is killed, a standby doesn't start a new
election until its own heartbeat timeout elapses (~10s), so it keeps
redirecting every candidate to the now-dead old primary -- exhausting
the whole candidate list and raising NoPrimaryError, which
_send_admin_request()'s existing transient-retry loop (bug-1713/1715)
never caught. That loop now retries on NoPrimaryError too, using its
own (longer) budget sized for the heartbeat-timeout-plus-election
window rather than the RPC-level "already electing" budget the
original retry logic was tuned for.

TDD RED->GREEN, 304/304 tests passing. Live-verified end to end: a
comma-list correctly skips a dead first candidate and reaches a live
one on a healthy cluster, and correctly rides out a real primary kill
and re-election (retry loop visibly cycling ~15s) to reach the newly-
elected primary rather than failing immediately.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC

src/xceadmin/tests/test_scheduler.py

@@ -36,7 +36,7 @@ import pytest
 from xcecommon import wire
 from xcecommon.errors import NoPrimaryError, ReplayFileError, ReplyDecodeError, SchedAdminError
 from xcecommon.schema import load_schema
-from xceadmin import scheduler
+from xceadmin import discovery, scheduler
 
 
 def _server_context(cert_file: Path, key_file: Path) -> ssl.SSLContext:
@@ -592,6 +592,76 @@ def test_record_status_retry_exhaustion_raises_distinguishable_error(
     assert f"{scheduler._MAX_TRANSIENT_ATTEMPTS} attempts" in exc_info.value.error
 
 
+def test_record_status_retries_on_no_primary_error_then_succeeds(monkeypatch, wrap_reply):
+    """Right after a primary is killed, every configured candidate can
+    still redirect to the now-dead old primary (a standby hasn't
+    necessarily noticed within the heartbeat timeout yet), so
+    follow_redirects_any exhausts the whole candidate list and raises
+    NoPrimaryError even though the cluster is only mid-election, not
+    genuinely unreachable. This must be retried using NoPrimaryError's own
+    (longer) budget/delay, _MAX_NO_PRIMARY_ATTEMPTS/
+    _NO_PRIMARY_RETRY_DELAY_S -- not treated as an immediate hard failure,
+    and not sharing _TRANSIENT_STATUSES' smaller budget. FAILS against the
+    pre-fix code (which let NoPrimaryError propagate out of the retry loop
+    on the very first attempt)."""
+    mod = load_schema()
+    sleep_calls: list[float] = []
+    monkeypatch.setattr(scheduler.time, "sleep", lambda s: sleep_calls.append(s))
+
+    reply = mod.RequestReply.new_message()
+    reply.status = 0
+    admin_reply = reply.data.init("schedadminreplymsg")
+    admin_reply.ok = True
+    admin_reply.error = ""
+    admin_reply.recording = True
+    admin_reply.file = "settled.replay"
+    admin_reply.eventCount = 0
+    admin_reply.startedAt = 6000
+    success_reply_bytes = wrap_reply(reply)
+
+    calls = {"n": 0}
+
+    def fake_follow_redirects_any(candidates, send_fn):
+        calls["n"] += 1
+        if calls["n"] < 3:
+            raise NoPrimaryError("no primary available: none of the configured controllers responded", "host1", 9991)
+        return "host2", 9992, success_reply_bytes
+
+    monkeypatch.setattr(scheduler, "follow_redirects_any", fake_follow_redirects_any)
+
+    result = scheduler.record_status("host1:9991,host2:9992,host3:9993")
+
+    assert result.file == "settled.replay"
+    assert calls["n"] == 3
+    assert sleep_calls == [scheduler._NO_PRIMARY_RETRY_DELAY_S] * 2
+
+
+def test_record_status_no_primary_error_retry_exhaustion_propagates(monkeypatch):
+    """If NoPrimaryError persists across the whole retry budget (the
+    cluster genuinely never settles, or every candidate really is down),
+    it must still propagate as NoPrimaryError -- not be silently
+    swallowed or converted into a less-informative error -- after exactly
+    _MAX_NO_PRIMARY_ATTEMPTS attempts (its own, longer budget -- NOT
+    _MAX_TRANSIENT_ATTEMPTS), matching the reply-status exhaustion test's
+    own proof shape."""
+    sleep_calls: list[float] = []
+    monkeypatch.setattr(scheduler.time, "sleep", lambda s: sleep_calls.append(s))
+
+    calls = {"n": 0}
+
+    def fake_follow_redirects_any(candidates, send_fn):
+        calls["n"] += 1
+        raise NoPrimaryError("no primary available: none of the configured controllers responded", "host1", 9991)
+
+    monkeypatch.setattr(scheduler, "follow_redirects_any", fake_follow_redirects_any)
+
+    with pytest.raises(NoPrimaryError):
+        scheduler.record_status("host1:9991,host2:9992,host3:9993")
+
+    assert calls["n"] == scheduler._MAX_NO_PRIMARY_ATTEMPTS
+    assert len(sleep_calls) == scheduler._MAX_NO_PRIMARY_ATTEMPTS - 1
+
+
 def test_record_status_standby_without_redirect_not_retried(mock_controller, wrap_reply, monkeypatch):
     """A standby rejection with NO populated redirect (the genuine "op is
     primary-only, no known primary to redirect to" case) is a hard,
@@ -822,18 +892,21 @@ def test_record_stop_not_recording_never_retried(sequential_mock_controller, wra
 
 
 # ---------------------------------------------------------------------------
-# _split_controller(): same default-port convention as push.py
+# scheduler.py delegates --controller parsing to xceadmin.discovery.
+# resolve_controllers (shared with push.py) -- see test_discovery.py for
+# that function's own dedicated coverage; these two just confirm
+# _send_admin_request's pipeline still reaches it for the single-host case.
 # ---------------------------------------------------------------------------
 
 
 def test_record_default_port_used_when_omitted():
-    host, port = scheduler._split_controller("somehost")
+    (host, port), = discovery.resolve_controllers("somehost")
     assert host == "somehost"
     assert port == 9991
 
 
 def test_record_explicit_port_parsed():
-    host, port = scheduler._split_controller("somehost:1234")
+    (host, port), = discovery.resolve_controllers("somehost:1234")
     assert host == "somehost"
     assert port == 1234