feat(xceadmin): accept a comma-separated --controller failover list (bug-1821, bug-1822)
xceadmin's --controller option (config push, and scheduler record start/stop/status) accepted only a single host:port. Passing every configured controller as a comma-separated list -- the natural way to query an HA cluster without knowing which node is alive/primary -- mis-parsed the whole joined string as one bogus hostname and failed with a DNS-resolution error. Adds xceadmin.discovery.resolve_controllers() (comma-split candidate parsing) and follow_redirects_any() (tries each candidate in order, skipping ones that are unreachable at the network level -- connection refused, DNS failure, or a discovery timeout -- mirroring xcecli/ xce_test's own client-side master-rotation failover). A candidate that IS reachable but returns a genuine protocol-level failure is not retried against a different node. push.py and scheduler.py's own duplicated single-host _split_controller helpers are removed in favor of this shared implementation. Also fixes a related gap bug-1821's own live verification surfaced: right after a primary is killed, a standby doesn't start a new election until its own heartbeat timeout elapses (~10s), so it keeps redirecting every candidate to the now-dead old primary -- exhausting the whole candidate list and raising NoPrimaryError, which _send_admin_request()'s existing transient-retry loop (bug-1713/1715) never caught. That loop now retries on NoPrimaryError too, using its own (longer) budget sized for the heartbeat-timeout-plus-election window rather than the RPC-level "already electing" budget the original retry logic was tuned for. TDD RED->GREEN, 304/304 tests passing. Live-verified end to end: a comma-list correctly skips a dead first candidate and reaches a live one on a healthy cluster, and correctly rides out a real primary kill and re-election (retry loop visibly cycling ~15s) to reach the newly- elected primary rather than failing immediately. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
parent
baa07d78db
commit
723fdb637b
8 changed files with 521 additions and 82 deletions
|
|
@ -24,7 +24,7 @@ import pytest
|
|||
from xcecommon import wire
|
||||
from xcecommon.errors import ConnectionTimeoutError, NoPrimaryError, ReplyDecodeError, TooManyRedirectsError
|
||||
from xcecommon.schema import load_schema
|
||||
from xceadmin import push
|
||||
from xceadmin import discovery, push
|
||||
|
||||
|
||||
def _server_context(cert_file: Path, key_file: Path) -> ssl.SSLContext:
|
||||
|
|
@ -228,13 +228,17 @@ def test_push_config_rejects_bare_requestreply_bytes_old_wire_shape_regression(
|
|||
|
||||
|
||||
def test_push_config_default_port_used_when_omitted():
|
||||
host, port = push._split_controller("somehost")
|
||||
# push.py delegates --controller parsing to xceadmin.discovery.
|
||||
# resolve_controllers (shared with scheduler.py) -- see test_discovery.py
|
||||
# for that function's own dedicated coverage; this test just confirms
|
||||
# push_config's pipeline still reaches it for the single-host case.
|
||||
(host, port), = discovery.resolve_controllers("somehost")
|
||||
assert host == "somehost"
|
||||
assert port == 9991
|
||||
|
||||
|
||||
def test_push_config_explicit_port_parsed():
|
||||
host, port = push._split_controller("somehost:1234")
|
||||
(host, port), = discovery.resolve_controllers("somehost:1234")
|
||||
assert host == "somehost"
|
||||
assert port == 1234
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ Requirements 3.1, 3.2, 3.3, 3.4, 7.3).
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
|
||||
from xcecommon.errors import ConnectionTimeoutError, NoPrimaryError, ReplyDecodeError, TooManyRedirectsError
|
||||
|
|
@ -218,3 +220,191 @@ def test_follow_redirects_propagates_reply_decode_error_on_corrupt_reply():
|
|||
|
||||
with pytest.raises(ReplyDecodeError):
|
||||
discovery.follow_redirects("node1", 9991, send_fn)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_controllers(): parses a --controller value (single "host:port",
|
||||
# or a comma-separated list of them) into an ordered candidate list.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resolve_controllers_single_bare_host_uses_default_port():
|
||||
assert discovery.resolve_controllers("somehost") == [("somehost", 9991)]
|
||||
|
||||
|
||||
def test_resolve_controllers_single_host_with_explicit_port():
|
||||
assert discovery.resolve_controllers("somehost:1234") == [("somehost", 1234)]
|
||||
|
||||
|
||||
def test_resolve_controllers_comma_separated_list_parsed_in_order():
|
||||
assert discovery.resolve_controllers("host1:9991,host2:9992,host3:9993") == [
|
||||
("host1", 9991),
|
||||
("host2", 9992),
|
||||
("host3", 9993),
|
||||
]
|
||||
|
||||
|
||||
def test_resolve_controllers_list_mixes_bare_hosts_and_explicit_ports():
|
||||
assert discovery.resolve_controllers("host1,host2:1234") == [
|
||||
("host1", 9991),
|
||||
("host2", 1234),
|
||||
]
|
||||
|
||||
|
||||
def test_resolve_controllers_strips_whitespace_and_drops_empty_entries():
|
||||
# A stray trailing comma or accidental spaces around each entry (e.g.
|
||||
# copy-pasted from a config file) must not produce a bogus "" candidate
|
||||
# or an unparsed leading/trailing space in a hostname.
|
||||
assert discovery.resolve_controllers(" host1:9991 , host2:9992 ,") == [
|
||||
("host1", 9991),
|
||||
("host2", 9992),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# follow_redirects_any(): tries each (host, port) candidate in order,
|
||||
# failing over past ones that are unreachable at the network level.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_follow_redirects_any_returns_first_candidate_when_reachable():
|
||||
reply = _reply_bytes()
|
||||
calls = []
|
||||
|
||||
def send_fn(host, port):
|
||||
calls.append((host, port))
|
||||
return reply
|
||||
|
||||
final_host, final_port, reply_bytes = discovery.follow_redirects_any(
|
||||
[("host1", 9991), ("host2", 9992)], send_fn
|
||||
)
|
||||
|
||||
assert (final_host, final_port) == ("host1", 9991)
|
||||
assert reply_bytes == reply
|
||||
# The second candidate must never even be attempted when the first
|
||||
# answers -- failover is a last resort, not a fan-out.
|
||||
assert calls == [("host1", 9991)]
|
||||
|
||||
|
||||
def test_follow_redirects_any_skips_connection_refused_and_tries_next(capsys):
|
||||
"""A dead controller raises a plain OSError (ConnectionRefusedError is
|
||||
a subclass) from wire.connect, propagated unwrapped through
|
||||
follow_redirects by design (xcecommon.wire's own docstring) --
|
||||
follow_redirects_any must catch it at the per-candidate level and move
|
||||
on, not let it escape as a raw OSError.
|
||||
"""
|
||||
reply = _reply_bytes()
|
||||
|
||||
def send_fn(host, port):
|
||||
if host == "dead-host":
|
||||
raise ConnectionRefusedError(111, "Connection refused")
|
||||
return reply
|
||||
|
||||
final_host, final_port, reply_bytes = discovery.follow_redirects_any(
|
||||
[("dead-host", 9901), ("live-host", 9902)], send_fn
|
||||
)
|
||||
|
||||
assert (final_host, final_port) == ("live-host", 9902)
|
||||
assert reply_bytes == reply
|
||||
assert "dead-host:9901" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_follow_redirects_any_skips_dns_failure_and_tries_next():
|
||||
"""socket.gaierror ("Name or service not known") is also a plain
|
||||
OSError subclass -- the exact failure mode from the original bug
|
||||
report (a comma-separated --controller value mis-parsed as one bogus
|
||||
hostname). Confirms a per-candidate DNS failure is skipped too, not
|
||||
just connection-refused.
|
||||
"""
|
||||
reply = _reply_bytes()
|
||||
|
||||
def send_fn(host, port):
|
||||
if host == "bogus.invalid":
|
||||
raise socket.gaierror(-2, "Name or service not known")
|
||||
return reply
|
||||
|
||||
final_host, final_port, reply_bytes = discovery.follow_redirects_any(
|
||||
[("bogus.invalid", 9901), ("real-host", 9902)], send_fn
|
||||
)
|
||||
|
||||
assert (final_host, final_port) == ("real-host", 9902)
|
||||
assert reply_bytes == reply
|
||||
|
||||
|
||||
def test_follow_redirects_any_skips_no_primary_error_and_tries_next():
|
||||
"""A discovery-level timeout on one candidate (NoPrimaryError, already
|
||||
raised by follow_redirects itself) is also skipped in favor of the
|
||||
next candidate, not just a raw OSError.
|
||||
"""
|
||||
reply = _reply_bytes()
|
||||
|
||||
def send_fn(host, port):
|
||||
if host == "slow-host":
|
||||
raise ConnectionTimeoutError("timeout connecting to slow-host:9901", host, port)
|
||||
return reply
|
||||
|
||||
final_host, final_port, reply_bytes = discovery.follow_redirects_any(
|
||||
[("slow-host", 9901), ("fast-host", 9902)], send_fn
|
||||
)
|
||||
|
||||
assert (final_host, final_port) == ("fast-host", 9902)
|
||||
assert reply_bytes == reply
|
||||
|
||||
|
||||
def test_follow_redirects_any_raises_no_primary_error_when_all_candidates_fail():
|
||||
def send_fn(host, port):
|
||||
raise ConnectionRefusedError(111, "Connection refused")
|
||||
|
||||
with pytest.raises(NoPrimaryError) as exc_info:
|
||||
discovery.follow_redirects_any(
|
||||
[("host1", 9991), ("host2", 9992), ("host3", 9993)], send_fn
|
||||
)
|
||||
|
||||
message = str(exc_info.value)
|
||||
assert "host1:9991" in message
|
||||
assert "host2:9992" in message
|
||||
assert "host3:9993" in message
|
||||
|
||||
|
||||
def test_follow_redirects_any_follows_redirect_from_whichever_candidate_answers():
|
||||
"""A candidate that answers with a redirect still gets the normal
|
||||
follow_redirects() hop-following applied -- failover across candidates
|
||||
and redirect-following within one candidate compose cleanly.
|
||||
"""
|
||||
reply_from_standby = _reply_bytes(redirect_host="primary", redirect_port=9991)
|
||||
reply_from_primary = _reply_bytes()
|
||||
|
||||
def send_fn(host, port):
|
||||
if host == "dead-host":
|
||||
raise ConnectionRefusedError(111, "Connection refused")
|
||||
if (host, port) == ("standby", 9992):
|
||||
return reply_from_standby
|
||||
if (host, port) == ("primary", 9991):
|
||||
return reply_from_primary
|
||||
raise AssertionError(f"unexpected send_fn call: {host}:{port}")
|
||||
|
||||
final_host, final_port, reply_bytes = discovery.follow_redirects_any(
|
||||
[("dead-host", 9901), ("standby", 9992)], send_fn
|
||||
)
|
||||
|
||||
assert (final_host, final_port) == ("primary", 9991)
|
||||
assert reply_bytes == reply_from_primary
|
||||
|
||||
|
||||
def test_follow_redirects_any_does_not_retry_a_reachable_candidates_decode_error():
|
||||
"""A candidate that IS reachable but sends back garbage is a real
|
||||
protocol failure, not a connectivity problem -- must propagate
|
||||
immediately, never fall through to the next candidate (a different
|
||||
node wouldn't fix a decode error from THIS one).
|
||||
"""
|
||||
calls = []
|
||||
|
||||
def send_fn(host, port):
|
||||
calls.append((host, port))
|
||||
return b"\x00" * 4 # too short to be a valid capnp frame
|
||||
|
||||
with pytest.raises(ReplyDecodeError):
|
||||
discovery.follow_redirects_any([("host1", 9991), ("host2", 9992)], send_fn)
|
||||
|
||||
# Only the first candidate was ever attempted.
|
||||
assert calls == [("host1", 9991)]
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ import pytest
|
|||
from xcecommon import wire
|
||||
from xcecommon.errors import NoPrimaryError, ReplayFileError, ReplyDecodeError, SchedAdminError
|
||||
from xcecommon.schema import load_schema
|
||||
from xceadmin import scheduler
|
||||
from xceadmin import discovery, scheduler
|
||||
|
||||
|
||||
def _server_context(cert_file: Path, key_file: Path) -> ssl.SSLContext:
|
||||
|
|
@ -592,6 +592,76 @@ def test_record_status_retry_exhaustion_raises_distinguishable_error(
|
|||
assert f"{scheduler._MAX_TRANSIENT_ATTEMPTS} attempts" in exc_info.value.error
|
||||
|
||||
|
||||
def test_record_status_retries_on_no_primary_error_then_succeeds(monkeypatch, wrap_reply):
|
||||
"""Right after a primary is killed, every configured candidate can
|
||||
still redirect to the now-dead old primary (a standby hasn't
|
||||
necessarily noticed within the heartbeat timeout yet), so
|
||||
follow_redirects_any exhausts the whole candidate list and raises
|
||||
NoPrimaryError even though the cluster is only mid-election, not
|
||||
genuinely unreachable. This must be retried using NoPrimaryError's own
|
||||
(longer) budget/delay, _MAX_NO_PRIMARY_ATTEMPTS/
|
||||
_NO_PRIMARY_RETRY_DELAY_S -- not treated as an immediate hard failure,
|
||||
and not sharing _TRANSIENT_STATUSES' smaller budget. FAILS against the
|
||||
pre-fix code (which let NoPrimaryError propagate out of the retry loop
|
||||
on the very first attempt)."""
|
||||
mod = load_schema()
|
||||
sleep_calls: list[float] = []
|
||||
monkeypatch.setattr(scheduler.time, "sleep", lambda s: sleep_calls.append(s))
|
||||
|
||||
reply = mod.RequestReply.new_message()
|
||||
reply.status = 0
|
||||
admin_reply = reply.data.init("schedadminreplymsg")
|
||||
admin_reply.ok = True
|
||||
admin_reply.error = ""
|
||||
admin_reply.recording = True
|
||||
admin_reply.file = "settled.replay"
|
||||
admin_reply.eventCount = 0
|
||||
admin_reply.startedAt = 6000
|
||||
success_reply_bytes = wrap_reply(reply)
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_follow_redirects_any(candidates, send_fn):
|
||||
calls["n"] += 1
|
||||
if calls["n"] < 3:
|
||||
raise NoPrimaryError("no primary available: none of the configured controllers responded", "host1", 9991)
|
||||
return "host2", 9992, success_reply_bytes
|
||||
|
||||
monkeypatch.setattr(scheduler, "follow_redirects_any", fake_follow_redirects_any)
|
||||
|
||||
result = scheduler.record_status("host1:9991,host2:9992,host3:9993")
|
||||
|
||||
assert result.file == "settled.replay"
|
||||
assert calls["n"] == 3
|
||||
assert sleep_calls == [scheduler._NO_PRIMARY_RETRY_DELAY_S] * 2
|
||||
|
||||
|
||||
def test_record_status_no_primary_error_retry_exhaustion_propagates(monkeypatch):
|
||||
"""If NoPrimaryError persists across the whole retry budget (the
|
||||
cluster genuinely never settles, or every candidate really is down),
|
||||
it must still propagate as NoPrimaryError -- not be silently
|
||||
swallowed or converted into a less-informative error -- after exactly
|
||||
_MAX_NO_PRIMARY_ATTEMPTS attempts (its own, longer budget -- NOT
|
||||
_MAX_TRANSIENT_ATTEMPTS), matching the reply-status exhaustion test's
|
||||
own proof shape."""
|
||||
sleep_calls: list[float] = []
|
||||
monkeypatch.setattr(scheduler.time, "sleep", lambda s: sleep_calls.append(s))
|
||||
|
||||
calls = {"n": 0}
|
||||
|
||||
def fake_follow_redirects_any(candidates, send_fn):
|
||||
calls["n"] += 1
|
||||
raise NoPrimaryError("no primary available: none of the configured controllers responded", "host1", 9991)
|
||||
|
||||
monkeypatch.setattr(scheduler, "follow_redirects_any", fake_follow_redirects_any)
|
||||
|
||||
with pytest.raises(NoPrimaryError):
|
||||
scheduler.record_status("host1:9991,host2:9992,host3:9993")
|
||||
|
||||
assert calls["n"] == scheduler._MAX_NO_PRIMARY_ATTEMPTS
|
||||
assert len(sleep_calls) == scheduler._MAX_NO_PRIMARY_ATTEMPTS - 1
|
||||
|
||||
|
||||
def test_record_status_standby_without_redirect_not_retried(mock_controller, wrap_reply, monkeypatch):
|
||||
"""A standby rejection with NO populated redirect (the genuine "op is
|
||||
primary-only, no known primary to redirect to" case) is a hard,
|
||||
|
|
@ -822,18 +892,21 @@ def test_record_stop_not_recording_never_retried(sequential_mock_controller, wra
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _split_controller(): same default-port convention as push.py
|
||||
# scheduler.py delegates --controller parsing to xceadmin.discovery.
|
||||
# resolve_controllers (shared with push.py) -- see test_discovery.py for
|
||||
# that function's own dedicated coverage; these two just confirm
|
||||
# _send_admin_request's pipeline still reaches it for the single-host case.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_record_default_port_used_when_omitted():
|
||||
host, port = scheduler._split_controller("somehost")
|
||||
(host, port), = discovery.resolve_controllers("somehost")
|
||||
assert host == "somehost"
|
||||
assert port == 9991
|
||||
|
||||
|
||||
def test_record_explicit_port_parsed():
|
||||
host, port = scheduler._split_controller("somehost:1234")
|
||||
(host, port), = discovery.resolve_controllers("somehost:1234")
|
||||
assert host == "somehost"
|
||||
assert port == 1234
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue