wip(admin-config-tool): task 6.3 HA E2E test (blocked, flaky primary-push)
Standby-redirect half is done and correct: OP_upgradeConfig has no server-side redirect wiring today, matches design.md's own documented "Out of Boundary" gap, and this test locks in the real current rejection behavior. Primary-push success-path half is unreliable (~24/25 live fails in one investigation round, see bug-1485) -- the elected primary was observed as a <defunct> zombie shortly after promotion, but a dedicated debug pass could not reproduce it in 26+ clean runs on an idle host and captured no crash evidence either way. tasks.md marks 6.3 _Blocked:_ pending re-verification on an idle host with real stderr capture.
This commit is contained in:
parent
0206b9e166
commit
34bd934bc7
3 changed files with 603 additions and 53 deletions
99
src/xceadmin/tests/_live_e2e_common.py
Normal file
99
src/xceadmin/tests/_live_e2e_common.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
"""Shared helpers for xceadmin's live E2E test suite (task 6.2's
|
||||
``test_e2e_live_push.py`` and task 6.3's ``test_e2e_live_ha_push.py``) --
|
||||
binary discovery, ephemeral port allocation, waiting for a listener to come
|
||||
up, and the ``versionTag=.../ackedCount=...`` stdout parser every live
|
||||
``config push`` invocation needs to check.
|
||||
|
||||
Extracted here (task 6.3) rather than duplicated a second time: task 6.3's
|
||||
own text asks whether task 6.2's fixture/helpers can be reused for the
|
||||
multi-node case rather than duplicated. These four helpers (binary
|
||||
discovery, port allocation, port-wait, the versionTag/ackedCount regex) are
|
||||
identical text in both files pre-extraction -- genuine repetition, not
|
||||
speculative generality -- so they move here and both test files import from
|
||||
it. The two files' *fixtures* (``live_controller`` vs. ``live_ha_cluster``)
|
||||
stay separate: single-node vs. 3-node deploy/launch/election-wait/cleanup
|
||||
sequences differ enough (peer strings, staggered multi-process start,
|
||||
primary-election polling) that merging them would trade a real readability
|
||||
loss for a smaller one.
|
||||
|
||||
Not a test module itself (no ``test_*`` functions/classes), so pytest's
|
||||
default collection (``test_*.py``/``*_test.py``) leaves it alone; it is
|
||||
picked up as an ordinary same-directory import instead (this project's
|
||||
xceadmin/tests/ has no ``__init__.py``, so pytest's default "prepend"
|
||||
import mode puts this directory on ``sys.path`` for every test module in
|
||||
it -- the same mechanism that already lets every test file here import
|
||||
``conftest.py``'s fixtures without a package-qualified path).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
SRC_DIR = Path(__file__).resolve().parents[2]
|
||||
REPO_CONFIGS_DIR = SRC_DIR / "configs"
|
||||
DEPLOY_HA_SH = SRC_DIR / "tools" / "deploy-ha.sh"
|
||||
|
||||
# Mirrors deploy-ha.sh's own xceadmin/controller discovery order
|
||||
# (src/tools/deploy-ha.sh) -- xce.build (asan, the default profile) first,
|
||||
# then release, then debug.
|
||||
CONTROLLER_CANDIDATES = [
|
||||
SRC_DIR / "xce.build" / "controller" / "xce_controller",
|
||||
SRC_DIR / "build.release" / "controller" / "xce_controller",
|
||||
SRC_DIR / "build.dbg" / "controller" / "xce_controller",
|
||||
]
|
||||
XCEADMIN_CANDIDATES = [
|
||||
SRC_DIR / "xce.build" / "xceadmin" / ".venv" / "bin" / "xceadmin",
|
||||
SRC_DIR / "build.release" / "xceadmin" / ".venv" / "bin" / "xceadmin",
|
||||
SRC_DIR / "build.dbg" / "xceadmin" / ".venv" / "bin" / "xceadmin",
|
||||
]
|
||||
|
||||
READY_TIMEOUT_S = 20.0
|
||||
VERSION_TAG_RE = re.compile(r"versionTag=(\d+)\s+ackedCount=(\d+)")
|
||||
|
||||
|
||||
def find_first_executable(candidates: list[Path]) -> Path | None:
|
||||
for candidate in candidates:
|
||||
if candidate.is_file() and (candidate.stat().st_mode & 0o111):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def free_tcp_port() -> int:
|
||||
"""A port that was free at bind-time -- the same best-effort ephemeral-
|
||||
port trick used to avoid hardcoding a fixed port a test could collide
|
||||
on. The bind-then-immediately-close TOCTOU window is acceptable for a
|
||||
single-shot test deploy.
|
||||
"""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
return probe.getsockname()[1]
|
||||
|
||||
|
||||
def free_tcp_ports(count: int) -> list[int]:
|
||||
"""``count`` distinct ports from :func:`free_tcp_port`, de-duplicated
|
||||
across the batch (a multi-node deploy needs several at once; calling
|
||||
:func:`free_tcp_port` in a bare loop can occasionally return the same
|
||||
port twice since each call binds-then-immediately-releases before the
|
||||
next one binds). Returned in ascending order for a stable, readable
|
||||
ctrl-1/ctrl-2/ctrl-3 assignment.
|
||||
"""
|
||||
ports: set[int] = set()
|
||||
while len(ports) < count:
|
||||
ports.add(free_tcp_port())
|
||||
return sorted(ports)
|
||||
|
||||
|
||||
def wait_for_port(host: str, port: int, timeout_s: float) -> None:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
last_err: OSError | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=1.0):
|
||||
return
|
||||
except OSError as exc:
|
||||
last_err = exc
|
||||
time.sleep(0.2)
|
||||
raise TimeoutError(f"controller never opened {host}:{port}: {last_err}")
|
||||
Loading…
Add table
Add a link
Reference in a new issue