test(admin-config-tool): add live single-node E2E test for config push

Deploys a real xce_controller (no HA peers) and runs real xceadmin
config build/push over actual TLS+capnp -- no mock server -- regression
locking task 6.1's envelope fix. Skips cleanly when no controller binary
is available. Surfaced a real, out-of-scope finding: single-node pushes
always report ackedCount==0/exit 1 even on full server-side success,
since there are no standbys to ACK.

src/xceadmin/tests/test_e2e_live_push.py

@@ -0,0 +1,245 @@
+"""Live single-node E2E coverage for `config push` (task 6.2; Requirements
+2.1, 2.5, 2.6, 3.1).
+
+Every other push test in this suite (``test_config_push_mock.py``) talks to
+a mock TLS server standing in for the controller. This file is the one
+place the real ``xce_controller`` binary and the real ``xceadmin`` CLI are
+exercised together end to end over the actual TLS+capnp wire protocol --
+no mock anywhere. It exists specifically to regression-lock task 6.1's
+reply-envelope decode fix (``xceadmin.discovery.decode_reply``) against a
+REAL controller reply: task 6.1 itself discovered that this suite's mock
+fixtures sent the wrong envelope shape and would never have caught the
+crash that only showed up against a live controller.
+
+Skip convention: an imperative ``pytest.skip`` inside the fixture, mirroring
+``conftest.py``'s ``tls_certs`` openssl-missing skip (task 5.3's precedent)
+-- not a module-level ``skipif``, since the discovery walk has to happen at
+fixture-setup time. Skipping is the NORMAL outcome when this suite runs via
+its usual path, ``xce-build.sh xceadmin``: that meson target builds only the
+Python package (never the C controller) and runs pytest from an isolated
+``repo-mirror`` copy of the source tree with no sibling
+``xce.build``/``build.dbg``/``build.release`` controller binaries to find.
+This test only runs for real when invoked directly against a real repo
+checkout that has already built a controller (``xce-build.sh controller
+[profile]``) -- the same "slow, on-demand live E2E" split this project
+already uses for ``deploy-ha.sh``/``run-ha-soak.sh``/the ``xce-test-ha``
+skill, which are likewise not part of the fast ``meson test`` path.
+
+REAL finding, not a test bug (see the task's own CONCERNS writeup): a
+genuinely single-node deployment (no HA peers -- ``deploy-ha.sh``'s
+single-entry peer-list mode) has ZERO standbys to ACK, so
+``push_config_to_standbys`` always returns ``acked=0`` even when the
+config-change marker commits and activates successfully server-side
+(verified live: the controller log shows "Config change committed",
+"Config upgraded to tag=...", and "config activated from committed marker
+tag=..." on every push). ``xceadmin.cli``'s postcondition (``acked_count
+<= 0`` => ``ClickException``, Requirements 2.6/2.7) cannot distinguish "no
+standbys exist" from "push genuinely failed", so a real single-node
+``config push`` ALWAYS exits 1 with "push rejected (ackedCount == 0)" --
+this is real, current, verified ``cli.py``/``push.py`` behavior against a
+real controller, not a mock artifact or a test mistake. This test asserts
+that REAL behavior (exit 1, ackedCount == 0) instead of a hypothetical
+"exit 0", and separately verifies via the controller's own log file that
+the push, commit, and activation all genuinely succeeded server-side --
+the actual signal this test cares about (task 6.1's crash regression
+lock), independent of the CLI's single-node-blind success heuristic (a
+pre-existing gap in ``cli.py``/``push.py``, out of this task's boundary to
+fix).
+"""
+
+from __future__ import annotations
+
+import re
+import shutil
+import socket
+import subprocess
+import time
+from pathlib import Path
+
+import pytest
+
+SRC_DIR = Path(__file__).resolve().parents[2]
+REPO_CONFIGS_DIR = SRC_DIR / "configs"
+DEPLOY_HA_SH = SRC_DIR / "tools" / "deploy-ha.sh"
+
+# Mirrors deploy-ha.sh's own xceadmin discovery order (src/tools/deploy-ha.sh)
+# -- xce.build (asan, the default profile) first, then release, then debug.
+_CONTROLLER_CANDIDATES = [
+    SRC_DIR / "xce.build" / "controller" / "xce_controller",
+    SRC_DIR / "build.release" / "controller" / "xce_controller",
+    SRC_DIR / "build.dbg" / "controller" / "xce_controller",
+]
+_XCEADMIN_CANDIDATES = [
+    SRC_DIR / "xce.build" / "xceadmin" / ".venv" / "bin" / "xceadmin",
+    SRC_DIR / "build.release" / "xceadmin" / ".venv" / "bin" / "xceadmin",
+    SRC_DIR / "build.dbg" / "xceadmin" / ".venv" / "bin" / "xceadmin",
+]
+
+_READY_TIMEOUT_S = 20.0
+_VERSION_TAG_RE = re.compile(r"versionTag=(\d+)\s+ackedCount=(\d+)")
+
+
+def _find_first_executable(candidates: list[Path]) -> Path | None:
+    for candidate in candidates:
+        if candidate.is_file() and (candidate.stat().st_mode & 0o111):
+            return candidate
+    return None
+
+
+def _free_tcp_port() -> int:
+    """A port that was free at bind-time -- the same best-effort ephemeral-
+    port trick used to avoid hardcoding a fixed port this test could
+    collide on. The bind-then-immediately-close TOCTOU window is
+    acceptable for a single-shot test deploy.
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        return probe.getsockname()[1]
+
+
+def _wait_for_port(host: str, port: int, timeout_s: float) -> None:
+    deadline = time.monotonic() + timeout_s
+    last_err: OSError | None = None
+    while time.monotonic() < deadline:
+        try:
+            with socket.create_connection((host, port), timeout=1.0):
+                return
+        except OSError as exc:
+            last_err = exc
+            time.sleep(0.2)
+    raise TimeoutError(f"controller never opened {host}:{port}: {last_err}")
+
+
+@pytest.fixture
+def live_controller(tmp_path: Path):
+    """Deploy + start one real, single-node ``xce_controller`` via the
+    project's own ``deploy-ha.sh`` (module docstring: a single-entry peer
+    list, so zero standbys -- not "no HA machinery at all"). Skips cleanly
+    if no controller binary or ``openssl`` is available. Always kills the
+    controller process and removes the scratch deploy directory, on both
+    success and failure.
+
+    Yields ``(host_port_str, log_file_path)``.
+
+    Deliberately does NOT launch the controller through deploy-ha.sh's
+    generated ``scripts/start.sh``: that script runs ``$BINARY ...`` as a
+    plain (non-``exec``'d) simple command, so the controller is bash's
+    CHILD, not bash itself. ``Popen.terminate()`` only signals the bash
+    wrapper; a non-interactive bash's default SIGTERM handling exits
+    immediately without waiting on/forwarding to its child, orphaning a
+    live ``xce_controller`` process on every test run (caught empirically
+    -- four orphaned controllers accumulated across four manual runs
+    during this task's own verification, still bound to their ports).
+    Launching the binary directly, with the exact args ``start.sh`` would
+    have used, makes ``proc`` the real controller process so
+    ``terminate()``/``kill()`` target it directly.
+    """
+    controller_bin = _find_first_executable(_CONTROLLER_CANDIDATES)
+    if controller_bin is None:
+        pytest.skip(
+            "no xce_controller binary found under xce.build/build.release/"
+            "build.dbg -- run xce-build.sh controller [profile] first to "
+            "exercise this live E2E test"
+        )
+    if shutil.which("openssl") is None:
+        pytest.skip("openssl not found on PATH -- required by deploy-ha.sh")
+
+    deploy_dir = tmp_path / "deploy"
+    port = _free_tcp_port()
+    deploy = subprocess.run(
+        ["bash", str(DEPLOY_HA_SH), str(controller_bin), str(deploy_dir), f"localhost:{port}"],
+        capture_output=True,
+        text=True,
+    )
+    assert deploy.returncode == 0, f"deploy-ha.sh failed: {deploy.stdout}\n{deploy.stderr}"
+
+    log_file = deploy_dir / "logs" / "ctrl-1.log"
+    proc = subprocess.Popen(
+        [
+            str(controller_bin),
+            "--myname", "ctrl-1",
+            "--peers", f"ctrl-1=localhost:{port}",
+            "--workdir", str(deploy_dir / "controller"),
+            "--certdir", str(deploy_dir / "certs"),
+            "--logfile", str(log_file),
+            "--vcpus", "4",
+        ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    try:
+        _wait_for_port("localhost", port, _READY_TIMEOUT_S)
+        yield f"localhost:{port}", log_file
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait(timeout=10)
+        shutil.rmtree(deploy_dir, ignore_errors=True)
+
+
+def test_live_config_push_against_real_single_node_controller(live_controller, tmp_path: Path) -> None:
+    """Real ``xceadmin config build`` + real ``xceadmin config push`` against
+    a real deployed ``xce_controller`` over actual TLS+capnp -- no mock
+    anywhere (Requirements 2.1, 2.5, 3.1).
+
+    Asserts (module docstring has the full rationale):
+    - no crash decoding the real reply (task 6.1's regression lock -- the
+      actual point of this test)
+    - a parsed ``versionTag``/``ackedCount`` are present in the output
+      (2.5)
+    - the REAL observed single-node exit/ackedCount semantics (ackedCount
+      == 0, exit 1, "push rejected") rather than a hypothetical exit 0
+    - the controller's own log independently confirms the config
+      genuinely committed, upgraded, and activated server-side (2.6),
+      decoupled from the CLI's single-node-blind ackedCount heuristic
+    """
+    controller_addr, log_file = live_controller
+    xceadmin_bin = _find_first_executable(_XCEADMIN_CANDIDATES)
+    if xceadmin_bin is None:
+        pytest.skip("no xceadmin binary found under xce.build/build.release/build.dbg")
+
+    cluster_bin = tmp_path / "cluster.bin"
+    build_result = subprocess.run(
+        [str(xceadmin_bin), "config", "build", str(REPO_CONFIGS_DIR), "-o", str(cluster_bin)],
+        capture_output=True,
+        text=True,
+    )
+    assert build_result.returncode == 0, build_result.stderr
+    assert cluster_bin.is_file()
+
+    push_result = subprocess.run(
+        [str(xceadmin_bin), "config", "push", str(cluster_bin), "--controller", controller_addr],
+        capture_output=True,
+        text=True,
+    )
+
+    # Task 6.1's regression lock: the real controller reply must decode
+    # cleanly through the Request{body.replymsg} envelope unwrap -- no
+    # capnp KjException, no unhandled traceback, against REAL wire bytes
+    # (not a mock fixture that could silently drift from the real shape).
+    assert "KjException" not in push_result.stderr, push_result.stderr
+    assert "Traceback" not in push_result.stderr, push_result.stderr
+
+    match = _VERSION_TAG_RE.search(push_result.stdout)
+    assert match, f"no versionTag/ackedCount found in stdout: {push_result.stdout!r}"
+    version_tag, acked_count = int(match.group(1)), int(match.group(2))
+    assert version_tag > 0
+
+    # REAL, live-verified single-node behavior (module docstring): a
+    # genuinely peer-less controller has no standbys to ACK, so
+    # acked_count is always 0 and xceadmin.cli's `acked_count > 0`
+    # postcondition reports this as a rejection -- even though, per the
+    # controller-log assertions below, the push itself fully succeeded.
+    assert acked_count == 0
+    assert push_result.returncode == 1
+    assert "push rejected" in push_result.stderr
+
+    log_text = log_file.read_text()
+    version_tag_hex = f"{version_tag:016x}"
+    assert f"tag={version_tag_hex}" in log_text, log_text
+    assert "Config change committed" in log_text
+    assert f"Config upgraded to tag={version_tag_hex}" in log_text
+    assert f"config activated from committed marker tag={version_tag_hex}" in log_text