myxce/src/xceadmin/tests/test_e2e_live_push.py

"""Live single-node E2E coverage for `config push` (task 6.2; Requirements
2.1, 2.5, 2.6, 3.1).

Every other push test in this suite (``test_config_push_mock.py``) talks to
a mock TLS server standing in for the controller. This file is the one
place the real ``xce_controller`` binary and the real ``xceadmin`` CLI are
exercised together end to end over the actual TLS+capnp wire protocol --
no mock anywhere. It exists specifically to regression-lock task 6.1's
reply-envelope decode fix (``xceadmin.discovery.decode_reply``) against a
REAL controller reply: task 6.1 itself discovered that this suite's mock
fixtures sent the wrong envelope shape and would never have caught the
crash that only showed up against a live controller.

Skip convention: an imperative ``pytest.skip`` inside the fixture, mirroring
``conftest.py``'s ``tls_certs`` openssl-missing skip (task 5.3's precedent)
-- not a module-level ``skipif``, since the discovery walk has to happen at
fixture-setup time. Skipping is the NORMAL outcome when this suite runs via
its usual path, ``xce-build.sh xceadmin``: that meson target builds only the
Python package (never the C controller) and runs pytest from an isolated
``repo-mirror`` copy of the source tree with no sibling
``xce.build``/``build.dbg``/``build.release`` controller binaries to find.
This test only runs for real when invoked directly against a real repo
checkout that has already built a controller (``xce-build.sh controller
[profile]``) -- the same "slow, on-demand live E2E" split this project
already uses for ``deploy-ha.sh``/``run-ha-soak.sh``/the ``xce-test-ha``
skill, which are likewise not part of the fast ``meson test`` path.

REAL finding, not a test bug (see the task's own CONCERNS writeup): a
genuinely single-node deployment (no HA peers -- ``deploy-ha.sh``'s
single-entry peer-list mode) has ZERO standbys to ACK, so
``push_config_to_standbys`` always returns ``acked=0`` even when the
config-change marker commits and activates successfully server-side
(verified live: the controller log shows "Config change committed",
"Config upgraded to tag=...", and "config activated from committed marker
tag=..." on every push). ``xceadmin.cli``'s postcondition (``acked_count
<= 0`` => ``ClickException``, Requirements 2.6/2.7) cannot distinguish "no
standbys exist" from "push genuinely failed", so a real single-node
``config push`` ALWAYS exits 1 with "push rejected (ackedCount == 0)" --
this is real, current, verified ``cli.py``/``push.py`` behavior against a
real controller, not a mock artifact or a test mistake. This test asserts
that REAL behavior (exit 1, ackedCount == 0) instead of a hypothetical
"exit 0", and separately verifies via the controller's own log file that
the push, commit, and activation all genuinely succeeded server-side --
the actual signal this test cares about (task 6.1's crash regression
lock), independent of the CLI's single-node-blind success heuristic (a
pre-existing gap in ``cli.py``/``push.py``, out of this task's boundary to
fix).
"""

from __future__ import annotations

import shutil
import subprocess
from pathlib import Path

import pytest

from _live_e2e_common import CONTROLLER_CANDIDATES as _CONTROLLER_CANDIDATES
from _live_e2e_common import DEPLOY_HA_SH, REPO_CONFIGS_DIR
from _live_e2e_common import READY_TIMEOUT_S as _READY_TIMEOUT_S
from _live_e2e_common import VERSION_TAG_RE as _VERSION_TAG_RE
from _live_e2e_common import XCEADMIN_CANDIDATES as _XCEADMIN_CANDIDATES
from _live_e2e_common import find_first_executable as _find_first_executable
from _live_e2e_common import free_tcp_port as _free_tcp_port
from _live_e2e_common import wait_for_port as _wait_for_port


@pytest.fixture
def live_controller(tmp_path: Path):
    """Deploy + start one real, single-node ``xce_controller`` via the
    project's own ``deploy-ha.sh`` (module docstring: a single-entry peer
    list, so zero standbys -- not "no HA machinery at all"). Skips cleanly
    if no controller binary or ``openssl`` is available. Always kills the
    controller process and removes the scratch deploy directory, on both
    success and failure.

    Yields ``(host_port_str, log_file_path)``.

    Deliberately does NOT launch the controller through deploy-ha.sh's
    generated ``scripts/start.sh``: that script runs ``$BINARY ...`` as a
    plain (non-``exec``'d) simple command, so the controller is bash's
    CHILD, not bash itself. ``Popen.terminate()`` only signals the bash
    wrapper; a non-interactive bash's default SIGTERM handling exits
    immediately without waiting on/forwarding to its child, orphaning a
    live ``xce_controller`` process on every test run (caught empirically
    -- four orphaned controllers accumulated across four manual runs
    during this task's own verification, still bound to their ports).
    Launching the binary directly, with the exact args ``start.sh`` would
    have used, makes ``proc`` the real controller process so
    ``terminate()``/``kill()`` target it directly.
    """
    controller_bin = _find_first_executable(_CONTROLLER_CANDIDATES)
    if controller_bin is None:
        pytest.skip(
            "no xce_controller binary found under xce.build/build.release/"
            "build.dbg -- run xce-build.sh controller [profile] first to "
            "exercise this live E2E test"
        )
    if shutil.which("openssl") is None:
        pytest.skip("openssl not found on PATH -- required by deploy-ha.sh")

    deploy_dir = tmp_path / "deploy"
    port = _free_tcp_port()
    deploy = subprocess.run(
        ["bash", str(DEPLOY_HA_SH), str(controller_bin), str(deploy_dir), f"localhost:{port}"],
        capture_output=True,
        text=True,
    )
    assert deploy.returncode == 0, f"deploy-ha.sh failed: {deploy.stdout}\n{deploy.stderr}"

    log_file = deploy_dir / "logs" / "ctrl-1.log"
    proc = subprocess.Popen(
        [
            str(controller_bin),
            "--myname", "ctrl-1",
            "--peers", f"ctrl-1=localhost:{port}",
            "--workdir", str(deploy_dir / "controller"),
            "--certdir", str(deploy_dir / "certs"),
            "--logfile", str(log_file),
            "--vcpus", "4",
        ],
        stdout=subprocess.DEVNULL,
        stderr=subprocess.DEVNULL,
    )
    try:
        _wait_for_port("localhost", port, _READY_TIMEOUT_S)
        yield f"localhost:{port}", log_file
    finally:
        proc.terminate()
        try:
            proc.wait(timeout=10)
        except subprocess.TimeoutExpired:
            proc.kill()
            proc.wait(timeout=10)
        shutil.rmtree(deploy_dir, ignore_errors=True)


def test_live_config_push_against_real_single_node_controller(live_controller, tmp_path: Path) -> None:
    """Real ``xceadmin config build`` + real ``xceadmin config push`` against
    a real deployed ``xce_controller`` over actual TLS+capnp -- no mock
    anywhere (Requirements 2.1, 2.5, 3.1).

    Asserts (module docstring has the full rationale):
    - no crash decoding the real reply (task 6.1's regression lock -- the
      actual point of this test)
    - a parsed ``versionTag``/``ackedCount`` are present in the output
      (2.5)
    - the REAL observed single-node exit/ackedCount semantics (ackedCount
      == 0, exit 1, "push rejected") rather than a hypothetical exit 0
    - the controller's own log independently confirms the config
      genuinely committed, upgraded, and activated server-side (2.6),
      decoupled from the CLI's single-node-blind ackedCount heuristic
    """
    controller_addr, log_file = live_controller
    xceadmin_bin = _find_first_executable(_XCEADMIN_CANDIDATES)
    if xceadmin_bin is None:
        pytest.skip("no xceadmin binary found under xce.build/build.release/build.dbg")

    cluster_bin = tmp_path / "cluster.bin"
    build_result = subprocess.run(
        [str(xceadmin_bin), "config", "build", str(REPO_CONFIGS_DIR), "-o", str(cluster_bin)],
        capture_output=True,
        text=True,
    )
    assert build_result.returncode == 0, build_result.stderr
    assert cluster_bin.is_file()

    push_result = subprocess.run(
        [str(xceadmin_bin), "config", "push", str(cluster_bin), "--controller", controller_addr],
        capture_output=True,
        text=True,
    )

    # Task 6.1's regression lock: the real controller reply must decode
    # cleanly through the Request{body.replymsg} envelope unwrap -- no
    # capnp KjException, no unhandled traceback, against REAL wire bytes
    # (not a mock fixture that could silently drift from the real shape).
    assert "KjException" not in push_result.stderr, push_result.stderr
    assert "Traceback" not in push_result.stderr, push_result.stderr

    match = _VERSION_TAG_RE.search(push_result.stdout)
    assert match, f"no versionTag/ackedCount found in stdout: {push_result.stdout!r}"
    version_tag, acked_count = int(match.group(1)), int(match.group(2))
    assert version_tag > 0

    # REAL, live-verified single-node behavior (module docstring): a
    # genuinely peer-less controller has no standbys to ACK, so
    # acked_count is always 0 and xceadmin.cli's `acked_count > 0`
    # postcondition reports this as a rejection -- even though, per the
    # controller-log assertions below, the push itself fully succeeded.
    assert acked_count == 0
    assert push_result.returncode == 1
    assert "push rejected" in push_result.stderr

    log_text = log_file.read_text()
    version_tag_hex = f"{version_tag:016x}"
    assert f"tag={version_tag_hex}" in log_text, log_text
    assert "Config change committed" in log_text
    assert f"Config upgraded to tag={version_tag_hex}" in log_text
    assert f"config activated from committed marker tag={version_tag_hex}" in log_text