feat(deploy,xceadmin): standalone xceadmin packaging + deploy-ha usability (bug-1566)

Three related enhancements to make /xce-deploy-ha's output a
self-contained, directly-usable cluster for manual testing:

- xceadmin now builds an installable wheel (meson-build.sh) with
  correct dependencies (click/pycapnp/PyYAML/jsonschema, verified
  against actual imports -- pyproject.toml was missing several).
- deploy-ha.sh resolves the deploy environment's own bash
  ($BASH_BIN, baked into every generated script's shebang -- this
  environment's deploy-ha.sh had drifted to #!/bin/sh while still
  using bash-only syntax) and generates scripts/start-worker.sh /
  scripts/start-scheduler.sh sharing the controllers' --peers, plus
  a standalone xceadmin install (wheel extracted directly, resolved
  $PYTHONPATH baked into a wrapper script -- nix devShell packages
  are $PYTHONPATH entries, not merged into python3's own
  site-packages, so a --system-site-packages venv can't see them).
- bug-1566: xceadmin's --controller option had no relationship to a
  deployment's actual peers (hardcoded localhost:9991 default) --
  every command required an explicit --controller or silently tried
  the wrong port. Fixed via a $XCEADMIN_CONTROLLER envvar fallback
  (click precedence: explicit flag > envvar > bare default),
  exported by deploy-ha.sh's generated scripts/xceadmin wrapper.

Fixed/validated via TDD + live reproduction against a real deployed
cluster in a fully stripped environment.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/xceadmin/meson-build.sh

@@ -60,5 +60,22 @@ rm -rf "$VENV_DIR"
 echo "=== xceadmin (meson): running tests ==="
 "$VENV_DIR/bin/python" -m pytest "$WORK_DIR/tests" -v
 
+# Build an installable wheel from the SAME schema-synced $WORK_DIR the pinned
+# install above just tested -- not a second, separately-assembled tree --
+# so the wheel is guaranteed to carry the identical schema_files/ package
+# data (Requirement 5.5) the just-passed test run actually exercised. Only
+# reached after `set -e` has let the test run above succeed, so a wheel is
+# never produced (or left stale) from a red test run. --no-deps: this is
+# xceadmin's own wheel only, not a vendored bundle of click/pycapnp/etc --
+# installing it still needs those from PyPI or (preferably) the nix shell,
+# same as the pinned install just above.
+DIST_DIR="$BUILD_DIR/dist"
+echo "=== xceadmin (meson): building installable wheel ==="
+rm -rf "$DIST_DIR"
+mkdir -p "$DIST_DIR"
+"$VENV_DIR/bin/pip" wheel "$WORK_DIR" --no-deps -w "$DIST_DIR" --quiet
+WHEEL_FILE=$(ls "$DIST_DIR"/xceadmin-*.whl)
+echo "=== xceadmin (meson): wheel built at $WHEEL_FILE ==="
+
 touch "$STAMP_FILE"
 echo "=== xceadmin (meson): done ==="

src/xceadmin/pyproject.toml

@@ -9,7 +9,22 @@ description = "XCE cluster configuration admin CLI (config build / config push)"
 requires-python = ">=3.10"
 dependencies = [
     "click>=8.0",
+    "pycapnp>=1.3",
+    "PyYAML>=6.0",
+    "jsonschema>=4.0",
 ]
+# All four are real runtime imports (capnp/click/jsonschema/yaml — grep
+# src/xceadmin src/xcecommon for `^import |^from `), not just dev/test
+# tooling: an installed-but-undeclared dependency here would let `pip
+# install` "succeed" while leaving the CLI broken at first real use
+# (`ModuleNotFoundError` the moment `config build` touches capnp/jsonschema/
+# yaml) — exactly the class of gap a wheel built for install-outside-this-
+# repo needs to not have. pycapnp specifically: the nix dev shell's
+# python3Packages.pycapnp (src/nixSupport/xce.nix) is preferred when
+# available because it links against the same capnproto C++ runtime this
+# repo's C code uses; PyPI's prebuilt pycapnp wheels are usually fine too
+# for a plain `pip install` outside the nix shell, but that's a "usually,"
+# not the guaranteed-consistent link this repo's own build gets.
 
 # Wired into meson (src/xceadmin/meson.build custom_target, pulled in by
 # src/meson.build's subdir('xceadmin')) so `nix build .#xcecore[_debug]`
@@ -31,10 +46,13 @@ include = ["xceadmin*", "xcecommon*"]
 "" = "src"
 
 # Bundled copies of controller.capnp/c.capnp and the 7 JSON validation
-# schemas (synced from src/schema/ and src/configs/schema/ by
-# src/tools/xce-build.sh's `xceadmin)` case, see Requirement 5.5). Without
-# this, a non-editable `pip install .` builds a wheel containing only *.py
-# files and silently drops schema_files/ — an editable install would never
-# catch this since it resolves straight into the checked-out source tree.
+# schemas (synced from src/schema/ and src/configs/schema/ into
+# xcecommon/schema_files/ by src/xceadmin/meson-build.sh, the meson
+# custom_target xce-build.sh's `xceadmin)` case now delegates to, before
+# either the pinned test-install or the wheel build run against that same
+# tree — see Requirement 5.5). Without this, a non-editable `pip install .`
+# (or a wheel build) produces a package containing only *.py files and
+# silently drops schema_files/ — an editable install would never catch this
+# since it resolves straight into the checked-out source tree.
 [tool.setuptools.package-data]
 xcecommon = ["schema_files/*.capnp", "schema_files/json/*.schema.json"]

src/xceadmin/src/xceadmin/cli.py

@@ -39,6 +39,14 @@ from xceadmin.scheduler import RecordResult, dump, record_start, record_status,
 
 _DEFAULT_OUTPUT = "cluster.bin"
 _DEFAULT_CONTROLLER = "localhost:9991"
+# Every --controller option below also reads this env var (click precedence:
+# explicit --controller flag > $XCEADMIN_CONTROLLER > _DEFAULT_CONTROLLER).
+# _DEFAULT_CONTROLLER has no relationship to any REAL deployment's actual
+# ports -- it's a bare fallback for a totally bare invocation. A deployment
+# that knows its own real controller ports (e.g. /xce-deploy-ha's generated
+# scripts/xceadmin wrapper) should export this instead of requiring every
+# command to be typed with an explicit --controller.
+_CONTROLLER_ENV_VAR = "XCEADMIN_CONTROLLER"
 _STATUS_LABEL_WIDTH = 13  # len("started at:") + 2 -- requirements.md's "Status output" example
 
 
@@ -68,7 +76,11 @@ def config_build(configs_dir: str, output: str) -> None:
 @config.command("push")
 @click.argument("cluster_bin_path")
 @click.option(
-    "--controller", default=_DEFAULT_CONTROLLER, help="host:port of any controller node (2.2)."
+    "--controller",
+    default=_DEFAULT_CONTROLLER,
+    envvar=_CONTROLLER_ENV_VAR,
+    help=f"host:port of any controller node (2.2). Falls back to ${_CONTROLLER_ENV_VAR} "
+    "if set, then the bare default.",
 )
 @click.option("--dry-run", is_flag=True, help="Probe acceptance without committing (Requirement 6).")
 def config_push(cluster_bin_path: str, controller: str, dry_run: bool) -> None:
@@ -195,7 +207,11 @@ def _run_scheduler_admin_command(controller: str, call: Callable[[], RecordResul
 @scheduler_record.command("start")
 @click.argument("file", required=False)
 @click.option(
-    "--controller", default=_DEFAULT_CONTROLLER, help="host:port of any controller node (2.2)."
+    "--controller",
+    default=_DEFAULT_CONTROLLER,
+    envvar=_CONTROLLER_ENV_VAR,
+    help=f"host:port of any controller node (2.2). Falls back to ${_CONTROLLER_ENV_VAR} "
+    "if set, then the bare default.",
 )
 def scheduler_record_start(file: str | None, controller: str) -> None:
     """Start scheduler event recording, optionally to FILE (auto-generated when omitted)."""
@@ -205,7 +221,11 @@ def scheduler_record_start(file: str | None, controller: str) -> None:
 
 @scheduler_record.command("stop")
 @click.option(
-    "--controller", default=_DEFAULT_CONTROLLER, help="host:port of any controller node (2.2)."
+    "--controller",
+    default=_DEFAULT_CONTROLLER,
+    envvar=_CONTROLLER_ENV_VAR,
+    help=f"host:port of any controller node (2.2). Falls back to ${_CONTROLLER_ENV_VAR} "
+    "if set, then the bare default.",
 )
 def scheduler_record_stop(controller: str) -> None:
     """Stop scheduler event recording."""
@@ -215,7 +235,11 @@ def scheduler_record_stop(controller: str) -> None:
 
 @scheduler_record.command("status")
 @click.option(
-    "--controller", default=_DEFAULT_CONTROLLER, help="host:port of any controller node (2.2)."
+    "--controller",
+    default=_DEFAULT_CONTROLLER,
+    envvar=_CONTROLLER_ENV_VAR,
+    help=f"host:port of any controller node (2.2). Falls back to ${_CONTROLLER_ENV_VAR} "
+    "if set, then the bare default.",
 )
 def scheduler_record_status(controller: str) -> None:
     """Report current scheduler event-recording state."""

src/xceadmin/tests/test_cli.py

@@ -498,6 +498,55 @@ def test_scheduler_record_start_passes_filename_and_controller_arguments(monkeyp
     assert captured["filename"] == "myfile.replay"
 
 
+# ---------------------------------------------------------------------------
+# --controller falls back to $XCEADMIN_CONTROLLER, not the hardcoded
+# _DEFAULT_CONTROLLER, when the env var is set (a deployment whose cluster
+# doesn't happen to use localhost:9991 -- e.g. /xce-deploy-ha's
+# scripts/xceadmin wrapper exports this to the actual first --peers
+# candidate -- must not silently point at the wrong port). Explicit
+# --controller on the command line still wins over the env var (click's
+# built-in precedence: CLI flag > envvar > default).
+# ---------------------------------------------------------------------------
+
+
+def test_controller_option_falls_back_to_xceadmin_controller_env_var(monkeypatch) -> None:
+    captured = {}
+
+    def _fake_record_status(controller):
+        captured["controller"] = controller
+        return RecordResult(recording=False, file="", event_count=0, started_at=0)
+
+    monkeypatch.setattr(cli_module, "record_status", _fake_record_status)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        cli, ["scheduler", "record", "status"], env={"XCEADMIN_CONTROLLER": "localhost:9901"}
+    )
+
+    assert result.exit_code == 0, result.output
+    assert captured["controller"] == "localhost:9901"
+
+
+def test_controller_flag_overrides_xceadmin_controller_env_var(monkeypatch) -> None:
+    captured = {}
+
+    def _fake_record_status(controller):
+        captured["controller"] = controller
+        return RecordResult(recording=False, file="", event_count=0, started_at=0)
+
+    monkeypatch.setattr(cli_module, "record_status", _fake_record_status)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        ["scheduler", "record", "status", "--controller", "127.0.0.1:9999"],
+        env={"XCEADMIN_CONTROLLER": "localhost:9901"},
+    )
+
+    assert result.exit_code == 0, result.output
+    assert captured["controller"] == "127.0.0.1:9999"
+
+
 def test_scheduler_record_stop_success_prints_formatted_result(monkeypatch) -> None:
     def _fake_record_stop(controller):
         return RecordResult(recording=False, file="final.replay", event_count=14273, started_at=1000)