#!/bin/sh
set -e

SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)

BINARY=$1
TOP=$2
PEERS=$3

if [ -z "$BINARY" ] || [ -z "$TOP" ] || [ -z "$PEERS" ]; then
    echo "Usage: $0 <binary> <top-dir> <peers>"
    echo "  peers: comma-separated host:port list (e.g. host1:port1,host2:port2)"
    echo "  TLS cert/key and cluster.bin are generated fresh into <top-dir> every"
    echo "  run (openssl self-signed cert; cluster.bin via xceadmin from"
    echo "  src/configs/*.yaml) -- no pre-existing data directory needed."
    exit 1
fi

BINARY=$(realpath "$BINARY")

# xceadmin is meson-built now (src/xceadmin/meson.build), not a plain
# `pip install -e .` venv checked out under src/xceadmin/ -- its build
# output lives under whichever meson build dir xce-build.sh's profile
# picked, or under a nix-installed prefix's bin/, and each candidate is a
# STATIC SNAPSHOT (meson-build.sh copies src/xceadmin out-of-tree and does
# a plain, non-editable `pip install`) taken whenever that profile's
# `xceadmin` target was last built -- editing src/xceadmin/ source does
# NOT retroactively update an already-built snapshot.
#
# Correctness, not just freshness, matters here: xceadmin builds the
# cluster.bin that $BINARY (the controller daemon being deployed) will
# decode, so the two must come from the SAME build generation -- e.g. if
# the capnp wire schema changed between builds, an xceadmin from one
# generation and a controller from another can silently mis-encode/
# mis-decode a field (this is exactly how a stale-xceadmin deploy bug
# went undiagnosed: a fresher-but-DIFFERENT-build xceadmin could just as
# easily mismatch $BINARY as a staler one). Prefer the xceadmin living in
# $BINARY's OWN build dir (<builddir>/controller/xce_controller ->
# <builddir>/xceadmin/.venv/bin/xceadmin) -- guaranteed same-generation.
# Only if that specific one doesn't exist (e.g. xceadmin was never built
# in that profile) fall back to whichever other candidate is newest, with
# a loud warning that same-generation consistency isn't guaranteed.
BINARY_BUILD_DIR=$(dirname "$(dirname "$BINARY")")
BINARY_DIR=$(dirname "$BINARY")
XCEADMIN=""
if [ -x "$BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin" ]; then
    # meson layout: <builddir>/controller/xce_controller + <builddir>/xceadmin/.venv/...
    XCEADMIN="$BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin"
    echo "Using xceadmin: $XCEADMIN (same build dir as $BINARY)"
elif [ -x "$BINARY_DIR/xceadmin" ]; then
    # nix result layout: result/bin/xce_controller + result/bin/xceadmin (flat, no
    # per-component subdir) -- BINARY_BUILD_DIR's two-levels-up guess doesn't apply.
    XCEADMIN="$BINARY_DIR/xceadmin"
    echo "Using xceadmin: $XCEADMIN (same directory as $BINARY)"
else
    echo "Warning: no xceadmin found alongside $BINARY (checked $BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin and $BINARY_DIR/xceadmin) -- falling back to the newest xceadmin found elsewhere, which is NOT guaranteed to be the same build generation as $BINARY."
    XCEADMIN_MTIME=0
    for candidate in \
        "$SCRIPT_DIR/../xce.build/xceadmin/.venv/bin/xceadmin" \
        "$SCRIPT_DIR/../build.release/xceadmin/.venv/bin/xceadmin" \
        "$SCRIPT_DIR/../build.dbg/xceadmin/.venv/bin/xceadmin" \
        "$SCRIPT_DIR/../result/bin/xceadmin" \
        "$(command -v xceadmin 2>/dev/null || true)"
    do
        if [ -n "$candidate" ] && [ -x "$candidate" ]; then
            candidate_mtime=$(stat -c %Y "$candidate" 2>/dev/null || stat -f %m "$candidate" 2>/dev/null || echo 0)
            if [ "$candidate_mtime" -gt "$XCEADMIN_MTIME" ]; then
                XCEADMIN="$candidate"
                XCEADMIN_MTIME="$candidate_mtime"
            fi
        fi
    done
    if [ -n "$XCEADMIN" ]; then
        echo "Using xceadmin: $XCEADMIN (built $(date -d "@$XCEADMIN_MTIME" 2>/dev/null || date -r "$XCEADMIN_MTIME" 2>/dev/null))"
    fi
fi
if [ -z "$XCEADMIN" ]; then
    echo "Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first"
    exit 1
fi

# Worker/scheduler binaries, same-build-generation sibling lookup as
# xceadmin above (<builddir>/worker/xcecd_virtual, <builddir>/scheduler/
# xce_scheduler next to <builddir>/controller/xce_controller). Soft-check
# only: xce-test-ha's soak driver also calls this script and never builds
# either, so a missing worker/scheduler binary must not fail the deploy --
# it just skips generating that one start script, with a warning.
#
# Which of the two sibling worker binaries this deploy's start-worker.sh
# targets is auto-detected by default, overridable with XCE_REAL_WORKER=0/1:
#   xcecd_virtual: skips the root-only sqlite db-recovery and domain-socket
#     subsystems, runs as an ordinary user, no sudo needed.
#   xcecd (the production daemon): worker/src/main.c's check_user()
#     unconditionally requires geteuid()==0, so the generated
#     start-worker.sh below wraps its exec in `sudo` and preflights
#     `sudo mkdir -p /var/run` (xcecd hardcodes db_path=/var/run/xcecd.db
#     and domain_sock_file=/var/run/.xcecd.sock -- neither is a flag).
#     Also, actually dispatching a job goes through sd_bus_open_system() ->
#     org.freedesktop.systemd1.Manager.StartTransientUnit
#     (worker/src/xcecd_cgroups.c) -- it needs a real systemd manager
#     answering on the system dbus, not just root.
#
# Auto-detection: /run/systemd/system existing is systemd's own sd_booted(3)
# signal for "a real systemd instance is running as this host's manager" --
# true on an actual host/VM, false in sandboxes that boot under something
# else (verified 2026-08-19: this dev sandbox has no /run at all, PID 1 is
# s6-svscan, no dbus-daemon/systemctl even under `nix develop`). Real job
# dispatch needs that systemd manager reachable regardless of root, so
# picking xcecd_virtual when it's absent -- and xcecd when it's present --
# means this one deploy-ha.sh works unmodified on both a throwaway sandbox
# and a real HA test host. XCE_REAL_WORKER=0/1 overrides the auto-detect in
# either direction (e.g. force xcecd_virtual on a real host, or force xcecd
# on a sandbox to exercise its root-gated startup path even though job
# dispatch there will still fail).
if [ -n "${XCE_REAL_WORKER:-}" ]; then
    USE_REAL_WORKER="$XCE_REAL_WORKER"
elif [ -d /run/systemd/system ]; then
    USE_REAL_WORKER=1
else
    USE_REAL_WORKER=0
fi
if [ "$USE_REAL_WORKER" = "1" ]; then
    WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd"
else
    WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd_virtual"
fi
echo "Worker binary: $([ "$USE_REAL_WORKER" = "1" ] && echo "xcecd (real, root via sudo)" || echo "xcecd_virtual (no root)") -- override with XCE_REAL_WORKER=0/1"
if [ ! -x "$WORKER_BIN" ]; then
    echo "Warning: no $(basename "$WORKER_BIN") found at $WORKER_BIN -- skipping scripts/start-worker.sh generation (run '/xce-build worker' first if you need it)"
    WORKER_BIN=""
fi
SCHED_BIN="$BINARY_BUILD_DIR/scheduler/xce_scheduler"
if [ ! -x "$SCHED_BIN" ]; then
    echo "Warning: no xce_scheduler found at $SCHED_BIN -- skipping scripts/start-scheduler.sh generation (run '/xce-build scheduler' first if you need it)"
    SCHED_BIN=""
fi

# Apiserver binary check -- same opt-in-by-presence spirit as WORKER_BIN/
# SCHED_BIN above. Prefer the meson-built binary, via `tools/go_build`
# (`/xce-build apiserver` or `/xce-build all`) -- lands in
# $BINARY_BUILD_DIR like every other component this script resolves, so
# it IS a same-build-generation sibling lookup, same as
# xceadmin/worker/scheduler above. Fall back to a binary already present
# at the fixed legacy path `/tmp/apiserver` if the preferred location is
# empty, with a caveat echoed: that path is NOT guaranteed to be the same
# build generation as $BINARY.
# deploy-ha.sh does NOT invoke either build itself -- building remains a
# caller prerequisite (same "run /xce-build controller first if stale"
# convention SKILL.md's Prerequisites section documents for the controller).
# Soft-check only, same reasoning as worker/scheduler: xce-test-ha never
# builds an apiserver and calls this same script, so a missing binary must
# not fail the deploy -- it just skips generating scripts/start-apiserver.sh,
# with a warning.
APISERVER_BIN=""
if [ -x "$BINARY_BUILD_DIR/frontend/apiserver/xce_apiserver" ]; then
    APISERVER_BIN="$BINARY_BUILD_DIR/frontend/apiserver/xce_apiserver"
    echo "Using apiserver: $APISERVER_BIN (same build dir as $BINARY)"
elif [ -x "/tmp/apiserver" ]; then
    APISERVER_BIN="/tmp/apiserver"
    echo "Using apiserver: $APISERVER_BIN (legacy /tmp/apiserver path; NOT guaranteed to be the same build generation as $BINARY)"
else
    echo "Warning: no apiserver binary found at $BINARY_BUILD_DIR/frontend/apiserver/xce_apiserver or /tmp/apiserver -- skipping scripts/start-apiserver.sh generation (run '/xce-build apiserver' first if you need it)"
fi

# xceadmin wheel (same-build-generation sibling again): /xce-build xceadmin
# leaves one at <builddir>/xceadmin/dist/xceadmin-*.whl (meson-build.sh).
# Installed below into its OWN venv under $TOP, not just pointed at
# $XCEADMIN's build-dir .venv above -- a live deployment can outlive the
# build dir (a later `/xce-build xceadmin` run rm -rf's and rebuilds that
# .venv out from under anything still depending on it), so this deploy gets
# a genuinely self-contained copy. Soft-skip (warn), same reasoning as
# worker/scheduler above: a missing wheel must not fail the deploy.
XCEADMIN_WHEEL=""
if [ -d "$BINARY_BUILD_DIR/xceadmin/dist" ]; then
    XCEADMIN_WHEEL=$(ls "$BINARY_BUILD_DIR/xceadmin/dist"/xceadmin-*.whl 2>/dev/null | head -1 || true)
fi
if [ -z "$XCEADMIN_WHEEL" ]; then
    echo "Warning: no xceadmin wheel found under $BINARY_BUILD_DIR/xceadmin/dist -- skipping xceadmin-venv install into \$TOP (run '/xce-build xceadmin' first, same build dir as $BINARY, if you need it)"
fi

if ! command -v openssl >/dev/null 2>&1; then
    echo "Error: openssl not found -- run under 'nix develop' (provides openssl on PATH)"
    exit 1
fi

IFS=',' read -ra PEER_ARRAY <<< "$PEERS"
N=${#PEER_ARRAY[@]}

# Create directory structure
mkdir -p "$TOP/certs"
mkdir -p "$TOP/logs"
mkdir -p "$TOP/scripts"
mkdir -p "$TOP/run"
for i in $(seq 1 $N); do
    mkdir -p "$TOP/controller/ctrl-$i/conf"
    mkdir -p "$TOP/controller/ctrl-$i/data"
done

# Self-signed dev/test TLS cert+key, generated fresh directly into the
# deployment dir every run -- nothing checked into git, nothing shared
# across deploys. Long-lived (10y) since it's regenerated per deploy
# anyway; TLS verification is SSL_VERIFY_NONE everywhere in this codebase
# (no hostname/identity check), so a fresh self-signed pair per run loses
# no real security property versus a long-lived shared one.
openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
    -keyout "$TOP/certs/xce.key" -out "$TOP/certs/xce.crt" \
    -subj "/CN=xce-dev-test" 2>/dev/null

# Build cluster.bin from src/configs/*.yaml via xceadmin, straight into
# this deploy's own ctrl-1/conf/ (no intermediate data-dir copy).
CLUSTER_BIN_TMP="$TOP/.cluster.bin.tmp"
"$XCEADMIN" config build "$SCRIPT_DIR/../configs" -o "$CLUSTER_BIN_TMP"
TAG=$(sha256sum "$CLUSTER_BIN_TMP" | cut -c1-12)
mv "$CLUSTER_BIN_TMP" "$TOP/controller/ctrl-1/conf/cluster.bin.$TAG"
echo "cluster.bin.$TAG" > "$TOP/controller/ctrl-1/conf/current"

# Build peers string: ctrl-1=host1:port1,ctrl-2=host2:port2,...
PEERS_STR=""
for i in "${!PEER_ARRAY[@]}"; do
    n=$((i+1))
    [ -n "$PEERS_STR" ] && PEERS_STR+=","
    PEERS_STR+="ctrl-$n=${PEER_ARRAY[$i]}"
done

# Resolve the bash actually present in THIS invocation's environment (e.g.
# the nix dev shell's bash-interactive package) instead of hardcoding
# #!/bin/bash into every generated script -- a static shebang can silently
# point at the wrong (or a missing) interpreter once these scripts leave
# the environment they were generated in, and "which bash" run right here,
# in the same shell that just built cluster.bin/certs, is the actual source
# of truth for which bash this deploy is meant to run under. Every script
# below is also always invoked explicitly as `"$BASH_BIN" script.sh ...`
# (never `./script.sh` relying on the shebang) by both this skill and
# xce-test-ha -- the shebang is belt-and-suspenders for a human running one
# directly, not load-bearing for either automated caller.
GEN_BASH=$(command -v bash)
if [ -z "$GEN_BASH" ]; then
    echo "Error: bash not found on PATH -- run under 'nix develop' (provides bash-interactive)"
    exit 1
fi

# Generate start script
cat > "$TOP/scripts/start.sh" << EOF
#!$GEN_BASH
set -e
NAME=\$1
if [ -z "\$NAME" ]; then
    echo "Usage: \$0 <ctrl-name>"
    exit 1
fi
shift
CTRL_DIR=$TOP/controller/\$NAME
if [ ! -d "\$CTRL_DIR/conf" ] || [ ! -d "\$CTRL_DIR/data" ]; then
    echo "Error: \$CTRL_DIR/conf or \$CTRL_DIR/data does not exist"
    exit 1
fi
PIDFILE=$TOP/run/\$NAME.pid
if [ -f "\$PIDFILE" ]; then
    OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
    if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
       tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "--myname \$NAME " && \\
       tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "--workdir $TOP/controller "; then
        echo "Error: \$NAME already running as PID \$OLD_PID (workdir $TOP/controller) -- stop it first (kill \$OLD_PID) before restarting"
        exit 1
    fi
fi
echo \$\$ > "\$PIDFILE"
exec $BINARY --myname \$NAME --peers "$PEERS_STR" --workdir $TOP/controller --certdir $TOP/certs --logfile $TOP/logs/\$NAME.log --vcpus 4 \$*
EOF
chmod +x "$TOP/scripts/start.sh"

# Generate cleanup script
cat > "$TOP/scripts/cleanup.sh" << EOF
#!$GEN_BASH
set -e
rm -f $TOP/logs/*.log
rm -f $TOP/run/*.pid
for i in \$(seq 1 $N); do
    rm -rf $TOP/controller/ctrl-\$i/data/*
done
for i in \$(seq 2 $N); do
    rm -rf $TOP/controller/ctrl-\$i/conf/*
done
EOF
chmod +x "$TOP/scripts/cleanup.sh"

# Generate worker start script (component-failover-adoption's --peers path):
# same $PEERS_STR the controllers themselves were just given, so a worker
# started via this script fails over across the identical candidate list.
# Which binary WORKER_BIN points at (xcecd_virtual or xcecd -- auto-detected
# or XCE_REAL_WORKER-overridden above) was decided once, at deploy time;
# XCE_WORKER_BIN can still override the path at run time (e.g. to point at a
# differently-built xcecd), but does NOT flip the sudo/preflight branch below
# -- that follows USE_REAL_WORKER as resolved for THIS deploy, since a
# run-time path swap doesn't change whether the binary needs root.
if [ -n "$WORKER_BIN" ]; then
    if [ "$USE_REAL_WORKER" = "1" ]; then
        cat > "$TOP/scripts/start-worker.sh" << EOF
#!$GEN_BASH
set -e
NAME=\${1:-real-worker}
[ \$# -gt 0 ] && shift
WORKER_BIN="\${XCE_WORKER_BIN:-$WORKER_BIN}"
mkdir -p $TOP/logs
PIDFILE=$TOP/run/\$NAME.pid
if [ -f "\$PIDFILE" ]; then
    OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
    if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
       tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "-n \$NAME "; then
        echo "Error: worker \$NAME already running as PID \$OLD_PID -- stop it first (sudo kill \$OLD_PID) before restarting"
        exit 1
    fi
fi
echo \$\$ > "\$PIDFILE"
# xcecd (production) requires root (worker/src/main.c:check_user()) and
# hardcodes db_path=/var/run/xcecd.db + domain_sock_file=/var/run/.xcecd.sock
# -- neither is a flag, so /var/run must exist before it will start.
sudo mkdir -p /var/run
exec sudo "\$WORKER_BIN" --peers "$PEERS_STR" -n "\$NAME" -l $TOP/logs \$*
EOF
    else
        cat > "$TOP/scripts/start-worker.sh" << EOF
#!$GEN_BASH
set -e
NAME=\${1:-worker-1}
[ \$# -gt 0 ] && shift
WORKER_BIN="\${XCE_WORKER_BIN:-$WORKER_BIN}"
mkdir -p $TOP/logs
PIDFILE=$TOP/run/\$NAME.pid
if [ -f "\$PIDFILE" ]; then
    OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
    if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
       tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "-n \$NAME "; then
        echo "Error: worker \$NAME already running as PID \$OLD_PID -- stop it first (kill \$OLD_PID) before restarting"
        exit 1
    fi
fi
echo \$\$ > "\$PIDFILE"
exec "\$WORKER_BIN" --peers "$PEERS_STR" -n "\$NAME" -l $TOP/logs \$*
EOF
    fi
    chmod +x "$TOP/scripts/start-worker.sh"
fi

# Generate scheduler start script -- same $PEERS_STR again. xce_scheduler is
# a cluster singleton (no --myname / identity flag; its identity is fixed
# internally to "xce_scheduler"), so this script takes no name argument.
if [ -n "$SCHED_BIN" ]; then
    cat > "$TOP/scripts/start-scheduler.sh" << EOF
#!$GEN_BASH
set -e
mkdir -p $TOP/logs
PIDFILE=$TOP/run/scheduler.pid
if [ -f "\$PIDFILE" ]; then
    OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
    if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
       tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "--logfile $TOP/logs/scheduler.log "; then
        echo "Error: scheduler already running as PID \$OLD_PID -- stop it first (kill \$OLD_PID) before restarting"
        exit 1
    fi
fi
echo \$\$ > "\$PIDFILE"
exec $SCHED_BIN --peers "$PEERS_STR" --logfile $TOP/logs/scheduler.log -v \$*
EOF
    chmod +x "$TOP/scripts/start-scheduler.sh"
fi

# Generate apiserver config + start script -- only if a pre-built apiserver
# binary was found above. This is NOT an exact mirror of the worker/
# scheduler conditional: those two just launch an existing meson-build
# binary with --peers; the apiserver additionally needs its own topDir
# (`-d`) carrying config/{app.yaml,auth.yaml,rbac.yaml,logger.yaml} before
# it will even boot (utils.PromptForTopDir/InitConf/InitLogger all read
# from "$TOPDIR/config/*.yaml"), so this block also generates that config,
# not just a launcher script.
#
# The apiserver's own topDir is $TOP/apiserver (its own config/ subdir,
# separate from the controller's $TOP/controller/ctrl-N/conf -- they must
# not collide). Its log file, however, is pointed via logger.yaml's
# outputPath at an ABSOLUTE path directly under $TOP/logs -- logger.go only
# prepends "<topdir>/logs/" to a RELATIVE outputPath, so an absolute one
# here lands the apiserver's log in the SAME shared logs/ directory
# ctrl-N.log/scheduler.log already use, matching this deploy's existing
# logs/ convention (this is also where a later verification skill greps
# for the apiserver's resource-update trace line).
if [ -n "$APISERVER_BIN" ]; then
    APISERVER_SRC_CONFIG="$SCRIPT_DIR/../frontend/apiserver/config"
    mkdir -p "$TOP/apiserver/config"

    # auth.yaml / rbac.yaml: copied verbatim from the repo's own shipped
    # dev-mode templates (config_file provider, built-in admin/basic_user
    # roles) -- functional auth/RBAC is NOT required for this deploy (no
    # REST query is ever issued against this apiserver instance), only
    # enough for auth.InitAuthProvider()/controllers.LoadRBACConfig() to
    # load successfully at boot without erroring.
    cp "$APISERVER_SRC_CONFIG/auth.yaml" "$TOP/apiserver/config/auth.yaml"
    cp "$APISERVER_SRC_CONFIG/rbac.yaml" "$TOP/apiserver/config/rbac.yaml"

    # app.yaml: copied verbatim from the shipped template (keeps every
    # required field -- e.g. file_upload.max_size, which InitConf()
    # strictly validates and errors out on if missing/malformed -- in sync
    # with whatever the repo's own default currently is), then its
    # trailing `backend:` section (the last top-level section in the
    # shipped file today) is stripped and replaced with one pointing at
    # THIS deploy's own controller cluster: the identical
    # ctrl-N=host:port candidate list every other generated script here
    # already uses, in the exact "[name=]host:port" syntax
    # RegisterToBackend's xce_parse_peers expects (backend.peers takes
    # priority over the legacy single address/port fallback -- see
    # apiserver/backend/config.go). Without this, BackendYamlConfig.Address
    # stays empty and main.go never calls RegisterToBackend at all --
    # the apiserver would boot in permanent stub mode and
    # OnResourceLocationUpdated would never fire.
    sed '/^backend:/,$d' "$APISERVER_SRC_CONFIG/app.yaml" > "$TOP/apiserver/config/app.yaml"
    APISERVER_PEERS_YAML=""
    for i in "${!PEER_ARRAY[@]}"; do
        n=$((i+1))
        APISERVER_PEERS_YAML+="    - \"ctrl-$n=${PEER_ARRAY[$i]}\"
"
    done
    # address/port are still populated (first --peers candidate), even
    # though `peers` (non-empty) is what RegisterToBackend actually uses to
    # pick a connection candidate -- confirmed live: main.go's own gate for
    # whether to call RegisterToBackend AT ALL is `BackendYamlConfig.Address
    # != ""`, checked BEFORE Peers is ever consulted (cmd/main.go:58). A
    # blank address here (even with a fully-populated non-empty peers list)
    # left the apiserver silently booting in permanent stub mode ("Backend
    # not configured") with RegisterToBackend never called -- caught during
    # this task's own live verification, not a hypothetical.
    APISERVER_FIRST_HOST="${PEER_ARRAY[0]%%:*}"
    APISERVER_FIRST_PORT="${PEER_ARRAY[0]##*:}"
    cat >> "$TOP/apiserver/config/app.yaml" << EOF
backend:
  address: "$APISERVER_FIRST_HOST"
  port: $APISERVER_FIRST_PORT
  from: "xce_apiserver"
  key: "keepsecret\$"
  peers:
$APISERVER_PEERS_YAML
EOF

    # logger.yaml: NOT copied from the shipped template -- generated fresh
    # so `logging.level: debug` is explicit and load-bearing here, rather
    # than incidentally inherited from whatever the shipped template's
    # default happens to be today. A later trace-log addition at the
    # apiserver's resource-update storage point is only ever emitted via
    # logger.Debug(...) (zap); at any less-verbose configured level it is
    # silently dropped, which would look identical to a genuine
    # propagation failure to anything polling this log.
    cat > "$TOP/apiserver/config/logger.yaml" << EOF
logging:
  level: "debug"
  encoding: console
  outputPath: "$TOP/logs/apiserver.log"
  file:
    maxSize: 100
    maxBackups: 10
    maxAge: 14
    compress: true
  encoderConfig:
    messageKey: "message"
    levelKey: "level"
    timeKey: "timestamp"
    nameKey: "logger"
    callerKey: "caller"
    functionKey: "func"
    stacktraceKey: "stacktrace"
    timeEncoder: "ISO8601"
    levelEncoder: "capital"
EOF

    cat > "$TOP/scripts/start-apiserver.sh" << EOF
#!$GEN_BASH
set -e
mkdir -p $TOP/logs
PIDFILE=$TOP/run/apiserver.pid
if [ -f "\$PIDFILE" ]; then
    OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
    if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
       tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "-d $TOP/apiserver "; then
        echo "Error: apiserver already running as PID \$OLD_PID -- stop it first (kill \$OLD_PID) before restarting"
        exit 1
    fi
fi
echo \$\$ > "\$PIDFILE"
exec $APISERVER_BIN -d $TOP/apiserver \$*
EOF
    chmod +x "$TOP/scripts/start-apiserver.sh"
fi

# Install xceadmin into THIS deployment ($TOP/xceadmin-site), not just
# pointed at $XCEADMIN's build-dir .venv resolved above -- a real, standalone
# copy, so the deployment keeps a working xceadmin long after the build dir
# it came from is rebuilt/cleaned.
#
# NOT a venv + `pip install`: a `python3 -m venv --system-site-packages`
# venv only inherits the BASE python's OWN site-packages directory -- and in
# this nix devShell, python3 itself is the bare interpreter derivation with
# an empty site-packages; click/pycapnp/PyYAML/jsonschema are each a
# SEPARATE nix store path made importable via $PYTHONPATH (verified: `nix
# develop … --command sh -c 'echo $PYTHONPATH'` lists them individually),
# not merged into python3's own site-packages tree the way a traditional
# system python + apt/dnf-installed packages would be. A --system-site-
# packages venv silently CANNOT see any of that (proven: an earlier version
# of this script's xceadmin-venv approach failed `ModuleNotFoundError:
# No module named 'click'` from the generated wrapper) -- $PYTHONPATH has to
# be captured explicitly. This mirrors meson-install.sh's own nix-packaging
# wrapper exactly (same "every path baked in below is a real /nix/store
# path" reasoning), just scoped to $TOP instead of a nix $out.
#
# `python3 -m zipfile -e` unzips the wheel (a wheel IS a zip file) straight
# into $TOP/xceadmin-site -- import-ready, no pip/venv machinery needed,
# since dependency resolution here is "capture $PYTHONPATH", not "install
# packages".
if [ -n "$XCEADMIN_WHEEL" ]; then
    echo "=== Installing xceadmin ($XCEADMIN_WHEEL) into $TOP/xceadmin-site ==="
    rm -rf "$TOP/xceadmin-site"
    mkdir -p "$TOP/xceadmin-site"
    python3 -m zipfile -e "$XCEADMIN_WHEEL" "$TOP/xceadmin-site"

    # Thin wrapper at scripts/xceadmin, same invocation convention as every
    # other generated script (`"$BASH_BIN" "$TOP/scripts/xceadmin" ...`).
    # $PYTHONPATH is expanded NOW (this script's own nix devShell
    # environment), baking the resolved nix store paths in as a literal
    # string -- not left as a `${PYTHONPATH:-}` runtime reference, which
    # would silently go back to empty/wrong if scripts/xceadmin is ever
    # invoked outside `nix develop`.
    #
    # XCEADMIN_CONTROLLER defaults every --controller-accepting xceadmin
    # subcommand (config push, scheduler record start/stop/status) to this
    # deploy's own first candidate -- xceadmin's baked-in fallback
    # (_DEFAULT_CONTROLLER = localhost:9991) has no relationship to THIS
    # deployment's actual ports (a custom --peers list, e.g. localhost:9901
    # here, silently left `./scripts/xceadmin scheduler record status`
    # connecting to the wrong port with no override given). An explicit
    # `--controller ...` on the command line still wins (click's own
    # envvar-fallback precedence, xceadmin/cli.py).
    cat > "$TOP/scripts/xceadmin" << EOF
#!$GEN_BASH
export PYTHONPATH="$TOP/xceadmin-site:$PYTHONPATH"
export XCEADMIN_CONTROLLER="${PEER_ARRAY[0]}"
exec $(command -v python3) -m xceadmin.cli "\$@"
EOF
    chmod +x "$TOP/scripts/xceadmin"
fi

# Generate start-all.sh -- ALWAYS (unlike start-worker.sh/start-scheduler.sh/
# start-apiserver.sh above, this one is not conditional on any optional
# binary's presence). It automates the exact manual recipe SKILL.md's Steps
# 4/4b/4c already document a human running by hand: start every controller
# node, wait for the same primary-election health-check Step 5 documents,
# then start whichever of start-worker.sh/start-scheduler.sh/
# start-apiserver.sh were actually generated for THIS deploy -- checked at
# the generated script's OWN runtime via [ -x ... ], not baked in as a
# deploy-time boolean, so start-all.sh is a single self-contained artifact
# that correctly no-ops a step whose binary wasn't available, with no
# regeneration needed if e.g. a stale copy of start-all.sh is run after
# manually deleting one of the optional scripts.
#
# Every component is started via `setsid nohup ... </dev/null >/dev/null
# 2>&1 & disown`, not a plain `&` -- reparents each process to init so it
# survives start-all.sh's own exit (needed since this script is meant to be
# run once and return, leaving the cluster running for later interactive
# use, exactly like this skill's own SKILL.md already recommends for
# Agent-subagent dispatch of Steps 1-5 and xce-test-ha's soak driver).
#
# The scheduler is started with -vvv specifically (not the -v this file's
# own start-scheduler.sh body already bakes in unconditionally) so its
# resulting verbosity is >= 3 (`debug3`) -- verbose_to_loglevel() caps at
# "debug3" for any count above 3, so stacking with the wrapper's own -v is
# harmless, but passing anything less than vvv here risks the trace log a
# later resource-verification feature depends on being silently never
# written at the daemon's default verbosity.
cat > "$TOP/scripts/start-all.sh" << EOF
#!$GEN_BASH
set -e

echo "=== start-all.sh: checking for already-running processes ==="
ALREADY_RUNNING=0
for i in \$(seq 1 $N); do
    PF=$TOP/run/ctrl-\$i.pid
    if [ -f "\$PF" ]; then
        PID=\$(cat "\$PF" 2>/dev/null || true)
        if [ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null; then
            echo "Error: ctrl-\$i already running as PID \$PID"
            ALREADY_RUNNING=1
        fi
    fi
done
for PF in $TOP/run/worker-1.pid $TOP/run/scheduler.pid $TOP/run/apiserver.pid; do
    if [ -f "\$PF" ]; then
        PID=\$(cat "\$PF" 2>/dev/null || true)
        if [ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null; then
            echo "Error: \$(basename "\$PF" .pid) already running as PID \$PID"
            ALREADY_RUNNING=1
        fi
    fi
done
if [ "\$ALREADY_RUNNING" -eq 1 ]; then
    echo "Refusing to start -- one or more components from a previous deploy are still running against $TOP. Stop them first (kill <pid>, or pkill -f $TOP), then re-run."
    exit 1
fi

echo "=== start-all.sh: starting controller node(s) ==="
for i in \$(seq 1 $N); do
    setsid nohup "$GEN_BASH" "$TOP/scripts/start.sh" ctrl-\$i -vvv </dev/null >/dev/null 2>&1 &
    disown
    sleep 0.5
done

echo "=== start-all.sh: waiting for controller health-check (up to 30s) ==="
HEALTHY=0
for i in \$(seq 1 30); do
    if grep -lE "CTRL_PRIMARY" $TOP/logs/ctrl-*.log >/dev/null 2>&1; then
        HEALTHY=1
        break
    fi
    sleep 1
done
if [ "\$HEALTHY" -eq 0 ]; then
    echo "Warning: no primary election observed in $TOP/logs/ctrl-*.log within 30s -- proceeding anyway (worker/scheduler/apiserver may fail to connect)"
else
    echo "Controller cluster healthy (primary elected)."
fi

if [ -x "$TOP/scripts/start-worker.sh" ]; then
    echo "=== start-all.sh: starting worker ==="
    setsid nohup "$GEN_BASH" "$TOP/scripts/start-worker.sh" worker-1 -vvv </dev/null >/dev/null 2>&1 &
    disown
fi

if [ -x "$TOP/scripts/start-scheduler.sh" ]; then
    echo "=== start-all.sh: starting scheduler (-vvv) ==="
    setsid nohup "$GEN_BASH" "$TOP/scripts/start-scheduler.sh" -vvv </dev/null >/dev/null 2>&1 &
    disown
fi

if [ -x "$TOP/scripts/start-apiserver.sh" ]; then
    echo "=== start-all.sh: starting apiserver ==="
    setsid nohup "$GEN_BASH" "$TOP/scripts/start-apiserver.sh" </dev/null >/dev/null 2>&1 &
    disown
fi

echo "=== start-all.sh: done -- cluster left running ==="
EOF
chmod +x "$TOP/scripts/start-all.sh"

echo "Deployed $N-node cluster to $TOP (tag: $TAG)"
if [ -n "$XCEADMIN_WHEEL" ]; then
    echo "xceadmin installed at $TOP/xceadmin-site -- use it via $TOP/scripts/xceadmin, e.g.:"
    echo "  $TOP/scripts/xceadmin config push $TOP/controller/ctrl-1/conf/cluster.bin.$TAG --controller ${PEER_ARRAY[0]}"
fi
echo "Run \"$GEN_BASH\" $TOP/scripts/start-all.sh to start controller(s) plus whichever of worker/scheduler/apiserver were generated for this deploy."