Rongsong Shen
6b10512502
Rewords deploy-ha.sh's block comment and two fallback echo lines, and run.sh's XCE-not-found error, to stop naming the now-deleted Makefile's build-server/build-cli targets, pointing at /xce-build apiserver instead. Discovery/control-flow logic in both files is unchanged (verified line-by-line: no if/elif/else lines touched); run.sh's XCE default placeholder is untouched by design. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XYdtrEWWZfKRRGED7raEYg
653 lines
30 KiB
Bash
Executable file
653 lines
30 KiB
Bash
Executable file
#!/bin/sh
|
|
set -e
|
|
|
|
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
|
|
|
BINARY=$1
|
|
TOP=$2
|
|
PEERS=$3
|
|
|
|
if [ -z "$BINARY" ] || [ -z "$TOP" ] || [ -z "$PEERS" ]; then
|
|
echo "Usage: $0 <binary> <top-dir> <peers>"
|
|
echo " peers: comma-separated host:port list (e.g. host1:port1,host2:port2)"
|
|
echo " TLS cert/key and cluster.bin are generated fresh into <top-dir> every"
|
|
echo " run (openssl self-signed cert; cluster.bin via xceadmin from"
|
|
echo " src/configs/*.yaml) -- no pre-existing data directory needed."
|
|
exit 1
|
|
fi
|
|
|
|
BINARY=$(realpath "$BINARY")
|
|
|
|
# xceadmin is meson-built now (src/xceadmin/meson.build), not a plain
|
|
# `pip install -e .` venv checked out under src/xceadmin/ -- its build
|
|
# output lives under whichever meson build dir xce-build.sh's profile
|
|
# picked, or under a nix-installed prefix's bin/, and each candidate is a
|
|
# STATIC SNAPSHOT (meson-build.sh copies src/xceadmin out-of-tree and does
|
|
# a plain, non-editable `pip install`) taken whenever that profile's
|
|
# `xceadmin` target was last built -- editing src/xceadmin/ source does
|
|
# NOT retroactively update an already-built snapshot.
|
|
#
|
|
# Correctness, not just freshness, matters here: xceadmin builds the
|
|
# cluster.bin that $BINARY (the controller daemon being deployed) will
|
|
# decode, so the two must come from the SAME build generation -- e.g. if
|
|
# the capnp wire schema changed between builds, an xceadmin from one
|
|
# generation and a controller from another can silently mis-encode/
|
|
# mis-decode a field (this is exactly how a stale-xceadmin deploy bug
|
|
# went undiagnosed: a fresher-but-DIFFERENT-build xceadmin could just as
|
|
# easily mismatch $BINARY as a staler one). Prefer the xceadmin living in
|
|
# $BINARY's OWN build dir (<builddir>/controller/xce_controller ->
|
|
# <builddir>/xceadmin/.venv/bin/xceadmin) -- guaranteed same-generation.
|
|
# Only if that specific one doesn't exist (e.g. xceadmin was never built
|
|
# in that profile) fall back to whichever other candidate is newest, with
|
|
# a loud warning that same-generation consistency isn't guaranteed.
|
|
BINARY_BUILD_DIR=$(dirname "$(dirname "$BINARY")")
|
|
BINARY_DIR=$(dirname "$BINARY")
|
|
XCEADMIN=""
|
|
if [ -x "$BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin" ]; then
|
|
# meson layout: <builddir>/controller/xce_controller + <builddir>/xceadmin/.venv/...
|
|
XCEADMIN="$BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin"
|
|
echo "Using xceadmin: $XCEADMIN (same build dir as $BINARY)"
|
|
elif [ -x "$BINARY_DIR/xceadmin" ]; then
|
|
# nix result layout: result/bin/xce_controller + result/bin/xceadmin (flat, no
|
|
# per-component subdir) -- BINARY_BUILD_DIR's two-levels-up guess doesn't apply.
|
|
XCEADMIN="$BINARY_DIR/xceadmin"
|
|
echo "Using xceadmin: $XCEADMIN (same directory as $BINARY)"
|
|
else
|
|
echo "Warning: no xceadmin found alongside $BINARY (checked $BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin and $BINARY_DIR/xceadmin) -- falling back to the newest xceadmin found elsewhere, which is NOT guaranteed to be the same build generation as $BINARY."
|
|
XCEADMIN_MTIME=0
|
|
for candidate in \
|
|
"$SCRIPT_DIR/../xce.build/xceadmin/.venv/bin/xceadmin" \
|
|
"$SCRIPT_DIR/../build.release/xceadmin/.venv/bin/xceadmin" \
|
|
"$SCRIPT_DIR/../build.dbg/xceadmin/.venv/bin/xceadmin" \
|
|
"$SCRIPT_DIR/../result/bin/xceadmin" \
|
|
"$(command -v xceadmin 2>/dev/null || true)"
|
|
do
|
|
if [ -n "$candidate" ] && [ -x "$candidate" ]; then
|
|
candidate_mtime=$(stat -c %Y "$candidate" 2>/dev/null || stat -f %m "$candidate" 2>/dev/null || echo 0)
|
|
if [ "$candidate_mtime" -gt "$XCEADMIN_MTIME" ]; then
|
|
XCEADMIN="$candidate"
|
|
XCEADMIN_MTIME="$candidate_mtime"
|
|
fi
|
|
fi
|
|
done
|
|
if [ -n "$XCEADMIN" ]; then
|
|
echo "Using xceadmin: $XCEADMIN (built $(date -d "@$XCEADMIN_MTIME" 2>/dev/null || date -r "$XCEADMIN_MTIME" 2>/dev/null))"
|
|
fi
|
|
fi
|
|
if [ -z "$XCEADMIN" ]; then
|
|
echo "Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first"
|
|
exit 1
|
|
fi
|
|
|
|
# Worker/scheduler binaries, same-build-generation sibling lookup as
|
|
# xceadmin above (<builddir>/worker/xcecd_virtual, <builddir>/scheduler/
|
|
# xce_scheduler next to <builddir>/controller/xce_controller). Soft-check
|
|
# only: xce-test-ha's soak driver also calls this script and never builds
|
|
# either, so a missing worker/scheduler binary must not fail the deploy --
|
|
# it just skips generating that one start script, with a warning.
|
|
#
|
|
# Which of the two sibling worker binaries this deploy's start-worker.sh
|
|
# targets is auto-detected by default, overridable with XCE_REAL_WORKER=0/1:
|
|
# xcecd_virtual: skips the root-only sqlite db-recovery and domain-socket
|
|
# subsystems, runs as an ordinary user, no sudo needed.
|
|
# xcecd (the production daemon): worker/src/main.c's check_user()
|
|
# unconditionally requires geteuid()==0, so the generated
|
|
# start-worker.sh below wraps its exec in `sudo` and preflights
|
|
# `sudo mkdir -p /var/run` (xcecd hardcodes db_path=/var/run/xcecd.db
|
|
# and domain_sock_file=/var/run/.xcecd.sock -- neither is a flag).
|
|
# Also, actually dispatching a job goes through sd_bus_open_system() ->
|
|
# org.freedesktop.systemd1.Manager.StartTransientUnit
|
|
# (worker/src/xcecd_cgroups.c) -- it needs a real systemd manager
|
|
# answering on the system dbus, not just root.
|
|
#
|
|
# Auto-detection: /run/systemd/system existing is systemd's own sd_booted(3)
|
|
# signal for "a real systemd instance is running as this host's manager" --
|
|
# true on an actual host/VM, false in sandboxes that boot under something
|
|
# else (verified 2026-08-19: this dev sandbox has no /run at all, PID 1 is
|
|
# s6-svscan, no dbus-daemon/systemctl even under `nix develop`). Real job
|
|
# dispatch needs that systemd manager reachable regardless of root, so
|
|
# picking xcecd_virtual when it's absent -- and xcecd when it's present --
|
|
# means this one deploy-ha.sh works unmodified on both a throwaway sandbox
|
|
# and a real HA test host. XCE_REAL_WORKER=0/1 overrides the auto-detect in
|
|
# either direction (e.g. force xcecd_virtual on a real host, or force xcecd
|
|
# on a sandbox to exercise its root-gated startup path even though job
|
|
# dispatch there will still fail).
|
|
if [ -n "${XCE_REAL_WORKER:-}" ]; then
|
|
USE_REAL_WORKER="$XCE_REAL_WORKER"
|
|
elif [ -d /run/systemd/system ]; then
|
|
USE_REAL_WORKER=1
|
|
else
|
|
USE_REAL_WORKER=0
|
|
fi
|
|
if [ "$USE_REAL_WORKER" = "1" ]; then
|
|
WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd"
|
|
else
|
|
WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd_virtual"
|
|
fi
|
|
echo "Worker binary: $([ "$USE_REAL_WORKER" = "1" ] && echo "xcecd (real, root via sudo)" || echo "xcecd_virtual (no root)") -- override with XCE_REAL_WORKER=0/1"
|
|
if [ ! -x "$WORKER_BIN" ]; then
|
|
echo "Warning: no $(basename "$WORKER_BIN") found at $WORKER_BIN -- skipping scripts/start-worker.sh generation (run '/xce-build worker' first if you need it)"
|
|
WORKER_BIN=""
|
|
fi
|
|
SCHED_BIN="$BINARY_BUILD_DIR/scheduler/xce_scheduler"
|
|
if [ ! -x "$SCHED_BIN" ]; then
|
|
echo "Warning: no xce_scheduler found at $SCHED_BIN -- skipping scripts/start-scheduler.sh generation (run '/xce-build scheduler' first if you need it)"
|
|
SCHED_BIN=""
|
|
fi
|
|
|
|
# Apiserver binary check -- same opt-in-by-presence spirit as WORKER_BIN/
|
|
# SCHED_BIN above. Prefer the meson-built binary, via `tools/go_build`
|
|
# (`/xce-build apiserver` or `/xce-build all`) -- lands in
|
|
# $BINARY_BUILD_DIR like every other component this script resolves, so
|
|
# it IS a same-build-generation sibling lookup, same as
|
|
# xceadmin/worker/scheduler above. Fall back to a binary already present
|
|
# at the fixed legacy path `/tmp/apiserver` if the preferred location is
|
|
# empty, with a caveat echoed: that path is NOT guaranteed to be the same
|
|
# build generation as $BINARY.
|
|
# deploy-ha.sh does NOT invoke either build itself -- building remains a
|
|
# caller prerequisite (same "run /xce-build controller first if stale"
|
|
# convention SKILL.md's Prerequisites section documents for the controller).
|
|
# Soft-check only, same reasoning as worker/scheduler: xce-test-ha never
|
|
# builds an apiserver and calls this same script, so a missing binary must
|
|
# not fail the deploy -- it just skips generating scripts/start-apiserver.sh,
|
|
# with a warning.
|
|
APISERVER_BIN=""
|
|
if [ -x "$BINARY_BUILD_DIR/frontend/apiserver/xce_apiserver" ]; then
|
|
APISERVER_BIN="$BINARY_BUILD_DIR/frontend/apiserver/xce_apiserver"
|
|
echo "Using apiserver: $APISERVER_BIN (same build dir as $BINARY)"
|
|
elif [ -x "/tmp/apiserver" ]; then
|
|
APISERVER_BIN="/tmp/apiserver"
|
|
echo "Using apiserver: $APISERVER_BIN (legacy /tmp/apiserver path; NOT guaranteed to be the same build generation as $BINARY)"
|
|
else
|
|
echo "Warning: no apiserver binary found at $BINARY_BUILD_DIR/frontend/apiserver/xce_apiserver or /tmp/apiserver -- skipping scripts/start-apiserver.sh generation (run '/xce-build apiserver' first if you need it)"
|
|
fi
|
|
|
|
# xceadmin wheel (same-build-generation sibling again): /xce-build xceadmin
|
|
# leaves one at <builddir>/xceadmin/dist/xceadmin-*.whl (meson-build.sh).
|
|
# Installed below into its OWN venv under $TOP, not just pointed at
|
|
# $XCEADMIN's build-dir .venv above -- a live deployment can outlive the
|
|
# build dir (a later `/xce-build xceadmin` run rm -rf's and rebuilds that
|
|
# .venv out from under anything still depending on it), so this deploy gets
|
|
# a genuinely self-contained copy. Soft-skip (warn), same reasoning as
|
|
# worker/scheduler above: a missing wheel must not fail the deploy.
|
|
XCEADMIN_WHEEL=""
|
|
if [ -d "$BINARY_BUILD_DIR/xceadmin/dist" ]; then
|
|
XCEADMIN_WHEEL=$(ls "$BINARY_BUILD_DIR/xceadmin/dist"/xceadmin-*.whl 2>/dev/null | head -1 || true)
|
|
fi
|
|
if [ -z "$XCEADMIN_WHEEL" ]; then
|
|
echo "Warning: no xceadmin wheel found under $BINARY_BUILD_DIR/xceadmin/dist -- skipping xceadmin-venv install into \$TOP (run '/xce-build xceadmin' first, same build dir as $BINARY, if you need it)"
|
|
fi
|
|
|
|
if ! command -v openssl >/dev/null 2>&1; then
|
|
echo "Error: openssl not found -- run under 'nix develop' (provides openssl on PATH)"
|
|
exit 1
|
|
fi
|
|
|
|
IFS=',' read -ra PEER_ARRAY <<< "$PEERS"
|
|
N=${#PEER_ARRAY[@]}
|
|
|
|
# Create directory structure
|
|
mkdir -p "$TOP/certs"
|
|
mkdir -p "$TOP/logs"
|
|
mkdir -p "$TOP/scripts"
|
|
mkdir -p "$TOP/run"
|
|
for i in $(seq 1 $N); do
|
|
mkdir -p "$TOP/controller/ctrl-$i/conf"
|
|
mkdir -p "$TOP/controller/ctrl-$i/data"
|
|
done
|
|
|
|
# Self-signed dev/test TLS cert+key, generated fresh directly into the
|
|
# deployment dir every run -- nothing checked into git, nothing shared
|
|
# across deploys. Long-lived (10y) since it's regenerated per deploy
|
|
# anyway; TLS verification is SSL_VERIFY_NONE everywhere in this codebase
|
|
# (no hostname/identity check), so a fresh self-signed pair per run loses
|
|
# no real security property versus a long-lived shared one.
|
|
openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
|
|
-keyout "$TOP/certs/xce.key" -out "$TOP/certs/xce.crt" \
|
|
-subj "/CN=xce-dev-test" 2>/dev/null
|
|
|
|
# Build cluster.bin from src/configs/*.yaml via xceadmin, straight into
|
|
# this deploy's own ctrl-1/conf/ (no intermediate data-dir copy).
|
|
CLUSTER_BIN_TMP="$TOP/.cluster.bin.tmp"
|
|
"$XCEADMIN" config build "$SCRIPT_DIR/../configs" -o "$CLUSTER_BIN_TMP"
|
|
TAG=$(sha256sum "$CLUSTER_BIN_TMP" | cut -c1-12)
|
|
mv "$CLUSTER_BIN_TMP" "$TOP/controller/ctrl-1/conf/cluster.bin.$TAG"
|
|
echo "cluster.bin.$TAG" > "$TOP/controller/ctrl-1/conf/current"
|
|
|
|
# Build peers string: ctrl-1=host1:port1,ctrl-2=host2:port2,...
|
|
PEERS_STR=""
|
|
for i in "${!PEER_ARRAY[@]}"; do
|
|
n=$((i+1))
|
|
[ -n "$PEERS_STR" ] && PEERS_STR+=","
|
|
PEERS_STR+="ctrl-$n=${PEER_ARRAY[$i]}"
|
|
done
|
|
|
|
# Resolve the bash actually present in THIS invocation's environment (e.g.
|
|
# the nix dev shell's bash-interactive package) instead of hardcoding
|
|
# #!/bin/bash into every generated script -- a static shebang can silently
|
|
# point at the wrong (or a missing) interpreter once these scripts leave
|
|
# the environment they were generated in, and "which bash" run right here,
|
|
# in the same shell that just built cluster.bin/certs, is the actual source
|
|
# of truth for which bash this deploy is meant to run under. Every script
|
|
# below is also always invoked explicitly as `"$BASH_BIN" script.sh ...`
|
|
# (never `./script.sh` relying on the shebang) by both this skill and
|
|
# xce-test-ha -- the shebang is belt-and-suspenders for a human running one
|
|
# directly, not load-bearing for either automated caller.
|
|
GEN_BASH=$(command -v bash)
|
|
if [ -z "$GEN_BASH" ]; then
|
|
echo "Error: bash not found on PATH -- run under 'nix develop' (provides bash-interactive)"
|
|
exit 1
|
|
fi
|
|
|
|
# Generate start script
|
|
cat > "$TOP/scripts/start.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
NAME=\$1
|
|
if [ -z "\$NAME" ]; then
|
|
echo "Usage: \$0 <ctrl-name>"
|
|
exit 1
|
|
fi
|
|
shift
|
|
CTRL_DIR=$TOP/controller/\$NAME
|
|
if [ ! -d "\$CTRL_DIR/conf" ] || [ ! -d "\$CTRL_DIR/data" ]; then
|
|
echo "Error: \$CTRL_DIR/conf or \$CTRL_DIR/data does not exist"
|
|
exit 1
|
|
fi
|
|
PIDFILE=$TOP/run/\$NAME.pid
|
|
if [ -f "\$PIDFILE" ]; then
|
|
OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
|
|
if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
|
|
tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "--myname \$NAME " && \\
|
|
tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "--workdir $TOP/controller "; then
|
|
echo "Error: \$NAME already running as PID \$OLD_PID (workdir $TOP/controller) -- stop it first (kill \$OLD_PID) before restarting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo \$\$ > "\$PIDFILE"
|
|
exec $BINARY --myname \$NAME --peers "$PEERS_STR" --workdir $TOP/controller --certdir $TOP/certs --logfile $TOP/logs/\$NAME.log --vcpus 4 \$*
|
|
EOF
|
|
chmod +x "$TOP/scripts/start.sh"
|
|
|
|
# Generate cleanup script
|
|
cat > "$TOP/scripts/cleanup.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
rm -f $TOP/logs/*.log
|
|
rm -f $TOP/run/*.pid
|
|
for i in \$(seq 1 $N); do
|
|
rm -rf $TOP/controller/ctrl-\$i/data/*
|
|
done
|
|
for i in \$(seq 2 $N); do
|
|
rm -rf $TOP/controller/ctrl-\$i/conf/*
|
|
done
|
|
EOF
|
|
chmod +x "$TOP/scripts/cleanup.sh"
|
|
|
|
# Generate worker start script (component-failover-adoption's --peers path):
|
|
# same $PEERS_STR the controllers themselves were just given, so a worker
|
|
# started via this script fails over across the identical candidate list.
|
|
# Which binary WORKER_BIN points at (xcecd_virtual or xcecd -- auto-detected
|
|
# or XCE_REAL_WORKER-overridden above) was decided once, at deploy time;
|
|
# XCE_WORKER_BIN can still override the path at run time (e.g. to point at a
|
|
# differently-built xcecd), but does NOT flip the sudo/preflight branch below
|
|
# -- that follows USE_REAL_WORKER as resolved for THIS deploy, since a
|
|
# run-time path swap doesn't change whether the binary needs root.
|
|
if [ -n "$WORKER_BIN" ]; then
|
|
if [ "$USE_REAL_WORKER" = "1" ]; then
|
|
cat > "$TOP/scripts/start-worker.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
NAME=\${1:-real-worker}
|
|
[ \$# -gt 0 ] && shift
|
|
WORKER_BIN="\${XCE_WORKER_BIN:-$WORKER_BIN}"
|
|
mkdir -p $TOP/logs
|
|
PIDFILE=$TOP/run/\$NAME.pid
|
|
if [ -f "\$PIDFILE" ]; then
|
|
OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
|
|
if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
|
|
tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "-n \$NAME "; then
|
|
echo "Error: worker \$NAME already running as PID \$OLD_PID -- stop it first (sudo kill \$OLD_PID) before restarting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo \$\$ > "\$PIDFILE"
|
|
# xcecd (production) requires root (worker/src/main.c:check_user()) and
|
|
# hardcodes db_path=/var/run/xcecd.db + domain_sock_file=/var/run/.xcecd.sock
|
|
# -- neither is a flag, so /var/run must exist before it will start.
|
|
sudo mkdir -p /var/run
|
|
exec sudo "\$WORKER_BIN" --peers "$PEERS_STR" -n "\$NAME" -l $TOP/logs \$*
|
|
EOF
|
|
else
|
|
cat > "$TOP/scripts/start-worker.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
NAME=\${1:-worker-1}
|
|
[ \$# -gt 0 ] && shift
|
|
WORKER_BIN="\${XCE_WORKER_BIN:-$WORKER_BIN}"
|
|
mkdir -p $TOP/logs
|
|
PIDFILE=$TOP/run/\$NAME.pid
|
|
if [ -f "\$PIDFILE" ]; then
|
|
OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
|
|
if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
|
|
tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "-n \$NAME "; then
|
|
echo "Error: worker \$NAME already running as PID \$OLD_PID -- stop it first (kill \$OLD_PID) before restarting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo \$\$ > "\$PIDFILE"
|
|
exec "\$WORKER_BIN" --peers "$PEERS_STR" -n "\$NAME" -l $TOP/logs \$*
|
|
EOF
|
|
fi
|
|
chmod +x "$TOP/scripts/start-worker.sh"
|
|
fi
|
|
|
|
# Generate scheduler start script -- same $PEERS_STR again. xce_scheduler is
|
|
# a cluster singleton (no --myname / identity flag; its identity is fixed
|
|
# internally to "xce_scheduler"), so this script takes no name argument.
|
|
if [ -n "$SCHED_BIN" ]; then
|
|
cat > "$TOP/scripts/start-scheduler.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
mkdir -p $TOP/logs
|
|
PIDFILE=$TOP/run/scheduler.pid
|
|
if [ -f "\$PIDFILE" ]; then
|
|
OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
|
|
if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
|
|
tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "--logfile $TOP/logs/scheduler.log "; then
|
|
echo "Error: scheduler already running as PID \$OLD_PID -- stop it first (kill \$OLD_PID) before restarting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo \$\$ > "\$PIDFILE"
|
|
exec $SCHED_BIN --peers "$PEERS_STR" --logfile $TOP/logs/scheduler.log -v \$*
|
|
EOF
|
|
chmod +x "$TOP/scripts/start-scheduler.sh"
|
|
fi
|
|
|
|
# Generate apiserver config + start script -- only if a pre-built apiserver
|
|
# binary was found above. This is NOT an exact mirror of the worker/
|
|
# scheduler conditional: those two just launch an existing meson-build
|
|
# binary with --peers; the apiserver additionally needs its own topDir
|
|
# (`-d`) carrying config/{app.yaml,auth.yaml,rbac.yaml,logger.yaml} before
|
|
# it will even boot (utils.PromptForTopDir/InitConf/InitLogger all read
|
|
# from "$TOPDIR/config/*.yaml"), so this block also generates that config,
|
|
# not just a launcher script.
|
|
#
|
|
# The apiserver's own topDir is $TOP/apiserver (its own config/ subdir,
|
|
# separate from the controller's $TOP/controller/ctrl-N/conf -- they must
|
|
# not collide). Its log file, however, is pointed via logger.yaml's
|
|
# outputPath at an ABSOLUTE path directly under $TOP/logs -- logger.go only
|
|
# prepends "<topdir>/logs/" to a RELATIVE outputPath, so an absolute one
|
|
# here lands the apiserver's log in the SAME shared logs/ directory
|
|
# ctrl-N.log/scheduler.log already use, matching this deploy's existing
|
|
# logs/ convention (this is also where a later verification skill greps
|
|
# for the apiserver's resource-update trace line).
|
|
if [ -n "$APISERVER_BIN" ]; then
|
|
APISERVER_SRC_CONFIG="$SCRIPT_DIR/../frontend/apiserver/config"
|
|
mkdir -p "$TOP/apiserver/config"
|
|
|
|
# auth.yaml / rbac.yaml: copied verbatim from the repo's own shipped
|
|
# dev-mode templates (config_file provider, built-in admin/basic_user
|
|
# roles) -- functional auth/RBAC is NOT required for this deploy (no
|
|
# REST query is ever issued against this apiserver instance), only
|
|
# enough for auth.InitAuthProvider()/controllers.LoadRBACConfig() to
|
|
# load successfully at boot without erroring.
|
|
cp "$APISERVER_SRC_CONFIG/auth.yaml" "$TOP/apiserver/config/auth.yaml"
|
|
cp "$APISERVER_SRC_CONFIG/rbac.yaml" "$TOP/apiserver/config/rbac.yaml"
|
|
|
|
# app.yaml: copied verbatim from the shipped template (keeps every
|
|
# required field -- e.g. file_upload.max_size, which InitConf()
|
|
# strictly validates and errors out on if missing/malformed -- in sync
|
|
# with whatever the repo's own default currently is), then its
|
|
# trailing `backend:` section (the last top-level section in the
|
|
# shipped file today) is stripped and replaced with one pointing at
|
|
# THIS deploy's own controller cluster: the identical
|
|
# ctrl-N=host:port candidate list every other generated script here
|
|
# already uses, in the exact "[name=]host:port" syntax
|
|
# RegisterToBackend's xce_parse_peers expects (backend.peers takes
|
|
# priority over the legacy single address/port fallback -- see
|
|
# apiserver/backend/config.go). Without this, BackendYamlConfig.Address
|
|
# stays empty and main.go never calls RegisterToBackend at all --
|
|
# the apiserver would boot in permanent stub mode and
|
|
# OnResourceLocationUpdated would never fire.
|
|
sed '/^backend:/,$d' "$APISERVER_SRC_CONFIG/app.yaml" > "$TOP/apiserver/config/app.yaml"
|
|
APISERVER_PEERS_YAML=""
|
|
for i in "${!PEER_ARRAY[@]}"; do
|
|
n=$((i+1))
|
|
APISERVER_PEERS_YAML+=" - \"ctrl-$n=${PEER_ARRAY[$i]}\"
|
|
"
|
|
done
|
|
# address/port are still populated (first --peers candidate), even
|
|
# though `peers` (non-empty) is what RegisterToBackend actually uses to
|
|
# pick a connection candidate -- confirmed live: main.go's own gate for
|
|
# whether to call RegisterToBackend AT ALL is `BackendYamlConfig.Address
|
|
# != ""`, checked BEFORE Peers is ever consulted (cmd/main.go:58). A
|
|
# blank address here (even with a fully-populated non-empty peers list)
|
|
# left the apiserver silently booting in permanent stub mode ("Backend
|
|
# not configured") with RegisterToBackend never called -- caught during
|
|
# this task's own live verification, not a hypothetical.
|
|
APISERVER_FIRST_HOST="${PEER_ARRAY[0]%%:*}"
|
|
APISERVER_FIRST_PORT="${PEER_ARRAY[0]##*:}"
|
|
cat >> "$TOP/apiserver/config/app.yaml" << EOF
|
|
backend:
|
|
address: "$APISERVER_FIRST_HOST"
|
|
port: $APISERVER_FIRST_PORT
|
|
from: "xce_apiserver"
|
|
key: "keepsecret\$"
|
|
peers:
|
|
$APISERVER_PEERS_YAML
|
|
EOF
|
|
|
|
# logger.yaml: NOT copied from the shipped template -- generated fresh
|
|
# so `logging.level: debug` is explicit and load-bearing here, rather
|
|
# than incidentally inherited from whatever the shipped template's
|
|
# default happens to be today. A later trace-log addition at the
|
|
# apiserver's resource-update storage point is only ever emitted via
|
|
# logger.Debug(...) (zap); at any less-verbose configured level it is
|
|
# silently dropped, which would look identical to a genuine
|
|
# propagation failure to anything polling this log.
|
|
cat > "$TOP/apiserver/config/logger.yaml" << EOF
|
|
logging:
|
|
level: "debug"
|
|
encoding: console
|
|
outputPath: "$TOP/logs/apiserver.log"
|
|
file:
|
|
maxSize: 100
|
|
maxBackups: 10
|
|
maxAge: 14
|
|
compress: true
|
|
encoderConfig:
|
|
messageKey: "message"
|
|
levelKey: "level"
|
|
timeKey: "timestamp"
|
|
nameKey: "logger"
|
|
callerKey: "caller"
|
|
functionKey: "func"
|
|
stacktraceKey: "stacktrace"
|
|
timeEncoder: "ISO8601"
|
|
levelEncoder: "capital"
|
|
EOF
|
|
|
|
cat > "$TOP/scripts/start-apiserver.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
mkdir -p $TOP/logs
|
|
PIDFILE=$TOP/run/apiserver.pid
|
|
if [ -f "\$PIDFILE" ]; then
|
|
OLD_PID=\$(cat "\$PIDFILE" 2>/dev/null || true)
|
|
if [ -n "\$OLD_PID" ] && kill -0 "\$OLD_PID" 2>/dev/null && \\
|
|
tr '\0' ' ' < /proc/\$OLD_PID/cmdline 2>/dev/null | grep -q -- "-d $TOP/apiserver "; then
|
|
echo "Error: apiserver already running as PID \$OLD_PID -- stop it first (kill \$OLD_PID) before restarting"
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo \$\$ > "\$PIDFILE"
|
|
exec $APISERVER_BIN -d $TOP/apiserver \$*
|
|
EOF
|
|
chmod +x "$TOP/scripts/start-apiserver.sh"
|
|
fi
|
|
|
|
# Install xceadmin into THIS deployment ($TOP/xceadmin-site), not just
|
|
# pointed at $XCEADMIN's build-dir .venv resolved above -- a real, standalone
|
|
# copy, so the deployment keeps a working xceadmin long after the build dir
|
|
# it came from is rebuilt/cleaned.
|
|
#
|
|
# NOT a venv + `pip install`: a `python3 -m venv --system-site-packages`
|
|
# venv only inherits the BASE python's OWN site-packages directory -- and in
|
|
# this nix devShell, python3 itself is the bare interpreter derivation with
|
|
# an empty site-packages; click/pycapnp/PyYAML/jsonschema are each a
|
|
# SEPARATE nix store path made importable via $PYTHONPATH (verified: `nix
|
|
# develop … --command sh -c 'echo $PYTHONPATH'` lists them individually),
|
|
# not merged into python3's own site-packages tree the way a traditional
|
|
# system python + apt/dnf-installed packages would be. A --system-site-
|
|
# packages venv silently CANNOT see any of that (proven: an earlier version
|
|
# of this script's xceadmin-venv approach failed `ModuleNotFoundError:
|
|
# No module named 'click'` from the generated wrapper) -- $PYTHONPATH has to
|
|
# be captured explicitly. This mirrors meson-install.sh's own nix-packaging
|
|
# wrapper exactly (same "every path baked in below is a real /nix/store
|
|
# path" reasoning), just scoped to $TOP instead of a nix $out.
|
|
#
|
|
# `python3 -m zipfile -e` unzips the wheel (a wheel IS a zip file) straight
|
|
# into $TOP/xceadmin-site -- import-ready, no pip/venv machinery needed,
|
|
# since dependency resolution here is "capture $PYTHONPATH", not "install
|
|
# packages".
|
|
if [ -n "$XCEADMIN_WHEEL" ]; then
|
|
echo "=== Installing xceadmin ($XCEADMIN_WHEEL) into $TOP/xceadmin-site ==="
|
|
rm -rf "$TOP/xceadmin-site"
|
|
mkdir -p "$TOP/xceadmin-site"
|
|
python3 -m zipfile -e "$XCEADMIN_WHEEL" "$TOP/xceadmin-site"
|
|
|
|
# Thin wrapper at scripts/xceadmin, same invocation convention as every
|
|
# other generated script (`"$BASH_BIN" "$TOP/scripts/xceadmin" ...`).
|
|
# $PYTHONPATH is expanded NOW (this script's own nix devShell
|
|
# environment), baking the resolved nix store paths in as a literal
|
|
# string -- not left as a `${PYTHONPATH:-}` runtime reference, which
|
|
# would silently go back to empty/wrong if scripts/xceadmin is ever
|
|
# invoked outside `nix develop`.
|
|
#
|
|
# XCEADMIN_CONTROLLER defaults every --controller-accepting xceadmin
|
|
# subcommand (config push, scheduler record start/stop/status) to this
|
|
# deploy's own first candidate -- xceadmin's baked-in fallback
|
|
# (_DEFAULT_CONTROLLER = localhost:9991) has no relationship to THIS
|
|
# deployment's actual ports (a custom --peers list, e.g. localhost:9901
|
|
# here, silently left `./scripts/xceadmin scheduler record status`
|
|
# connecting to the wrong port with no override given). An explicit
|
|
# `--controller ...` on the command line still wins (click's own
|
|
# envvar-fallback precedence, xceadmin/cli.py).
|
|
cat > "$TOP/scripts/xceadmin" << EOF
|
|
#!$GEN_BASH
|
|
export PYTHONPATH="$TOP/xceadmin-site:$PYTHONPATH"
|
|
export XCEADMIN_CONTROLLER="${PEER_ARRAY[0]}"
|
|
exec $(command -v python3) -m xceadmin.cli "\$@"
|
|
EOF
|
|
chmod +x "$TOP/scripts/xceadmin"
|
|
fi
|
|
|
|
# Generate start-all.sh -- ALWAYS (unlike start-worker.sh/start-scheduler.sh/
|
|
# start-apiserver.sh above, this one is not conditional on any optional
|
|
# binary's presence). It automates the exact manual recipe SKILL.md's Steps
|
|
# 4/4b/4c already document a human running by hand: start every controller
|
|
# node, wait for the same primary-election health-check Step 5 documents,
|
|
# then start whichever of start-worker.sh/start-scheduler.sh/
|
|
# start-apiserver.sh were actually generated for THIS deploy -- checked at
|
|
# the generated script's OWN runtime via [ -x ... ], not baked in as a
|
|
# deploy-time boolean, so start-all.sh is a single self-contained artifact
|
|
# that correctly no-ops a step whose binary wasn't available, with no
|
|
# regeneration needed if e.g. a stale copy of start-all.sh is run after
|
|
# manually deleting one of the optional scripts.
|
|
#
|
|
# Every component is started via `setsid nohup ... </dev/null >/dev/null
|
|
# 2>&1 & disown`, not a plain `&` -- reparents each process to init so it
|
|
# survives start-all.sh's own exit (needed since this script is meant to be
|
|
# run once and return, leaving the cluster running for later interactive
|
|
# use, exactly like this skill's own SKILL.md already recommends for
|
|
# Agent-subagent dispatch of Steps 1-5 and xce-test-ha's soak driver).
|
|
#
|
|
# The scheduler is started with -vvv specifically (not the -v this file's
|
|
# own start-scheduler.sh body already bakes in unconditionally) so its
|
|
# resulting verbosity is >= 3 (`debug3`) -- verbose_to_loglevel() caps at
|
|
# "debug3" for any count above 3, so stacking with the wrapper's own -v is
|
|
# harmless, but passing anything less than vvv here risks the trace log a
|
|
# later resource-verification feature depends on being silently never
|
|
# written at the daemon's default verbosity.
|
|
cat > "$TOP/scripts/start-all.sh" << EOF
|
|
#!$GEN_BASH
|
|
set -e
|
|
|
|
echo "=== start-all.sh: checking for already-running processes ==="
|
|
ALREADY_RUNNING=0
|
|
for i in \$(seq 1 $N); do
|
|
PF=$TOP/run/ctrl-\$i.pid
|
|
if [ -f "\$PF" ]; then
|
|
PID=\$(cat "\$PF" 2>/dev/null || true)
|
|
if [ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null; then
|
|
echo "Error: ctrl-\$i already running as PID \$PID"
|
|
ALREADY_RUNNING=1
|
|
fi
|
|
fi
|
|
done
|
|
for PF in $TOP/run/worker-1.pid $TOP/run/scheduler.pid $TOP/run/apiserver.pid; do
|
|
if [ -f "\$PF" ]; then
|
|
PID=\$(cat "\$PF" 2>/dev/null || true)
|
|
if [ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null; then
|
|
echo "Error: \$(basename "\$PF" .pid) already running as PID \$PID"
|
|
ALREADY_RUNNING=1
|
|
fi
|
|
fi
|
|
done
|
|
if [ "\$ALREADY_RUNNING" -eq 1 ]; then
|
|
echo "Refusing to start -- one or more components from a previous deploy are still running against $TOP. Stop them first (kill <pid>, or pkill -f $TOP), then re-run."
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== start-all.sh: starting controller node(s) ==="
|
|
for i in \$(seq 1 $N); do
|
|
setsid nohup "$GEN_BASH" "$TOP/scripts/start.sh" ctrl-\$i -vvv </dev/null >/dev/null 2>&1 &
|
|
disown
|
|
sleep 0.5
|
|
done
|
|
|
|
echo "=== start-all.sh: waiting for controller health-check (up to 30s) ==="
|
|
HEALTHY=0
|
|
for i in \$(seq 1 30); do
|
|
if grep -lE "CTRL_PRIMARY" $TOP/logs/ctrl-*.log >/dev/null 2>&1; then
|
|
HEALTHY=1
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
if [ "\$HEALTHY" -eq 0 ]; then
|
|
echo "Warning: no primary election observed in $TOP/logs/ctrl-*.log within 30s -- proceeding anyway (worker/scheduler/apiserver may fail to connect)"
|
|
else
|
|
echo "Controller cluster healthy (primary elected)."
|
|
fi
|
|
|
|
if [ -x "$TOP/scripts/start-worker.sh" ]; then
|
|
echo "=== start-all.sh: starting worker ==="
|
|
setsid nohup "$GEN_BASH" "$TOP/scripts/start-worker.sh" worker-1 -vvv </dev/null >/dev/null 2>&1 &
|
|
disown
|
|
fi
|
|
|
|
if [ -x "$TOP/scripts/start-scheduler.sh" ]; then
|
|
echo "=== start-all.sh: starting scheduler (-vvv) ==="
|
|
setsid nohup "$GEN_BASH" "$TOP/scripts/start-scheduler.sh" -vvv </dev/null >/dev/null 2>&1 &
|
|
disown
|
|
fi
|
|
|
|
if [ -x "$TOP/scripts/start-apiserver.sh" ]; then
|
|
echo "=== start-all.sh: starting apiserver ==="
|
|
setsid nohup "$GEN_BASH" "$TOP/scripts/start-apiserver.sh" </dev/null >/dev/null 2>&1 &
|
|
disown
|
|
fi
|
|
|
|
echo "=== start-all.sh: done -- cluster left running ==="
|
|
EOF
|
|
chmod +x "$TOP/scripts/start-all.sh"
|
|
|
|
echo "Deployed $N-node cluster to $TOP (tag: $TAG)"
|
|
if [ -n "$XCEADMIN_WHEEL" ]; then
|
|
echo "xceadmin installed at $TOP/xceadmin-site -- use it via $TOP/scripts/xceadmin, e.g.:"
|
|
echo " $TOP/scripts/xceadmin config push $TOP/controller/ctrl-1/conf/cluster.bin.$TAG --controller ${PEER_ARRAY[0]}"
|
|
fi
|
|
echo "Run \"$GEN_BASH\" $TOP/scripts/start-all.sh to start controller(s) plus whichever of worker/scheduler/apiserver were generated for this deploy."
|