feat(resource-value-verification): extend xce-deploy-ha with apiserver step + start-all.sh (task 3.1)
deploy-ha.sh Step 3 gains an opt-in-by-presence apiserver step (gated on a pre-built /tmp/apiserver binary, not built by this script) that generates start-apiserver.sh with a debug-level logger.yaml, plus an always-generated start-all.sh sequencing controller-health-check-then-worker/scheduler (-vvv)/apiserver. Purely additive - callers without a built apiserver (e.g. xce-test-ha) see zero behavior change. Live-confirmed both scenarios end-to-end: no-apiserver correctly skips the step; with-apiserver reaches full backend registration with debug logging active and scheduler emitting DEBUG3 output. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
fd5ab8b20d
commit
8f542a384a
SSH key fingerprint:
SHA256:8Vf2pZ3h6rJa7EhGK+WaAQ9Mta/apZvteW6g9U/WKKk
3 changed files with 291 additions and 1 deletions
|
|
@ -96,6 +96,27 @@ if [ ! -x "$SCHED_BIN" ]; then
|
|||
SCHED_BIN=""
|
||||
fi
|
||||
|
||||
# Apiserver binary check -- same opt-in-by-presence spirit as WORKER_BIN/
|
||||
# SCHED_BIN above, but NOT a same-build-generation sibling lookup: the
|
||||
# apiserver is a SEPARATE build system (Go, `make build-server` in
|
||||
# src/frontend/Makefile), not part of the meson build dir $BINARY lives in,
|
||||
# so there is no "$BINARY_BUILD_DIR/apiserver/..." to check. Its own build
|
||||
# output always lands at a fixed path (`/tmp/apiserver`, build-server's
|
||||
# `go build -o` target) regardless of which meson profile/build dir $BINARY
|
||||
# came from. deploy-ha.sh does NOT invoke `make build-server` itself --
|
||||
# building it remains a caller prerequisite (same "run /xce-build controller
|
||||
# first if stale" convention SKILL.md's Prerequisites section already
|
||||
# documents for the controller, just a different build command). Soft-check
|
||||
# only, same reasoning as worker/scheduler: xce-test-ha never builds an
|
||||
# apiserver and calls this same script, so a missing binary must not fail
|
||||
# the deploy -- it just skips generating scripts/start-apiserver.sh, with a
|
||||
# warning.
|
||||
APISERVER_BIN="/tmp/apiserver"
|
||||
if [ ! -x "$APISERVER_BIN" ]; then
|
||||
echo "Warning: no apiserver binary found at $APISERVER_BIN -- skipping scripts/start-apiserver.sh generation (run 'cd src/frontend && make build-server' first if you need it)"
|
||||
APISERVER_BIN=""
|
||||
fi
|
||||
|
||||
# xceadmin wheel (same-build-generation sibling again): /xce-build xceadmin
|
||||
# leaves one at <builddir>/xceadmin/dist/xceadmin-*.whl (meson-build.sh).
|
||||
# Installed below into its OWN venv under $TOP, not just pointed at
|
||||
|
|
@ -241,6 +262,119 @@ EOF
|
|||
chmod +x "$TOP/scripts/start-scheduler.sh"
|
||||
fi
|
||||
|
||||
# Generate apiserver config + start script -- only if a pre-built apiserver
|
||||
# binary was found above. This is NOT an exact mirror of the worker/
|
||||
# scheduler conditional: those two just launch an existing meson-build
|
||||
# binary with --peers; the apiserver additionally needs its own topDir
|
||||
# (`-d`) carrying config/{app.yaml,auth.yaml,rbac.yaml,logger.yaml} before
|
||||
# it will even boot (utils.PromptForTopDir/InitConf/InitLogger all read
|
||||
# from "$TOPDIR/config/*.yaml"), so this block also generates that config,
|
||||
# not just a launcher script.
|
||||
#
|
||||
# The apiserver's own topDir is $TOP/apiserver (its own config/ subdir,
|
||||
# separate from the controller's $TOP/controller/ctrl-N/conf -- they must
|
||||
# not collide). Its log file, however, is pointed via logger.yaml's
|
||||
# outputPath at an ABSOLUTE path directly under $TOP/logs -- logger.go only
|
||||
# prepends "<topdir>/logs/" to a RELATIVE outputPath, so an absolute one
|
||||
# here lands the apiserver's log in the SAME shared logs/ directory
|
||||
# ctrl-N.log/scheduler.log already use, matching this deploy's existing
|
||||
# logs/ convention (this is also where a later verification skill greps
|
||||
# for the apiserver's resource-update trace line).
|
||||
if [ -n "$APISERVER_BIN" ]; then
|
||||
APISERVER_SRC_CONFIG="$SCRIPT_DIR/../frontend/apiserver/config"
|
||||
mkdir -p "$TOP/apiserver/config"
|
||||
|
||||
# auth.yaml / rbac.yaml: copied verbatim from the repo's own shipped
|
||||
# dev-mode templates (config_file provider, built-in admin/basic_user
|
||||
# roles) -- functional auth/RBAC is NOT required for this deploy (no
|
||||
# REST query is ever issued against this apiserver instance), only
|
||||
# enough for auth.InitAuthProvider()/controllers.LoadRBACConfig() to
|
||||
# load successfully at boot without erroring.
|
||||
cp "$APISERVER_SRC_CONFIG/auth.yaml" "$TOP/apiserver/config/auth.yaml"
|
||||
cp "$APISERVER_SRC_CONFIG/rbac.yaml" "$TOP/apiserver/config/rbac.yaml"
|
||||
|
||||
# app.yaml: copied verbatim from the shipped template (keeps every
|
||||
# required field -- e.g. file_upload.max_size, which InitConf()
|
||||
# strictly validates and errors out on if missing/malformed -- in sync
|
||||
# with whatever the repo's own default currently is), then its
|
||||
# trailing `backend:` section (the last top-level section in the
|
||||
# shipped file today) is stripped and replaced with one pointing at
|
||||
# THIS deploy's own controller cluster: the identical
|
||||
# ctrl-N=host:port candidate list every other generated script here
|
||||
# already uses, in the exact "[name=]host:port" syntax
|
||||
# RegisterToBackend's xce_parse_peers expects (backend.peers takes
|
||||
# priority over the legacy single address/port fallback -- see
|
||||
# apiserver/backend/config.go). Without this, BackendYamlConfig.Address
|
||||
# stays empty and main.go never calls RegisterToBackend at all --
|
||||
# the apiserver would boot in permanent stub mode and
|
||||
# OnResourceLocationUpdated would never fire.
|
||||
sed '/^backend:/,$d' "$APISERVER_SRC_CONFIG/app.yaml" > "$TOP/apiserver/config/app.yaml"
|
||||
APISERVER_PEERS_YAML=""
|
||||
for i in "${!PEER_ARRAY[@]}"; do
|
||||
n=$((i+1))
|
||||
APISERVER_PEERS_YAML+=" - \"ctrl-$n=${PEER_ARRAY[$i]}\"
|
||||
"
|
||||
done
|
||||
# address/port are still populated (first --peers candidate), even
|
||||
# though `peers` (non-empty) is what RegisterToBackend actually uses to
|
||||
# pick a connection candidate -- confirmed live: main.go's own gate for
|
||||
# whether to call RegisterToBackend AT ALL is `BackendYamlConfig.Address
|
||||
# != ""`, checked BEFORE Peers is ever consulted (cmd/main.go:58). A
|
||||
# blank address here (even with a fully-populated non-empty peers list)
|
||||
# left the apiserver silently booting in permanent stub mode ("Backend
|
||||
# not configured") with RegisterToBackend never called -- caught during
|
||||
# this task's own live verification, not a hypothetical.
|
||||
APISERVER_FIRST_HOST="${PEER_ARRAY[0]%%:*}"
|
||||
APISERVER_FIRST_PORT="${PEER_ARRAY[0]##*:}"
|
||||
cat >> "$TOP/apiserver/config/app.yaml" << EOF
|
||||
backend:
|
||||
address: "$APISERVER_FIRST_HOST"
|
||||
port: $APISERVER_FIRST_PORT
|
||||
from: "xce_apiserver"
|
||||
key: "keepsecret\$"
|
||||
peers:
|
||||
$APISERVER_PEERS_YAML
|
||||
EOF
|
||||
|
||||
# logger.yaml: NOT copied from the shipped template -- generated fresh
|
||||
# so `logging.level: debug` is explicit and load-bearing here, rather
|
||||
# than incidentally inherited from whatever the shipped template's
|
||||
# default happens to be today. A later trace-log addition at the
|
||||
# apiserver's resource-update storage point is only ever emitted via
|
||||
# logger.Debug(...) (zap); at any less-verbose configured level it is
|
||||
# silently dropped, which would look identical to a genuine
|
||||
# propagation failure to anything polling this log.
|
||||
cat > "$TOP/apiserver/config/logger.yaml" << EOF
|
||||
logging:
|
||||
level: "debug"
|
||||
encoding: console
|
||||
outputPath: "$TOP/logs/apiserver.log"
|
||||
file:
|
||||
maxSize: 100
|
||||
maxBackups: 10
|
||||
maxAge: 14
|
||||
compress: true
|
||||
encoderConfig:
|
||||
messageKey: "message"
|
||||
levelKey: "level"
|
||||
timeKey: "timestamp"
|
||||
nameKey: "logger"
|
||||
callerKey: "caller"
|
||||
functionKey: "func"
|
||||
stacktraceKey: "stacktrace"
|
||||
timeEncoder: "ISO8601"
|
||||
levelEncoder: "capital"
|
||||
EOF
|
||||
|
||||
cat > "$TOP/scripts/start-apiserver.sh" << EOF
|
||||
#!$GEN_BASH
|
||||
set -e
|
||||
mkdir -p $TOP/logs
|
||||
$APISERVER_BIN -d $TOP/apiserver \$*
|
||||
EOF
|
||||
chmod +x "$TOP/scripts/start-apiserver.sh"
|
||||
fi
|
||||
|
||||
# Install xceadmin into THIS deployment ($TOP/xceadmin-site), not just
|
||||
# pointed at $XCEADMIN's build-dir .venv resolved above -- a real, standalone
|
||||
# copy, so the deployment keeps a working xceadmin long after the build dir
|
||||
|
|
@ -297,8 +431,84 @@ EOF
|
|||
chmod +x "$TOP/scripts/xceadmin"
|
||||
fi
|
||||
|
||||
# Generate start-all.sh -- ALWAYS (unlike start-worker.sh/start-scheduler.sh/
|
||||
# start-apiserver.sh above, this one is not conditional on any optional
|
||||
# binary's presence). It automates the exact manual recipe SKILL.md's Steps
|
||||
# 4/4b/4c already document a human running by hand: start every controller
|
||||
# node, wait for the same primary-election health-check Step 5 documents,
|
||||
# then start whichever of start-worker.sh/start-scheduler.sh/
|
||||
# start-apiserver.sh were actually generated for THIS deploy -- checked at
|
||||
# the generated script's OWN runtime via [ -x ... ], not baked in as a
|
||||
# deploy-time boolean, so start-all.sh is a single self-contained artifact
|
||||
# that correctly no-ops a step whose binary wasn't available, with no
|
||||
# regeneration needed if e.g. a stale copy of start-all.sh is run after
|
||||
# manually deleting one of the optional scripts.
|
||||
#
|
||||
# Every component is started via `setsid nohup ... </dev/null >/dev/null
|
||||
# 2>&1 & disown`, not a plain `&` -- reparents each process to init so it
|
||||
# survives start-all.sh's own exit (needed since this script is meant to be
|
||||
# run once and return, leaving the cluster running for later interactive
|
||||
# use, exactly like this skill's own SKILL.md already recommends for
|
||||
# Agent-subagent dispatch of Steps 1-5 and xce-test-ha's soak driver).
|
||||
#
|
||||
# The scheduler is started with -vvv specifically (not the -v this file's
|
||||
# own start-scheduler.sh body already bakes in unconditionally) so its
|
||||
# resulting verbosity is >= 3 (`debug3`) -- verbose_to_loglevel() caps at
|
||||
# "debug3" for any count above 3, so stacking with the wrapper's own -v is
|
||||
# harmless, but passing anything less than vvv here risks the trace log a
|
||||
# later resource-verification feature depends on being silently never
|
||||
# written at the daemon's default verbosity.
|
||||
cat > "$TOP/scripts/start-all.sh" << EOF
|
||||
#!$GEN_BASH
|
||||
set -e
|
||||
|
||||
echo "=== start-all.sh: starting controller node(s) ==="
|
||||
for i in \$(seq 1 $N); do
|
||||
setsid nohup "$GEN_BASH" "$TOP/scripts/start.sh" ctrl-\$i -vvv </dev/null >/dev/null 2>&1 &
|
||||
disown
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "=== start-all.sh: waiting for controller health-check (up to 30s) ==="
|
||||
HEALTHY=0
|
||||
for i in \$(seq 1 30); do
|
||||
if grep -lE "CTRL_PRIMARY" $TOP/logs/ctrl-*.log >/dev/null 2>&1; then
|
||||
HEALTHY=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [ "\$HEALTHY" -eq 0 ]; then
|
||||
echo "Warning: no primary election observed in $TOP/logs/ctrl-*.log within 30s -- proceeding anyway (worker/scheduler/apiserver may fail to connect)"
|
||||
else
|
||||
echo "Controller cluster healthy (primary elected)."
|
||||
fi
|
||||
|
||||
if [ -x "$TOP/scripts/start-worker.sh" ]; then
|
||||
echo "=== start-all.sh: starting worker ==="
|
||||
setsid nohup "$GEN_BASH" "$TOP/scripts/start-worker.sh" worker-1 -vvv </dev/null >/dev/null 2>&1 &
|
||||
disown
|
||||
fi
|
||||
|
||||
if [ -x "$TOP/scripts/start-scheduler.sh" ]; then
|
||||
echo "=== start-all.sh: starting scheduler (-vvv) ==="
|
||||
setsid nohup "$GEN_BASH" "$TOP/scripts/start-scheduler.sh" -vvv </dev/null >/dev/null 2>&1 &
|
||||
disown
|
||||
fi
|
||||
|
||||
if [ -x "$TOP/scripts/start-apiserver.sh" ]; then
|
||||
echo "=== start-all.sh: starting apiserver ==="
|
||||
setsid nohup "$GEN_BASH" "$TOP/scripts/start-apiserver.sh" </dev/null >/dev/null 2>&1 &
|
||||
disown
|
||||
fi
|
||||
|
||||
echo "=== start-all.sh: done -- cluster left running ==="
|
||||
EOF
|
||||
chmod +x "$TOP/scripts/start-all.sh"
|
||||
|
||||
echo "Deployed $N-node cluster to $TOP (tag: $TAG)"
|
||||
if [ -n "$XCEADMIN_WHEEL" ]; then
|
||||
echo "xceadmin installed at $TOP/xceadmin-site -- use it via $TOP/scripts/xceadmin, e.g.:"
|
||||
echo " $TOP/scripts/xceadmin config push $TOP/controller/ctrl-1/conf/cluster.bin.$TAG --controller ${PEER_ARRAY[0]}"
|
||||
fi
|
||||
echo "Run \"$GEN_BASH\" $TOP/scripts/start-all.sh to start controller(s) plus whichever of worker/scheduler/apiserver were generated for this deploy."
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue