feat(controller-peers-env-fallback): create the regression script skeleton (task 1.3)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
d4858e1fe9
commit
241ed3a142
GPG key ID:
80533536B0EB651C
2 changed files with 363 additions and 1 deletions
356
src/tools/verify-controller-peers-env.sh
Executable file
356
src/tools/verify-controller-peers-env.sh
Executable file
|
|
@ -0,0 +1,356 @@
|
|||
#!/usr/bin/env bash
|
||||
# Helpers and the *_BIN variables are used by the cases later tasks add (and
|
||||
# on_exit via trap), which shellcheck cannot see while the case list is empty.
|
||||
# shellcheck disable=SC2329,SC2034
|
||||
# verify-controller-peers-env.sh -- controller-peers-env-fallback regression
|
||||
# script (.kiro/specs/controller-peers-env-fallback, design.md "Testing
|
||||
# Strategy -> Scripted Regression Verification").
|
||||
#
|
||||
# Drives the real built binaries (controller, scheduler, xcecd_virtual worker,
|
||||
# xce_test harness) and asserts on their startup log / settings-dump / stdout
|
||||
# output only. No live cluster is needed or wanted: resolving the controller
|
||||
# candidate list happens, and is observable, before any connection attempt.
|
||||
#
|
||||
# Usage:
|
||||
# src/tools/verify-controller-peers-env.sh [--keep] [--only REGEX] [--list] [BUILD_DIR]
|
||||
#
|
||||
# BUILD_DIR build directory holding all four binaries (default:
|
||||
# $XCE_BUILD_DIR, else src/xce.build -- where
|
||||
# `bash src/tools/xce-build.sh <component> --incremental` builds).
|
||||
# Build every component into the SAME directory with
|
||||
# `/xce-build <component> --incremental`; without --incremental a
|
||||
# component build wipes the directory, removing the others.
|
||||
# --keep keep the scratch root afterwards (also: XCE_VERIFY_KEEP=1).
|
||||
# --only RE run only the cases whose function name matches RE (grep -E).
|
||||
# --list print the registered cases and exit.
|
||||
#
|
||||
# Exit status: 0 all cases passed (or none registered); 1 a case failed;
|
||||
# 2 usage error or pre-flight refusal (local cluster up, binary missing).
|
||||
#
|
||||
# Adding a case (later tasks):
|
||||
# case_controller_env_list() {
|
||||
# local d; d="$(new_scratch controller-env-list)"
|
||||
# run_to_completion "$d" 10 XCE_CONTROLLER_PEERS=a=localhost:1,... \
|
||||
# "$CONTROLLER_BIN" --myname a --workdir "$d/work" ...
|
||||
# assert_grep "$d/controller.log" 'HA mode: 3 controller candidates' || return 1
|
||||
# }
|
||||
# add_case case_controller_env_list "controller: env list used with no --peers"
|
||||
#
|
||||
# A case is a function that returns 0 on pass and non-zero on fail. It runs in
|
||||
# the main shell (not a subshell) inside an `if`, so `set -e` does NOT abort on
|
||||
# a failing command inside it: every check must `|| return 1` explicitly (the
|
||||
# assert_* helpers print why). Background processes a case starts with
|
||||
# start_bg are stopped after the case even if it forgets to.
|
||||
#
|
||||
# Isolation: every daemon run gets its own scratch directory (new_scratch);
|
||||
# the controller must always be given --workdir/--certdir/--logfile inside it,
|
||||
# the scheduler --logfile and the worker -l. Two things a scratch directory
|
||||
# cannot isolate -- the default controller address localhost:9990 that the
|
||||
# scheduler/worker fall back to, and the worker's hardcoded socket
|
||||
# /var/run/.xcecd.sock -- are checked by the pre-flight, which refuses to run
|
||||
# when a local cluster is up.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
PEER_PORT=9990
|
||||
WORKER_SOCK=/var/run/.xcecd.sock
|
||||
|
||||
# ── output helpers (same style as the other src/tools/test-*.sh scripts) ────
|
||||
green() { printf '\033[32m%s\033[0m\n' "$*"; }
|
||||
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
||||
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
|
||||
log() { echo "[verify-peers-env] $*"; }
|
||||
|
||||
usage() {
|
||||
sed -n '/^# Usage:/,/^# Exit status/p' "$0" | sed 's/^# \{0,1\}//' | sed '$d'
|
||||
}
|
||||
|
||||
# ── arguments ───────────────────────────────────────────────────────────────
|
||||
BUILD_DIR="${XCE_BUILD_DIR:-$SRC_DIR/xce.build}"
|
||||
KEEP="${XCE_VERIFY_KEEP:-0}"
|
||||
ONLY=""
|
||||
LIST_ONLY=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--keep) KEEP=1 ;;
|
||||
--only)
|
||||
[ $# -ge 2 ] || { red "--only needs a regex" >&2; exit 2; }
|
||||
ONLY="$2"; shift ;;
|
||||
--list) LIST_ONLY=1 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
-*) red "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
||||
*) BUILD_DIR="$1" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# The variable under test must never leak in from the invoking shell: every
|
||||
# case sets it explicitly (or deliberately leaves it unset).
|
||||
unset XCE_CONTROLLER_PEERS
|
||||
|
||||
# ── binaries ────────────────────────────────────────────────────────────────
|
||||
CONTROLLER_BIN=""
|
||||
SCHEDULER_BIN=""
|
||||
WORKER_BIN=""
|
||||
HARNESS_BIN=""
|
||||
|
||||
# component:relative-path:variable
|
||||
BINARIES=(
|
||||
"controller:controller/xce_controller:CONTROLLER_BIN"
|
||||
"scheduler:scheduler/xce_scheduler:SCHEDULER_BIN"
|
||||
"worker:worker/xcecd_virtual:WORKER_BIN"
|
||||
"harness:harness/xce_test:HARNESS_BIN"
|
||||
)
|
||||
|
||||
# ── case registry ───────────────────────────────────────────────────────────
|
||||
CASES=() # function names, in registration order
|
||||
CASE_DESC=() # parallel array of one-line descriptions
|
||||
PASSED=()
|
||||
FAILED=()
|
||||
|
||||
add_case() { # add_case <function> [description]
|
||||
declare -F "$1" >/dev/null || { red "add_case: no such function: $1" >&2; exit 2; }
|
||||
CASES+=("$1")
|
||||
CASE_DESC+=("${2:-$1}")
|
||||
}
|
||||
|
||||
# ── scratch directories & process tracking ─────────────────────────────────
|
||||
SCRATCH_ROOT=""
|
||||
STARTED_PIDS=() # PIDs started by start_bg and not yet stopped
|
||||
LAST_PID=""
|
||||
RUN_RC=0
|
||||
|
||||
# new_scratch <name>: print the path of a fresh, empty directory for one run.
|
||||
new_scratch() {
|
||||
mktemp -d "$SCRATCH_ROOT/${1:-run}.XXXXXX"
|
||||
}
|
||||
|
||||
_forget_pid() {
|
||||
local p keep=()
|
||||
for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do
|
||||
[ "$p" = "$1" ] || keep+=("$p")
|
||||
done
|
||||
STARTED_PIDS=("${keep[@]+"${keep[@]}"}")
|
||||
}
|
||||
|
||||
# start_bg <dir> [env-args...] <binary> [args...]
|
||||
# Start a process in the background via env(1), so leading NAME=value
|
||||
# assignments and `-u NAME` set/unset its environment. stdout/stderr go to
|
||||
# <dir>/stdout and <dir>/stderr; stdin is /dev/null so the process can never
|
||||
# eat the script's input. Sets LAST_PID (do not call it in $(...)).
|
||||
start_bg() {
|
||||
local dir="$1"; shift
|
||||
env "$@" </dev/null >"$dir/stdout" 2>"$dir/stderr" &
|
||||
LAST_PID=$!
|
||||
STARTED_PIDS+=("$LAST_PID")
|
||||
echo "$LAST_PID" >"$dir/pid"
|
||||
}
|
||||
|
||||
# stop_proc <pid>: SIGTERM, then SIGKILL after ~3s, and reap it.
|
||||
stop_proc() {
|
||||
local pid="$1" i
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -TERM "$pid" 2>/dev/null || true
|
||||
for i in $(seq 1 30); do
|
||||
kill -0 "$pid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
wait "$pid" 2>/dev/null || true
|
||||
_forget_pid "$pid"
|
||||
}
|
||||
|
||||
stop_all() {
|
||||
local p
|
||||
for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do
|
||||
stop_proc "$p"
|
||||
done
|
||||
STARTED_PIDS=()
|
||||
}
|
||||
|
||||
# run_to_completion <dir> <timeout-seconds> [env-args...] <binary> [args...]
|
||||
# Run in the foreground with a timeout (SIGKILL 2s after the
|
||||
# SIGTERM). stdout/stderr go to <dir>/stdout, <dir>/stderr; the exit code is
|
||||
# left in RUN_RC and <dir>/rc (timed out: 124 with coreutils timeout, 143
|
||||
# with busybox timeout, 137 if it had to be SIGKILLed). `-k` rather than
|
||||
# `--kill-after` because busybox timeout only accepts the short form.
|
||||
run_to_completion() {
|
||||
local dir="$1" secs="$2"; shift 2
|
||||
RUN_RC=0
|
||||
timeout -k 2 "$secs" env "$@" </dev/null \
|
||||
>"$dir/stdout" 2>"$dir/stderr" || RUN_RC=$?
|
||||
echo "$RUN_RC" >"$dir/rc"
|
||||
}
|
||||
|
||||
# wait_for_line <file> <regex> <timeout-seconds> [pid]
|
||||
# Poll until a line matching <regex> (grep -E) appears in <file>. Returns 0
|
||||
# when found, 1 on timeout -- or as soon as [pid] has exited without the line
|
||||
# having appeared (after one last look at the file).
|
||||
wait_for_line() {
|
||||
local file="$1" re="$2" secs="$3" pid="${4:-}" deadline
|
||||
deadline=$(( $(date +%s) + secs ))
|
||||
while :; do
|
||||
if [ -f "$file" ] && grep -Eq -- "$re" "$file"; then
|
||||
return 0
|
||||
fi
|
||||
if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
|
||||
[ -f "$file" ] && grep -Eq -- "$re" "$file" && return 0
|
||||
return 1
|
||||
fi
|
||||
[ "$(date +%s)" -lt "$deadline" ] || return 1
|
||||
sleep 0.1
|
||||
done
|
||||
}
|
||||
|
||||
# ── assertions (print the reason on failure; use as `assert_x ... || return 1`)
|
||||
assert_grep() { # assert_grep <file> <regex> [what]
|
||||
if [ -f "$1" ] && grep -Eq -- "$2" "$1"; then return 0; fi
|
||||
red " expected /$2/ in ${3:-$1}"
|
||||
_show_tail "$1"
|
||||
return 1
|
||||
}
|
||||
|
||||
assert_no_grep() { # assert_no_grep <file> <regex> [what]
|
||||
if [ ! -f "$1" ] || ! grep -Eq -- "$2" "$1"; then return 0; fi
|
||||
red " unexpected /$2/ in ${3:-$1}:"
|
||||
grep -E -- "$2" "$1" | head -5 | sed 's/^/ /'
|
||||
return 1
|
||||
}
|
||||
|
||||
assert_rc() { # assert_rc <expected|nonzero> [actual=$RUN_RC]
|
||||
local want="$1" got="${2:-$RUN_RC}"
|
||||
if [ "$want" = nonzero ]; then
|
||||
[ "$got" -ne 0 ] && return 0
|
||||
elif [ "$got" -eq "$want" ]; then
|
||||
return 0
|
||||
fi
|
||||
red " expected exit status $want, got $got"
|
||||
return 1
|
||||
}
|
||||
|
||||
_show_tail() {
|
||||
[ -f "$1" ] || { red " ($1 does not exist)"; return 0; }
|
||||
echo " --- last lines of $1 ---"
|
||||
tail -n 15 "$1" | sed 's/^/ /'
|
||||
}
|
||||
|
||||
# ── pre-flight ──────────────────────────────────────────────────────────────
|
||||
port_listening() { # port_listening <port>: 0 if something listens on it
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -tln 2>/dev/null | awk -v p=":$1\$" '$4 ~ p {f=1} END{exit !f}'
|
||||
else
|
||||
(exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
preflight() {
|
||||
local problems=0 entry comp rel var path
|
||||
|
||||
if [ ! -d "$BUILD_DIR" ]; then
|
||||
red "pre-flight: build directory $BUILD_DIR does not exist"
|
||||
problems=1
|
||||
fi
|
||||
for entry in "${BINARIES[@]}"; do
|
||||
IFS=: read -r comp rel var <<<"$entry"
|
||||
path="$BUILD_DIR/$rel"
|
||||
if [ -x "$path" ]; then
|
||||
printf -v "$var" '%s' "$path"
|
||||
else
|
||||
red "pre-flight: missing binary $path (component: $comp) -- build it with: /xce-build $comp --incremental"
|
||||
problems=1
|
||||
fi
|
||||
done
|
||||
|
||||
if port_listening "$PEER_PORT"; then
|
||||
red "pre-flight: something is listening on port $PEER_PORT (a local controller?); the scheduler/worker cases would talk to it."
|
||||
ss -tlnp 2>/dev/null | awk -v p=":$PEER_PORT\$" '$4 ~ p' | sed 's/^/ /' || true
|
||||
problems=1
|
||||
fi
|
||||
if [ -e "$WORKER_SOCK" ]; then
|
||||
red "pre-flight: worker socket $WORKER_SOCK exists (a local worker running?); the worker cases would collide with it."
|
||||
problems=1
|
||||
fi
|
||||
|
||||
if [ "$problems" -ne 0 ]; then
|
||||
red "pre-flight failed: refusing to run. Stop the local cluster and/or build the missing component(s) into $BUILD_DIR, then re-run."
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
# ── cleanup ─────────────────────────────────────────────────────────────────
|
||||
on_exit() {
|
||||
local rc=$?
|
||||
stop_all
|
||||
if [ -n "$SCRATCH_ROOT" ] && [ -d "$SCRATCH_ROOT" ]; then
|
||||
if [ "$KEEP" = 1 ]; then
|
||||
yellow "scratch root kept: $SCRATCH_ROOT"
|
||||
else
|
||||
rm -rf "$SCRATCH_ROOT"
|
||||
fi
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# Cases. Later tasks (2.2-3.3, 5.1) define case_* functions here and register
|
||||
# them with add_case, in the order they should run.
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# main
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
main() {
|
||||
local i fn
|
||||
|
||||
if [ "$LIST_ONLY" = 1 ]; then
|
||||
for i in "${!CASES[@]}"; do
|
||||
printf '%s\t%s\n' "${CASES[$i]}" "${CASE_DESC[$i]}"
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
preflight
|
||||
|
||||
trap on_exit EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
SCRATCH_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/verify-peers-env.XXXXXX")"
|
||||
|
||||
log "build dir: $BUILD_DIR"
|
||||
log "scratch root: $SCRATCH_ROOT"
|
||||
log "${#CASES[@]} case(s) registered${ONLY:+, filtered by /$ONLY/}"
|
||||
|
||||
for i in "${!CASES[@]}"; do
|
||||
fn="${CASES[$i]}"
|
||||
if [ -n "$ONLY" ] && ! grep -Eq -- "$ONLY" <<<"$fn"; then
|
||||
continue
|
||||
fi
|
||||
echo "--- $fn: ${CASE_DESC[$i]}"
|
||||
if "$fn"; then
|
||||
green "PASS $fn"
|
||||
PASSED+=("$fn")
|
||||
else
|
||||
red "FAIL $fn"
|
||||
FAILED+=("$fn")
|
||||
fi
|
||||
stop_all
|
||||
done
|
||||
|
||||
echo
|
||||
log "summary: ${#PASSED[@]} passed, ${#FAILED[@]} failed"
|
||||
for fn in "${FAILED[@]+"${FAILED[@]}"}"; do
|
||||
red " FAILED: $fn"
|
||||
done
|
||||
if [ "${#FAILED[@]}" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
green "ALL PASS"
|
||||
exit 0
|
||||
}
|
||||
|
||||
main
|
||||
Loading…
Add table
Add a link
Reference in a new issue