diff --git a/.kiro/specs/controller-peers-env-fallback/tasks.md b/.kiro/specs/controller-peers-env-fallback/tasks.md index f17b3180..2420b571 100644 --- a/.kiro/specs/controller-peers-env-fallback/tasks.md +++ b/.kiro/specs/controller-peers-env-fallback/tasks.md @@ -14,7 +14,7 @@ - Done when the libxce suite passes with all seven cases - _Requirements: 1.6_ -- [ ] 1.3 Create the regression script skeleton +- [x] 1.3 Create the regression script skeleton - Test infrastructure for the scripted cases in later tasks; it satisfies no acceptance criterion by itself - Create the repeatable regression script that later tasks add their cases to: take the build directory as an input and locate the built binaries in it, create a fresh scratch directory per daemon run, and provide helpers that run a binary with a given environment and arguments, wait for a line in a log file or captured stdout/stderr with a timeout, kill the process, and record pass/fail - Pre-flight: fail with a message, rather than run, when a local cluster is up (something listening on port 9990, or the worker's fixed socket present) or a needed binary is missing @@ -126,3 +126,9 @@ - Done when both skill documents carry the notes at the listed locations - _Requirements: 3.2, 3.3_ - _Boundary: skill docs_ + +## Implementation Notes +- (1.3) `verify-controller-peers-env.sh` unsets `XCE_CONTROLLER_PEERS` at startup; every case sets it explicitly. Exit 2 = usage/pre-flight refusal, 1 = a case failed. BusyBox `timeout` is in use here: a timed-out run exits 124 (coreutils) or 143 (BusyBox) — accept either. +- (1.3 → 3.3) `start_bg`/`run_to_completion` feed stdin from `/dev/null`, but `xce_test` reads its script only from stdin; add an optional stdin file to `run_to_completion` (e.g. `RUN_STDIN`) rather than the `sh -c 'exec "$0" --norc <"$1"'` workaround. +- (1.3 → 3.2) The worker's early "HA cluster config"/"No --peers flag" lines go to stdout before its log file is set up; confirm they are not stdio-buffered when stdout is a file (else `wait_for_line` on stdout times out while it runs) — wait for exit or use line buffering if so. +- (1.3) Drop the file-wide `shellcheck disable=SC2329,SC2034` once cases call the helpers, so real unused-variable warnings surface. diff --git a/src/tools/verify-controller-peers-env.sh b/src/tools/verify-controller-peers-env.sh new file mode 100755 index 00000000..b7add110 --- /dev/null +++ b/src/tools/verify-controller-peers-env.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash +# Helpers and the *_BIN variables are used by the cases later tasks add (and +# on_exit via trap), which shellcheck cannot see while the case list is empty. +# shellcheck disable=SC2329,SC2034 +# verify-controller-peers-env.sh -- controller-peers-env-fallback regression +# script (.kiro/specs/controller-peers-env-fallback, design.md "Testing +# Strategy -> Scripted Regression Verification"). +# +# Drives the real built binaries (controller, scheduler, xcecd_virtual worker, +# xce_test harness) and asserts on their startup log / settings-dump / stdout +# output only. No live cluster is needed or wanted: resolving the controller +# candidate list happens, and is observable, before any connection attempt. +# +# Usage: +# src/tools/verify-controller-peers-env.sh [--keep] [--only REGEX] [--list] [BUILD_DIR] +# +# BUILD_DIR build directory holding all four binaries (default: +# $XCE_BUILD_DIR, else src/xce.build -- where +# `bash src/tools/xce-build.sh --incremental` builds). +# Build every component into the SAME directory with +# `/xce-build --incremental`; without --incremental a +# component build wipes the directory, removing the others. +# --keep keep the scratch root afterwards (also: XCE_VERIFY_KEEP=1). +# --only RE run only the cases whose function name matches RE (grep -E). +# --list print the registered cases and exit. +# +# Exit status: 0 all cases passed (or none registered); 1 a case failed; +# 2 usage error or pre-flight refusal (local cluster up, binary missing). +# +# Adding a case (later tasks): +# case_controller_env_list() { +# local d; d="$(new_scratch controller-env-list)" +# run_to_completion "$d" 10 XCE_CONTROLLER_PEERS=a=localhost:1,... \ +# "$CONTROLLER_BIN" --myname a --workdir "$d/work" ... +# assert_grep "$d/controller.log" 'HA mode: 3 controller candidates' || return 1 +# } +# add_case case_controller_env_list "controller: env list used with no --peers" +# +# A case is a function that returns 0 on pass and non-zero on fail. It runs in +# the main shell (not a subshell) inside an `if`, so `set -e` does NOT abort on +# a failing command inside it: every check must `|| return 1` explicitly (the +# assert_* helpers print why). Background processes a case starts with +# start_bg are stopped after the case even if it forgets to. +# +# Isolation: every daemon run gets its own scratch directory (new_scratch); +# the controller must always be given --workdir/--certdir/--logfile inside it, +# the scheduler --logfile and the worker -l. Two things a scratch directory +# cannot isolate -- the default controller address localhost:9990 that the +# scheduler/worker fall back to, and the worker's hardcoded socket +# /var/run/.xcecd.sock -- are checked by the pre-flight, which refuses to run +# when a local cluster is up. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +PEER_PORT=9990 +WORKER_SOCK=/var/run/.xcecd.sock + +# ── output helpers (same style as the other src/tools/test-*.sh scripts) ──── +green() { printf '\033[32m%s\033[0m\n' "$*"; } +red() { printf '\033[31m%s\033[0m\n' "$*"; } +yellow() { printf '\033[33m%s\033[0m\n' "$*"; } +log() { echo "[verify-peers-env] $*"; } + +usage() { + sed -n '/^# Usage:/,/^# Exit status/p' "$0" | sed 's/^# \{0,1\}//' | sed '$d' +} + +# ── arguments ─────────────────────────────────────────────────────────────── +BUILD_DIR="${XCE_BUILD_DIR:-$SRC_DIR/xce.build}" +KEEP="${XCE_VERIFY_KEEP:-0}" +ONLY="" +LIST_ONLY=0 +while [ $# -gt 0 ]; do + case "$1" in + --keep) KEEP=1 ;; + --only) + [ $# -ge 2 ] || { red "--only needs a regex" >&2; exit 2; } + ONLY="$2"; shift ;; + --list) LIST_ONLY=1 ;; + -h|--help) usage; exit 0 ;; + -*) red "Unknown option: $1" >&2; usage >&2; exit 2 ;; + *) BUILD_DIR="$1" ;; + esac + shift +done + +# The variable under test must never leak in from the invoking shell: every +# case sets it explicitly (or deliberately leaves it unset). +unset XCE_CONTROLLER_PEERS + +# ── binaries ──────────────────────────────────────────────────────────────── +CONTROLLER_BIN="" +SCHEDULER_BIN="" +WORKER_BIN="" +HARNESS_BIN="" + +# component:relative-path:variable +BINARIES=( + "controller:controller/xce_controller:CONTROLLER_BIN" + "scheduler:scheduler/xce_scheduler:SCHEDULER_BIN" + "worker:worker/xcecd_virtual:WORKER_BIN" + "harness:harness/xce_test:HARNESS_BIN" +) + +# ── case registry ─────────────────────────────────────────────────────────── +CASES=() # function names, in registration order +CASE_DESC=() # parallel array of one-line descriptions +PASSED=() +FAILED=() + +add_case() { # add_case [description] + declare -F "$1" >/dev/null || { red "add_case: no such function: $1" >&2; exit 2; } + CASES+=("$1") + CASE_DESC+=("${2:-$1}") +} + +# ── scratch directories & process tracking ───────────────────────────────── +SCRATCH_ROOT="" +STARTED_PIDS=() # PIDs started by start_bg and not yet stopped +LAST_PID="" +RUN_RC=0 + +# new_scratch : print the path of a fresh, empty directory for one run. +new_scratch() { + mktemp -d "$SCRATCH_ROOT/${1:-run}.XXXXXX" +} + +_forget_pid() { + local p keep=() + for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do + [ "$p" = "$1" ] || keep+=("$p") + done + STARTED_PIDS=("${keep[@]+"${keep[@]}"}") +} + +# start_bg [env-args...] [args...] +# Start a process in the background via env(1), so leading NAME=value +# assignments and `-u NAME` set/unset its environment. stdout/stderr go to +# /stdout and /stderr; stdin is /dev/null so the process can never +# eat the script's input. Sets LAST_PID (do not call it in $(...)). +start_bg() { + local dir="$1"; shift + env "$@" "$dir/stdout" 2>"$dir/stderr" & + LAST_PID=$! + STARTED_PIDS+=("$LAST_PID") + echo "$LAST_PID" >"$dir/pid" +} + +# stop_proc : SIGTERM, then SIGKILL after ~3s, and reap it. +stop_proc() { + local pid="$1" i + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + for i in $(seq 1 30); do + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + done + kill -KILL "$pid" 2>/dev/null || true + fi + wait "$pid" 2>/dev/null || true + _forget_pid "$pid" +} + +stop_all() { + local p + for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do + stop_proc "$p" + done + STARTED_PIDS=() +} + +# run_to_completion [env-args...] [args...] +# Run in the foreground with a timeout (SIGKILL 2s after the +# SIGTERM). stdout/stderr go to /stdout, /stderr; the exit code is +# left in RUN_RC and /rc (timed out: 124 with coreutils timeout, 143 +# with busybox timeout, 137 if it had to be SIGKILLed). `-k` rather than +# `--kill-after` because busybox timeout only accepts the short form. +run_to_completion() { + local dir="$1" secs="$2"; shift 2 + RUN_RC=0 + timeout -k 2 "$secs" env "$@" "$dir/stdout" 2>"$dir/stderr" || RUN_RC=$? + echo "$RUN_RC" >"$dir/rc" +} + +# wait_for_line [pid] +# Poll until a line matching (grep -E) appears in . Returns 0 +# when found, 1 on timeout -- or as soon as [pid] has exited without the line +# having appeared (after one last look at the file). +wait_for_line() { + local file="$1" re="$2" secs="$3" pid="${4:-}" deadline + deadline=$(( $(date +%s) + secs )) + while :; do + if [ -f "$file" ] && grep -Eq -- "$re" "$file"; then + return 0 + fi + if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then + [ -f "$file" ] && grep -Eq -- "$re" "$file" && return 0 + return 1 + fi + [ "$(date +%s)" -lt "$deadline" ] || return 1 + sleep 0.1 + done +} + +# ── assertions (print the reason on failure; use as `assert_x ... || return 1`) +assert_grep() { # assert_grep [what] + if [ -f "$1" ] && grep -Eq -- "$2" "$1"; then return 0; fi + red " expected /$2/ in ${3:-$1}" + _show_tail "$1" + return 1 +} + +assert_no_grep() { # assert_no_grep [what] + if [ ! -f "$1" ] || ! grep -Eq -- "$2" "$1"; then return 0; fi + red " unexpected /$2/ in ${3:-$1}:" + grep -E -- "$2" "$1" | head -5 | sed 's/^/ /' + return 1 +} + +assert_rc() { # assert_rc [actual=$RUN_RC] + local want="$1" got="${2:-$RUN_RC}" + if [ "$want" = nonzero ]; then + [ "$got" -ne 0 ] && return 0 + elif [ "$got" -eq "$want" ]; then + return 0 + fi + red " expected exit status $want, got $got" + return 1 +} + +_show_tail() { + [ -f "$1" ] || { red " ($1 does not exist)"; return 0; } + echo " --- last lines of $1 ---" + tail -n 15 "$1" | sed 's/^/ /' +} + +# ── pre-flight ────────────────────────────────────────────────────────────── +port_listening() { # port_listening : 0 if something listens on it + if command -v ss >/dev/null 2>&1; then + ss -tln 2>/dev/null | awk -v p=":$1\$" '$4 ~ p {f=1} END{exit !f}' + else + (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null + fi +} + +preflight() { + local problems=0 entry comp rel var path + + if [ ! -d "$BUILD_DIR" ]; then + red "pre-flight: build directory $BUILD_DIR does not exist" + problems=1 + fi + for entry in "${BINARIES[@]}"; do + IFS=: read -r comp rel var <<<"$entry" + path="$BUILD_DIR/$rel" + if [ -x "$path" ]; then + printf -v "$var" '%s' "$path" + else + red "pre-flight: missing binary $path (component: $comp) -- build it with: /xce-build $comp --incremental" + problems=1 + fi + done + + if port_listening "$PEER_PORT"; then + red "pre-flight: something is listening on port $PEER_PORT (a local controller?); the scheduler/worker cases would talk to it." + ss -tlnp 2>/dev/null | awk -v p=":$PEER_PORT\$" '$4 ~ p' | sed 's/^/ /' || true + problems=1 + fi + if [ -e "$WORKER_SOCK" ]; then + red "pre-flight: worker socket $WORKER_SOCK exists (a local worker running?); the worker cases would collide with it." + problems=1 + fi + + if [ "$problems" -ne 0 ]; then + red "pre-flight failed: refusing to run. Stop the local cluster and/or build the missing component(s) into $BUILD_DIR, then re-run." + exit 2 + fi +} + +# ── cleanup ───────────────────────────────────────────────────────────────── +on_exit() { + local rc=$? + stop_all + if [ -n "$SCRATCH_ROOT" ] && [ -d "$SCRATCH_ROOT" ]; then + if [ "$KEEP" = 1 ]; then + yellow "scratch root kept: $SCRATCH_ROOT" + else + rm -rf "$SCRATCH_ROOT" + fi + fi + exit "$rc" +} + +# ════════════════════════════════════════════════════════════════════════════ +# Cases. Later tasks (2.2-3.3, 5.1) define case_* functions here and register +# them with add_case, in the order they should run. +# ════════════════════════════════════════════════════════════════════════════ + + +# ════════════════════════════════════════════════════════════════════════════ +# main +# ════════════════════════════════════════════════════════════════════════════ +main() { + local i fn + + if [ "$LIST_ONLY" = 1 ]; then + for i in "${!CASES[@]}"; do + printf '%s\t%s\n' "${CASES[$i]}" "${CASE_DESC[$i]}" + done + exit 0 + fi + + preflight + + trap on_exit EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + SCRATCH_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/verify-peers-env.XXXXXX")" + + log "build dir: $BUILD_DIR" + log "scratch root: $SCRATCH_ROOT" + log "${#CASES[@]} case(s) registered${ONLY:+, filtered by /$ONLY/}" + + for i in "${!CASES[@]}"; do + fn="${CASES[$i]}" + if [ -n "$ONLY" ] && ! grep -Eq -- "$ONLY" <<<"$fn"; then + continue + fi + echo "--- $fn: ${CASE_DESC[$i]}" + if "$fn"; then + green "PASS $fn" + PASSED+=("$fn") + else + red "FAIL $fn" + FAILED+=("$fn") + fi + stop_all + done + + echo + log "summary: ${#PASSED[@]} passed, ${#FAILED[@]} failed" + for fn in "${FAILED[@]+"${FAILED[@]}"}"; do + red " FAILED: $fn" + done + if [ "${#FAILED[@]}" -ne 0 ]; then + exit 1 + fi + green "ALL PASS" + exit 0 +} + +main