853 lines
36 KiB
Bash
Executable file
853 lines
36 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# main() runs every case through the registry ("$fn"), an indirect call that
|
|
# the linter cannot follow, so it reports each case_* function -- and every
|
|
# helper only cases call -- as never invoked (SC2329).
|
|
# shellcheck disable=SC2329
|
|
# verify-controller-peers-env.sh -- controller-peers-env-fallback regression
|
|
# script (.kiro/specs/controller-peers-env-fallback, design.md "Testing
|
|
# Strategy -> Scripted Regression Verification").
|
|
#
|
|
# Drives the real built binaries (controller, scheduler, xcecd_virtual worker,
|
|
# xce_test harness) and asserts on their startup log / settings-dump / stdout
|
|
# output only. No live cluster is needed or wanted: resolving the controller
|
|
# candidate list happens, and is observable, before any connection attempt.
|
|
#
|
|
# Usage:
|
|
# src/tools/verify-controller-peers-env.sh [--keep] [--only REGEX] [--list] [BUILD_DIR]
|
|
#
|
|
# BUILD_DIR build directory holding all four binaries (default:
|
|
# $XCE_BUILD_DIR, else src/xce.build -- where
|
|
# `bash src/tools/xce-build.sh <component> --incremental` builds).
|
|
# Build every component into the SAME directory with
|
|
# `/xce-build <component> --incremental`; without --incremental a
|
|
# component build wipes the directory, removing the others.
|
|
# --keep keep the scratch root afterwards (also: XCE_VERIFY_KEEP=1).
|
|
# --only RE run only the cases whose function name matches RE (grep -E).
|
|
# --list print the registered cases and exit.
|
|
#
|
|
# Exit status: 0 all cases passed (or none registered); 1 a case failed;
|
|
# 2 usage error or pre-flight refusal (local cluster up, binary missing).
|
|
#
|
|
# Adding a case (later tasks):
|
|
# case_controller_env_list() {
|
|
# local d; d="$(new_scratch controller-env-list)"
|
|
# run_to_completion "$d" 10 XCE_CONTROLLER_PEERS=a=localhost:1,... \
|
|
# "$CONTROLLER_BIN" --myname a --workdir "$d/work" ...
|
|
# assert_grep "$d/controller.log" 'HA mode: 3 controller candidates' || return 1
|
|
# }
|
|
# add_case case_controller_env_list "controller: env list used with no --peers"
|
|
#
|
|
# A case is a function that returns 0 on pass and non-zero on fail. It runs in
|
|
# the main shell (not a subshell) inside an `if`, so `set -e` does NOT abort on
|
|
# a failing command inside it: every check must `|| return 1` explicitly (the
|
|
# assert_* helpers print why). Background processes a case starts with
|
|
# start_bg are stopped after the case even if it forgets to.
|
|
#
|
|
# Isolation: every daemon run gets its own scratch directory (new_scratch);
|
|
# the controller must always be given --workdir/--certdir/--logfile inside it,
|
|
# the scheduler --logfile and the worker -l. Two things a scratch directory
|
|
# cannot isolate -- the default controller address localhost:9990 that the
|
|
# scheduler/worker fall back to, and the worker's hardcoded socket
|
|
# /var/run/.xcecd.sock -- are checked by the pre-flight, which refuses to run
|
|
# when a local cluster is up.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
PEER_PORT=9990
|
|
WORKER_SOCK=/var/run/.xcecd.sock
|
|
|
|
# ── output helpers (same style as the other src/tools/test-*.sh scripts) ────
|
|
green() { printf '\033[32m%s\033[0m\n' "$*"; }
|
|
red() { printf '\033[31m%s\033[0m\n' "$*"; }
|
|
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
|
|
log() { echo "[verify-peers-env] $*"; }
|
|
|
|
usage() {
|
|
sed -n '/^# Usage:/,/^# Exit status/p' "$0" | sed 's/^# \{0,1\}//' | sed '$d'
|
|
}
|
|
|
|
# ── arguments ───────────────────────────────────────────────────────────────
|
|
BUILD_DIR="${XCE_BUILD_DIR:-$SRC_DIR/xce.build}"
|
|
KEEP="${XCE_VERIFY_KEEP:-0}"
|
|
ONLY=""
|
|
LIST_ONLY=0
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--keep) KEEP=1 ;;
|
|
--only)
|
|
[ $# -ge 2 ] || { red "--only needs a regex" >&2; exit 2; }
|
|
ONLY="$2"; shift ;;
|
|
--list) LIST_ONLY=1 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
-*) red "Unknown option: $1" >&2; usage >&2; exit 2 ;;
|
|
*) BUILD_DIR="$1" ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# The variable under test must never leak in from the invoking shell: every
|
|
# case sets it explicitly (or deliberately leaves it unset).
|
|
unset XCE_CONTROLLER_PEERS
|
|
|
|
# ── binaries ────────────────────────────────────────────────────────────────
|
|
CONTROLLER_BIN=""
|
|
SCHEDULER_BIN=""
|
|
WORKER_BIN=""
|
|
# Set by preflight; the harness cases (task 3.3) are its only reader.
|
|
# shellcheck disable=SC2034
|
|
HARNESS_BIN=""
|
|
|
|
# component:relative-path:variable
|
|
BINARIES=(
|
|
"controller:controller/xce_controller:CONTROLLER_BIN"
|
|
"scheduler:scheduler/xce_scheduler:SCHEDULER_BIN"
|
|
"worker:worker/xcecd_virtual:WORKER_BIN"
|
|
"harness:harness/xce_test:HARNESS_BIN"
|
|
)
|
|
|
|
# ── case registry ───────────────────────────────────────────────────────────
|
|
CASES=() # function names, in registration order
|
|
CASE_DESC=() # parallel array of one-line descriptions
|
|
PASSED=()
|
|
FAILED=()
|
|
|
|
add_case() { # add_case <function> [description]
|
|
declare -F "$1" >/dev/null || { red "add_case: no such function: $1" >&2; exit 2; }
|
|
CASES+=("$1")
|
|
CASE_DESC+=("${2:-$1}")
|
|
}
|
|
|
|
# ── scratch directories & process tracking ─────────────────────────────────
|
|
SCRATCH_ROOT=""
|
|
STARTED_PIDS=() # PIDs started by start_bg and not yet stopped
|
|
LAST_PID=""
|
|
RUN_RC=0
|
|
|
|
# new_scratch <name>: print the path of a fresh, empty directory for one run.
|
|
new_scratch() {
|
|
mktemp -d "$SCRATCH_ROOT/${1:-run}.XXXXXX"
|
|
}
|
|
|
|
_forget_pid() {
|
|
local p keep=()
|
|
for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do
|
|
[ "$p" = "$1" ] || keep+=("$p")
|
|
done
|
|
STARTED_PIDS=("${keep[@]+"${keep[@]}"}")
|
|
}
|
|
|
|
# start_bg <dir> [env-args...] <binary> [args...]
|
|
# Start a process in the background via env(1), so leading NAME=value
|
|
# assignments and `-u NAME` set/unset its environment. stdout/stderr go to
|
|
# <dir>/stdout and <dir>/stderr; stdin is /dev/null so the process can never
|
|
# eat the script's input. Sets LAST_PID (do not call it in $(...)).
|
|
start_bg() {
|
|
local dir="$1"; shift
|
|
env "$@" </dev/null >"$dir/stdout" 2>"$dir/stderr" &
|
|
LAST_PID=$!
|
|
STARTED_PIDS+=("$LAST_PID")
|
|
echo "$LAST_PID" >"$dir/pid"
|
|
}
|
|
|
|
# stop_proc <pid>: SIGTERM, then SIGKILL after ~3s, and reap it.
|
|
stop_proc() {
|
|
local pid="$1" i
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
kill -TERM "$pid" 2>/dev/null || true
|
|
for i in $(seq 1 30); do
|
|
kill -0 "$pid" 2>/dev/null || break
|
|
sleep 0.1
|
|
done
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
fi
|
|
wait "$pid" 2>/dev/null || true
|
|
_forget_pid "$pid"
|
|
}
|
|
|
|
stop_all() {
|
|
local p
|
|
for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do
|
|
stop_proc "$p"
|
|
done
|
|
STARTED_PIDS=()
|
|
}
|
|
|
|
# run_to_completion <dir> <timeout-seconds> [env-args...] <binary> [args...]
|
|
# Run in the foreground with a timeout (SIGKILL 2s after the
|
|
# SIGTERM). stdout/stderr go to <dir>/stdout, <dir>/stderr; the exit code is
|
|
# left in RUN_RC and <dir>/rc (timed out: 124 with coreutils timeout, 143
|
|
# with busybox timeout, 137 if it had to be SIGKILLed). `-k` rather than
|
|
# `--kill-after` because busybox timeout only accepts the short form.
|
|
run_to_completion() {
|
|
local dir="$1" secs="$2"; shift 2
|
|
RUN_RC=0
|
|
timeout -k 2 "$secs" env "$@" </dev/null \
|
|
>"$dir/stdout" 2>"$dir/stderr" || RUN_RC=$?
|
|
echo "$RUN_RC" >"$dir/rc"
|
|
}
|
|
|
|
# wait_for_line <file> <regex> <timeout-seconds> [pid]
|
|
# Poll until a line matching <regex> (grep -E) appears in <file>. Returns 0
|
|
# when found, 1 on timeout -- or as soon as [pid] has exited without the line
|
|
# having appeared (after one last look at the file).
|
|
wait_for_line() {
|
|
local file="$1" re="$2" secs="$3" pid="${4:-}" deadline
|
|
deadline=$(( $(date +%s) + secs ))
|
|
while :; do
|
|
if [ -f "$file" ] && grep -Eq -- "$re" "$file"; then
|
|
return 0
|
|
fi
|
|
if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
|
|
[ -f "$file" ] && grep -Eq -- "$re" "$file" && return 0
|
|
return 1
|
|
fi
|
|
[ "$(date +%s)" -lt "$deadline" ] || return 1
|
|
sleep 0.1
|
|
done
|
|
}
|
|
|
|
# ── assertions (print the reason on failure; use as `assert_x ... || return 1`)
|
|
assert_grep() { # assert_grep <file> <regex> [what]
|
|
if [ -f "$1" ] && grep -Eq -- "$2" "$1"; then return 0; fi
|
|
red " expected /$2/ in ${3:-$1}"
|
|
_show_tail "$1"
|
|
return 1
|
|
}
|
|
|
|
assert_no_grep() { # assert_no_grep <file> <regex> [what]
|
|
if [ ! -f "$1" ] || ! grep -Eq -- "$2" "$1"; then return 0; fi
|
|
red " unexpected /$2/ in ${3:-$1}:"
|
|
grep -E -- "$2" "$1" | head -5 | sed 's/^/ /'
|
|
return 1
|
|
}
|
|
|
|
assert_rc() { # assert_rc <expected|nonzero> [actual=$RUN_RC]
|
|
local want="$1" got="${2:-$RUN_RC}"
|
|
if [ "$want" = nonzero ]; then
|
|
[ "$got" -ne 0 ] && return 0
|
|
elif [ "$got" -eq "$want" ]; then
|
|
return 0
|
|
fi
|
|
red " expected exit status $want, got $got"
|
|
return 1
|
|
}
|
|
|
|
_show_tail() {
|
|
[ -f "$1" ] || { red " ($1 does not exist)"; return 0; }
|
|
echo " --- last lines of $1 ---"
|
|
tail -n 15 "$1" | sed 's/^/ /'
|
|
}
|
|
|
|
# ── pre-flight ──────────────────────────────────────────────────────────────
|
|
port_listening() { # port_listening <port>: 0 if something listens on it
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ss -tln 2>/dev/null | awk -v p=":$1\$" '$4 ~ p {f=1} END{exit !f}'
|
|
else
|
|
(exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null
|
|
fi
|
|
}
|
|
|
|
preflight() {
|
|
local problems=0 entry comp rel var path
|
|
|
|
if [ ! -d "$BUILD_DIR" ]; then
|
|
red "pre-flight: build directory $BUILD_DIR does not exist"
|
|
problems=1
|
|
fi
|
|
for entry in "${BINARIES[@]}"; do
|
|
IFS=: read -r comp rel var <<<"$entry"
|
|
path="$BUILD_DIR/$rel"
|
|
if [ -x "$path" ]; then
|
|
printf -v "$var" '%s' "$path"
|
|
else
|
|
red "pre-flight: missing binary $path (component: $comp) -- build it with: /xce-build $comp --incremental"
|
|
problems=1
|
|
fi
|
|
done
|
|
|
|
if port_listening "$PEER_PORT"; then
|
|
red "pre-flight: something is listening on port $PEER_PORT (a local controller?); the scheduler/worker cases would talk to it."
|
|
ss -tlnp 2>/dev/null | awk -v p=":$PEER_PORT\$" '$4 ~ p' | sed 's/^/ /' || true
|
|
problems=1
|
|
fi
|
|
if [ -e "$WORKER_SOCK" ]; then
|
|
red "pre-flight: worker socket $WORKER_SOCK exists (a local worker running?); the worker cases would collide with it."
|
|
problems=1
|
|
fi
|
|
|
|
if [ "$problems" -ne 0 ]; then
|
|
red "pre-flight failed: refusing to run. Stop the local cluster and/or build the missing component(s) into $BUILD_DIR, then re-run."
|
|
exit 2
|
|
fi
|
|
}
|
|
|
|
# ── cleanup ─────────────────────────────────────────────────────────────────
|
|
on_exit() {
|
|
local rc=$?
|
|
stop_all
|
|
if [ -n "$SCRATCH_ROOT" ] && [ -d "$SCRATCH_ROOT" ]; then
|
|
if [ "$KEEP" = 1 ]; then
|
|
yellow "scratch root kept: $SCRATCH_ROOT"
|
|
else
|
|
rm -rf "$SCRATCH_ROOT"
|
|
fi
|
|
fi
|
|
exit "$rc"
|
|
}
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# Cases. Later tasks (2.2-3.3, 5.1) define case_* functions here and register
|
|
# them with add_case, in the order they should run.
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
|
|
# Task 2.2 (Req 1.3): the controller logs through tlog's background writer,
|
|
# so a line logged just before an early return is only in the log file if the
|
|
# return flushes the log first. With an empty --certdir and no --peers (and
|
|
# no XCE_CONTROLLER_PEERS) the controller stops at the certificate pre-flight
|
|
# right after its "No --peers flag" line. The loss is a race, so repeat the
|
|
# run and fail if any single run is missing the line.
|
|
FLUSH_RUNS=20
|
|
case_controller_log_flushed_on_early_exit() {
|
|
local i d missing=0
|
|
for i in $(seq 1 "$FLUSH_RUNS"); do
|
|
d="$(new_scratch controller-flush)"
|
|
mkdir -p "$d/work" "$d/certs"
|
|
run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS \
|
|
"$CONTROLLER_BIN" --myname ctrl-flush --workdir "$d/work" \
|
|
--certdir "$d/certs" --logfile "$d/controller.log"
|
|
if [ "$RUN_RC" -eq 0 ] || [ "$RUN_RC" -ge 124 ]; then
|
|
red " run $i: expected the certificate pre-flight to stop the controller, exit status $RUN_RC"
|
|
_show_tail "$d/stderr"
|
|
return 1
|
|
fi
|
|
if ! grep -Eq -- 'No --peers flag and XCE_CONTROLLER_PEERS unset or empty; running as single-node primary' \
|
|
"$d/controller.log" 2>/dev/null; then
|
|
missing=$((missing + 1))
|
|
red " run $i: last startup line missing from $d/controller.log"
|
|
fi
|
|
done
|
|
if [ "$missing" -ne 0 ]; then
|
|
red " $missing/$FLUSH_RUNS run(s) lost the last startup line"
|
|
return 1
|
|
fi
|
|
log " all $FLUSH_RUNS runs had the last startup line in the log file"
|
|
}
|
|
add_case case_controller_log_flushed_on_early_exit \
|
|
"controller: log flushed before the certificate pre-flight exit ($FLUSH_RUNS runs)"
|
|
|
|
# ── Task 2.3 (Req 1.1-1.5, 1.7, 1.8): controller resolves its candidate list
|
|
# from --peers, else XCE_CONTROLLER_PEERS, else runs standalone.
|
|
#
|
|
# run_controller <dir> <env-args...> -- [controller args...]
|
|
# Run the controller with scratch --workdir/--certdir (empty)/--logfile in
|
|
# <dir>, so it stops at the certificate pre-flight. <env-args> are env(1)
|
|
# arguments (NAME=value or `-u NAME`) placed before the binary.
|
|
run_controller() {
|
|
local dir="$1"; shift
|
|
local envargs=()
|
|
while [ $# -gt 0 ] && [ "$1" != -- ]; do
|
|
envargs+=("$1"); shift
|
|
done
|
|
[ $# -gt 0 ] && shift
|
|
mkdir -p "$dir/work" "$dir/certs"
|
|
run_to_completion "$dir" 10 "${envargs[@]}" "$CONTROLLER_BIN" \
|
|
--workdir "$dir/work" --certdir "$dir/certs" \
|
|
--logfile "$dir/controller.log" "$@"
|
|
}
|
|
|
|
# The cert pre-flight (or an argument error) must have stopped the run: a
|
|
# zero status or a timeout means it went further than these cases expect.
|
|
_assert_stopped_early() {
|
|
if [ "$RUN_RC" -eq 0 ] || [ "$RUN_RC" -ge 124 ]; then
|
|
red " expected the controller to stop early, exit status $RUN_RC"
|
|
_show_tail "$1/stderr"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
ENV3="a=localhost:1,b=localhost:2,c=localhost:3"
|
|
STANDALONE_RE='No --peers flag and XCE_CONTROLLER_PEERS unset or empty; running as single-node primary'
|
|
|
|
case_controller_env_list() {
|
|
local d; d="$(new_scratch controller-env-list)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname a
|
|
_assert_stopped_early "$d" || return 1
|
|
assert_grep "$d/controller.log" 'HA mode: 3 controller candidates, identity=a' || return 1
|
|
assert_no_grep "$d/controller.log" 'No --peers flag' || return 1
|
|
}
|
|
add_case case_controller_env_list \
|
|
"controller: XCE_CONTROLLER_PEERS used when --peers is absent (1.2)"
|
|
|
|
case_controller_peers_wins() {
|
|
local d; d="$(new_scratch controller-peers-wins)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- \
|
|
--myname solo --peers solo=localhost:7
|
|
_assert_stopped_early "$d" || return 1
|
|
assert_grep "$d/controller.log" 'HA mode: 1 controller candidates, identity=solo' || return 1
|
|
assert_no_grep "$d/controller.log" 'HA mode: 3 controller candidates' || return 1
|
|
assert_no_grep "$d/controller.log" 'from XCE_CONTROLLER_PEERS' || return 1
|
|
}
|
|
add_case case_controller_peers_wins \
|
|
"controller: explicit --peers wins over XCE_CONTROLLER_PEERS (1.1)"
|
|
|
|
case_controller_neither_set() {
|
|
local d; d="$(new_scratch controller-neither)"
|
|
run_controller "$d" -u XCE_CONTROLLER_PEERS -- --myname ctrl-none
|
|
_assert_stopped_early "$d" || return 1
|
|
assert_grep "$d/controller.log" "$STANDALONE_RE" || return 1
|
|
assert_no_grep "$d/controller.log" 'HA mode' || return 1
|
|
}
|
|
add_case case_controller_neither_set \
|
|
"controller: neither --peers nor the variable -> standalone, line names the variable (1.3, 1.8)"
|
|
|
|
case_controller_env_empty() {
|
|
local d; d="$(new_scratch controller-env-empty)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=" -- --myname ctrl-empty
|
|
_assert_stopped_early "$d" || return 1
|
|
assert_grep "$d/controller.log" "$STANDALONE_RE" || return 1
|
|
assert_no_grep "$d/controller.log" 'HA mode' || return 1
|
|
}
|
|
add_case case_controller_env_empty \
|
|
"controller: empty XCE_CONTROLLER_PEERS behaves as unset (1.7, 1.8)"
|
|
|
|
case_controller_parse_error_source() {
|
|
local d
|
|
d="$(new_scratch controller-parse-env)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=a=nocolon" -- --myname a
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'XCE_CONTROLLER_PEERS parse error' "stderr (env value)" || return 1
|
|
|
|
d="$(new_scratch controller-parse-flag)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname a --peers a=nocolon
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" -- '--peers parse error' "stderr (--peers value)" || return 1
|
|
assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
|
|
}
|
|
add_case case_controller_parse_error_source \
|
|
"controller: parse error names --peers or XCE_CONTROLLER_PEERS (1.4)"
|
|
|
|
case_controller_even_count_source() {
|
|
local d
|
|
d="$(new_scratch controller-even-env)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=a=localhost:1,b=localhost:2" -- --myname a
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'XCE_CONTROLLER_PEERS: even peer count \(2\)' "stderr (env value)" || return 1
|
|
|
|
d="$(new_scratch controller-even-flag)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname a \
|
|
--peers a=localhost:1,b=localhost:2
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" -- '--peers: even peer count \(2\)' "stderr (--peers value)" || return 1
|
|
assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
|
|
}
|
|
add_case case_controller_even_count_source \
|
|
"controller: even-count error names --peers or XCE_CONTROLLER_PEERS (1.4)"
|
|
|
|
case_controller_help_names_variable() {
|
|
local d; d="$(new_scratch controller-help)"
|
|
run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS "$CONTROLLER_BIN" --help
|
|
assert_rc 0 || return 1
|
|
# argp wraps help text, so join the lines before matching.
|
|
tr '\n' ' ' <"$d/stdout" | tr -s ' ' >"$d/help.txt"
|
|
assert_grep "$d/help.txt" 'XCE_CONTROLLER_PEERS' "--help output" || return 1
|
|
assert_grep "$d/help.txt" 'unset or empty' "--help output" || return 1
|
|
assert_grep "$d/help.txt" 'standalone' "--help output" || return 1
|
|
}
|
|
add_case case_controller_help_names_variable \
|
|
"controller: --help documents XCE_CONTROLLER_PEERS and the opt-out (1.5, 1.7)"
|
|
|
|
# ── Task 2.4 (Req 1.6): an XCE_CONTROLLER_PEERS list that has no candidate for
|
|
# the controller's own name is refused on stderr before the certificate
|
|
# pre-flight. Without the check these runs still exit non-zero at the
|
|
# pre-flight, so the cases assert on the message, not just the exit status.
|
|
OWN_NAME_RE="has no candidate named"
|
|
|
|
case_controller_env_own_name_missing() {
|
|
local d; d="$(new_scratch controller-own-missing)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname zz
|
|
assert_rc nonzero || return 1
|
|
assert_grep "$d/stderr" "XCE_CONTROLLER_PEERS \\($ENV3\\) $OWN_NAME_RE 'zz'" "stderr" || return 1
|
|
assert_grep "$d/stderr" -- '--myname <one of: a, b, c>' "stderr" || return 1
|
|
assert_grep "$d/stderr" 'unset XCE_CONTROLLER_PEERS' "stderr" || return 1
|
|
assert_no_grep "$d/stderr" 'cert|key' "stderr (must stop before the certificate pre-flight)" || return 1
|
|
}
|
|
add_case case_controller_env_own_name_missing \
|
|
"controller: env list without the own name is refused on stderr (1.6)"
|
|
|
|
case_controller_env_own_name_unnamed() {
|
|
local d; d="$(new_scratch controller-own-unnamed)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=localhost:1" -- --myname zz
|
|
assert_rc nonzero || return 1
|
|
assert_grep "$d/stderr" "$OWN_NAME_RE 'zz'" "stderr" || return 1
|
|
assert_grep "$d/stderr" -- '--myname <one of: localhost>' "stderr (unnamed candidate listed by hostname)" || return 1
|
|
}
|
|
add_case case_controller_env_own_name_unnamed \
|
|
"controller: unnamed env candidates are listed by hostname in the 1.6 message"
|
|
|
|
case_controller_env_own_name_present() {
|
|
local d; d="$(new_scratch controller-own-present)"
|
|
run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname b
|
|
_assert_stopped_early "$d" || return 1
|
|
assert_grep "$d/controller.log" 'HA mode: 3 controller candidates, identity=b' || return 1
|
|
assert_no_grep "$d/stderr" "$OWN_NAME_RE" "stderr" || return 1
|
|
}
|
|
add_case case_controller_env_own_name_present \
|
|
"controller: env list containing the own name is accepted (1.6)"
|
|
|
|
case_controller_flag_own_name_unchecked() {
|
|
local d; d="$(new_scratch controller-own-flag)"
|
|
run_controller "$d" -u XCE_CONTROLLER_PEERS -- --myname zz --peers "$ENV3"
|
|
_assert_stopped_early "$d" || return 1
|
|
assert_grep "$d/controller.log" 'HA mode: 3 controller candidates, identity=zz' || return 1
|
|
assert_no_grep "$d/stderr" "$OWN_NAME_RE" "stderr" || return 1
|
|
}
|
|
add_case case_controller_flag_own_name_unchecked \
|
|
"controller: explicit --peers without the own name is not checked at startup (1.6)"
|
|
|
|
# ── Task 3.1 (Req 1.1-1.5, 1.7, 1.8): the scheduler resolves its candidate
|
|
# list from --peers, else XCE_CONTROLLER_PEERS, else the single-controller
|
|
# path. A valid list keeps the scheduler running (it retries its controller),
|
|
# so those runs are started in the background, stopped once the startup line
|
|
# is in the log, and only then asserted on. Argument errors exit on their own.
|
|
|
|
# Either startup line; every successful scheduler start logs exactly one.
|
|
SCHED_ANCHOR_RE='HA cluster config|No --peers flag'
|
|
SCHED_STANDALONE_RE='No --peers flag and XCE_CONTROLLER_PEERS unset or empty; using single-controller path, version '
|
|
|
|
# start_scheduler <dir> <env-args...> -- [scheduler args...]
|
|
# Start the scheduler in the background with --logfile <dir>/scheduler.log,
|
|
# wait for its startup line, then stop it. Fails (with the log tail) when
|
|
# the line never appears.
|
|
start_scheduler() {
|
|
local dir="$1"; shift
|
|
local envargs=() pid
|
|
while [ $# -gt 0 ] && [ "$1" != -- ]; do
|
|
envargs+=("$1"); shift
|
|
done
|
|
[ $# -gt 0 ] && shift
|
|
start_bg "$dir" "${envargs[@]}" "$SCHEDULER_BIN" \
|
|
--logfile "$dir/scheduler.log" "$@"
|
|
pid="$LAST_PID"
|
|
if ! wait_for_line "$dir/scheduler.log" "$SCHED_ANCHOR_RE" 10 "$pid"; then
|
|
stop_proc "$pid"
|
|
red " scheduler never logged its startup line"
|
|
_show_tail "$dir/scheduler.log"
|
|
_show_tail "$dir/stderr"
|
|
return 1
|
|
fi
|
|
stop_proc "$pid"
|
|
}
|
|
|
|
# run_scheduler <dir> <env-args...> -- [scheduler args...]
|
|
# Run the scheduler to completion (argument errors, --sim, --help).
|
|
run_scheduler() {
|
|
local dir="$1"; shift
|
|
local envargs=()
|
|
while [ $# -gt 0 ] && [ "$1" != -- ]; do
|
|
envargs+=("$1"); shift
|
|
done
|
|
[ $# -gt 0 ] && shift
|
|
run_to_completion "$dir" 10 "${envargs[@]}" "$SCHEDULER_BIN" \
|
|
--logfile "$dir/scheduler.log" "$@"
|
|
}
|
|
|
|
case_scheduler_env_list() {
|
|
local d; d="$(new_scratch scheduler-env-list)"
|
|
start_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- || return 1
|
|
assert_grep "$d/scheduler.log" \
|
|
'HA cluster config: 3 candidate\(s\), identity=xce_scheduler \(from XCE_CONTROLLER_PEERS\), version ' || return 1
|
|
assert_no_grep "$d/scheduler.log" 'No --peers flag' || return 1
|
|
}
|
|
add_case case_scheduler_env_list \
|
|
"scheduler: XCE_CONTROLLER_PEERS used when --peers is absent (1.2)"
|
|
|
|
case_scheduler_peers_wins() {
|
|
local d; d="$(new_scratch scheduler-peers-wins)"
|
|
start_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- \
|
|
--peers solo=localhost:7 || return 1
|
|
assert_grep "$d/scheduler.log" \
|
|
'HA cluster config: 1 candidate\(s\), identity=xce_scheduler, version ' || return 1
|
|
assert_no_grep "$d/scheduler.log" '3 candidate|from XCE_CONTROLLER_PEERS' || return 1
|
|
}
|
|
add_case case_scheduler_peers_wins \
|
|
"scheduler: explicit --peers wins over XCE_CONTROLLER_PEERS (1.1)"
|
|
|
|
case_scheduler_neither_set() {
|
|
local d; d="$(new_scratch scheduler-neither)"
|
|
start_scheduler "$d" -u XCE_CONTROLLER_PEERS -- || return 1
|
|
assert_grep "$d/scheduler.log" "$SCHED_STANDALONE_RE" || return 1
|
|
assert_no_grep "$d/scheduler.log" 'HA cluster config' || return 1
|
|
}
|
|
add_case case_scheduler_neither_set \
|
|
"scheduler: neither --peers nor the variable -> single-controller path, line names the variable (1.3, 1.8)"
|
|
|
|
case_scheduler_env_empty() {
|
|
local d; d="$(new_scratch scheduler-env-empty)"
|
|
start_scheduler "$d" "XCE_CONTROLLER_PEERS=" -- || return 1
|
|
assert_grep "$d/scheduler.log" "$SCHED_STANDALONE_RE" || return 1
|
|
assert_no_grep "$d/scheduler.log" 'HA cluster config' || return 1
|
|
}
|
|
add_case case_scheduler_env_empty \
|
|
"scheduler: empty XCE_CONTROLLER_PEERS behaves as unset (1.7, 1.8)"
|
|
|
|
case_scheduler_parse_error_source() {
|
|
local d
|
|
d="$(new_scratch scheduler-parse-env)"
|
|
run_scheduler "$d" "XCE_CONTROLLER_PEERS=a=nocolon" --
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xce_scheduler: XCE_CONTROLLER_PEERS parse error' "stderr (env value)" || return 1
|
|
|
|
d="$(new_scratch scheduler-parse-flag)"
|
|
run_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --peers a=nocolon
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xce_scheduler: --peers parse error' "stderr (--peers value)" || return 1
|
|
assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
|
|
}
|
|
add_case case_scheduler_parse_error_source \
|
|
"scheduler: parse error names --peers or XCE_CONTROLLER_PEERS (1.4)"
|
|
|
|
case_scheduler_validate_error_source() {
|
|
local d
|
|
d="$(new_scratch scheduler-port0-env)"
|
|
run_scheduler "$d" "XCE_CONTROLLER_PEERS=a=localhost:0" --
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xce_scheduler: XCE_CONTROLLER_PEERS: invalid cluster config' "stderr (env value)" || return 1
|
|
|
|
d="$(new_scratch scheduler-port0-flag)"
|
|
run_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --peers a=localhost:0
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xce_scheduler: --peers: invalid cluster config' "stderr (--peers value)" || return 1
|
|
assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
|
|
}
|
|
add_case case_scheduler_validate_error_source \
|
|
"scheduler: cluster-config validation error (port 0) names --peers or XCE_CONTROLLER_PEERS (1.4)"
|
|
|
|
case_scheduler_help_names_variable() {
|
|
local d; d="$(new_scratch scheduler-help)"
|
|
run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS "$SCHEDULER_BIN" --help
|
|
assert_rc 0 || return 1
|
|
# argp wraps help text, so join the lines before matching.
|
|
tr '\n' ' ' <"$d/stdout" | tr -s ' ' >"$d/help.txt"
|
|
assert_grep "$d/help.txt" 'XCE_CONTROLLER_PEERS' "--help output" || return 1
|
|
assert_grep "$d/help.txt" 'unset or empty' "--help output" || return 1
|
|
assert_grep "$d/help.txt" 'single-controller' "--help output" || return 1
|
|
}
|
|
add_case case_scheduler_help_names_variable \
|
|
"scheduler: --help documents XCE_CONTROLLER_PEERS and the opt-out (1.5, 1.7)"
|
|
|
|
# Guard (passes before and after 3.1): --sim returns before the peers block,
|
|
# so the variable must not reach simulation mode. The "--sim startup failed"
|
|
# line, which only the --sim branch prints, proves that branch ran.
|
|
case_scheduler_sim_ignores_variable() {
|
|
local d; d="$(new_scratch scheduler-sim)"
|
|
run_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- \
|
|
--sim "$d/does-not-exist.replay"
|
|
assert_rc nonzero || return 1
|
|
assert_grep "$d/stderr" 'xce_scheduler: --sim startup failed' "stderr" || return 1
|
|
assert_no_grep "$d/scheduler.log" "$SCHED_ANCHOR_RE" || return 1
|
|
assert_no_grep "$d/stderr" "$SCHED_ANCHOR_RE" "stderr" || return 1
|
|
}
|
|
add_case case_scheduler_sim_ignores_variable \
|
|
"scheduler: --sim never reaches the peers block, even with the variable set"
|
|
|
|
# ── Task 3.2 (Req 1.1-1.5, 1.7, 1.8): the worker (xcecd_virtual, which needs
|
|
# no root) resolves its candidate list from --peers, else XCE_CONTROLLER_PEERS,
|
|
# else the single-controller path. Its startup line is logged before the log
|
|
# file is set up, so it lands on STDOUT. tlog writes it there unbuffered (the
|
|
# line is in the redirected stdout file within ~0.3s while the worker keeps
|
|
# running), so it is waited for while the worker runs, like the scheduler's.
|
|
# The worker ignores SIGTERM for longer than stop_proc's 3s grace, so it is
|
|
# SIGKILLed once the line is there. Argument errors exit on their own, before
|
|
# the log directory is created; in an ASAN build those early returns must not
|
|
# leak (the -l directory string), which the env-value error cases check.
|
|
|
|
WORKER_ANCHOR_RE='HA cluster config|No --peers flag'
|
|
WORKER_STANDALONE_RE='No --peers flag and XCE_CONTROLLER_PEERS unset or empty; using single-controller path \(host=localhost port=9990\), version '
|
|
|
|
# _split_env_args: shared by start_worker/run_worker -- fills WENV with the
|
|
# env(1) arguments before `--` and WARGS with the worker arguments after it.
|
|
WENV=()
|
|
WARGS=()
|
|
_split_env_args() {
|
|
WENV=(); WARGS=()
|
|
while [ $# -gt 0 ] && [ "$1" != -- ]; do
|
|
WENV+=("$1"); shift
|
|
done
|
|
[ $# -gt 0 ] && shift
|
|
WARGS=("$@")
|
|
}
|
|
|
|
# start_worker <dir> <env-args...> -- [worker args...]
|
|
# Start xcecd_virtual in the background with -l <dir>/logs, wait for its
|
|
# startup line on stdout, then kill it. Fails (with the output tails) when
|
|
# the line never appears.
|
|
start_worker() {
|
|
local dir="$1" pid; shift
|
|
_split_env_args "$@"
|
|
start_bg "$dir" "${WENV[@]}" "$WORKER_BIN" -l "$dir/logs" \
|
|
"${WARGS[@]+"${WARGS[@]}"}"
|
|
pid="$LAST_PID"
|
|
if ! wait_for_line "$dir/stdout" "$WORKER_ANCHOR_RE" 10 "$pid"; then
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
stop_proc "$pid"
|
|
red " worker never printed its startup line"
|
|
_show_tail "$dir/stdout"
|
|
_show_tail "$dir/stderr"
|
|
return 1
|
|
fi
|
|
kill -KILL "$pid" 2>/dev/null || true
|
|
stop_proc "$pid"
|
|
}
|
|
|
|
# run_worker <dir> <env-args...> -- [worker args...]
|
|
# Run xcecd_virtual to completion (argument errors, -h) with -l <dir>/logs.
|
|
run_worker() {
|
|
local dir="$1"; shift
|
|
_split_env_args "$@"
|
|
run_to_completion "$dir" 10 "${WENV[@]}" "$WORKER_BIN" -l "$dir/logs" \
|
|
"${WARGS[@]+"${WARGS[@]}"}"
|
|
}
|
|
|
|
case_worker_env_list() {
|
|
local d; d="$(new_scratch worker-env-list)"
|
|
start_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-env || return 1
|
|
assert_grep "$d/stdout" \
|
|
'HA cluster config: 3 candidate\(s\), identity=w-env \(from XCE_CONTROLLER_PEERS\), version ' "stdout" || return 1
|
|
assert_no_grep "$d/stdout" 'No --peers flag' "stdout" || return 1
|
|
}
|
|
add_case case_worker_env_list \
|
|
"worker: XCE_CONTROLLER_PEERS used when --peers is absent (1.2)"
|
|
|
|
case_worker_peers_wins() {
|
|
local d; d="$(new_scratch worker-peers-wins)"
|
|
start_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-flag \
|
|
--peers solo=localhost:7 || return 1
|
|
assert_grep "$d/stdout" \
|
|
'HA cluster config: 1 candidate\(s\), identity=w-flag, version ' "stdout" || return 1
|
|
assert_no_grep "$d/stdout" '3 candidate|from XCE_CONTROLLER_PEERS' "stdout" || return 1
|
|
}
|
|
add_case case_worker_peers_wins \
|
|
"worker: explicit --peers wins over XCE_CONTROLLER_PEERS (1.1)"
|
|
|
|
case_worker_neither_set() {
|
|
local d; d="$(new_scratch worker-neither)"
|
|
start_worker "$d" -u XCE_CONTROLLER_PEERS -- -n w-none || return 1
|
|
assert_grep "$d/stdout" "$WORKER_STANDALONE_RE" "stdout" || return 1
|
|
assert_no_grep "$d/stdout" 'HA cluster config' "stdout" || return 1
|
|
}
|
|
add_case case_worker_neither_set \
|
|
"worker: neither --peers nor the variable -> single-controller path, line names the variable (1.3, 1.8)"
|
|
|
|
case_worker_env_empty() {
|
|
local d; d="$(new_scratch worker-env-empty)"
|
|
start_worker "$d" "XCE_CONTROLLER_PEERS=" -- -n w-empty || return 1
|
|
assert_grep "$d/stdout" "$WORKER_STANDALONE_RE" "stdout" || return 1
|
|
assert_no_grep "$d/stdout" 'HA cluster config' "stdout" || return 1
|
|
}
|
|
add_case case_worker_env_empty \
|
|
"worker: empty XCE_CONTROLLER_PEERS behaves as unset (1.7, 1.8)"
|
|
|
|
case_worker_parse_error_source() {
|
|
local d
|
|
d="$(new_scratch worker-parse-env)"
|
|
run_worker "$d" "XCE_CONTROLLER_PEERS=a=nocolon" -- -n w-perr
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xcecd: XCE_CONTROLLER_PEERS parse error' "stderr (env value)" || return 1
|
|
assert_no_grep "$d/stderr" 'LeakSanitizer' "stderr (ASAN build: early return must free)" || return 1
|
|
|
|
d="$(new_scratch worker-parse-flag)"
|
|
run_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-perr --peers a=nocolon
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xcecd: --peers parse error' "stderr (--peers value)" || return 1
|
|
assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
|
|
}
|
|
add_case case_worker_parse_error_source \
|
|
"worker: parse error names --peers or XCE_CONTROLLER_PEERS (1.4)"
|
|
|
|
case_worker_validate_error_source() {
|
|
local d
|
|
d="$(new_scratch worker-port0-env)"
|
|
run_worker "$d" "XCE_CONTROLLER_PEERS=a=localhost:0" -- -n w-verr
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xcecd: XCE_CONTROLLER_PEERS: invalid cluster config .* in "a=localhost:0"' "stderr (env value)" || return 1
|
|
assert_no_grep "$d/stderr" 'LeakSanitizer' "stderr (ASAN build: early return must free)" || return 1
|
|
|
|
d="$(new_scratch worker-port0-flag)"
|
|
run_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-verr --peers a=localhost:0
|
|
assert_rc 1 || return 1
|
|
assert_grep "$d/stderr" 'xcecd: --peers: invalid cluster config .* in "a=localhost:0"' "stderr (--peers value)" || return 1
|
|
assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
|
|
}
|
|
add_case case_worker_validate_error_source \
|
|
"worker: cluster-config validation error (port 0) names --peers or XCE_CONTROLLER_PEERS (1.4)"
|
|
|
|
case_worker_help_names_variable() {
|
|
local d; d="$(new_scratch worker-help)"
|
|
run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS "$WORKER_BIN" -h
|
|
assert_rc 0 || return 1
|
|
tr '\n' ' ' <"$d/stdout" | tr -s ' ' >"$d/help.txt"
|
|
assert_grep "$d/help.txt" 'XCE_CONTROLLER_PEERS' "-h output" || return 1
|
|
assert_grep "$d/help.txt" 'unset or empty' "-h output" || return 1
|
|
assert_grep "$d/help.txt" 'single-controller' "-h output" || return 1
|
|
assert_grep "$d/help.txt" 'sudo --preserve-env=XCE_CONTROLLER_PEERS' "-h output" || return 1
|
|
}
|
|
add_case case_worker_help_names_variable \
|
|
"worker: -h documents XCE_CONTROLLER_PEERS, the opt-out and that sudo drops it (1.5, 1.7)"
|
|
|
|
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
# main
|
|
# ════════════════════════════════════════════════════════════════════════════
|
|
main() {
|
|
local i fn
|
|
|
|
if [ "$LIST_ONLY" = 1 ]; then
|
|
for i in "${!CASES[@]}"; do
|
|
printf '%s\t%s\n' "${CASES[$i]}" "${CASE_DESC[$i]}"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
preflight
|
|
|
|
trap on_exit EXIT
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
SCRATCH_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/verify-peers-env.XXXXXX")"
|
|
|
|
log "build dir: $BUILD_DIR"
|
|
log "scratch root: $SCRATCH_ROOT"
|
|
log "${#CASES[@]} case(s) registered${ONLY:+, filtered by /$ONLY/}"
|
|
|
|
for i in "${!CASES[@]}"; do
|
|
fn="${CASES[$i]}"
|
|
if [ -n "$ONLY" ] && ! grep -Eq -- "$ONLY" <<<"$fn"; then
|
|
continue
|
|
fi
|
|
echo "--- $fn: ${CASE_DESC[$i]}"
|
|
if "$fn"; then
|
|
green "PASS $fn"
|
|
PASSED+=("$fn")
|
|
else
|
|
red "FAIL $fn"
|
|
FAILED+=("$fn")
|
|
fi
|
|
stop_all
|
|
done
|
|
|
|
echo
|
|
log "summary: ${#PASSED[@]} passed, ${#FAILED[@]} failed"
|
|
for fn in "${FAILED[@]+"${FAILED[@]}"}"; do
|
|
red " FAILED: $fn"
|
|
done
|
|
if [ "${#FAILED[@]}" -ne 0 ]; then
|
|
exit 1
|
|
fi
|
|
green "ALL PASS"
|
|
exit 0
|
|
}
|
|
|
|
main
|