#!/usr/bin/env bash
# main() runs every case through the registry ("$fn"), an indirect call that
# the linter cannot follow, so it reports each case_* function -- and every
# helper only cases call -- as never invoked (SC2329).
# shellcheck disable=SC2329
# verify-controller-peers-env.sh -- controller-peers-env-fallback regression
# script (.kiro/specs/controller-peers-env-fallback, design.md "Testing
# Strategy -> Scripted Regression Verification").
#
# Drives the real built binaries (controller, scheduler, xcecd_virtual worker,
# xce_test harness) and asserts on their startup log / settings-dump / stdout
# output only. No live cluster is needed or wanted: resolving the controller
# candidate list happens, and is observable, before any connection attempt.
#
# Usage:
#   src/tools/verify-controller-peers-env.sh [--keep] [--only REGEX] [--list] [BUILD_DIR]
#
#   BUILD_DIR   build directory holding all four binaries (default:
#               $XCE_BUILD_DIR, else src/xce.build -- where
#               `bash src/tools/xce-build.sh <component> --incremental` builds).
#               Build every component into the SAME directory with
#               `/xce-build <component> --incremental`; without --incremental a
#               component build wipes the directory, removing the others.
#   --keep      keep the scratch root afterwards (also: XCE_VERIFY_KEEP=1).
#   --only RE   run only the cases whose function name matches RE (grep -E).
#   --list      print the registered cases and exit.
#
# Exit status: 0 all cases passed (or none registered); 1 a case failed;
# 2 usage error or pre-flight refusal (local cluster up, binary missing).
#
# Adding a case (later tasks):
#   case_controller_env_list() {
#       local d; d="$(new_scratch controller-env-list)"
#       run_to_completion "$d" 10 XCE_CONTROLLER_PEERS=a=localhost:1,... \
#           "$CONTROLLER_BIN" --myname a --workdir "$d/work" ...
#       assert_grep "$d/controller.log" 'HA mode: 3 controller candidates' || return 1
#   }
#   add_case case_controller_env_list "controller: env list used with no --peers"
#
# A case is a function that returns 0 on pass and non-zero on fail. It runs in
# the main shell (not a subshell) inside an `if`, so `set -e` does NOT abort on
# a failing command inside it: every check must `|| return 1` explicitly (the
# assert_* helpers print why). Background processes a case starts with
# start_bg are stopped after the case even if it forgets to.
#
# Isolation: every daemon run gets its own scratch directory (new_scratch);
# the controller must always be given --workdir/--certdir/--logfile inside it,
# the scheduler --logfile and the worker -l. Two things a scratch directory
# cannot isolate -- the default controller address localhost:9990 that the
# scheduler/worker fall back to, and the worker's hardcoded socket
# /var/run/.xcecd.sock -- are checked by the pre-flight, which refuses to run
# when a local cluster is up.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"

PEER_PORT=9990
WORKER_SOCK=/var/run/.xcecd.sock

# ── output helpers (same style as the other src/tools/test-*.sh scripts) ────
green()  { printf '\033[32m%s\033[0m\n' "$*"; }
red()    { printf '\033[31m%s\033[0m\n' "$*"; }
yellow() { printf '\033[33m%s\033[0m\n' "$*"; }
log()    { echo "[verify-peers-env] $*"; }

usage() {
    sed -n '/^# Usage:/,/^# Exit status/p' "$0" | sed 's/^# \{0,1\}//' | sed '$d'
}

# ── arguments ───────────────────────────────────────────────────────────────
BUILD_DIR="${XCE_BUILD_DIR:-$SRC_DIR/xce.build}"
KEEP="${XCE_VERIFY_KEEP:-0}"
ONLY=""
LIST_ONLY=0
while [ $# -gt 0 ]; do
    case "$1" in
        --keep) KEEP=1 ;;
        --only)
            [ $# -ge 2 ] || { red "--only needs a regex" >&2; exit 2; }
            ONLY="$2"; shift ;;
        --list) LIST_ONLY=1 ;;
        -h|--help) usage; exit 0 ;;
        -*) red "Unknown option: $1" >&2; usage >&2; exit 2 ;;
        *) BUILD_DIR="$1" ;;
    esac
    shift
done

# The variable under test must never leak in from the invoking shell: every
# case sets it explicitly (or deliberately leaves it unset).
unset XCE_CONTROLLER_PEERS

# ── binaries ────────────────────────────────────────────────────────────────
CONTROLLER_BIN=""
SCHEDULER_BIN=""
WORKER_BIN=""
# Set by preflight; the harness cases (task 3.3) are its only reader.
# shellcheck disable=SC2034
HARNESS_BIN=""

# component:relative-path:variable
BINARIES=(
    "controller:controller/xce_controller:CONTROLLER_BIN"
    "scheduler:scheduler/xce_scheduler:SCHEDULER_BIN"
    "worker:worker/xcecd_virtual:WORKER_BIN"
    "harness:harness/xce_test:HARNESS_BIN"
)

# ── case registry ───────────────────────────────────────────────────────────
CASES=()          # function names, in registration order
CASE_DESC=()      # parallel array of one-line descriptions
PASSED=()
FAILED=()

add_case() {      # add_case <function> [description]
    declare -F "$1" >/dev/null || { red "add_case: no such function: $1" >&2; exit 2; }
    CASES+=("$1")
    CASE_DESC+=("${2:-$1}")
}

# ── scratch directories & process tracking ─────────────────────────────────
SCRATCH_ROOT=""
STARTED_PIDS=()   # PIDs started by start_bg and not yet stopped
LAST_PID=""
RUN_RC=0

# new_scratch <name>: print the path of a fresh, empty directory for one run.
new_scratch() {
    mktemp -d "$SCRATCH_ROOT/${1:-run}.XXXXXX"
}

_forget_pid() {
    local p keep=()
    for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do
        [ "$p" = "$1" ] || keep+=("$p")
    done
    STARTED_PIDS=("${keep[@]+"${keep[@]}"}")
}

# start_bg <dir> [env-args...] <binary> [args...]
#   Start a process in the background via env(1), so leading NAME=value
#   assignments and `-u NAME` set/unset its environment. stdout/stderr go to
#   <dir>/stdout and <dir>/stderr; stdin is /dev/null so the process can never
#   eat the script's input. Sets LAST_PID (do not call it in $(...)).
start_bg() {
    local dir="$1"; shift
    env "$@" </dev/null >"$dir/stdout" 2>"$dir/stderr" &
    LAST_PID=$!
    STARTED_PIDS+=("$LAST_PID")
    echo "$LAST_PID" >"$dir/pid"
}

# stop_proc <pid>: SIGTERM, then SIGKILL after ~3s, and reap it.
stop_proc() {
    local pid="$1" i
    if kill -0 "$pid" 2>/dev/null; then
        kill -TERM "$pid" 2>/dev/null || true
        for i in $(seq 1 30); do
            kill -0 "$pid" 2>/dev/null || break
            sleep 0.1
        done
        kill -KILL "$pid" 2>/dev/null || true
    fi
    wait "$pid" 2>/dev/null || true
    _forget_pid "$pid"
}

stop_all() {
    local p
    for p in "${STARTED_PIDS[@]+"${STARTED_PIDS[@]}"}"; do
        stop_proc "$p"
    done
    STARTED_PIDS=()
}

# run_to_completion <dir> <timeout-seconds> [env-args...] <binary> [args...]
#   Run in the foreground with a timeout (SIGKILL 2s after the
#   SIGTERM). stdout/stderr go to <dir>/stdout, <dir>/stderr; the exit code is
#   left in RUN_RC and <dir>/rc (timed out: 124 with coreutils timeout, 143
#   with busybox timeout, 137 if it had to be SIGKILLed). `-k` rather than
#   `--kill-after` because busybox timeout only accepts the short form.
run_to_completion() {
    local dir="$1" secs="$2"; shift 2
    RUN_RC=0
    timeout -k 2 "$secs" env "$@" </dev/null \
        >"$dir/stdout" 2>"$dir/stderr" || RUN_RC=$?
    echo "$RUN_RC" >"$dir/rc"
}

# wait_for_line <file> <regex> <timeout-seconds> [pid]
#   Poll until a line matching <regex> (grep -E) appears in <file>. Returns 0
#   when found, 1 on timeout -- or as soon as [pid] has exited without the line
#   having appeared (after one last look at the file).
wait_for_line() {
    local file="$1" re="$2" secs="$3" pid="${4:-}" deadline
    deadline=$(( $(date +%s) + secs ))
    while :; do
        if [ -f "$file" ] && grep -Eq -- "$re" "$file"; then
            return 0
        fi
        if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
            [ -f "$file" ] && grep -Eq -- "$re" "$file" && return 0
            return 1
        fi
        [ "$(date +%s)" -lt "$deadline" ] || return 1
        sleep 0.1
    done
}

# ── assertions (print the reason on failure; use as `assert_x ... || return 1`)
assert_grep() {      # assert_grep <file> <regex> [what]
    if [ -f "$1" ] && grep -Eq -- "$2" "$1"; then return 0; fi
    red "    expected /$2/ in ${3:-$1}"
    _show_tail "$1"
    return 1
}

assert_no_grep() {   # assert_no_grep <file> <regex> [what]
    if [ ! -f "$1" ] || ! grep -Eq -- "$2" "$1"; then return 0; fi
    red "    unexpected /$2/ in ${3:-$1}:"
    grep -E -- "$2" "$1" | head -5 | sed 's/^/      /'
    return 1
}

assert_rc() {        # assert_rc <expected|nonzero> [actual=$RUN_RC]
    local want="$1" got="${2:-$RUN_RC}"
    if [ "$want" = nonzero ]; then
        [ "$got" -ne 0 ] && return 0
    elif [ "$got" -eq "$want" ]; then
        return 0
    fi
    red "    expected exit status $want, got $got"
    return 1
}

_show_tail() {
    [ -f "$1" ] || { red "      ($1 does not exist)"; return 0; }
    echo "      --- last lines of $1 ---"
    tail -n 15 "$1" | sed 's/^/      /'
}

# ── pre-flight ──────────────────────────────────────────────────────────────
port_listening() {   # port_listening <port>: 0 if something listens on it
    if command -v ss >/dev/null 2>&1; then
        ss -tln 2>/dev/null | awk -v p=":$1\$" '$4 ~ p {f=1} END{exit !f}'
    else
        (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null
    fi
}

preflight() {
    local problems=0 entry comp rel var path

    if [ ! -d "$BUILD_DIR" ]; then
        red "pre-flight: build directory $BUILD_DIR does not exist"
        problems=1
    fi
    for entry in "${BINARIES[@]}"; do
        IFS=: read -r comp rel var <<<"$entry"
        path="$BUILD_DIR/$rel"
        if [ -x "$path" ]; then
            printf -v "$var" '%s' "$path"
        else
            red "pre-flight: missing binary $path (component: $comp) -- build it with: /xce-build $comp --incremental"
            problems=1
        fi
    done

    if port_listening "$PEER_PORT"; then
        red "pre-flight: something is listening on port $PEER_PORT (a local controller?); the scheduler/worker cases would talk to it."
        ss -tlnp 2>/dev/null | awk -v p=":$PEER_PORT\$" '$4 ~ p' | sed 's/^/    /' || true
        problems=1
    fi
    if [ -e "$WORKER_SOCK" ]; then
        red "pre-flight: worker socket $WORKER_SOCK exists (a local worker running?); the worker cases would collide with it."
        problems=1
    fi

    if [ "$problems" -ne 0 ]; then
        red "pre-flight failed: refusing to run. Stop the local cluster and/or build the missing component(s) into $BUILD_DIR, then re-run."
        exit 2
    fi
}

# ── cleanup ─────────────────────────────────────────────────────────────────
on_exit() {
    local rc=$?
    stop_all
    if [ -n "$SCRATCH_ROOT" ] && [ -d "$SCRATCH_ROOT" ]; then
        if [ "$KEEP" = 1 ]; then
            yellow "scratch root kept: $SCRATCH_ROOT"
        else
            rm -rf "$SCRATCH_ROOT"
        fi
    fi
    exit "$rc"
}

# ════════════════════════════════════════════════════════════════════════════
# Cases. Later tasks (2.2-3.3, 5.1) define case_* functions here and register
# them with add_case, in the order they should run.
# ════════════════════════════════════════════════════════════════════════════

# Task 2.2 (Req 1.3): the controller logs through tlog's background writer,
# so a line logged just before an early return is only in the log file if the
# return flushes the log first. With an empty --certdir and no --peers (and
# no XCE_CONTROLLER_PEERS) the controller stops at the certificate pre-flight
# right after its "No --peers flag" line. The loss is a race, so repeat the
# run and fail if any single run is missing the line.
FLUSH_RUNS=20
case_controller_log_flushed_on_early_exit() {
    local i d missing=0
    for i in $(seq 1 "$FLUSH_RUNS"); do
        d="$(new_scratch controller-flush)"
        mkdir -p "$d/work" "$d/certs"
        run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS \
            "$CONTROLLER_BIN" --myname ctrl-flush --workdir "$d/work" \
            --certdir "$d/certs" --logfile "$d/controller.log"
        if [ "$RUN_RC" -eq 0 ] || [ "$RUN_RC" -ge 124 ]; then
            red "    run $i: expected the certificate pre-flight to stop the controller, exit status $RUN_RC"
            _show_tail "$d/stderr"
            return 1
        fi
        if ! grep -Eq -- 'No --peers flag and XCE_CONTROLLER_PEERS unset or empty; running as single-node primary' \
                "$d/controller.log" 2>/dev/null; then
            missing=$((missing + 1))
            red "    run $i: last startup line missing from $d/controller.log"
        fi
    done
    if [ "$missing" -ne 0 ]; then
        red "    $missing/$FLUSH_RUNS run(s) lost the last startup line"
        return 1
    fi
    log "    all $FLUSH_RUNS runs had the last startup line in the log file"
}
add_case case_controller_log_flushed_on_early_exit \
    "controller: log flushed before the certificate pre-flight exit ($FLUSH_RUNS runs)"

# ── Task 2.3 (Req 1.1-1.5, 1.7, 1.8): controller resolves its candidate list
# from --peers, else XCE_CONTROLLER_PEERS, else runs standalone.
#
# run_controller <dir> <env-args...> -- [controller args...]
#   Run the controller with scratch --workdir/--certdir (empty)/--logfile in
#   <dir>, so it stops at the certificate pre-flight. <env-args> are env(1)
#   arguments (NAME=value or `-u NAME`) placed before the binary.
run_controller() {
    local dir="$1"; shift
    local envargs=()
    while [ $# -gt 0 ] && [ "$1" != -- ]; do
        envargs+=("$1"); shift
    done
    [ $# -gt 0 ] && shift
    mkdir -p "$dir/work" "$dir/certs"
    run_to_completion "$dir" 10 "${envargs[@]}" "$CONTROLLER_BIN" \
        --workdir "$dir/work" --certdir "$dir/certs" \
        --logfile "$dir/controller.log" "$@"
}

# The cert pre-flight (or an argument error) must have stopped the run: a
# zero status or a timeout means it went further than these cases expect.
_assert_stopped_early() {
    if [ "$RUN_RC" -eq 0 ] || [ "$RUN_RC" -ge 124 ]; then
        red "    expected the controller to stop early, exit status $RUN_RC"
        _show_tail "$1/stderr"
        return 1
    fi
}

ENV3="a=localhost:1,b=localhost:2,c=localhost:3"
STANDALONE_RE='No --peers flag and XCE_CONTROLLER_PEERS unset or empty; running as single-node primary'

case_controller_env_list() {
    local d; d="$(new_scratch controller-env-list)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname a
    _assert_stopped_early "$d" || return 1
    assert_grep "$d/controller.log" 'HA mode: 3 controller candidates, identity=a' || return 1
    assert_no_grep "$d/controller.log" 'No --peers flag' || return 1
}
add_case case_controller_env_list \
    "controller: XCE_CONTROLLER_PEERS used when --peers is absent (1.2)"

case_controller_peers_wins() {
    local d; d="$(new_scratch controller-peers-wins)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- \
        --myname solo --peers solo=localhost:7
    _assert_stopped_early "$d" || return 1
    assert_grep "$d/controller.log" 'HA mode: 1 controller candidates, identity=solo' || return 1
    assert_no_grep "$d/controller.log" 'HA mode: 3 controller candidates' || return 1
    assert_no_grep "$d/controller.log" 'from XCE_CONTROLLER_PEERS' || return 1
}
add_case case_controller_peers_wins \
    "controller: explicit --peers wins over XCE_CONTROLLER_PEERS (1.1)"

case_controller_neither_set() {
    local d; d="$(new_scratch controller-neither)"
    run_controller "$d" -u XCE_CONTROLLER_PEERS -- --myname ctrl-none
    _assert_stopped_early "$d" || return 1
    assert_grep "$d/controller.log" "$STANDALONE_RE" || return 1
    assert_no_grep "$d/controller.log" 'HA mode' || return 1
}
add_case case_controller_neither_set \
    "controller: neither --peers nor the variable -> standalone, line names the variable (1.3, 1.8)"

case_controller_env_empty() {
    local d; d="$(new_scratch controller-env-empty)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=" -- --myname ctrl-empty
    _assert_stopped_early "$d" || return 1
    assert_grep "$d/controller.log" "$STANDALONE_RE" || return 1
    assert_no_grep "$d/controller.log" 'HA mode' || return 1
}
add_case case_controller_env_empty \
    "controller: empty XCE_CONTROLLER_PEERS behaves as unset (1.7, 1.8)"

case_controller_parse_error_source() {
    local d
    d="$(new_scratch controller-parse-env)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=a=nocolon" -- --myname a
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'XCE_CONTROLLER_PEERS parse error' "stderr (env value)" || return 1

    d="$(new_scratch controller-parse-flag)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname a --peers a=nocolon
    assert_rc 1 || return 1
    assert_grep "$d/stderr" -- '--peers parse error' "stderr (--peers value)" || return 1
    assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
}
add_case case_controller_parse_error_source \
    "controller: parse error names --peers or XCE_CONTROLLER_PEERS (1.4)"

case_controller_even_count_source() {
    local d
    d="$(new_scratch controller-even-env)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=a=localhost:1,b=localhost:2" -- --myname a
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'XCE_CONTROLLER_PEERS: even peer count \(2\)' "stderr (env value)" || return 1

    d="$(new_scratch controller-even-flag)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname a \
        --peers a=localhost:1,b=localhost:2
    assert_rc 1 || return 1
    assert_grep "$d/stderr" -- '--peers: even peer count \(2\)' "stderr (--peers value)" || return 1
    assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
}
add_case case_controller_even_count_source \
    "controller: even-count error names --peers or XCE_CONTROLLER_PEERS (1.4)"

case_controller_help_names_variable() {
    local d; d="$(new_scratch controller-help)"
    run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS "$CONTROLLER_BIN" --help
    assert_rc 0 || return 1
    # argp wraps help text, so join the lines before matching.
    tr '\n' ' ' <"$d/stdout" | tr -s ' ' >"$d/help.txt"
    assert_grep "$d/help.txt" 'XCE_CONTROLLER_PEERS' "--help output" || return 1
    assert_grep "$d/help.txt" 'unset or empty' "--help output" || return 1
    assert_grep "$d/help.txt" 'standalone' "--help output" || return 1
}
add_case case_controller_help_names_variable \
    "controller: --help documents XCE_CONTROLLER_PEERS and the opt-out (1.5, 1.7)"

# ── Task 2.4 (Req 1.6): an XCE_CONTROLLER_PEERS list that has no candidate for
# the controller's own name is refused on stderr before the certificate
# pre-flight. Without the check these runs still exit non-zero at the
# pre-flight, so the cases assert on the message, not just the exit status.
OWN_NAME_RE="has no candidate named"

case_controller_env_own_name_missing() {
    local d; d="$(new_scratch controller-own-missing)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname zz
    assert_rc nonzero || return 1
    assert_grep "$d/stderr" "XCE_CONTROLLER_PEERS \\($ENV3\\) $OWN_NAME_RE 'zz'" "stderr" || return 1
    assert_grep "$d/stderr" -- '--myname <one of: a, b, c>' "stderr" || return 1
    assert_grep "$d/stderr" 'unset XCE_CONTROLLER_PEERS' "stderr" || return 1
    assert_no_grep "$d/stderr" 'cert|key' "stderr (must stop before the certificate pre-flight)" || return 1
}
add_case case_controller_env_own_name_missing \
    "controller: env list without the own name is refused on stderr (1.6)"

case_controller_env_own_name_unnamed() {
    local d; d="$(new_scratch controller-own-unnamed)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=localhost:1" -- --myname zz
    assert_rc nonzero || return 1
    assert_grep "$d/stderr" "$OWN_NAME_RE 'zz'" "stderr" || return 1
    assert_grep "$d/stderr" -- '--myname <one of: localhost>' "stderr (unnamed candidate listed by hostname)" || return 1
}
add_case case_controller_env_own_name_unnamed \
    "controller: unnamed env candidates are listed by hostname in the 1.6 message"

case_controller_env_own_name_present() {
    local d; d="$(new_scratch controller-own-present)"
    run_controller "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --myname b
    _assert_stopped_early "$d" || return 1
    assert_grep "$d/controller.log" 'HA mode: 3 controller candidates, identity=b' || return 1
    assert_no_grep "$d/stderr" "$OWN_NAME_RE" "stderr" || return 1
}
add_case case_controller_env_own_name_present \
    "controller: env list containing the own name is accepted (1.6)"

case_controller_flag_own_name_unchecked() {
    local d; d="$(new_scratch controller-own-flag)"
    run_controller "$d" -u XCE_CONTROLLER_PEERS -- --myname zz --peers "$ENV3"
    _assert_stopped_early "$d" || return 1
    assert_grep "$d/controller.log" 'HA mode: 3 controller candidates, identity=zz' || return 1
    assert_no_grep "$d/stderr" "$OWN_NAME_RE" "stderr" || return 1
}
add_case case_controller_flag_own_name_unchecked \
    "controller: explicit --peers without the own name is not checked at startup (1.6)"

# ── Task 3.1 (Req 1.1-1.5, 1.7, 1.8): the scheduler resolves its candidate
# list from --peers, else XCE_CONTROLLER_PEERS, else the single-controller
# path. A valid list keeps the scheduler running (it retries its controller),
# so those runs are started in the background, stopped once the startup line
# is in the log, and only then asserted on. Argument errors exit on their own.

# Either startup line; every successful scheduler start logs exactly one.
SCHED_ANCHOR_RE='HA cluster config|No --peers flag'
SCHED_STANDALONE_RE='No --peers flag and XCE_CONTROLLER_PEERS unset or empty; using single-controller path, version '

# start_scheduler <dir> <env-args...> -- [scheduler args...]
#   Start the scheduler in the background with --logfile <dir>/scheduler.log,
#   wait for its startup line, then stop it. Fails (with the log tail) when
#   the line never appears.
start_scheduler() {
    local dir="$1"; shift
    local envargs=() pid
    while [ $# -gt 0 ] && [ "$1" != -- ]; do
        envargs+=("$1"); shift
    done
    [ $# -gt 0 ] && shift
    start_bg "$dir" "${envargs[@]}" "$SCHEDULER_BIN" \
        --logfile "$dir/scheduler.log" "$@"
    pid="$LAST_PID"
    if ! wait_for_line "$dir/scheduler.log" "$SCHED_ANCHOR_RE" 10 "$pid"; then
        stop_proc "$pid"
        red "    scheduler never logged its startup line"
        _show_tail "$dir/scheduler.log"
        _show_tail "$dir/stderr"
        return 1
    fi
    stop_proc "$pid"
}

# run_scheduler <dir> <env-args...> -- [scheduler args...]
#   Run the scheduler to completion (argument errors, --sim, --help).
run_scheduler() {
    local dir="$1"; shift
    local envargs=()
    while [ $# -gt 0 ] && [ "$1" != -- ]; do
        envargs+=("$1"); shift
    done
    [ $# -gt 0 ] && shift
    run_to_completion "$dir" 10 "${envargs[@]}" "$SCHEDULER_BIN" \
        --logfile "$dir/scheduler.log" "$@"
}

case_scheduler_env_list() {
    local d; d="$(new_scratch scheduler-env-list)"
    start_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- || return 1
    assert_grep "$d/scheduler.log" \
        'HA cluster config: 3 candidate\(s\), identity=xce_scheduler \(from XCE_CONTROLLER_PEERS\), version ' || return 1
    assert_no_grep "$d/scheduler.log" 'No --peers flag' || return 1
}
add_case case_scheduler_env_list \
    "scheduler: XCE_CONTROLLER_PEERS used when --peers is absent (1.2)"

case_scheduler_peers_wins() {
    local d; d="$(new_scratch scheduler-peers-wins)"
    start_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- \
        --peers solo=localhost:7 || return 1
    assert_grep "$d/scheduler.log" \
        'HA cluster config: 1 candidate\(s\), identity=xce_scheduler, version ' || return 1
    assert_no_grep "$d/scheduler.log" '3 candidate|from XCE_CONTROLLER_PEERS' || return 1
}
add_case case_scheduler_peers_wins \
    "scheduler: explicit --peers wins over XCE_CONTROLLER_PEERS (1.1)"

case_scheduler_neither_set() {
    local d; d="$(new_scratch scheduler-neither)"
    start_scheduler "$d" -u XCE_CONTROLLER_PEERS -- || return 1
    assert_grep "$d/scheduler.log" "$SCHED_STANDALONE_RE" || return 1
    assert_no_grep "$d/scheduler.log" 'HA cluster config' || return 1
}
add_case case_scheduler_neither_set \
    "scheduler: neither --peers nor the variable -> single-controller path, line names the variable (1.3, 1.8)"

case_scheduler_env_empty() {
    local d; d="$(new_scratch scheduler-env-empty)"
    start_scheduler "$d" "XCE_CONTROLLER_PEERS=" -- || return 1
    assert_grep "$d/scheduler.log" "$SCHED_STANDALONE_RE" || return 1
    assert_no_grep "$d/scheduler.log" 'HA cluster config' || return 1
}
add_case case_scheduler_env_empty \
    "scheduler: empty XCE_CONTROLLER_PEERS behaves as unset (1.7, 1.8)"

case_scheduler_parse_error_source() {
    local d
    d="$(new_scratch scheduler-parse-env)"
    run_scheduler "$d" "XCE_CONTROLLER_PEERS=a=nocolon" --
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xce_scheduler: XCE_CONTROLLER_PEERS parse error' "stderr (env value)" || return 1

    d="$(new_scratch scheduler-parse-flag)"
    run_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --peers a=nocolon
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xce_scheduler: --peers parse error' "stderr (--peers value)" || return 1
    assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
}
add_case case_scheduler_parse_error_source \
    "scheduler: parse error names --peers or XCE_CONTROLLER_PEERS (1.4)"

case_scheduler_validate_error_source() {
    local d
    d="$(new_scratch scheduler-port0-env)"
    run_scheduler "$d" "XCE_CONTROLLER_PEERS=a=localhost:0" --
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xce_scheduler: XCE_CONTROLLER_PEERS: invalid cluster config' "stderr (env value)" || return 1

    d="$(new_scratch scheduler-port0-flag)"
    run_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- --peers a=localhost:0
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xce_scheduler: --peers: invalid cluster config' "stderr (--peers value)" || return 1
    assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
}
add_case case_scheduler_validate_error_source \
    "scheduler: cluster-config validation error (port 0) names --peers or XCE_CONTROLLER_PEERS (1.4)"

case_scheduler_help_names_variable() {
    local d; d="$(new_scratch scheduler-help)"
    run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS "$SCHEDULER_BIN" --help
    assert_rc 0 || return 1
    # argp wraps help text, so join the lines before matching.
    tr '\n' ' ' <"$d/stdout" | tr -s ' ' >"$d/help.txt"
    assert_grep "$d/help.txt" 'XCE_CONTROLLER_PEERS' "--help output" || return 1
    assert_grep "$d/help.txt" 'unset or empty' "--help output" || return 1
    assert_grep "$d/help.txt" 'single-controller' "--help output" || return 1
}
add_case case_scheduler_help_names_variable \
    "scheduler: --help documents XCE_CONTROLLER_PEERS and the opt-out (1.5, 1.7)"

# Guard (passes before and after 3.1): --sim returns before the peers block,
# so the variable must not reach simulation mode. The "--sim startup failed"
# line, which only the --sim branch prints, proves that branch ran.
case_scheduler_sim_ignores_variable() {
    local d; d="$(new_scratch scheduler-sim)"
    run_scheduler "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- \
        --sim "$d/does-not-exist.replay"
    assert_rc nonzero || return 1
    assert_grep "$d/stderr" 'xce_scheduler: --sim startup failed' "stderr" || return 1
    assert_no_grep "$d/scheduler.log" "$SCHED_ANCHOR_RE" || return 1
    assert_no_grep "$d/stderr" "$SCHED_ANCHOR_RE" "stderr" || return 1
}
add_case case_scheduler_sim_ignores_variable \
    "scheduler: --sim never reaches the peers block, even with the variable set"

# ── Task 3.2 (Req 1.1-1.5, 1.7, 1.8): the worker (xcecd_virtual, which needs
# no root) resolves its candidate list from --peers, else XCE_CONTROLLER_PEERS,
# else the single-controller path. Its startup line is logged before the log
# file is set up, so it lands on STDOUT. tlog writes it there unbuffered (the
# line is in the redirected stdout file within ~0.3s while the worker keeps
# running), so it is waited for while the worker runs, like the scheduler's.
# The worker ignores SIGTERM for longer than stop_proc's 3s grace, so it is
# SIGKILLed once the line is there. Argument errors exit on their own, before
# the log directory is created; in an ASAN build those early returns must not
# leak (the -l directory string), which the env-value error cases check.

WORKER_ANCHOR_RE='HA cluster config|No --peers flag'
WORKER_STANDALONE_RE='No --peers flag and XCE_CONTROLLER_PEERS unset or empty; using single-controller path \(host=localhost port=9990\), version '

# _split_env_args: shared by start_worker/run_worker -- fills WENV with the
# env(1) arguments before `--` and WARGS with the worker arguments after it.
WENV=()
WARGS=()
_split_env_args() {
    WENV=(); WARGS=()
    while [ $# -gt 0 ] && [ "$1" != -- ]; do
        WENV+=("$1"); shift
    done
    [ $# -gt 0 ] && shift
    WARGS=("$@")
}

# start_worker <dir> <env-args...> -- [worker args...]
#   Start xcecd_virtual in the background with -l <dir>/logs, wait for its
#   startup line on stdout, then kill it. Fails (with the output tails) when
#   the line never appears.
start_worker() {
    local dir="$1" pid; shift
    _split_env_args "$@"
    start_bg "$dir" "${WENV[@]}" "$WORKER_BIN" -l "$dir/logs" \
        "${WARGS[@]+"${WARGS[@]}"}"
    pid="$LAST_PID"
    if ! wait_for_line "$dir/stdout" "$WORKER_ANCHOR_RE" 10 "$pid"; then
        kill -KILL "$pid" 2>/dev/null || true
        stop_proc "$pid"
        red "    worker never printed its startup line"
        _show_tail "$dir/stdout"
        _show_tail "$dir/stderr"
        return 1
    fi
    kill -KILL "$pid" 2>/dev/null || true
    stop_proc "$pid"
}

# run_worker <dir> <env-args...> -- [worker args...]
#   Run xcecd_virtual to completion (argument errors, -h) with -l <dir>/logs.
run_worker() {
    local dir="$1"; shift
    _split_env_args "$@"
    run_to_completion "$dir" 10 "${WENV[@]}" "$WORKER_BIN" -l "$dir/logs" \
        "${WARGS[@]+"${WARGS[@]}"}"
}

case_worker_env_list() {
    local d; d="$(new_scratch worker-env-list)"
    start_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-env || return 1
    assert_grep "$d/stdout" \
        'HA cluster config: 3 candidate\(s\), identity=w-env \(from XCE_CONTROLLER_PEERS\), version ' "stdout" || return 1
    assert_no_grep "$d/stdout" 'No --peers flag' "stdout" || return 1
}
add_case case_worker_env_list \
    "worker: XCE_CONTROLLER_PEERS used when --peers is absent (1.2)"

case_worker_peers_wins() {
    local d; d="$(new_scratch worker-peers-wins)"
    start_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-flag \
        --peers solo=localhost:7 || return 1
    assert_grep "$d/stdout" \
        'HA cluster config: 1 candidate\(s\), identity=w-flag, version ' "stdout" || return 1
    assert_no_grep "$d/stdout" '3 candidate|from XCE_CONTROLLER_PEERS' "stdout" || return 1
}
add_case case_worker_peers_wins \
    "worker: explicit --peers wins over XCE_CONTROLLER_PEERS (1.1)"

case_worker_neither_set() {
    local d; d="$(new_scratch worker-neither)"
    start_worker "$d" -u XCE_CONTROLLER_PEERS -- -n w-none || return 1
    assert_grep "$d/stdout" "$WORKER_STANDALONE_RE" "stdout" || return 1
    assert_no_grep "$d/stdout" 'HA cluster config' "stdout" || return 1
}
add_case case_worker_neither_set \
    "worker: neither --peers nor the variable -> single-controller path, line names the variable (1.3, 1.8)"

case_worker_env_empty() {
    local d; d="$(new_scratch worker-env-empty)"
    start_worker "$d" "XCE_CONTROLLER_PEERS=" -- -n w-empty || return 1
    assert_grep "$d/stdout" "$WORKER_STANDALONE_RE" "stdout" || return 1
    assert_no_grep "$d/stdout" 'HA cluster config' "stdout" || return 1
}
add_case case_worker_env_empty \
    "worker: empty XCE_CONTROLLER_PEERS behaves as unset (1.7, 1.8)"

case_worker_parse_error_source() {
    local d
    d="$(new_scratch worker-parse-env)"
    run_worker "$d" "XCE_CONTROLLER_PEERS=a=nocolon" -- -n w-perr
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xcecd: XCE_CONTROLLER_PEERS parse error' "stderr (env value)" || return 1
    assert_no_grep "$d/stderr" 'LeakSanitizer' "stderr (ASAN build: early return must free)" || return 1

    d="$(new_scratch worker-parse-flag)"
    run_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-perr --peers a=nocolon
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xcecd: --peers parse error' "stderr (--peers value)" || return 1
    assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
}
add_case case_worker_parse_error_source \
    "worker: parse error names --peers or XCE_CONTROLLER_PEERS (1.4)"

case_worker_validate_error_source() {
    local d
    d="$(new_scratch worker-port0-env)"
    run_worker "$d" "XCE_CONTROLLER_PEERS=a=localhost:0" -- -n w-verr
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xcecd: XCE_CONTROLLER_PEERS: invalid cluster config .* in "a=localhost:0"' "stderr (env value)" || return 1
    assert_no_grep "$d/stderr" 'LeakSanitizer' "stderr (ASAN build: early return must free)" || return 1

    d="$(new_scratch worker-port0-flag)"
    run_worker "$d" "XCE_CONTROLLER_PEERS=$ENV3" -- -n w-verr --peers a=localhost:0
    assert_rc 1 || return 1
    assert_grep "$d/stderr" 'xcecd: --peers: invalid cluster config .* in "a=localhost:0"' "stderr (--peers value)" || return 1
    assert_no_grep "$d/stderr" 'XCE_CONTROLLER_PEERS' "stderr (--peers value)" || return 1
}
add_case case_worker_validate_error_source \
    "worker: cluster-config validation error (port 0) names --peers or XCE_CONTROLLER_PEERS (1.4)"

case_worker_help_names_variable() {
    local d; d="$(new_scratch worker-help)"
    run_to_completion "$d" 10 -u XCE_CONTROLLER_PEERS "$WORKER_BIN" -h
    assert_rc 0 || return 1
    tr '\n' ' ' <"$d/stdout" | tr -s ' ' >"$d/help.txt"
    assert_grep "$d/help.txt" 'XCE_CONTROLLER_PEERS' "-h output" || return 1
    assert_grep "$d/help.txt" 'unset or empty' "-h output" || return 1
    assert_grep "$d/help.txt" 'single-controller' "-h output" || return 1
    assert_grep "$d/help.txt" 'sudo --preserve-env=XCE_CONTROLLER_PEERS' "-h output" || return 1
}
add_case case_worker_help_names_variable \
    "worker: -h documents XCE_CONTROLLER_PEERS, the opt-out and that sudo drops it (1.5, 1.7)"


# ════════════════════════════════════════════════════════════════════════════
# main
# ════════════════════════════════════════════════════════════════════════════
main() {
    local i fn

    if [ "$LIST_ONLY" = 1 ]; then
        for i in "${!CASES[@]}"; do
            printf '%s\t%s\n' "${CASES[$i]}" "${CASE_DESC[$i]}"
        done
        exit 0
    fi

    preflight

    trap on_exit EXIT
    trap 'exit 130' INT
    trap 'exit 143' TERM
    SCRATCH_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/verify-peers-env.XXXXXX")"

    log "build dir: $BUILD_DIR"
    log "scratch root: $SCRATCH_ROOT"
    log "${#CASES[@]} case(s) registered${ONLY:+, filtered by /$ONLY/}"

    for i in "${!CASES[@]}"; do
        fn="${CASES[$i]}"
        if [ -n "$ONLY" ] && ! grep -Eq -- "$ONLY" <<<"$fn"; then
            continue
        fi
        echo "--- $fn: ${CASE_DESC[$i]}"
        if "$fn"; then
            green "PASS $fn"
            PASSED+=("$fn")
        else
            red "FAIL $fn"
            FAILED+=("$fn")
        fi
        stop_all
    done

    echo
    log "summary: ${#PASSED[@]} passed, ${#FAILED[@]} failed"
    for fn in "${FAILED[@]+"${FAILED[@]}"}"; do
        red "  FAILED: $fn"
    done
    if [ "${#FAILED[@]}" -ne 0 ]; then
        exit 1
    fi
    green "ALL PASS"
    exit 0
}

main