#!/usr/bin/env bash # test-build-version-regression.sh — build-time-versioning task 6.1 # (.kiro/specs/build-time-versioning/tasks.md, Requirements 4.1, 4.2, 4.3, # 5.1, 5.2) # # MOTIVATING INCIDENT this guards against: a live deployment kept running a # stale debug-profile xce_scheduler binary that lacked a just-landed fix, # and diagnosing that required manual `ls -la` mtime checks plus # `git merge-base --is-ancestor` -- there was no way to tell, from the # binary or its logs, which commit it was actually built from. Tasks 1-5 of # this spec fixed that end to end (xce-build.sh now computes and passes # real commit_id/build_time; --version and the startup log both render # them). THIS script is the automated regression guard proving that fix # holds -- via a REAL `/xce-build`-equivalent invocation, not a unit test # against mocked build output -- and that it keeps holding for any future # change to xce-build.sh or the version-composition code path. # # What it does, for BOTH the asan and debug profiles (Requirement 4.3 -- # every C build profile must carry accurate version info, and the # motivating incident was specifically a debug-profile binary): # 1. Runs a REAL `nix develop --command bash src/tools/xce-build.sh # scheduler ` -- the exact command the # /xce-build skill itself runs (.claude/skills/xce-build/SKILL.md). # 2. Runs the resulting xce_scheduler binary with --version. # 3. Parses the commit id out of that output. # 4. Compares it against a live `git rev-parse --short HEAD` (accounting # for the -dirty suffix xce-build.sh itself applies when the tree has # uncommitted changes). # 5. FAILS LOUDLY (non-zero exit, clear diagnostic on stderr) if the # commit id is empty, missing, "unknown", or does not match. # It also runs a self-contained DEMONSTRATION (see demonstrate_detection_ # capability below) proving the comparison logic itself would have caught # the original bug: a deliberately-wrong expected hash, and synthetic # --version output with an empty/"unknown" commit id, must both be # rejected; genuinely-matching real output must be accepted. # # Usage: # bash src/tools/test-build-version-regression.sh [--keep-build-dirs] # # Exit code is the sole pass/fail signal (0 = pass, non-zero = regression # or a demonstration inconsistency). Suitable for CI or a developer running # it standalone. Two full clean builds happen (xce-build.sh always does a # `rm -rf` + fresh `meson setup` per profile -- see its own header comment) # -- budget several minutes of real wall-clock time. # # Component choice: xce_scheduler (controller uses the identical argp/ # xce_compose_version() mechanism and would work equally well -- scheduler # is chosen because the motivating incident was specifically a scheduler # binary). # # Why this file lives here: src/tools/ already hosts standalone, # CLI-runnable regression/integration scripts for cross-cutting concerns # that need a real build/deployment rather than a meson unit test -- # test-commit-durability.sh and test-heartbeat-tolerance.sh are the # existing precedent this script follows (same header-comment style, # colour helpers, pass/fail accounting). This is deliberately NOT a meson # test() (two full `rm -rf` + `meson setup` + compile cycles per run is far # too expensive to run on every `meson test`) and deliberately does NOT # invent a new top-level test-framework directory. # # Revalidation trigger: VERSION_PREFIX below must match the literal string # literal passed to xce_compose_version() in scheduler/src/main.c (task # 3.2). If that literal ever changes, update VERSION_PREFIX here too. set -euo pipefail # ── paths ─────────────────────────────────────────────────────────────────── SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" WORKTREE="$(cd "$SCRIPT_DIR/../.." && pwd)" SRC_DIR="$WORKTREE/src" VERSION_PREFIX="xce v0.0.0.1-" BUILD_DIR_ASAN="versionregress.asan" BUILD_DIR_DEBUG="versionregress.debug" KEEP_BUILD_DIRS=0 for arg in "$@"; do case "$arg" in --keep-build-dirs) KEEP_BUILD_DIRS=1 ;; *) echo "Unknown argument: $arg" >&2 echo "Usage: $0 [--keep-build-dirs]" >&2 exit 2 ;; esac done # ── colour / pass-fail helpers (matches test-heartbeat-tolerance.sh) ──────── green() { printf '\033[32m%s\033[0m\n' "$*"; } red() { printf '\033[31m%s\033[0m\n' "$*"; } yellow() { printf '\033[33m%s\033[0m\n' "$*"; } trap 'red "FATAL: command failed (line $LINENO). See output above."' ERR cleanup() { if [ "$KEEP_BUILD_DIRS" = 1 ]; then yellow "--keep-build-dirs given: leaving $SRC_DIR/$BUILD_DIR_ASAN and $SRC_DIR/$BUILD_DIR_DEBUG in place." return 0 fi for d in "$SRC_DIR/$BUILD_DIR_ASAN" "$SRC_DIR/$BUILD_DIR_DEBUG"; do if [ -d "$d" ]; then rm -rf "$d" fi done } trap cleanup EXIT # ── expected commit id (same computation xce-build.sh itself does) ───────── compute_expected_commit_id() { local hash if ! hash="$(git -C "$SRC_DIR" rev-parse --short HEAD 2>/dev/null)"; then echo "" return 1 fi if [ -n "$(git -C "$SRC_DIR" status --porcelain 2>/dev/null)" ]; then echo "${hash}-dirty" else echo "$hash" fi } # ── build + version-string extraction ─────────────────────────────────────── # Deliberately does NOT let a non-zero xce-build.sh exit abort this whole # script (set -e is bypassed here via the if/else form, an intentional # exception). xce-build.sh's own `set -euo pipefail` means ANY failure in # its bundled `meson test` run (asan profile only) -- including a # pre-existing, unrelated flaky test -- makes the WHOLE invocation exit # non-zero, even though `meson compile` (and therefore the binary this # script actually needs) already succeeded. This script's job is build-time # VERSION embedding (Requirements 4.1-4.3, 5.1-5.2), not general test-suite # health, so a downstream test failure must not mask/prevent the version # check itself -- the binary-existence check right after this call is what # still catches a genuine compile failure. run_build() { local profile="$1" build_dir="$2" rc=0 yellow "=== [$profile] nix develop $SRC_DIR --command bash $SRC_DIR/tools/xce-build.sh scheduler $profile $build_dir ===" nix develop "$SRC_DIR" --command bash "$SRC_DIR/tools/xce-build.sh" scheduler "$profile" "$build_dir" || rc=$? if [ "$rc" != 0 ]; then yellow "WARNING [$profile]: xce-build.sh exited non-zero (rc=$rc). Most commonly this means its OWN bundled 'meson test' step hit a pre-existing/unrelated failing test, not that the compile itself failed. Proceeding to check whether the binary was still produced and whether its --version output is correct." fi return 0 } # Extracts "" from a raw --version string of the shape # " built ". Prints the commit id (may # be empty) on stdout; prints nothing and returns 1 if the expected prefix # is not even present (a stronger failure than a merely-wrong commit id). extract_commit_id() { local raw="$1" rest rest="${raw#*"$VERSION_PREFIX"}" if [ "$rest" = "$raw" ]; then return 1 fi printf '%s' "${rest%% built *}" } # check_result LABEL ACTUAL_VERSION_OUTPUT EXPECTED_COMMIT # Prints a PASS/FAIL line and returns 0/1. This is the exact comparison # logic Requirement 4.2/5.x require the build tooling to apply -- factored # into one function so both the real per-profile checks below AND # demonstrate_detection_capability() (the RED-phase proof) exercise the # identical code path. check_result() { local label="$1" actual="$2" expected="$3" got if ! got="$(extract_commit_id "$actual")"; then red "FAIL [$label]: --version output did not even contain the expected prefix '$VERSION_PREFIX' -- raw output: [$actual]" return 1 fi if [ -z "$got" ]; then red "FAIL [$label]: commit id is EMPTY in --version output -- this is exactly the motivating-incident regression. Raw output: [$actual]" return 1 fi if [ "$got" = "unknown" ]; then red "FAIL [$label]: commit id is the literal fallback 'unknown', not a real commit -- xce-build.sh should have computed a real value on this git checkout. Raw output: [$actual]" return 1 fi if [ "$got" != "$expected" ]; then red "FAIL [$label]: commit id MISMATCH -- --version reported '$got', expected (git rev-parse --short HEAD [+ -dirty]) '$expected'. Raw output: [$actual]" return 1 fi green "PASS [$label]: commit id '$got' matches expected '$expected'." return 0 } # Proves check_result's comparison logic itself would have caught the # original bug. Since task 2.1 (xce-build.sh's own fix) is already # committed, we cannot cheaply revert just it in isolation -- instead we # feed the SAME comparison function deliberately-wrong / synthetic-empty / # synthetic-"unknown" inputs and confirm it rejects every one of them, plus # a sanity check that genuinely-matching real output is still accepted. demonstrate_detection_capability() { local real_output="$1" real_commit="$2" yellow "=== DEMONSTRATION: proving the comparison logic catches the motivating-incident failure mode ===" yellow "--- (a) deliberately-WRONG expected commit id against REAL --version output ---" if check_result "DEMO-wrong-hash" "$real_output" "deadbeef"; then red "DEMONSTRATION FAILED: incorrectly PASSED against a wrong expected commit id." exit 1 fi green "Confirmed: a wrong expected commit id is correctly rejected." local synthetic_empty="${VERSION_PREFIX} built 2026-01-01T00:00:00Z" yellow "--- (b) SYNTHETIC --version output with an EMPTY commit id (the literal original bug) ---" if check_result "DEMO-empty-commit" "$synthetic_empty" "$real_commit"; then red "DEMONSTRATION FAILED: incorrectly PASSED against an empty commit id." exit 1 fi green "Confirmed: an empty/missing commit id is correctly rejected." local synthetic_unknown="${VERSION_PREFIX}unknown built 2026-01-01T00:00:00Z" yellow "--- (c) SYNTHETIC --version output with commit id 'unknown' ---" if check_result "DEMO-unknown-commit" "$synthetic_unknown" "$real_commit"; then red "DEMONSTRATION FAILED: incorrectly PASSED against commit id 'unknown'." exit 1 fi green "Confirmed: commit id 'unknown' is correctly rejected." yellow "--- (d) sanity: REAL output against the REAL expected commit id must PASS ---" if ! check_result "DEMO-sanity-real-match" "$real_output" "$real_commit"; then red "DEMONSTRATION FAILED: incorrectly FAILED against genuinely matching real output." exit 1 fi green "=== DEMONSTRATION complete: wrong / empty / unknown commit ids are rejected; a genuine match is accepted ===" } # ── main ───────────────────────────────────────────────────────────────── expected_commit="$(compute_expected_commit_id)" || { red "FATAL: could not compute expected commit id via 'git rev-parse --short HEAD' in $SRC_DIR -- is this a git checkout?" exit 1 } if [ -z "$expected_commit" ]; then red "FATAL: expected commit id resolved empty -- cannot run this regression check without a real git checkout." exit 1 fi yellow "Expected commit id for this run (git rev-parse --short HEAD [+ -dirty]): $expected_commit" overall_pass=1 first_profile_output="" for spec in "asan:$BUILD_DIR_ASAN" "debug:$BUILD_DIR_DEBUG"; do profile="${spec%%:*}" build_dir="${spec#*:}" run_build "$profile" "$build_dir" binary="$SRC_DIR/$build_dir/scheduler/xce_scheduler" if [ ! -x "$binary" ]; then red "FAIL [$profile]: expected binary not found/executable at $binary after a successful-looking build." overall_pass=0 continue fi actual_output="$("$binary" --version 2>&1)" || true yellow "[$profile] xce_scheduler --version raw output: [$actual_output]" if [ -z "$first_profile_output" ]; then first_profile_output="$actual_output" fi if ! check_result "$profile" "$actual_output" "$expected_commit"; then overall_pass=0 fi done if [ -n "$first_profile_output" ]; then demonstrate_detection_capability "$first_profile_output" "$expected_commit" fi if [ "$overall_pass" = 1 ]; then green "=== ALL PROFILES PASS: xce_scheduler --version commit id is real (non-empty, non-'unknown') and matches git HEAD, for both asan and debug ===" exit 0 else red "=== REGRESSION DETECTED: see FAIL line(s) above ===" exit 1 fi