#!/usr/bin/env bash # test-version-startup-consistency.sh — build-time-versioning task 6.2 # (.kiro/specs/build-time-versioning/tasks.md, Requirement 2.2: "The system # shall present the same commit identifier and build timestamp values # through the --version flag and through the startup log line ... for the # same build artifact, so the two never disagree.") # # Unlike task 6.1 (test-build-version-regression.sh), which checks a # binary's version info against a live `git rev-parse` (build-vs-source-tree # consistency), THIS script checks a binary's version info against ITSELF: # the exact same running artifact's --version/-version output must report # the identical commit id and build time as the version fields it logs at # INFO level when it starts up. A daemon that composed its --version string # once but its startup log from a stale/different computation (or vice # versa) would slip past 6.1 entirely (6.1 only ever inspects --version) -- # this is the check that specifically guards the "two never disagree" # half of Requirement 2.2. # # Components checked (one C daemon, one Go binary, per the task's own # scope -- controller/scheduler both already share the identical # xce_compose_version() mechanism per task 3.2, so either would equally # prove the C side; scheduler is chosen to match task 6.1's precedent and # because it reaches its startup INFO line without needing an actual # reachable controller): # - xce_scheduler (C daemon, task 3.2) # - xce_apiserver (Go binary, task 4.4) # # What "byte-identical" means here, precisely (the task's own text asks # the implementation to state its interpretation): the two captures are # NOT compared as whole strings/lines -- --version's output and the # startup log's line are, by design (design.md's Components and # Interfaces section), differently-shaped text: a composed # " built " string for the C daemon's # --version vs. that same string embedded mid-sentence in a larger INFO # log line; a plain "xce-apiserver built " line for # apiserver's -version vs. structured zap JSON fields # ("gitCommit"/"buildTime") on its startup log line. What Requirement 2.2 # actually constrains is the two DATA VALUES (commit id, build time) -- # so this script extracts just those two field values from each capture # and asserts the extracted VALUES are byte-identical between --version # and the startup log, for each binary. Surrounding text differing is # expected and not a failure. # # Build path notes: # - xce_scheduler is built via the exact same real `/xce-build`-equivalent # invocation task 6.1 established: `nix develop --command bash # src/tools/xce-build.sh scheduler `. # - xce_apiserver's two documented real build paths (src/frontend/ # Makefile's build-server target, and apiserver/meson.build's go_build- # driven custom_target invoked via `xce-build.sh apiserver`) are BOTH # pre-existing-broken/unusable in this environment, independently of # this task: build-server lacks CGO_CFLAGS/CGO_LDFLAGS entirely # (bug-1597, confirmed still present, unmodified here per this # task's constraints) and go_build (src/tools/go_build) hardcodes # GOPROXY=https://goproxy.cn plus a fresh-every-invocation # GOCACHE/GOPATH under meson's own $TMPDIR sandbox -- confirmed via # live reproduction (established but stalled TCP connection to # goproxy.cn:443, zero CPU progress for 8-10 minutes across three # separate attempts) that this specific combination hangs in this # sandbox even though goproxy.cn itself is directly reachable outside # that fresh-cache/custom-GOPROXY combination. Per bug-1597's own # documented workaround (its "fix" field), this script instead invokes # `go build` directly with the same -ldflags shape go_build/Makefile # use and the same CGO_CFLAGS/CGO_LDFLAGS pkg-config derivation # test-backend/apiserver's meson.build already use, against the # default (already-populated) module cache -- functionally the same # real build (same source, same compiler, same ldflags injection # mechanism), just not routed through the specific wrapper script that # hangs in this sandbox. Neither go_build nor the Makefile is modified. # `go build` is invoked with `-mod=readonly` explicitly (confirmed live: # without it, this build silently ADDED ~65 unrelated indirect- # dependency /go.mod checksum lines to the committed # src/frontend/apiserver/go.sum as a side effect of resolving the full # module graph -- go.sum can still be extended under readonly mode's # default in some Go versions, but never go.mod itself; -mod=readonly # makes any genuinely-missing checksum a loud build failure instead of # a silent repo mutation, which is what this task's "only ADDS a new # check" constraint requires. The build still succeeds under # -mod=readonly against this repo's current go.sum, so nothing was # actually missing -- the extra entries were superfluous). # # Usage: # bash src/tools/test-version-startup-consistency.sh [--keep-artifacts] # # Exit code is the sole pass/fail signal (0 = pass, non-zero = a real # --version/startup-log disagreement or a demonstration inconsistency). # Two real builds (scheduler + apiserver's C lib dependency) plus two real # daemon startups happen -- budget a few minutes of wall-clock time. # # Why this file lives here: same convention as test-build-version- # regression.sh / test-commit-durability.sh / test-heartbeat-tolerance.sh # -- a standalone, CLI-runnable, real-build-and-real-process check for a # cross-cutting concern, deliberately NOT a meson test() (real builds + # real daemon startups are too expensive for every `meson test` run). # # Revalidation triggers: # - SCHED_VERSION_PREFIX below must match the literal prefix passed to # xce_compose_version() in scheduler/src/main.c (task 3.2). # - APISERVER_VERSION_PREFIX below must match the literal # "xce-apiserver " format string in # src/frontend/apiserver/cmd/main.go's -version branch (task 4.4). # - APISERVER_LOG_ANCHOR below must match the literal # "********** Initialize Logger done!" log message in the same file. set -euo pipefail # ── paths ─────────────────────────────────────────────────────────────────── SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" WORKTREE="$(cd "$SCRIPT_DIR/../.." && pwd)" SRC_DIR="$WORKTREE/src" SCHED_VERSION_PREFIX="xce v0.0.0.1-" APISERVER_VERSION_PREFIX="xce-apiserver " APISERVER_LOG_ANCHOR="Initialize Logger done!" BUILD_DIR_SCHED="versionstartup.sched" BUILD_DIR_APILIB="versionstartup.apilib" KEEP_ARTIFACTS=0 for arg in "$@"; do case "$arg" in --keep-artifacts) KEEP_ARTIFACTS=1 ;; *) echo "Unknown argument: $arg" >&2 echo "Usage: $0 [--keep-artifacts]" >&2 exit 2 ;; esac done # ── colour / pass-fail helpers (matches test-build-version-regression.sh / # test-heartbeat-tolerance.sh) ─────────────────────────────────────────────── green() { printf '\033[32m%s\033[0m\n' "$*"; } red() { printf '\033[31m%s\033[0m\n' "$*"; } yellow() { printf '\033[33m%s\033[0m\n' "$*"; } trap 'red "FATAL: command failed (line $LINENO). See output above."' ERR SCHED_PID="" API_PID="" WORKDIR="" cleanup() { if [ -n "$SCHED_PID" ]; then kill -9 "$SCHED_PID" 2>/dev/null || true wait "$SCHED_PID" 2>/dev/null || true fi if [ -n "$API_PID" ]; then kill -9 "$API_PID" 2>/dev/null || true wait "$API_PID" 2>/dev/null || true fi if [ "$KEEP_ARTIFACTS" = 1 ]; then yellow "--keep-artifacts given: leaving $SRC_DIR/$BUILD_DIR_SCHED, $SRC_DIR/$BUILD_DIR_APILIB, and $WORKDIR in place." return 0 fi for d in "$SRC_DIR/$BUILD_DIR_SCHED" "$SRC_DIR/$BUILD_DIR_APILIB"; do [ -d "$d" ] && rm -rf "$d" done [ -n "$WORKDIR" ] && [ -d "$WORKDIR" ] && rm -rf "$WORKDIR" } trap cleanup EXIT WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/xce-version-startup-check.XXXXXX")" # ── field extraction ───────────────────────────────────────────────────── # extract_composed_fields TEXT PREFIX # TEXT is either a raw --version string, or a larger line of text (e.g. a # full log line) that CONTAINS " built " # somewhere in it. Prints "\t" on stdout (may contain # empty fields); returns 1 if PREFIX is not found at all -- a stronger # failure than a merely-empty field. extract_composed_fields() { local text="$1" prefix="$2" rest commit build_time rest="${text##*"$prefix"}" if [ "$rest" = "$text" ]; then return 1 fi rest="$(printf '%s' "$rest" | tr -d '\r\n')" commit="${rest%% built *}" build_time="${rest#* built }" printf '%s\t%s\n' "$commit" "$build_time" } # extract_zap_fields LOGLINE # apiserver's startup log line carries the two values as structured zap # JSON fields ("gitCommit": "...", "buildTime": "...") rather than a # composed string. Prints "\t"; returns 1 if either # field is absent. Deliberately POSIX BRE `sed` (not `grep -P`/PCRE) -- # this script's own real (non-interactive) invocation environment resolves # plain `grep` to a BusyBox build with no -P support (confirmed live: an # interactive shell alias shadows `grep` with a PCRE-capable binary, but # that alias is interactive-shell-only and is NOT inherited by `bash # thisscript.sh` run as its own process) -- BusyBox `sed` supports POSIX # BRE including [[:space:]] and -n/p, which is sufficient here and portable # to any POSIX sed. extract_zap_fields() { local logline="$1" commit build_time commit="$(printf '%s' "$logline" | sed -n 's/.*"gitCommit":[[:space:]]*"\([^"]*\)".*/\1/p')" build_time="$(printf '%s' "$logline" | sed -n 's/.*"buildTime":[[:space:]]*"\([^"]*\)".*/\1/p')" if [ -z "$commit" ] || [ -z "$build_time" ]; then return 1 fi printf '%s\t%s\n' "$commit" "$build_time" } # check_consistency LABEL VERSION_COMMIT VERSION_BUILD_TIME LOG_COMMIT LOG_BUILD_TIME # The exact comparison Requirement 2.2 requires: both the commit id AND the # build time must be identical (non-empty on both sides, and equal) between # the --version/-version capture and the startup-log capture. This is the # single function both the real per-binary checks below AND # demonstrate_detection_capability() (the discriminating-power proof) # exercise, so the demonstration genuinely proves this exact logic's # behaviour. check_consistency() { local label="$1" v_commit="$2" v_build="$3" l_commit="$4" l_build="$5" local ok=1 if [ -z "$v_commit" ] || [ -z "$l_commit" ]; then red "FAIL [$label]: commit id missing from one or both captures (--version='$v_commit' startup-log='$l_commit')" ok=0 elif [ "$v_commit" != "$l_commit" ]; then red "FAIL [$label]: commit id MISMATCH -- --version reports '$v_commit', startup log reports '$l_commit'. These must never disagree (Requirement 2.2)." ok=0 fi if [ -z "$v_build" ] || [ -z "$l_build" ]; then red "FAIL [$label]: build time missing from one or both captures (--version='$v_build' startup-log='$l_build')" ok=0 elif [ "$v_build" != "$l_build" ]; then red "FAIL [$label]: build time MISMATCH -- --version reports '$v_build', startup log reports '$l_build'. These must never disagree (Requirement 2.2)." ok=0 fi if [ "$ok" = 1 ]; then green "PASS [$label]: --version and startup log agree -- commit='$v_commit' build_time='$v_build'" return 0 fi return 1 } # Proves check_consistency's comparison logic itself would catch a genuine # disagreement, since (unlike task 6.1) there is no prior broken # implementation to revert against for a real RED phase: 3.2/4.4 already # guarantee --version and the startup log are read from the identical # in-memory value, so a live mismatch cannot currently be produced. Instead, # feed the SAME function deliberately-mismatched commit id / build time # pairs and confirm both are rejected, plus a sanity check that genuinely # matching real captures are still accepted. demonstrate_detection_capability() { local real_commit="$1" real_build="$2" yellow "=== DEMONSTRATION: proving check_consistency() catches a real --version/startup-log disagreement ===" yellow "--- (a) commit id mismatch (build time matches) ---" if check_consistency "DEMO-commit-mismatch" "$real_commit" "$real_build" "deadbeef0000" "$real_build"; then red "DEMONSTRATION FAILED: incorrectly PASSED a commit id mismatch." exit 1 fi green "Confirmed: a commit id mismatch is correctly rejected." yellow "--- (b) build time mismatch (commit id matches) ---" if check_consistency "DEMO-buildtime-mismatch" "$real_commit" "$real_build" "$real_commit" "1999-01-01T00:00:00Z"; then red "DEMONSTRATION FAILED: incorrectly PASSED a build time mismatch." exit 1 fi green "Confirmed: a build time mismatch is correctly rejected." yellow "--- (c) one side empty (the degenerate case a naive equality check could miss if BOTH sides were empty) ---" if check_consistency "DEMO-empty-side" "$real_commit" "$real_build" "" "$real_build"; then red "DEMONSTRATION FAILED: incorrectly PASSED an empty commit id on one side." exit 1 fi green "Confirmed: an empty field on one side is correctly rejected." yellow "--- (d) sanity: genuinely matching real values must PASS ---" if ! check_consistency "DEMO-sanity-real-match" "$real_commit" "$real_build" "$real_commit" "$real_build"; then red "DEMONSTRATION FAILED: incorrectly FAILED genuinely matching real values." exit 1 fi green "=== DEMONSTRATION complete: commit/build-time mismatches and missing fields are rejected; a genuine match is accepted ===" } overall_pass=1 # ══════════════════════════════════════════════════════════════════════════ # xce_scheduler (C daemon) # ══════════════════════════════════════════════════════════════════════════ yellow "=== [scheduler] nix develop $SRC_DIR --command bash $SRC_DIR/tools/xce-build.sh scheduler debug $BUILD_DIR_SCHED ===" nix develop "$SRC_DIR" --command bash "$SRC_DIR/tools/xce-build.sh" scheduler debug "$BUILD_DIR_SCHED" SCHED_BIN="$SRC_DIR/$BUILD_DIR_SCHED/scheduler/xce_scheduler" if [ ! -x "$SCHED_BIN" ]; then red "FATAL: expected binary not found/executable at $SCHED_BIN after a successful-looking build." exit 1 fi sched_version_raw="$("$SCHED_BIN" --version 2>&1)" yellow "[scheduler] --version raw output: [$sched_version_raw]" sched_log="$WORKDIR/scheduler.log" rm -f "$sched_log" "$SCHED_BIN" --logfile "$sched_log" >"$WORKDIR/scheduler.stdout" 2>&1 & SCHED_PID=$! sched_log_line="" waited=0 while [ "$waited" -lt 20 ]; do if [ -f "$sched_log" ] && grep -qF -- "$SCHED_VERSION_PREFIX" "$sched_log" 2>/dev/null; then sched_log_line="$(grep -F -- "$SCHED_VERSION_PREFIX" "$sched_log" | tail -1)" break fi sleep 0.5 waited=$((waited + 1)) done kill -9 "$SCHED_PID" 2>/dev/null || true wait "$SCHED_PID" 2>/dev/null || true SCHED_PID="" if [ -z "$sched_log_line" ]; then red "FAIL [scheduler]: startup log never produced a line containing '$SCHED_VERSION_PREFIX' within 10s. Log contents:" cat "$sched_log" 2>/dev/null >&2 || true overall_pass=0 sched_v_commit=""; sched_v_build="" else yellow "[scheduler] startup log line: [$sched_log_line]" if ! sched_v_fields="$(extract_composed_fields "$sched_version_raw" "$SCHED_VERSION_PREFIX")"; then red "FAIL [scheduler]: --version output did not contain the expected prefix '$SCHED_VERSION_PREFIX' -- raw: [$sched_version_raw]" overall_pass=0 sched_v_commit=""; sched_v_build="" else sched_v_commit="${sched_v_fields%%$'\t'*}" sched_v_build="${sched_v_fields#*$'\t'}" fi if ! sched_l_fields="$(extract_composed_fields "$sched_log_line" "$SCHED_VERSION_PREFIX")"; then red "FAIL [scheduler]: startup log line did not contain the expected prefix '$SCHED_VERSION_PREFIX' -- line: [$sched_log_line]" overall_pass=0 sched_l_commit=""; sched_l_build="" else sched_l_commit="${sched_l_fields%%$'\t'*}" sched_l_build="${sched_l_fields#*$'\t'}" fi if [ -n "${sched_v_commit:-}" ] || [ -n "${sched_l_commit:-}" ]; then if ! check_consistency "scheduler" "$sched_v_commit" "$sched_v_build" "$sched_l_commit" "$sched_l_build"; then overall_pass=0 fi fi fi # ══════════════════════════════════════════════════════════════════════════ # xce_apiserver (Go binary) # ══════════════════════════════════════════════════════════════════════════ yellow "=== [apiserver] building lib dependency: nix develop $SRC_DIR --command bash $SRC_DIR/tools/xce-build.sh lib debug $BUILD_DIR_APILIB ===" nix develop "$SRC_DIR" --command bash "$SRC_DIR/tools/xce-build.sh" lib debug "$BUILD_DIR_APILIB" APILIB_DIR="$SRC_DIR/$BUILD_DIR_APILIB" if [ ! -f "$APILIB_DIR/lib/libxce.a" ]; then red "FATAL: expected $APILIB_DIR/lib/libxce.a not found after 'lib' build." exit 1 fi API_COMMIT_ID="$(git -C "$SRC_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" if [ -n "$(git -C "$SRC_DIR" status --porcelain 2>/dev/null)" ]; then API_COMMIT_ID="${API_COMMIT_ID}-dirty" fi API_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" yellow "=== [apiserver] go build (direct invocation, see header comment re: bug-1597 / go_build's goproxy.cn hang in this sandbox) -- commit_id=$API_COMMIT_ID build_time=$API_BUILD_TIME ===" APISERVER_BIN="$APILIB_DIR/frontend/apiserver/xce_apiserver" mkdir -p "$(dirname "$APISERVER_BIN")" nix develop "$SRC_DIR" --command bash -c " set -e cd '$SRC_DIR/frontend/apiserver' CGO_CFLAGS=\"-g \$(pkg-config --cflags librtti c-capnproto openssl tinylog) -I$SRC_DIR/include -I$APILIB_DIR/lib\" CGO_LDFLAGS=\"-g \$(pkg-config --libs librtti c-capnproto openssl tinylog) -lm -lrt -L$APILIB_DIR/lib -lxce\" GOPROXY=off CGO_ENABLED=1 CGO_CFLAGS=\"\$CGO_CFLAGS\" CGO_LDFLAGS=\"\$CGO_LDFLAGS\" \ go build -mod=readonly -gcflags 'all=-N -l' -ldflags \"-X main.GitCommit=$API_COMMIT_ID -X main.BuildTime=$API_BUILD_TIME\" \ -o $APISERVER_BIN ./cmd/main.go " if [ ! -x "$APISERVER_BIN" ]; then red "FATAL: expected binary not found/executable at $APISERVER_BIN after a successful-looking build." exit 1 fi api_version_raw="$("$APISERVER_BIN" -version 2>&1)" yellow "[apiserver] -version raw output: [$api_version_raw]" # Minimal topdir so apiserver reaches its startup INFO log line without a # real controller cluster: config/{auth.yaml,rbac.yaml} copied verbatim # (same reasoning as deploy-ha.sh -- functional auth/RBAC is not required, # only enough for InitAuthProvider()/LoadRBACConfig() to load without # erroring), app.yaml with its trailing `backend:` section stripped (same # deploy-ha.sh technique) so main.go boots in permanent stub mode -- no # RegisterToBackend attempt, no real controller needed -- and a # freshly-generated logger.yaml pointing outputPath at this run's own # WORKDIR. This is the identical lightweight setup deploy-ha.sh already # uses for its own apiserver instance (src/tools/deploy-ha.sh:337-421), # minus the backend re-population deploy-ha.sh does for ITS OWN (different) # purpose of live resource-value propagation testing. APISERVER_SRC_CONFIG="$SRC_DIR/frontend/apiserver/config" API_TOPDIR="$WORKDIR/apiserver_topdir" mkdir -p "$API_TOPDIR/config" cp "$APISERVER_SRC_CONFIG/auth.yaml" "$API_TOPDIR/config/auth.yaml" cp "$APISERVER_SRC_CONFIG/rbac.yaml" "$API_TOPDIR/config/rbac.yaml" sed '/^backend:/,$d' "$APISERVER_SRC_CONFIG/app.yaml" > "$API_TOPDIR/config/app.yaml" cat > "$API_TOPDIR/config/logger.yaml" << EOF logging: level: "debug" encoding: console outputPath: "$WORKDIR/apiserver.log" file: maxSize: 100 maxBackups: 10 maxAge: 14 compress: true encoderConfig: messageKey: "message" levelKey: "level" timeKey: "timestamp" nameKey: "logger" callerKey: "caller" functionKey: "func" stacktraceKey: "stacktrace" timeEncoder: "ISO8601" levelEncoder: "capital" EOF api_log="$WORKDIR/apiserver.log" rm -f "$api_log" "$APISERVER_BIN" -d "$API_TOPDIR" >"$WORKDIR/apiserver.stdout" 2>&1 & API_PID=$! api_log_line="" waited=0 while [ "$waited" -lt 20 ]; do if [ -f "$api_log" ] && grep -qF -- "$APISERVER_LOG_ANCHOR" "$api_log" 2>/dev/null; then api_log_line="$(grep -F -- "$APISERVER_LOG_ANCHOR" "$api_log" | head -1)" break fi sleep 0.5 waited=$((waited + 1)) done kill -9 "$API_PID" 2>/dev/null || true wait "$API_PID" 2>/dev/null || true API_PID="" if [ -z "$api_log_line" ]; then red "FAIL [apiserver]: startup log never produced a line containing '$APISERVER_LOG_ANCHOR' within 10s. Log contents:" cat "$api_log" 2>/dev/null >&2 || true cat "$WORKDIR/apiserver.stdout" 2>/dev/null >&2 || true overall_pass=0 api_v_commit=""; api_v_build="" else yellow "[apiserver] startup log line: [$api_log_line]" if ! api_v_fields="$(extract_composed_fields "$api_version_raw" "$APISERVER_VERSION_PREFIX")"; then red "FAIL [apiserver]: -version output did not contain the expected prefix '$APISERVER_VERSION_PREFIX' -- raw: [$api_version_raw]" overall_pass=0 api_v_commit=""; api_v_build="" else api_v_commit="${api_v_fields%%$'\t'*}" api_v_build="${api_v_fields#*$'\t'}" fi if ! api_l_fields="$(extract_zap_fields "$api_log_line")"; then red "FAIL [apiserver]: startup log line did not contain both gitCommit/buildTime zap fields -- line: [$api_log_line]" overall_pass=0 api_l_commit=""; api_l_build="" else api_l_commit="${api_l_fields%%$'\t'*}" api_l_build="${api_l_fields#*$'\t'}" fi if [ -n "${api_v_commit:-}" ] || [ -n "${api_l_commit:-}" ]; then if ! check_consistency "apiserver" "$api_v_commit" "$api_v_build" "$api_l_commit" "$api_l_build"; then overall_pass=0 fi fi fi # ── demonstration (discriminating-power proof) ────────────────────────────── if [ -n "${sched_v_commit:-}" ] && [ -n "${sched_v_build:-}" ]; then demonstrate_detection_capability "$sched_v_commit" "$sched_v_build" elif [ -n "${api_v_commit:-}" ] && [ -n "${api_v_build:-}" ]; then demonstrate_detection_capability "$api_v_commit" "$api_v_build" else red "FATAL: neither binary produced usable real values to demonstrate the comparison logic against." exit 1 fi if [ "$overall_pass" = 1 ]; then green "=== ALL CHECKS PASS: xce_scheduler and xce_apiserver each report byte-identical commit id + build time between --version/-version and their startup log ===" exit 0 else red "=== INCONSISTENCY DETECTED: see FAIL line(s) above ===" exit 1 fi