Rongsong Shen
3c199f376a
Replaces both `cd frontend && make test` call sites (apiserver and all cases) with direct `go test` invocations covering the identical package set/flags, mirroring the xcecli case's already-Makefile-free style. Verified: apiserver case end-to-end, matching the captured pre-change baseline exactly; all case verified by proxy (byte- identical replacement line, apiserver instance fully verified) since a full /xce-build all run is currently blocked earlier in the pipeline by two pre-existing, unrelated C-test bugs (bug-1723, bug-1758) -- noted in Implementation Notes for task 4's awareness. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XYdtrEWWZfKRRGED7raEYg
334 lines
16 KiB
Bash
Executable file
334 lines
16 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# --incremental (bug-1565 escape hatch): opt-in, NOT the default. Every
|
|
# invocation normally does a full `rm -rf "$BUILD_DIR"` + fresh `meson
|
|
# setup` (see setup_build below) -- correct in isolation, but since
|
|
# `meson compile` afterward only rebuilds the ONE requested component, a
|
|
# single-component build silently deletes any OTHER already-built binary
|
|
# sharing that same dir (e.g. `/xce-build xceadmin` after `/xce-build
|
|
# controller` in xce.build/ wipes xce_controller). --incremental instead
|
|
# reconfigures the EXISTING dir in place (`meson setup --reconfigure`),
|
|
# preserving whatever else was already built there.
|
|
#
|
|
# Deliberately NOT made the default: this project's capnp-generation
|
|
# custom_target (lib/controller.capnp.h/.c) has documented, unfixed
|
|
# missing-dependency-edge races (buglog.json, "retry once before
|
|
# debugging"). A full wipe papers over that gap -- worst case it's a loud,
|
|
# self-evident, retry-fixable failure on a clean build. Without the wipe,
|
|
# the SAME gap can instead cause silent staleness: a `.capnp` schema edit
|
|
# not triggering regeneration, so the build reports success while linking
|
|
# against stale generated code. Use --incremental only when you specifically
|
|
# want a sibling component alongside an already-built one in the same dir;
|
|
# rebuild without it (a real clean build) if anything looks off afterward.
|
|
INCREMENTAL=0
|
|
ARGS=()
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--incremental) INCREMENTAL=1 ;;
|
|
*) ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
set -- "${ARGS[@]}"
|
|
|
|
COMPONENT="${1:-all}"
|
|
|
|
# Build PROFILE controls the meson buildtype + sanitizer (and the default build
|
|
# dir, so the variants never clobber each other):
|
|
# asan (default) debug + AddressSanitizer → xce.build (unit-test CI)
|
|
# debug debug, NO sanitizer (-O0 -g3 -fno-inline -fno-omit-frame-
|
|
# pointer) → build.dbg (gdb / core dumps)
|
|
# release release -O3, no sanitizer → build.release (soak / perf)
|
|
#
|
|
# Why `debug` exists (bug-1286): some crashes are heisenbugs that ASAN perturbs
|
|
# away (different allocator/layout → no SIGSEGV) and that gdb-on-release turns
|
|
# into a timeout/hang. The non-ASAN -O0 -g3 build reproduces them AND yields a
|
|
# fully symbolized backtrace under gdb — the only way to root-cause that class.
|
|
#
|
|
# Arg parsing is backward-compatible: arg2 may be a profile keyword OR a legacy
|
|
# build_dir; arg3 is an optional explicit build_dir override.
|
|
case "${2:-}" in
|
|
asan|debug|release) PROFILE="$2"; BUILD_DIR="${3:-}" ;;
|
|
"") PROFILE="asan"; BUILD_DIR="" ;;
|
|
*) PROFILE="asan"; BUILD_DIR="$2" ;; # legacy: arg2 = build_dir
|
|
esac
|
|
|
|
if [ -z "${BUILD_DIR:-}" ]; then
|
|
case "$PROFILE" in
|
|
asan) BUILD_DIR="xce.build" ;;
|
|
debug) BUILD_DIR="build.dbg" ;;
|
|
release) BUILD_DIR="build.release" ;;
|
|
esac
|
|
fi
|
|
|
|
# RUN_TESTS: the unit suites are ASAN-oriented (leak detection) and the meson
|
|
# test harness is fragile on this sandbox's overlay fs (bug-924 fdatasync). Only
|
|
# the `asan` profile runs them; `debug`/`release` are BUILD-ONLY — their purpose
|
|
# is a binary to drive under gdb / a soak, not a test run. Run a suite directly
|
|
# against those binaries when you want it (it is overlay-fs sensitive).
|
|
case "$PROFILE" in
|
|
asan) MESON_OPTS=(--buildtype=debug -Db_sanitize=address); RUN_TESTS=1 ;;
|
|
debug) MESON_OPTS=(--buildtype=debug -Db_sanitize=none); RUN_TESTS=0 ;;
|
|
release) MESON_OPTS=(--buildtype=release -Db_sanitize=none); RUN_TESTS=0 ;;
|
|
esac
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
cd "$SRC_DIR"
|
|
|
|
# build-time-versioning task 2.1: compute commit_id/build_time once, common to
|
|
# every profile, and append to MESON_OPTS below (before either `meson setup`
|
|
# call in setup_build() runs). This feeds src/meson.options' commit_id/
|
|
# build_time options -> VERSION_COMMIT_ID/VERSION_BUILD_TIME C defines
|
|
# (src/meson.build:30-31). This is the actual fix for the motivating incident:
|
|
# xce-build.sh never set -Dcommit_id= at all, so every local dev-loop build
|
|
# had an EMPTY (not "unknown", not real) commit id -- see
|
|
# .kiro/specs/build-time-versioning/requirements.md Requirement 1/4/5.
|
|
if commit_hash="$(git rev-parse --short HEAD 2>/dev/null)"; then
|
|
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
|
|
commit_id="${commit_hash}-dirty"
|
|
else
|
|
commit_id="$commit_hash"
|
|
fi
|
|
else
|
|
# git itself failed (e.g. no .git present, extracted source tarball) --
|
|
# Requirement 1.4: record as unknown rather than fail the build. Still
|
|
# surface this loudly (distinct from the "resolved truly empty" check
|
|
# below) -- a genuine git failure must never be silent, even though it
|
|
# isn't fatal.
|
|
commit_id="unknown"
|
|
echo "WARNING: xce-build.sh could not determine commit_id from git (git rev-parse --short HEAD failed) -- falling back to \"unknown\". If this is a real git checkout, investigate; version info for this build will not identify a source commit." >&2
|
|
fi
|
|
build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
# Requirement 4.2: the build tooling must not silently proceed with a value it
|
|
# cannot confirm. build_time should always be non-empty (date never
|
|
# legitimately returns ""); commit_id falls back to the literal "unknown"
|
|
# above only on a genuine git failure -- an empty result here means the
|
|
# computation itself broke in some new way, so warn loudly rather than
|
|
# silently handing meson an empty -Dcommit_id=/-Dbuild_time= (the exact bug
|
|
# this task fixes).
|
|
if [ -z "$commit_id" ]; then
|
|
echo "WARNING: xce-build.sh could not determine commit_id (resolved empty) -- version info will be missing/incorrect in this build!" >&2
|
|
fi
|
|
if [ -z "$build_time" ]; then
|
|
echo "WARNING: xce-build.sh could not determine build_time (resolved empty) -- version info will be missing/incorrect in this build!" >&2
|
|
fi
|
|
|
|
# dirty-content-fingerprint task 2.1: dirty_fingerprint, computed
|
|
# INDEPENDENTLY of commit_id above -- commit_id's own computation
|
|
# (including its -dirty suffix) is intentionally left untouched by this
|
|
# block; this only adds a new, parallel value. Empty when the tree is
|
|
# clean, or when the tree is dirty but the hash computation itself fails
|
|
# (Requirement 5.1 -- never fabricate a value; commit_id's own untouched
|
|
# -dirty suffix already signals "modified" in that fallback case, consumed
|
|
# by a later task's --version composition, not here). When the tree is
|
|
# dirty and the computation succeeds: a short sha256 prefix over `git diff
|
|
# HEAD`'s output (tracked modifications, staged or unstaged) concatenated
|
|
# with the raw content of every file `git ls-files --others
|
|
# --exclude-standard` lists (untracked, non-ignored -- Requirement 1.4),
|
|
# in a stable sorted order for determinism. Zero working-tree mutation --
|
|
# reads only, no git add/stash/anything that touches the index or working
|
|
# tree.
|
|
dirty_fingerprint=""
|
|
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
|
|
fp_ok=1
|
|
fp_tmp=""
|
|
list_tmp=""
|
|
if ! fp_tmp="$(mktemp 2>/dev/null)" || [ -z "$fp_tmp" ]; then
|
|
fp_ok=0
|
|
fi
|
|
if [ "$fp_ok" = 1 ] && ! git diff HEAD >"$fp_tmp" 2>/dev/null; then
|
|
fp_ok=0
|
|
fi
|
|
if [ "$fp_ok" = 1 ]; then
|
|
if ! list_tmp="$(mktemp 2>/dev/null)" || [ -z "$list_tmp" ]; then
|
|
fp_ok=0
|
|
elif ! git ls-files -z --others --exclude-standard 2>/dev/null | LC_ALL=C sort -z >"$list_tmp"; then
|
|
fp_ok=0
|
|
else
|
|
while IFS= read -r -d '' f; do
|
|
if ! cat -- "$f" >>"$fp_tmp" 2>/dev/null; then
|
|
fp_ok=0
|
|
break
|
|
fi
|
|
done <"$list_tmp"
|
|
fi
|
|
fi
|
|
if [ "$fp_ok" = 1 ]; then
|
|
if fp="$(sha256sum <"$fp_tmp" 2>/dev/null | cut -c1-8)" && [ -n "$fp" ]; then
|
|
dirty_fingerprint="$fp"
|
|
fi
|
|
fi
|
|
[ -n "$fp_tmp" ] && rm -f "$fp_tmp"
|
|
[ -n "$list_tmp" ] && rm -f "$list_tmp"
|
|
fi
|
|
|
|
MESON_OPTS+=(-Dcommit_id="$commit_id" -Dbuild_time="$build_time" -Ddirty_fingerprint="$dirty_fingerprint")
|
|
|
|
usage() {
|
|
echo "Usage: $0 [component] [profile|build_dir] [build_dir] [--incremental]"
|
|
echo "Components: all (default), lib, controller, scheduler, worker, apiserver, xcecli, xceadmin"
|
|
echo "Profiles: asan (default), debug (non-ASAN -O0 -g3, for gdb/cores), release"
|
|
echo "--incremental: reconfigure the existing build dir in place instead of a full"
|
|
echo " rm -rf + fresh setup -- preserves other already-built components in the same"
|
|
echo " dir (bug-1565). Not the default: see the comment at the top of this script"
|
|
echo " for the staleness tradeoff. Rebuild without it if anything looks wrong."
|
|
exit 1
|
|
}
|
|
|
|
# bug-924 (recurred 2026-06-22, 2026-08-26, 2026-08-30 -- each time at a
|
|
# larger scale, most recently 16238 dirs / 696G / disk 100% full): every
|
|
# ASAN test file in this project's testsuite/ directories that needs a
|
|
# scratch directory calls mkdtemp() on its own (70+ files, no shared
|
|
# fixture helper to retrofit centrally -- confirmed by grep), and NONE of
|
|
# them clean up their own /tmp/xce_* dir afterward. Left unchecked this
|
|
# fills the sandbox disk to ENOSPC, which then manifests as unrelated
|
|
# "Expected Non-NULL"-style cascade failures in whatever test happens to
|
|
# call mkdtemp()/persist_init() next -- easily mistaken for a real
|
|
# regression (see bug-924's own history). Retrofitting cleanup into each
|
|
# of the 70+ test files individually would be invasive and risks breaking
|
|
# tests that inspect a fixture's on-disk contents across multiple TEST_
|
|
# functions in the same file. Pruning here instead, once per build
|
|
# invocation, is the single lowest-risk choke point: every test run in
|
|
# this project goes through this script. Age-gated (not "delete
|
|
# everything") so it never touches a fixture another CONCURRENT session's
|
|
# test run just created -- this sandbox routinely has multiple sessions
|
|
# building/testing at once.
|
|
prune_stale_tmp_fixtures() {
|
|
local stale
|
|
stale="$(find /tmp -maxdepth 1 -name 'xce_*' -mmin +180 2>/dev/null)"
|
|
if [ -n "$stale" ]; then
|
|
local count
|
|
count="$(printf '%s\n' "$stale" | wc -l)"
|
|
echo "=== Pruning $count stale (>3h old) /tmp/xce_* test-fixture dir(s) from prior runs (bug-924) ==="
|
|
printf '%s\n' "$stale" | xargs -r rm -rf -- 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
setup_build() {
|
|
if [ "$INCREMENTAL" = 1 ] && [ -f "$BUILD_DIR/build.ninja" ]; then
|
|
echo "=== Incremental build: $BUILD_DIR (profile=$PROFILE) — reconfiguring in place, other components' binaries preserved. If anything looks wrong, rebuild without --incremental. ==="
|
|
meson setup --reconfigure "$BUILD_DIR" "${MESON_OPTS[@]}"
|
|
else
|
|
echo "=== Clean build: $BUILD_DIR (profile=$PROFILE) ==="
|
|
rm -rf "$BUILD_DIR"
|
|
meson setup "$BUILD_DIR" "${MESON_OPTS[@]}"
|
|
fi
|
|
}
|
|
|
|
prune_stale_tmp_fixtures
|
|
|
|
case "$COMPONENT" in
|
|
lib)
|
|
setup_build
|
|
echo "=== Building lib + tests ==="
|
|
meson compile -C "$BUILD_DIR" xcelib_test
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running lib tests ==="
|
|
meson test -C "$BUILD_DIR" "libxce test" --print-errorlogs
|
|
fi
|
|
;;
|
|
controller)
|
|
setup_build
|
|
echo "=== Building lib + controller + tests ==="
|
|
meson compile -C "$BUILD_DIR" xce_controller controller_unit_test controller/xce_test
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running controller tests ==="
|
|
meson test -C "$BUILD_DIR" --suite controller --print-errorlogs
|
|
fi
|
|
;;
|
|
scheduler)
|
|
setup_build
|
|
echo "=== Building lib + scheduler + tests ==="
|
|
meson compile -C "$BUILD_DIR" xce_scheduler sched_test
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running scheduler tests ==="
|
|
# bug-1559: run the whole 'scheduler' meson suite (meson test
|
|
# rebuilds by default), not just the sched_test binary directly
|
|
# by path -- that bypassed every other scheduler test target
|
|
# registered under the same suite tag (component-failover-
|
|
# adoption's 4 task-3.x binaries among them).
|
|
meson test -C "$BUILD_DIR" --suite scheduler --print-errorlogs
|
|
fi
|
|
;;
|
|
worker)
|
|
setup_build
|
|
echo "=== Building lib + worker + tests ==="
|
|
# bug-1732: xcecd_virtual (the no-root worker deploy-ha.sh/xce-deploy-ha
|
|
# picks whenever /run/systemd/system is absent, e.g. this sandbox) was
|
|
# missing from this target list -- /xce-build worker alone left it
|
|
# unbuilt, so deploy-ha.sh soft-skipped generating start-worker.sh.
|
|
meson compile -C "$BUILD_DIR" xcecd xcecd_worker xcecd_virtual xrun controller_connector_test
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running worker tests ==="
|
|
meson test -C "$BUILD_DIR" --suite worker --print-errorlogs
|
|
fi
|
|
;;
|
|
apiserver)
|
|
setup_build
|
|
echo "=== Building lib + apiserver + xcecli ==="
|
|
meson compile -C "$BUILD_DIR" apiserver xcecli
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running frontend Go unit tests ==="
|
|
cd frontend/apiserver && CGO_ENABLED=0 go test ./store/... ./utils/... ./controllers/... ./backend/... -count=1 && cd ../xcecli && CGO_ENABLED=0 go test ./... -count=1
|
|
fi
|
|
;;
|
|
xcecli)
|
|
setup_build
|
|
echo "=== Building xcecli ==="
|
|
meson compile -C "$BUILD_DIR" xcecli
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running xcecli Go unit tests ==="
|
|
cd frontend/xcecli && CGO_ENABLED=0 go test ./... -count=1
|
|
fi
|
|
;;
|
|
xceadmin)
|
|
# Meson-driven like every other component (src/xceadmin/meson.build,
|
|
# custom_target wrapping meson-build.sh) -- installs pinned/non-
|
|
# editable out-of-tree in the meson build dir and runs pytest,
|
|
# every invocation regardless of $PROFILE (xceadmin's suite is pure
|
|
# Python, unrelated to the ASAN/release C compiler flags $PROFILE
|
|
# otherwise controls, and fast enough — ~2s — that always running
|
|
# it costs nothing). This is a strictly stronger guarantee than the
|
|
# old editable-install-by-default behavior: every run now exercises
|
|
# real package-data resolution (bug-1460's class of bug), not just
|
|
# the `release` profile.
|
|
#
|
|
# For fast iterative dev without a full meson reconfigure, xceadmin
|
|
# can still be built standalone: cd xceadmin && pip install -e .
|
|
# (or see src/xceadmin/meson-build.sh for the exact out-of-tree
|
|
# steps this target runs).
|
|
setup_build
|
|
echo "=== Building + testing xceadmin (via meson) ==="
|
|
meson compile -C "$BUILD_DIR" xceadmin
|
|
WHEEL_FILE=$(ls "$BUILD_DIR"/xceadmin/dist/xceadmin-*.whl 2>/dev/null || true)
|
|
if [ -n "$WHEEL_FILE" ]; then
|
|
echo "=== xceadmin wheel: $WHEEL_FILE ==="
|
|
echo " Install it directly with: pip install \"$WHEEL_FILE\""
|
|
fi
|
|
;;
|
|
all)
|
|
setup_build
|
|
echo "=== Building all components ==="
|
|
meson compile -C "$BUILD_DIR"
|
|
if [ "$RUN_TESTS" = 1 ]; then
|
|
echo "=== Running all C tests ==="
|
|
meson test -C "$BUILD_DIR" --print-errorlogs
|
|
echo "=== Running scheduler test (manual) ==="
|
|
"./$BUILD_DIR/scheduler/testsuite/sched_test"
|
|
echo "=== Running frontend Go unit tests ==="
|
|
cd frontend/apiserver && CGO_ENABLED=0 go test ./store/... ./utils/... ./controllers/... ./backend/... -count=1 && cd ../xcecli && CGO_ENABLED=0 go test ./... -count=1
|
|
fi
|
|
;;
|
|
*)
|
|
usage
|
|
;;
|
|
esac
|
|
|
|
if [ "$RUN_TESTS" != 1 ]; then
|
|
echo "=== Skipped tests (profile=$PROFILE — build only). Binary in $BUILD_DIR; run a suite directly, e.g. ASAN_OPTIONS=detect_leaks=0 ./$BUILD_DIR/controller/controller_unit_test ==="
|
|
fi
|
|
|
|
echo "=== Done: $COMPONENT (profile=$PROFILE, dir=$BUILD_DIR) ==="
|