#!/usr/bin/env bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" # Self-wrap into the hermetic nix dev shell. meson/ninja/capnp/Go live only # there; a bare invocation outside it fails opaquely ("meson: command not # found") well past the point a caller can tell why. Re-exec once under # `nix develop` -- the same wrapper the /xce-build skill already documents # running by hand -- so every caller (the skill, a bare `bash xce-build.sh`, # CI, a human at a terminal) gets it automatically instead of relying on # remembering it. `nix develop` sets IN_NIX_SHELL, so a nested invocation # (e.g. already inside the shell) is a no-op past this check. if [ -z "${IN_NIX_SHELL:-}" ]; then exec nix develop "$SRC_DIR" --command "$0" "$@" fi # --incremental (bug-1565 escape hatch): opt-in, NOT the default. Every # invocation normally does a full `rm -rf "$BUILD_DIR"` + fresh `meson # setup` (see setup_build below) -- correct in isolation, but since # `meson compile` afterward only rebuilds the ONE requested component, a # single-component build silently deletes any OTHER already-built binary # sharing that same dir (e.g. `/xce-build xceadmin` after `/xce-build # controller` in xce.build/ wipes xce_controller). --incremental instead # reconfigures the EXISTING dir in place (`meson setup --reconfigure`), # preserving whatever else was already built there. # # Deliberately NOT made the default: this project's capnp-generation # custom_target (lib/controller.capnp.h/.c) has documented, unfixed # missing-dependency-edge races (buglog.json, "retry once before # debugging"). A full wipe papers over that gap -- worst case it's a loud, # self-evident, retry-fixable failure on a clean build. Without the wipe, # the SAME gap can instead cause silent staleness: a `.capnp` schema edit # not triggering regeneration, so the build reports success while linking # against stale generated code. Use --incremental only when you specifically # want a sibling component alongside an already-built one in the same dir; # rebuild without it (a real clean build) if anything looks off afterward. INCREMENTAL=0 JOBS="" ARGS=() while [ $# -gt 0 ]; do case "$1" in --incremental) INCREMENTAL=1; shift ;; --jobs) JOBS="${2:-}"; shift 2 ;; --jobs=*) JOBS="${1#--jobs=}"; shift ;; *) ARGS+=("$1"); shift ;; esac done set -- "${ARGS[@]}" # --jobs (run-queue-length limit): opt-in, no default -- when unset, # meson/ninja/go fall back to their own auto-parallelism (typically nproc or # nproc+2 workers), exactly as before this flag existed. Callers are expected # to compute a live, invocation-time value from current CPU count, available # memory, and current load average (this sandbox routinely runs multiple # CONCURRENT sessions building/testing at once -- see prune_stale_tmp_fixtures # below) rather than a fixed constant baked into this script; xce-build's own # skill does that computation before invoking this script. # # For meson compile (the ninja backend, and the memory-heaviest step -- ASAN+ # debug C/C++ compiles), the value is applied ONLY as ninja's -l LOAD_AVERAGE: # ninja re-checks the system's live load average -- literally its run-queue # length -- before launching EACH new job, throughout the whole build, and # holds off starting more while it's above this ceiling. That is deliberately # NOT the same as also pinning -j to this number: -j is a one-time snapshot # cap decided before the first job starts, while -l keeps adapting for the # entire build, including backing off if some OTHER concurrent session's # build drives load up partway through. -j is left at ninja's own default # (~nproc+2) as a loose ceiling; -l is the actual throttle. # # meson test and go test have no load-average equivalent (meson test's -j/ # --num-processes and go test's -p are both flat process-count caps), so for # those the same value is used as a job-count ceiling instead. if [ -n "$JOBS" ] && ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: --jobs value must be a positive integer, got: '$JOBS'" >&2 exit 1 fi MESON_COMPILE_JOB_OPTS=() MESON_TEST_JOB_OPTS=() GO_TEST_JOB_OPTS=() if [ -n "$JOBS" ]; then MESON_COMPILE_JOB_OPTS=(-l "$JOBS") MESON_TEST_JOB_OPTS=(-j "$JOBS") GO_TEST_JOB_OPTS=(-p "$JOBS") echo "=== Run-queue-length limit: $JOBS (meson compile -l load-average ceiling, meson test -j, go test -p) ===" fi COMPONENT="${1:-all}" # Build PROFILE controls the meson buildtype + sanitizer (and the default build # dir, so the variants never clobber each other): # asan (default) debug + AddressSanitizer → xce.build (unit-test CI) # debug debug, NO sanitizer (-O0 -g3 -fno-inline -fno-omit-frame- # pointer) → build.dbg (gdb / core dumps) # release release -O3, no sanitizer → build.release (soak / perf) # # Why `debug` exists (bug-1286): some crashes are heisenbugs that ASAN perturbs # away (different allocator/layout → no SIGSEGV) and that gdb-on-release turns # into a timeout/hang. The non-ASAN -O0 -g3 build reproduces them AND yields a # fully symbolized backtrace under gdb — the only way to root-cause that class. # # Arg parsing is backward-compatible: arg2 may be a profile keyword OR a legacy # build_dir; arg3 is an optional explicit build_dir override. case "${2:-}" in asan|debug|release) PROFILE="$2"; BUILD_DIR="${3:-}" ;; "") PROFILE="asan"; BUILD_DIR="" ;; *) PROFILE="asan"; BUILD_DIR="$2" ;; # legacy: arg2 = build_dir esac if [ -z "${BUILD_DIR:-}" ]; then case "$PROFILE" in asan) BUILD_DIR="xce.build" ;; debug) BUILD_DIR="build.dbg" ;; release) BUILD_DIR="build.release" ;; esac fi # RUN_TESTS: the unit suites are ASAN-oriented (leak detection) and the meson # test harness is fragile on this sandbox's overlay fs (bug-924 fdatasync). Only # the `asan` profile runs them; `debug`/`release` are BUILD-ONLY — their purpose # is a binary to drive under gdb / a soak, not a test run. Run a suite directly # against those binaries when you want it (it is overlay-fs sensitive). case "$PROFILE" in asan) MESON_OPTS=(--buildtype=debug -Db_sanitize=address); RUN_TESTS=1 ;; debug) MESON_OPTS=(--buildtype=debug -Db_sanitize=none); RUN_TESTS=0 ;; release) MESON_OPTS=(--buildtype=release -Db_sanitize=none); RUN_TESTS=0 ;; esac cd "$SRC_DIR" # build-time-versioning task 2.1: compute commit_id/build_time once, common to # every profile, and append to MESON_OPTS below (before either `meson setup` # call in setup_build() runs). This feeds src/meson.options' commit_id/ # build_time options -> VERSION_COMMIT_ID/VERSION_BUILD_TIME C defines # (src/meson.build:30-31). This is the actual fix for the motivating incident: # xce-build.sh never set -Dcommit_id= at all, so every local dev-loop build # had an EMPTY (not "unknown", not real) commit id -- see # .kiro/specs/build-time-versioning/requirements.md Requirement 1/4/5. if commit_hash="$(git rev-parse --short HEAD 2>/dev/null)"; then if [ -n "$(git status --porcelain 2>/dev/null)" ]; then commit_id="${commit_hash}-dirty" else commit_id="$commit_hash" fi else # git itself failed (e.g. no .git present, extracted source tarball) -- # Requirement 1.4: record as unknown rather than fail the build. Still # surface this loudly (distinct from the "resolved truly empty" check # below) -- a genuine git failure must never be silent, even though it # isn't fatal. commit_id="unknown" echo "WARNING: xce-build.sh could not determine commit_id from git (git rev-parse --short HEAD failed) -- falling back to \"unknown\". If this is a real git checkout, investigate; version info for this build will not identify a source commit." >&2 fi build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)" # Requirement 4.2: the build tooling must not silently proceed with a value it # cannot confirm. build_time should always be non-empty (date never # legitimately returns ""); commit_id falls back to the literal "unknown" # above only on a genuine git failure -- an empty result here means the # computation itself broke in some new way, so warn loudly rather than # silently handing meson an empty -Dcommit_id=/-Dbuild_time= (the exact bug # this task fixes). if [ -z "$commit_id" ]; then echo "WARNING: xce-build.sh could not determine commit_id (resolved empty) -- version info will be missing/incorrect in this build!" >&2 fi if [ -z "$build_time" ]; then echo "WARNING: xce-build.sh could not determine build_time (resolved empty) -- version info will be missing/incorrect in this build!" >&2 fi # dirty-content-fingerprint task 2.1: dirty_fingerprint, computed # INDEPENDENTLY of commit_id above -- commit_id's own computation # (including its -dirty suffix) is intentionally left untouched by this # block; this only adds a new, parallel value. Empty when the tree is # clean, or when the tree is dirty but the hash computation itself fails # (Requirement 5.1 -- never fabricate a value; commit_id's own untouched # -dirty suffix already signals "modified" in that fallback case, consumed # by a later task's --version composition, not here). When the tree is # dirty and the computation succeeds: a short sha256 prefix over `git diff # HEAD`'s output (tracked modifications, staged or unstaged) concatenated # with the raw content of every file `git ls-files --others # --exclude-standard` lists (untracked, non-ignored -- Requirement 1.4), # in a stable sorted order for determinism. Zero working-tree mutation -- # reads only, no git add/stash/anything that touches the index or working # tree. dirty_fingerprint="" if [ -n "$(git status --porcelain 2>/dev/null)" ]; then fp_ok=1 fp_tmp="" list_tmp="" if ! fp_tmp="$(mktemp 2>/dev/null)" || [ -z "$fp_tmp" ]; then fp_ok=0 fi if [ "$fp_ok" = 1 ] && ! git diff HEAD >"$fp_tmp" 2>/dev/null; then fp_ok=0 fi if [ "$fp_ok" = 1 ]; then if ! list_tmp="$(mktemp 2>/dev/null)" || [ -z "$list_tmp" ]; then fp_ok=0 elif ! git ls-files -z --others --exclude-standard 2>/dev/null | LC_ALL=C sort -z >"$list_tmp"; then fp_ok=0 else while IFS= read -r -d '' f; do if ! cat -- "$f" >>"$fp_tmp" 2>/dev/null; then fp_ok=0 break fi done <"$list_tmp" fi fi if [ "$fp_ok" = 1 ]; then if fp="$(sha256sum <"$fp_tmp" 2>/dev/null | cut -c1-8)" && [ -n "$fp" ]; then dirty_fingerprint="$fp" fi fi [ -n "$fp_tmp" ] && rm -f "$fp_tmp" [ -n "$list_tmp" ] && rm -f "$list_tmp" fi MESON_OPTS+=(-Dcommit_id="$commit_id" -Dbuild_time="$build_time" -Ddirty_fingerprint="$dirty_fingerprint") usage() { echo "Usage: $0 [component] [profile|build_dir] [build_dir] [--incremental] [--jobs N]" echo "Components: all (default), lib, controller, scheduler, worker, apiserver, xcecli, xceadmin" echo "Profiles: asan (default), debug (non-ASAN -O0 -g3, for gdb/cores), release" echo "--incremental: reconfigure the existing build dir in place instead of a full" echo " rm -rf + fresh setup -- preserves other already-built components in the same" echo " dir (bug-1565). Not the default: see the comment at the top of this script" echo " for the staleness tradeoff. Rebuild without it if anything looks wrong." echo "--jobs N: cap this build's run-queue length at N -- meson compile's -l N" echo " (ninja's load-average ceiling, re-checked live before every job launch for" echo " the whole build), meson test's -j N, go test's -p N. No default -- omit to" echo " let ninja/go auto-detect, as before this flag existed. N should be computed" echo " live from current CPU count, available memory, and current load average" echo " (see the xce-build skill), not a fixed constant, since this sandbox often" echo " runs multiple concurrent build/test sessions at once." exit 1 } # bug-924 (recurred 2026-06-22, 2026-08-26, 2026-08-30 -- each time at a # larger scale, most recently 16238 dirs / 696G / disk 100% full): every # ASAN test file in this project's testsuite/ directories that needs a # scratch directory calls mkdtemp() on its own (70+ files, no shared # fixture helper to retrofit centrally -- confirmed by grep), and NONE of # them clean up their own /tmp/xce_* dir afterward. Left unchecked this # fills the sandbox disk to ENOSPC, which then manifests as unrelated # "Expected Non-NULL"-style cascade failures in whatever test happens to # call mkdtemp()/persist_init() next -- easily mistaken for a real # regression (see bug-924's own history). Retrofitting cleanup into each # of the 70+ test files individually would be invasive and risks breaking # tests that inspect a fixture's on-disk contents across multiple TEST_ # functions in the same file. Pruning here instead, once per build # invocation, is the single lowest-risk choke point: every test run in # this project goes through this script. Age-gated (not "delete # everything") so it never touches a fixture another CONCURRENT session's # test run just created -- this sandbox routinely has multiple sessions # building/testing at once. prune_stale_tmp_fixtures() { local stale stale="$(find /tmp -maxdepth 1 -name 'xce_*' -mmin +180 2>/dev/null)" if [ -n "$stale" ]; then local count count="$(printf '%s\n' "$stale" | wc -l)" echo "=== Pruning $count stale (>3h old) /tmp/xce_* test-fixture dir(s) from prior runs (bug-924) ===" printf '%s\n' "$stale" | xargs -r rm -rf -- 2>/dev/null || true fi } setup_build() { if [ "$INCREMENTAL" = 1 ] && [ -f "$BUILD_DIR/build.ninja" ]; then echo "=== Incremental build: $BUILD_DIR (profile=$PROFILE) — reconfiguring in place, other components' binaries preserved. If anything looks wrong, rebuild without --incremental. ===" meson setup --reconfigure "$BUILD_DIR" "${MESON_OPTS[@]}" else echo "=== Clean build: $BUILD_DIR (profile=$PROFILE) ===" rm -rf "$BUILD_DIR" meson setup "$BUILD_DIR" "${MESON_OPTS[@]}" fi } prune_stale_tmp_fixtures case "$COMPONENT" in lib) setup_build echo "=== Building lib + tests ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xcelib_test if [ "$RUN_TESTS" = 1 ]; then echo "=== Running lib tests ===" meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" "libxce test" --print-errorlogs fi ;; controller) setup_build echo "=== Building lib + controller + tests ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xce_controller controller_unit_test controller/xce_test if [ "$RUN_TESTS" = 1 ]; then echo "=== Running controller tests ===" meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --suite controller --print-errorlogs fi ;; scheduler) setup_build echo "=== Building lib + scheduler + tests ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xce_scheduler sched_test if [ "$RUN_TESTS" = 1 ]; then echo "=== Running scheduler tests ===" # bug-1559: run the whole 'scheduler' meson suite (meson test # rebuilds by default), not just the sched_test binary directly # by path -- that bypassed every other scheduler test target # registered under the same suite tag (component-failover- # adoption's 4 task-3.x binaries among them). meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --suite scheduler --print-errorlogs fi ;; worker) setup_build echo "=== Building lib + worker + tests ===" # bug-1732: xcecd_virtual (the no-root worker deploy-ha.sh/xce-deploy-ha # picks whenever /run/systemd/system is absent, e.g. this sandbox) was # missing from this target list -- /xce-build worker alone left it # unbuilt, so deploy-ha.sh soft-skipped generating start-worker.sh. meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xcecd xcecd_worker xcecd_virtual xrun controller_connector_test if [ "$RUN_TESTS" = 1 ]; then echo "=== Running worker tests ===" meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --suite worker --print-errorlogs fi ;; apiserver) setup_build echo "=== Building lib + apiserver + xcecli ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" apiserver xcecli if [ "$RUN_TESTS" = 1 ]; then echo "=== Running frontend Go unit tests ===" cd frontend/apiserver && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./store/... ./utils/... ./controllers/... ./backend/... -count=1 && cd ../xcecli && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./... -count=1 fi ;; xcecli) setup_build echo "=== Building xcecli ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xcecli if [ "$RUN_TESTS" = 1 ]; then echo "=== Running xcecli Go unit tests ===" cd frontend/xcecli && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./... -count=1 fi ;; xceadmin) # Meson-driven like every other component (src/xceadmin/meson.build, # custom_target wrapping meson-build.sh) -- installs pinned/non- # editable out-of-tree in the meson build dir and runs pytest, # every invocation regardless of $PROFILE (xceadmin's suite is pure # Python, unrelated to the ASAN/release C compiler flags $PROFILE # otherwise controls, and fast enough — ~2s — that always running # it costs nothing). This is a strictly stronger guarantee than the # old editable-install-by-default behavior: every run now exercises # real package-data resolution (bug-1460's class of bug), not just # the `release` profile. # # For fast iterative dev without a full meson reconfigure, xceadmin # can still be built standalone: cd xceadmin && pip install -e . # (or see src/xceadmin/meson-build.sh for the exact out-of-tree # steps this target runs). setup_build echo "=== Building + testing xceadmin (via meson) ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xceadmin WHEEL_FILE=$(ls "$BUILD_DIR"/xceadmin/dist/xceadmin-*.whl 2>/dev/null || true) if [ -n "$WHEEL_FILE" ]; then echo "=== xceadmin wheel: $WHEEL_FILE ===" echo " Install it directly with: pip install \"$WHEEL_FILE\"" fi ;; all) setup_build echo "=== Building all components ===" meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" if [ "$RUN_TESTS" = 1 ]; then echo "=== Running all C tests ===" meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --print-errorlogs echo "=== Running scheduler test (manual) ===" "./$BUILD_DIR/scheduler/testsuite/sched_test" echo "=== Running frontend Go unit tests ===" cd frontend/apiserver && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./store/... ./utils/... ./controllers/... ./backend/... -count=1 && cd ../xcecli && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./... -count=1 fi ;; *) usage ;; esac if [ "$RUN_TESTS" != 1 ]; then echo "=== Skipped tests (profile=$PROFILE — build only). Binary in $BUILD_DIR; run a suite directly, e.g. ASAN_OPTIONS=detect_leaks=0 ./$BUILD_DIR/controller/controller_unit_test ===" fi echo "=== Done: $COMPONENT (profile=$PROFILE, dir=$BUILD_DIR) ==="