myxce/src/tools/xce-build.sh
Rongsong Shen ee3373af97
Merge branch 'better_xce_build' into scheduler-framework
# Conflicts:
#	.wolf/anatomy-index.json
#	.wolf/anatomy.md
#	.wolf/memory.md
2026-09-08 03:09:53 +00:00

393 lines
20 KiB
Bash
Executable file

#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
# Self-wrap into the hermetic nix dev shell. meson/ninja/capnp/Go live only
# there; a bare invocation outside it fails opaquely ("meson: command not
# found") well past the point a caller can tell why. Re-exec once under
# `nix develop` -- the same wrapper the /xce-build skill already documents
# running by hand -- so every caller (the skill, a bare `bash xce-build.sh`,
# CI, a human at a terminal) gets it automatically instead of relying on
# remembering it. `nix develop` sets IN_NIX_SHELL, so a nested invocation
# (e.g. already inside the shell) is a no-op past this check.
if [ -z "${IN_NIX_SHELL:-}" ]; then
exec nix develop "$SRC_DIR" --command "$0" "$@"
fi
# --incremental (bug-1565 escape hatch): opt-in, NOT the default. Every
# invocation normally does a full `rm -rf "$BUILD_DIR"` + fresh `meson
# setup` (see setup_build below) -- correct in isolation, but since
# `meson compile` afterward only rebuilds the ONE requested component, a
# single-component build silently deletes any OTHER already-built binary
# sharing that same dir (e.g. `/xce-build xceadmin` after `/xce-build
# controller` in xce.build/ wipes xce_controller). --incremental instead
# reconfigures the EXISTING dir in place (`meson setup --reconfigure`),
# preserving whatever else was already built there.
#
# Deliberately NOT made the default: this project's capnp-generation
# custom_target (lib/controller.capnp.h/.c) has documented, unfixed
# missing-dependency-edge races (buglog.json, "retry once before
# debugging"). A full wipe papers over that gap -- worst case it's a loud,
# self-evident, retry-fixable failure on a clean build. Without the wipe,
# the SAME gap can instead cause silent staleness: a `.capnp` schema edit
# not triggering regeneration, so the build reports success while linking
# against stale generated code. Use --incremental only when you specifically
# want a sibling component alongside an already-built one in the same dir;
# rebuild without it (a real clean build) if anything looks off afterward.
INCREMENTAL=0
JOBS=""
ARGS=()
while [ $# -gt 0 ]; do
case "$1" in
--incremental) INCREMENTAL=1; shift ;;
--jobs) JOBS="${2:-}"; shift 2 ;;
--jobs=*) JOBS="${1#--jobs=}"; shift ;;
*) ARGS+=("$1"); shift ;;
esac
done
set -- "${ARGS[@]}"
# --jobs (run-queue-length limit): opt-in, no default -- when unset,
# meson/ninja/go fall back to their own auto-parallelism (typically nproc or
# nproc+2 workers), exactly as before this flag existed. Callers are expected
# to compute a live, invocation-time value from current CPU count, available
# memory, and current load average (this sandbox routinely runs multiple
# CONCURRENT sessions building/testing at once -- see prune_stale_tmp_fixtures
# below) rather than a fixed constant baked into this script; xce-build's own
# skill does that computation before invoking this script.
#
# For meson compile (the ninja backend, and the memory-heaviest step -- ASAN+
# debug C/C++ compiles), the value is applied ONLY as ninja's -l LOAD_AVERAGE:
# ninja re-checks the system's live load average -- literally its run-queue
# length -- before launching EACH new job, throughout the whole build, and
# holds off starting more while it's above this ceiling. That is deliberately
# NOT the same as also pinning -j to this number: -j is a one-time snapshot
# cap decided before the first job starts, while -l keeps adapting for the
# entire build, including backing off if some OTHER concurrent session's
# build drives load up partway through. -j is left at ninja's own default
# (~nproc+2) as a loose ceiling; -l is the actual throttle.
#
# meson test and go test have no load-average equivalent (meson test's -j/
# --num-processes and go test's -p are both flat process-count caps), so for
# those the same value is used as a job-count ceiling instead.
if [ -n "$JOBS" ] && ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: --jobs value must be a positive integer, got: '$JOBS'" >&2
exit 1
fi
MESON_COMPILE_JOB_OPTS=()
MESON_TEST_JOB_OPTS=()
GO_TEST_JOB_OPTS=()
if [ -n "$JOBS" ]; then
MESON_COMPILE_JOB_OPTS=(-l "$JOBS")
MESON_TEST_JOB_OPTS=(-j "$JOBS")
GO_TEST_JOB_OPTS=(-p "$JOBS")
echo "=== Run-queue-length limit: $JOBS (meson compile -l load-average ceiling, meson test -j, go test -p) ==="
fi
COMPONENT="${1:-all}"
# Build PROFILE controls the meson buildtype + sanitizer (and the default build
# dir, so the variants never clobber each other):
# asan (default) debug + AddressSanitizer → xce.build (unit-test CI)
# debug debug, NO sanitizer (-O0 -g3 -fno-inline -fno-omit-frame-
# pointer) → build.dbg (gdb / core dumps)
# release release -O3, no sanitizer → build.release (soak / perf)
#
# Why `debug` exists (bug-1286): some crashes are heisenbugs that ASAN perturbs
# away (different allocator/layout → no SIGSEGV) and that gdb-on-release turns
# into a timeout/hang. The non-ASAN -O0 -g3 build reproduces them AND yields a
# fully symbolized backtrace under gdb — the only way to root-cause that class.
#
# Arg parsing is backward-compatible: arg2 may be a profile keyword OR a legacy
# build_dir; arg3 is an optional explicit build_dir override.
case "${2:-}" in
asan|debug|release) PROFILE="$2"; BUILD_DIR="${3:-}" ;;
"") PROFILE="asan"; BUILD_DIR="" ;;
*) PROFILE="asan"; BUILD_DIR="$2" ;; # legacy: arg2 = build_dir
esac
if [ -z "${BUILD_DIR:-}" ]; then
case "$PROFILE" in
asan) BUILD_DIR="xce.build" ;;
debug) BUILD_DIR="build.dbg" ;;
release) BUILD_DIR="build.release" ;;
esac
fi
# RUN_TESTS: the unit suites are ASAN-oriented (leak detection) and the meson
# test harness is fragile on this sandbox's overlay fs (bug-924 fdatasync). Only
# the `asan` profile runs them; `debug`/`release` are BUILD-ONLY — their purpose
# is a binary to drive under gdb / a soak, not a test run. Run a suite directly
# against those binaries when you want it (it is overlay-fs sensitive).
case "$PROFILE" in
asan) MESON_OPTS=(--buildtype=debug -Db_sanitize=address); RUN_TESTS=1 ;;
debug) MESON_OPTS=(--buildtype=debug -Db_sanitize=none); RUN_TESTS=0 ;;
release) MESON_OPTS=(--buildtype=release -Db_sanitize=none); RUN_TESTS=0 ;;
esac
cd "$SRC_DIR"
# build-time-versioning task 2.1: compute commit_id/build_time once, common to
# every profile, and append to MESON_OPTS below (before either `meson setup`
# call in setup_build() runs). This feeds src/meson.options' commit_id/
# build_time options -> VERSION_COMMIT_ID/VERSION_BUILD_TIME C defines
# (src/meson.build:30-31). This is the actual fix for the motivating incident:
# xce-build.sh never set -Dcommit_id= at all, so every local dev-loop build
# had an EMPTY (not "unknown", not real) commit id -- see
# .kiro/specs/build-time-versioning/requirements.md Requirement 1/4/5.
if commit_hash="$(git rev-parse --short HEAD 2>/dev/null)"; then
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
commit_id="${commit_hash}-dirty"
else
commit_id="$commit_hash"
fi
else
# git itself failed (e.g. no .git present, extracted source tarball) --
# Requirement 1.4: record as unknown rather than fail the build. Still
# surface this loudly (distinct from the "resolved truly empty" check
# below) -- a genuine git failure must never be silent, even though it
# isn't fatal.
commit_id="unknown"
echo "WARNING: xce-build.sh could not determine commit_id from git (git rev-parse --short HEAD failed) -- falling back to \"unknown\". If this is a real git checkout, investigate; version info for this build will not identify a source commit." >&2
fi
build_time="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
# Requirement 4.2: the build tooling must not silently proceed with a value it
# cannot confirm. build_time should always be non-empty (date never
# legitimately returns ""); commit_id falls back to the literal "unknown"
# above only on a genuine git failure -- an empty result here means the
# computation itself broke in some new way, so warn loudly rather than
# silently handing meson an empty -Dcommit_id=/-Dbuild_time= (the exact bug
# this task fixes).
if [ -z "$commit_id" ]; then
echo "WARNING: xce-build.sh could not determine commit_id (resolved empty) -- version info will be missing/incorrect in this build!" >&2
fi
if [ -z "$build_time" ]; then
echo "WARNING: xce-build.sh could not determine build_time (resolved empty) -- version info will be missing/incorrect in this build!" >&2
fi
# dirty-content-fingerprint task 2.1: dirty_fingerprint, computed
# INDEPENDENTLY of commit_id above -- commit_id's own computation
# (including its -dirty suffix) is intentionally left untouched by this
# block; this only adds a new, parallel value. Empty when the tree is
# clean, or when the tree is dirty but the hash computation itself fails
# (Requirement 5.1 -- never fabricate a value; commit_id's own untouched
# -dirty suffix already signals "modified" in that fallback case, consumed
# by a later task's --version composition, not here). When the tree is
# dirty and the computation succeeds: a short sha256 prefix over `git diff
# HEAD`'s output (tracked modifications, staged or unstaged) concatenated
# with the raw content of every file `git ls-files --others
# --exclude-standard` lists (untracked, non-ignored -- Requirement 1.4),
# in a stable sorted order for determinism. Zero working-tree mutation --
# reads only, no git add/stash/anything that touches the index or working
# tree.
dirty_fingerprint=""
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
fp_ok=1
fp_tmp=""
list_tmp=""
if ! fp_tmp="$(mktemp 2>/dev/null)" || [ -z "$fp_tmp" ]; then
fp_ok=0
fi
if [ "$fp_ok" = 1 ] && ! git diff HEAD >"$fp_tmp" 2>/dev/null; then
fp_ok=0
fi
if [ "$fp_ok" = 1 ]; then
if ! list_tmp="$(mktemp 2>/dev/null)" || [ -z "$list_tmp" ]; then
fp_ok=0
elif ! git ls-files -z --others --exclude-standard 2>/dev/null | LC_ALL=C sort -z >"$list_tmp"; then
fp_ok=0
else
while IFS= read -r -d '' f; do
if ! cat -- "$f" >>"$fp_tmp" 2>/dev/null; then
fp_ok=0
break
fi
done <"$list_tmp"
fi
fi
if [ "$fp_ok" = 1 ]; then
if fp="$(sha256sum <"$fp_tmp" 2>/dev/null | cut -c1-8)" && [ -n "$fp" ]; then
dirty_fingerprint="$fp"
fi
fi
[ -n "$fp_tmp" ] && rm -f "$fp_tmp"
[ -n "$list_tmp" ] && rm -f "$list_tmp"
fi
MESON_OPTS+=(-Dcommit_id="$commit_id" -Dbuild_time="$build_time" -Ddirty_fingerprint="$dirty_fingerprint")
usage() {
echo "Usage: $0 [component] [profile|build_dir] [build_dir] [--incremental] [--jobs N]"
echo "Components: all (default), lib, controller, scheduler, worker, apiserver, xcecli, xceadmin"
echo "Profiles: asan (default), debug (non-ASAN -O0 -g3, for gdb/cores), release"
echo "--incremental: reconfigure the existing build dir in place instead of a full"
echo " rm -rf + fresh setup -- preserves other already-built components in the same"
echo " dir (bug-1565). Not the default: see the comment at the top of this script"
echo " for the staleness tradeoff. Rebuild without it if anything looks wrong."
echo "--jobs N: cap this build's run-queue length at N -- meson compile's -l N"
echo " (ninja's load-average ceiling, re-checked live before every job launch for"
echo " the whole build), meson test's -j N, go test's -p N. No default -- omit to"
echo " let ninja/go auto-detect, as before this flag existed. N should be computed"
echo " live from current CPU count, available memory, and current load average"
echo " (see the xce-build skill), not a fixed constant, since this sandbox often"
echo " runs multiple concurrent build/test sessions at once."
exit 1
}
# bug-924 (recurred 2026-06-22, 2026-08-26, 2026-08-30 -- each time at a
# larger scale, most recently 16238 dirs / 696G / disk 100% full): every
# ASAN test file in this project's testsuite/ directories that needs a
# scratch directory calls mkdtemp() on its own (70+ files, no shared
# fixture helper to retrofit centrally -- confirmed by grep), and NONE of
# them clean up their own /tmp/xce_* dir afterward. Left unchecked this
# fills the sandbox disk to ENOSPC, which then manifests as unrelated
# "Expected Non-NULL"-style cascade failures in whatever test happens to
# call mkdtemp()/persist_init() next -- easily mistaken for a real
# regression (see bug-924's own history). Retrofitting cleanup into each
# of the 70+ test files individually would be invasive and risks breaking
# tests that inspect a fixture's on-disk contents across multiple TEST_
# functions in the same file. Pruning here instead, once per build
# invocation, is the single lowest-risk choke point: every test run in
# this project goes through this script. Age-gated (not "delete
# everything") so it never touches a fixture another CONCURRENT session's
# test run just created -- this sandbox routinely has multiple sessions
# building/testing at once.
prune_stale_tmp_fixtures() {
local stale
stale="$(find /tmp -maxdepth 1 -name 'xce_*' -mmin +180 2>/dev/null)"
if [ -n "$stale" ]; then
local count
count="$(printf '%s\n' "$stale" | wc -l)"
echo "=== Pruning $count stale (>3h old) /tmp/xce_* test-fixture dir(s) from prior runs (bug-924) ==="
printf '%s\n' "$stale" | xargs -r rm -rf -- 2>/dev/null || true
fi
}
setup_build() {
if [ "$INCREMENTAL" = 1 ] && [ -f "$BUILD_DIR/build.ninja" ]; then
echo "=== Incremental build: $BUILD_DIR (profile=$PROFILE) — reconfiguring in place, other components' binaries preserved. If anything looks wrong, rebuild without --incremental. ==="
meson setup --reconfigure "$BUILD_DIR" "${MESON_OPTS[@]}"
else
echo "=== Clean build: $BUILD_DIR (profile=$PROFILE) ==="
rm -rf "$BUILD_DIR"
meson setup "$BUILD_DIR" "${MESON_OPTS[@]}"
fi
}
prune_stale_tmp_fixtures
case "$COMPONENT" in
lib)
setup_build
echo "=== Building lib + tests ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xcelib_test
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running lib tests ==="
meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" "libxce test" --print-errorlogs
fi
;;
controller)
setup_build
echo "=== Building lib + controller + tests ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xce_controller controller_unit_test controller/xce_test
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running controller tests ==="
meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --suite controller --print-errorlogs
fi
;;
scheduler)
setup_build
echo "=== Building lib + scheduler + tests ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xce_scheduler sched_test
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running scheduler tests ==="
# bug-1559: run the whole 'scheduler' meson suite (meson test
# rebuilds by default), not just the sched_test binary directly
# by path -- that bypassed every other scheduler test target
# registered under the same suite tag (component-failover-
# adoption's 4 task-3.x binaries among them).
meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --suite scheduler --print-errorlogs
fi
;;
worker)
setup_build
echo "=== Building lib + worker + tests ==="
# bug-1732: xcecd_virtual (the no-root worker deploy-ha.sh/xce-deploy-ha
# picks whenever /run/systemd/system is absent, e.g. this sandbox) was
# missing from this target list -- /xce-build worker alone left it
# unbuilt, so deploy-ha.sh soft-skipped generating start-worker.sh.
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xcecd xcecd_worker xcecd_virtual xrun controller_connector_test
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running worker tests ==="
meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --suite worker --print-errorlogs
fi
;;
apiserver)
setup_build
echo "=== Building lib + apiserver + xcecli ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" apiserver xcecli
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running frontend Go unit tests ==="
cd frontend/apiserver && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./store/... ./utils/... ./controllers/... ./backend/... -count=1 && cd ../xcecli && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./... -count=1
fi
;;
xcecli)
setup_build
echo "=== Building xcecli ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xcecli
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running xcecli Go unit tests ==="
cd frontend/xcecli && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./... -count=1
fi
;;
xceadmin)
# Meson-driven like every other component (src/xceadmin/meson.build,
# custom_target wrapping meson-build.sh) -- installs pinned/non-
# editable out-of-tree in the meson build dir and runs pytest,
# every invocation regardless of $PROFILE (xceadmin's suite is pure
# Python, unrelated to the ASAN/release C compiler flags $PROFILE
# otherwise controls, and fast enough — ~2s — that always running
# it costs nothing). This is a strictly stronger guarantee than the
# old editable-install-by-default behavior: every run now exercises
# real package-data resolution (bug-1460's class of bug), not just
# the `release` profile.
#
# For fast iterative dev without a full meson reconfigure, xceadmin
# can still be built standalone: cd xceadmin && pip install -e .
# (or see src/xceadmin/meson-build.sh for the exact out-of-tree
# steps this target runs).
setup_build
echo "=== Building + testing xceadmin (via meson) ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}" xceadmin
WHEEL_FILE=$(ls "$BUILD_DIR"/xceadmin/dist/xceadmin-*.whl 2>/dev/null || true)
if [ -n "$WHEEL_FILE" ]; then
echo "=== xceadmin wheel: $WHEEL_FILE ==="
echo " Install it directly with: pip install \"$WHEEL_FILE\""
fi
;;
all)
setup_build
echo "=== Building all components ==="
meson compile -C "$BUILD_DIR" "${MESON_COMPILE_JOB_OPTS[@]}"
if [ "$RUN_TESTS" = 1 ]; then
echo "=== Running all C tests ==="
meson test -C "$BUILD_DIR" "${MESON_TEST_JOB_OPTS[@]}" --print-errorlogs
echo "=== Running scheduler test (manual) ==="
"./$BUILD_DIR/scheduler/testsuite/sched_test"
echo "=== Running frontend Go unit tests ==="
cd frontend/apiserver && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./store/... ./utils/... ./controllers/... ./backend/... -count=1 && cd ../xcecli && CGO_ENABLED=0 go test "${GO_TEST_JOB_OPTS[@]}" ./... -count=1
fi
;;
*)
usage
;;
esac
if [ "$RUN_TESTS" != 1 ]; then
echo "=== Skipped tests (profile=$PROFILE — build only). Binary in $BUILD_DIR; run a suite directly, e.g. ASAN_OPTIONS=detect_leaks=0 ./$BUILD_DIR/controller/controller_unit_test ==="
fi
echo "=== Done: $COMPONENT (profile=$PROFILE, dir=$BUILD_DIR) ==="