feat(xce-build): default --jobs to a live-computed value when omitted
xce-build.sh previously left --jobs unset with no fallback -- a bare/ direct invocation (CI, a manual `bash xce-build.sh`, anything bypassing the /xce-build skill) got meson/ninja/go's own unthrottled auto-parallelism (~nproc+2), risking starvation on this sandbox's routinely-concurrent build/test sessions. The script now computes the same live default the /xce-build skill has always computed by hand -- nproc, available memory, and load average read fresh at invocation, with a per-profile memory budget (1536 MB/job asan, 1024 MB/job debug/release) and process-factor discount (0.5 asan/ debug, 0.7 release) for the compiler driver's forked cc1/as children -- whenever --jobs isn't passed explicitly. An explicit --jobs N still always wins; skill-driven builds are unaffected since the skill already passes its own computed value. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
parent
d5ec1eb3ae
commit
76e6359241
5 changed files with 99 additions and 55 deletions
|
|
@ -49,42 +49,14 @@ while [ $# -gt 0 ]; do
|
|||
done
|
||||
set -- "${ARGS[@]}"
|
||||
|
||||
# --jobs (run-queue-length limit): opt-in, no default -- when unset,
|
||||
# meson/ninja/go fall back to their own auto-parallelism (typically nproc or
|
||||
# nproc+2 workers), exactly as before this flag existed. Callers are expected
|
||||
# to compute a live, invocation-time value from current CPU count, available
|
||||
# memory, and current load average (this sandbox routinely runs multiple
|
||||
# CONCURRENT sessions building/testing at once -- see prune_stale_tmp_fixtures
|
||||
# below) rather than a fixed constant baked into this script; xce-build's own
|
||||
# skill does that computation before invoking this script.
|
||||
#
|
||||
# For meson compile (the ninja backend, and the memory-heaviest step -- ASAN+
|
||||
# debug C/C++ compiles), the value is applied ONLY as ninja's -l LOAD_AVERAGE:
|
||||
# ninja re-checks the system's live load average -- literally its run-queue
|
||||
# length -- before launching EACH new job, throughout the whole build, and
|
||||
# holds off starting more while it's above this ceiling. That is deliberately
|
||||
# NOT the same as also pinning -j to this number: -j is a one-time snapshot
|
||||
# cap decided before the first job starts, while -l keeps adapting for the
|
||||
# entire build, including backing off if some OTHER concurrent session's
|
||||
# build drives load up partway through. -j is left at ninja's own default
|
||||
# (~nproc+2) as a loose ceiling; -l is the actual throttle.
|
||||
#
|
||||
# meson test and go test have no load-average equivalent (meson test's -j/
|
||||
# --num-processes and go test's -p are both flat process-count caps), so for
|
||||
# those the same value is used as a job-count ceiling instead.
|
||||
# --jobs (run-queue-length limit) validation only here -- the value isn't
|
||||
# actually applied until after PROFILE is known (below), since the live
|
||||
# default computation needs it. See that block's own comment for the full
|
||||
# rationale.
|
||||
if [ -n "$JOBS" ] && ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "ERROR: --jobs value must be a positive integer, got: '$JOBS'" >&2
|
||||
exit 1
|
||||
fi
|
||||
MESON_COMPILE_JOB_OPTS=()
|
||||
MESON_TEST_JOB_OPTS=()
|
||||
GO_TEST_JOB_OPTS=()
|
||||
if [ -n "$JOBS" ]; then
|
||||
MESON_COMPILE_JOB_OPTS=(-l "$JOBS")
|
||||
MESON_TEST_JOB_OPTS=(-j "$JOBS")
|
||||
GO_TEST_JOB_OPTS=(-p "$JOBS")
|
||||
echo "=== Run-queue-length limit: $JOBS (meson compile -l load-average ceiling, meson test -j, go test -p) ==="
|
||||
fi
|
||||
|
||||
COMPONENT="${1:-all}"
|
||||
|
||||
|
|
@ -116,6 +88,73 @@ if [ -z "${BUILD_DIR:-}" ]; then
|
|||
esac
|
||||
fi
|
||||
|
||||
# --jobs (run-queue-length limit): an explicit --jobs N on the command line
|
||||
# always wins outright. When omitted, this script computes a live default
|
||||
# itself instead of falling back to meson/ninja/go's own unthrottled
|
||||
# auto-parallelism (~nproc+2) -- this sandbox routinely runs multiple
|
||||
# CONCURRENT sessions building/testing at once (see prune_stale_tmp_fixtures
|
||||
# below), so unthrottled parallelism from a bare/direct invocation of this
|
||||
# script (CI, a manual `bash xce-build.sh`, any caller that isn't the
|
||||
# /xce-build skill) can starve, or get starved by, another session's build.
|
||||
# The /xce-build skill already computes and passes --jobs N explicitly using
|
||||
# this exact same formula (its own SKILL.md documents it) before ever
|
||||
# reaching this script, so a skill-driven build's behavior is unchanged --
|
||||
# this default only engages when --jobs was never passed at all.
|
||||
#
|
||||
# For meson compile (the ninja backend, and the memory-heaviest step -- ASAN+
|
||||
# debug C/C++ compiles), the value is applied ONLY as ninja's -l LOAD_AVERAGE:
|
||||
# ninja re-checks the system's live load average -- literally its run-queue
|
||||
# length -- before launching EACH new job, throughout the whole build, and
|
||||
# holds off starting more while it's above this ceiling. That is deliberately
|
||||
# NOT the same as also pinning -j to this number: -j is a one-time snapshot
|
||||
# cap decided before the first job starts, while -l keeps adapting for the
|
||||
# entire build, including backing off if some OTHER concurrent session's
|
||||
# build drives load up partway through. -j is left at ninja's own default
|
||||
# (~nproc+2) as a loose ceiling; -l is the actual throttle.
|
||||
#
|
||||
# meson test and go test have no load-average equivalent (meson test's -j/
|
||||
# --num-processes and go test's -p are both flat process-count caps), so for
|
||||
# those the same value is used as a job-count ceiling instead.
|
||||
if [ -n "$JOBS" ]; then
|
||||
echo "=== Run-queue-length limit (explicit --jobs): $JOBS (meson compile -l load-average ceiling, meson test -j, go test -p) ==="
|
||||
else
|
||||
# Live default -- same formula the /xce-build skill computes by hand: a
|
||||
# per-profile memory budget per job (ASAN+debug's sanitizer
|
||||
# instrumentation and full debug info are the heaviest), reserving ~15%
|
||||
# of available memory as headroom, and a per-profile PROCESS_FACTOR
|
||||
# discounting nproc-minus-load-average for the compiler driver's forked
|
||||
# cc1/cc1plus + as children (asan/debug keep those resident far longer
|
||||
# than release's leaner ones).
|
||||
default_nproc="$(nproc --all 2>/dev/null || echo 1)"
|
||||
default_mem_mb="$(awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null)"
|
||||
default_load="$(cut -d' ' -f1 /proc/loadavg 2>/dev/null)"
|
||||
: "${default_mem_mb:=0}"
|
||||
: "${default_load:=0}"
|
||||
|
||||
case "$PROFILE" in
|
||||
asan) default_per_job_mb=1536; default_process_factor=0.5 ;;
|
||||
debug) default_per_job_mb=1024; default_process_factor=0.5 ;;
|
||||
release) default_per_job_mb=1024; default_process_factor=0.7 ;;
|
||||
esac
|
||||
|
||||
JOBS="$(awk -v nproc="$default_nproc" -v mem="$default_mem_mb" \
|
||||
-v loadavg="$default_load" -v per_job="$default_per_job_mb" \
|
||||
-v factor="$default_process_factor" 'BEGIN {
|
||||
mem_jobs = int(mem * 0.85 / per_job)
|
||||
cpu_jobs = int((nproc - loadavg) * factor)
|
||||
if (cpu_jobs < 1) cpu_jobs = 1
|
||||
jobs = nproc
|
||||
if (mem_jobs < jobs) jobs = mem_jobs
|
||||
if (cpu_jobs < jobs) jobs = cpu_jobs
|
||||
if (jobs < 1) jobs = 1
|
||||
print jobs
|
||||
}')"
|
||||
echo "=== --jobs not passed -- computed live default: nproc=$default_nproc available=${default_mem_mb}MB load=$default_load profile=$PROFILE factor=$default_process_factor -> --jobs $JOBS ==="
|
||||
fi
|
||||
MESON_COMPILE_JOB_OPTS=(-l "$JOBS")
|
||||
MESON_TEST_JOB_OPTS=(-j "$JOBS")
|
||||
GO_TEST_JOB_OPTS=(-p "$JOBS")
|
||||
|
||||
# RUN_TESTS: the unit suites are ASAN-oriented (leak detection) and the meson
|
||||
# test harness is fragile on this sandbox's overlay fs (bug-924 fdatasync). Only
|
||||
# the `asan` profile runs them; `debug`/`release` are BUILD-ONLY — their purpose
|
||||
|
|
@ -229,11 +268,11 @@ usage() {
|
|||
echo " for the staleness tradeoff. Rebuild without it if anything looks wrong."
|
||||
echo "--jobs N: cap this build's run-queue length at N -- meson compile's -l N"
|
||||
echo " (ninja's load-average ceiling, re-checked live before every job launch for"
|
||||
echo " the whole build), meson test's -j N, go test's -p N. No default -- omit to"
|
||||
echo " let ninja/go auto-detect, as before this flag existed. N should be computed"
|
||||
echo " live from current CPU count, available memory, and current load average"
|
||||
echo " (see the xce-build skill), not a fixed constant, since this sandbox often"
|
||||
echo " runs multiple concurrent build/test sessions at once."
|
||||
echo " the whole build), meson test's -j N, go test's -p N. Omit to have this"
|
||||
echo " script compute a live default itself from current CPU count, available"
|
||||
echo " memory, current load average, and profile (asan/debug/release), instead of"
|
||||
echo " ninja/go's own unthrottled auto-detect -- this sandbox often runs multiple"
|
||||
echo " concurrent build/test sessions at once."
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue