diff --git a/.claude/skills/xce-build/SKILL.md b/.claude/skills/xce-build/SKILL.md index 15a0d2fe..8a0f5a40 100644 --- a/.claude/skills/xce-build/SKILL.md +++ b/.claude/skills/xce-build/SKILL.md @@ -11,7 +11,7 @@ Args: `[component] [profile|build_dir] [build_dir] [--incremental]` — passed a ## Run-queue-length limit (`--jobs N`) — compute it live before every build, don't skip this -`xce-build.sh` accepts a `--jobs N` flag that caps the build's run-queue length. For `meson compile` (the ninja backend, and the memory-heaviest step — ASAN+debug C/C++ compiles) it's passed as **`-l N`, ninja's load-average ceiling** — ninja re-checks the system's live load average, literally its run-queue length, before launching *every* new job for the *whole* build, and holds off while it's above N. That's the actual control lever, deliberately not paired with pinning `-j` to the same number: `-j` would just be a one-time snapshot cap decided before the first job starts, whereas `-l` keeps adapting for the entire build — including backing off if some *other* concurrent session's build drives load up partway through (this sandbox routinely runs multiple sessions building at once). `-j` is left at ninja's own default (~nproc+2) as a loose ceiling. `meson test` and `go test` have no load-average equivalent (only flat process-count knobs), so N is used there as a job-count ceiling instead — `-j`/`--num-processes N` for `meson test`, `-p N` for `go test`. The flag has no default — if you omit it, everything falls back to its own auto-parallelism, same as before this flag existed. +`xce-build.sh` accepts a `--jobs N` flag that caps the build's run-queue length. For `meson compile` (the ninja backend, and the memory-heaviest step — ASAN+debug C/C++ compiles) it's passed as **`-l N`, ninja's load-average ceiling** — ninja re-checks the system's live load average, literally its run-queue length, before launching *every* new job for the *whole* build, and holds off while it's above N. That's the actual control lever, deliberately not paired with pinning `-j` to the same number: `-j` would just be a one-time snapshot cap decided before the first job starts, whereas `-l` keeps adapting for the entire build — including backing off if some *other* concurrent session's build drives load up partway through (this sandbox routinely runs multiple sessions building at once). `-j` is left at ninja's own default (~nproc+2) as a loose ceiling. `meson test` and `go test` have no load-average equivalent (only flat process-count knobs), so N is used there as a job-count ceiling instead — `-j`/`--num-processes N` for `meson test`, `-p N` for `go test`. If you omit it, `xce-build.sh` now computes an equivalent live default itself (same formula, same inputs) rather than falling back to unthrottled auto-parallelism — but compute and pass it explicitly here anyway: this skill's own read happens at the moment `/xce-build` is invoked, which can be materially fresher than whatever instant the script gets around to its own read a bit further into setup, and an explicit value is easier to audit in the resolved command line. **Always compute N yourself, live, immediately before launching the build** — from the hardware/load state *at the moment the skill is invoked*, not a number remembered from an earlier run or a fixed constant. This sandbox routinely runs multiple concurrent sessions building/testing at once (see `xce-build.sh`'s own `prune_stale_tmp_fixtures` comment), so a stale or hardcoded value can either starve this build or overload a system something else is already using. diff --git a/.wolf/anatomy-index.json b/.wolf/anatomy-index.json index 69eaf43b..b604d66a 100644 --- a/.wolf/anatomy-index.json +++ b/.wolf/anatomy-index.json @@ -1,13 +1,13 @@ { "version": 1, "meta": { - "lastScanned": "2026-09-15T08:47:36.542Z", + "lastScanned": "2026-09-15T09:18:07.061Z", "fileCount": 572, "hits": 0, "misses": 0, - "renderedHash": "a6cf6257d3a03c11e763a6e34e7b004b90a51e62156a9ef766f127af914454ed", - "storeUpdatedAt": "2026-09-15T08:47:36.550Z", - "rootHash": "547dd1e8c0823374" + "renderedHash": "5820aa887fe384093984d9635ef98189a5e599ef21ffb579eb5088893310988f", + "storeUpdatedAt": "2026-09-15T09:18:07.069Z", + "rootHash": "8acb88a113abdee5" }, "files": { ".agent-shell/transcripts/2026-08-11-11-39-59.md": { @@ -9754,11 +9754,11 @@ }, ".claude/skills/xce-build/SKILL.md": { "description": "build xce source code", - "tokens": 4561, - "hash": "79eb234b59610c7d", - "size": 18353, - "mtimeMs": 1789388187356.294, - "updatedAt": "2026-09-14T12:16:27.397Z", + "tokens": 4647, + "hash": "2c0e8c19fb0b456c", + "size": 18698, + "mtimeMs": 1789463887014.313, + "updatedAt": "2026-09-15T09:18:07.061Z", "source": "hook" }, "src/configs/limits.yaml": { @@ -10132,11 +10132,11 @@ }, "src/tools/xce-build.sh": { "description": "", - "tokens": 5473, - "hash": "3e7647bc3a190f55", - "size": 20542, - "mtimeMs": 1789366341549.8474, - "updatedAt": "2026-09-14T06:12:21.585Z", + "tokens": 6044, + "hash": "8500665bd5c91423", + "size": 22684, + "mtimeMs": 1789463545195.4717, + "updatedAt": "2026-09-15T09:12:25.246Z", "source": "hook" }, ".claude/skills/xce-test-ha/run-ha-soak.sh": { diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index 48618622..552d60f0 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,6 +1,6 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T08:47:36.542Z +> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T09:18:07.061Z > Files: 572 tracked | Anatomy hits: 0 | Misses: 0 ## ./ @@ -27,7 +27,7 @@ ## .claude/skills/xce-build/ -- `SKILL.md` — build xce source code (~4561 tok) +- `SKILL.md` — build xce source code (~4647 tok) ## .claude/skills/xce-deploy-ha/ @@ -945,4 +945,4 @@ ## src/tools/ - `test-commit-durability.sh` — test-commit-durability.sh — bug-661 reproduction: committed jobs lost across failover. (~9874 tok) -- `xce-build.sh` (~5473 tok) +- `xce-build.sh` (~6044 tok) diff --git a/.wolf/memory.md b/.wolf/memory.md index 60bd56e2..ac7e0eca 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -4783,3 +4783,8 @@ | 01:46 | Edited src/scheduler/testsuite/test_job_array_wire_e2e.c | modified vtask_entry() | ~1067 | | 01:47 | Edited src/scheduler/testsuite/test_job_array_wire_e2e.c | modified pop_job_array_update() | ~216 | | 01:47 | Edited src/scheduler/testsuite/test_job_array_wire_e2e.c | modified test_one_push_enqueues_job_array_descriptor() | ~1041 | +| 02:10 | Edited src/tools/xce-build.sh | removed 38 lines | ~108 | +| 02:11 | Edited src/tools/xce-build.sh | added 4 condition(s) | ~1087 | +| 02:11 | Edited src/tools/xce-build.sh | 7→7 lines | ~152 | +| 02:12 | Edited src/tools/xce-build.sh | 5→5 lines | ~81 | +| 02:18 | Edited .claude/skills/xce-build/SKILL.md | inline fix | ~126 | diff --git a/src/tools/xce-build.sh b/src/tools/xce-build.sh index d929d1e5..131d6808 100755 --- a/src/tools/xce-build.sh +++ b/src/tools/xce-build.sh @@ -49,42 +49,14 @@ while [ $# -gt 0 ]; do done set -- "${ARGS[@]}" -# --jobs (run-queue-length limit): opt-in, no default -- when unset, -# meson/ninja/go fall back to their own auto-parallelism (typically nproc or -# nproc+2 workers), exactly as before this flag existed. Callers are expected -# to compute a live, invocation-time value from current CPU count, available -# memory, and current load average (this sandbox routinely runs multiple -# CONCURRENT sessions building/testing at once -- see prune_stale_tmp_fixtures -# below) rather than a fixed constant baked into this script; xce-build's own -# skill does that computation before invoking this script. -# -# For meson compile (the ninja backend, and the memory-heaviest step -- ASAN+ -# debug C/C++ compiles), the value is applied ONLY as ninja's -l LOAD_AVERAGE: -# ninja re-checks the system's live load average -- literally its run-queue -# length -- before launching EACH new job, throughout the whole build, and -# holds off starting more while it's above this ceiling. That is deliberately -# NOT the same as also pinning -j to this number: -j is a one-time snapshot -# cap decided before the first job starts, while -l keeps adapting for the -# entire build, including backing off if some OTHER concurrent session's -# build drives load up partway through. -j is left at ninja's own default -# (~nproc+2) as a loose ceiling; -l is the actual throttle. -# -# meson test and go test have no load-average equivalent (meson test's -j/ -# --num-processes and go test's -p are both flat process-count caps), so for -# those the same value is used as a job-count ceiling instead. +# --jobs (run-queue-length limit) validation only here -- the value isn't +# actually applied until after PROFILE is known (below), since the live +# default computation needs it. See that block's own comment for the full +# rationale. if [ -n "$JOBS" ] && ! [[ "$JOBS" =~ ^[1-9][0-9]*$ ]]; then echo "ERROR: --jobs value must be a positive integer, got: '$JOBS'" >&2 exit 1 fi -MESON_COMPILE_JOB_OPTS=() -MESON_TEST_JOB_OPTS=() -GO_TEST_JOB_OPTS=() -if [ -n "$JOBS" ]; then - MESON_COMPILE_JOB_OPTS=(-l "$JOBS") - MESON_TEST_JOB_OPTS=(-j "$JOBS") - GO_TEST_JOB_OPTS=(-p "$JOBS") - echo "=== Run-queue-length limit: $JOBS (meson compile -l load-average ceiling, meson test -j, go test -p) ===" -fi COMPONENT="${1:-all}" @@ -116,6 +88,73 @@ if [ -z "${BUILD_DIR:-}" ]; then esac fi +# --jobs (run-queue-length limit): an explicit --jobs N on the command line +# always wins outright. When omitted, this script computes a live default +# itself instead of falling back to meson/ninja/go's own unthrottled +# auto-parallelism (~nproc+2) -- this sandbox routinely runs multiple +# CONCURRENT sessions building/testing at once (see prune_stale_tmp_fixtures +# below), so unthrottled parallelism from a bare/direct invocation of this +# script (CI, a manual `bash xce-build.sh`, any caller that isn't the +# /xce-build skill) can starve, or get starved by, another session's build. +# The /xce-build skill already computes and passes --jobs N explicitly using +# this exact same formula (its own SKILL.md documents it) before ever +# reaching this script, so a skill-driven build's behavior is unchanged -- +# this default only engages when --jobs was never passed at all. +# +# For meson compile (the ninja backend, and the memory-heaviest step -- ASAN+ +# debug C/C++ compiles), the value is applied ONLY as ninja's -l LOAD_AVERAGE: +# ninja re-checks the system's live load average -- literally its run-queue +# length -- before launching EACH new job, throughout the whole build, and +# holds off starting more while it's above this ceiling. That is deliberately +# NOT the same as also pinning -j to this number: -j is a one-time snapshot +# cap decided before the first job starts, while -l keeps adapting for the +# entire build, including backing off if some OTHER concurrent session's +# build drives load up partway through. -j is left at ninja's own default +# (~nproc+2) as a loose ceiling; -l is the actual throttle. +# +# meson test and go test have no load-average equivalent (meson test's -j/ +# --num-processes and go test's -p are both flat process-count caps), so for +# those the same value is used as a job-count ceiling instead. +if [ -n "$JOBS" ]; then + echo "=== Run-queue-length limit (explicit --jobs): $JOBS (meson compile -l load-average ceiling, meson test -j, go test -p) ===" +else + # Live default -- same formula the /xce-build skill computes by hand: a + # per-profile memory budget per job (ASAN+debug's sanitizer + # instrumentation and full debug info are the heaviest), reserving ~15% + # of available memory as headroom, and a per-profile PROCESS_FACTOR + # discounting nproc-minus-load-average for the compiler driver's forked + # cc1/cc1plus + as children (asan/debug keep those resident far longer + # than release's leaner ones). + default_nproc="$(nproc --all 2>/dev/null || echo 1)" + default_mem_mb="$(awk '/MemAvailable/ {print int($2/1024)}' /proc/meminfo 2>/dev/null)" + default_load="$(cut -d' ' -f1 /proc/loadavg 2>/dev/null)" + : "${default_mem_mb:=0}" + : "${default_load:=0}" + + case "$PROFILE" in + asan) default_per_job_mb=1536; default_process_factor=0.5 ;; + debug) default_per_job_mb=1024; default_process_factor=0.5 ;; + release) default_per_job_mb=1024; default_process_factor=0.7 ;; + esac + + JOBS="$(awk -v nproc="$default_nproc" -v mem="$default_mem_mb" \ + -v loadavg="$default_load" -v per_job="$default_per_job_mb" \ + -v factor="$default_process_factor" 'BEGIN { + mem_jobs = int(mem * 0.85 / per_job) + cpu_jobs = int((nproc - loadavg) * factor) + if (cpu_jobs < 1) cpu_jobs = 1 + jobs = nproc + if (mem_jobs < jobs) jobs = mem_jobs + if (cpu_jobs < jobs) jobs = cpu_jobs + if (jobs < 1) jobs = 1 + print jobs + }')" + echo "=== --jobs not passed -- computed live default: nproc=$default_nproc available=${default_mem_mb}MB load=$default_load profile=$PROFILE factor=$default_process_factor -> --jobs $JOBS ===" +fi +MESON_COMPILE_JOB_OPTS=(-l "$JOBS") +MESON_TEST_JOB_OPTS=(-j "$JOBS") +GO_TEST_JOB_OPTS=(-p "$JOBS") + # RUN_TESTS: the unit suites are ASAN-oriented (leak detection) and the meson # test harness is fragile on this sandbox's overlay fs (bug-924 fdatasync). Only # the `asan` profile runs them; `debug`/`release` are BUILD-ONLY — their purpose @@ -229,11 +268,11 @@ usage() { echo " for the staleness tradeoff. Rebuild without it if anything looks wrong." echo "--jobs N: cap this build's run-queue length at N -- meson compile's -l N" echo " (ninja's load-average ceiling, re-checked live before every job launch for" - echo " the whole build), meson test's -j N, go test's -p N. No default -- omit to" - echo " let ninja/go auto-detect, as before this flag existed. N should be computed" - echo " live from current CPU count, available memory, and current load average" - echo " (see the xce-build skill), not a fixed constant, since this sandbox often" - echo " runs multiple concurrent build/test sessions at once." + echo " the whole build), meson test's -j N, go test's -p N. Omit to have this" + echo " script compute a live default itself from current CPU count, available" + echo " memory, current load average, and profile (asan/debug/release), instead of" + echo " ninja/go's own unthrottled auto-detect -- this sandbox often runs multiple" + echo " concurrent build/test sessions at once." exit 1 }