fix(xce-test-multihost): deploy with a fixture config set that clears limits (bug-1857)

Since bug-1790's fix (9caf1424) counts a job's full task total against
global limits, the repo's limit1 (all users except rshen share 3 slots)
pends xce_test's root-submitted 33-task job forever, so the multihost
e2e fixture failed even though the allocator is correct.

deploy-ha.sh gains an optional XCE_DEPLOY_CONFIG_DIR (default unchanged:
src/configs). run-multihost-verify.sh copies src/configs into its deploy
dir, replaces limits.yaml with an empty list, and deploys with that
override.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.claude/skills/xce-test-multihost/run-multihost-verify.sh

@@ -410,8 +410,18 @@ deploy() {
     mkdir -p "$TOP" || fail "could not create deploy dir $TOP"
     touch "$TOP/$MARKER_NAME"   # claim ownership so a later run may safely re-wipe it
 
-    log "deploying controller+scheduler+worker fixture into $TOP (peers=$PEERS)"
+    # bug-1857: fixture-owned config set. The repo's limits.yaml carries
-    if ! nix develop "$REPO/src" --command "$BASH_BIN" "$DEPLOY_HA" "$BINARY" "$TOP" "$PEERS" >"$TOP/deploy.log" 2>&1; then
+    # limit1 (every user except rshen shares 3 slots), and xce_test submits
+    # as root -- since bug-1790's fix counts a job's full task total against
+    # global limits, the 33-task job pends on limit1 forever. This test
+    # proves allocation, not limit enforcement, so copy src/configs and
+    # clear every limit rather than tuning around whichever one binds today.
+    local cfg_dir="$TOP/configs"
+    cp -r "$REPO/src/configs" "$cfg_dir" || fail "could not copy $REPO/src/configs into $cfg_dir"
+    printf -- '---\nlimits: []\n' >"$cfg_dir/limits.yaml"
+
+    log "deploying controller+scheduler+worker fixture into $TOP (peers=$PEERS, configs=$cfg_dir, limits cleared)"
+    if ! XCE_DEPLOY_CONFIG_DIR="$cfg_dir" nix develop "$REPO/src" --command "$BASH_BIN" "$DEPLOY_HA" "$BINARY" "$TOP" "$PEERS" >"$TOP/deploy.log" 2>&1; then
         fail "deployment failed at step deploy-ha.sh (see $TOP/deploy.log)"
     fi
 

.wolf/buglog.json

@@ -27824,6 +27824,26 @@
       ],
       "occurrences": 1,
       "last_seen": "2026-09-24T09:39:50Z"
+    },
+    {
+      "id": "bug-1857",
+      "status": "partial",
+      "status_updated": "2026-09-24",
+      "status_note": "fix written and verified, awaiting commit",
+      "timestamp": "2026-09-24T13:02:14Z",
+      "error_message": "xce-test-multihost FAIL: job never reached RUN within 30s (last observed state: PEND); scheduler.log: 'unsatisfy_ unmet resource 33 (limit=limit1) shortfall 30.000000'",
+      "file": ".claude/skills/xce-test-multihost/run-multihost-verify.sh,src/tools/deploy-ha.sh,src/configs/limits.yaml",
+      "root_cause": "run-multihost-verify.sh deploys with the repo's default src/configs/limits.yaml, whose limit1 caps all users except rshen at a shared 3 slots. xce_test submits as root, so the 33-task job is capped. The fixture passed 2026-09-06 only because global limits did not yet count a job's full task total; the bug-1790 fix (9caf1424, 2026-09-08) correctly started doing so, leaving the fixture stale. Allocator itself is fine: passes with limit1 raised to 100.",
+      "fix": "UPDATE (2026-09-24): Fix (uncommitted): deploy-ha.sh gains optional XCE_DEPLOY_CONFIG_DIR (default unchanged: src/configs); run-multihost-verify.sh copies src/configs into $TOP/configs, replaces limits.yaml with 'limits: []', and deploys with that override. Verified 2026-09-24: multihost e2e PASS (plain 33-task job + array element both split across 2 hosts, DONE), 0 limit1 rejections; default deploy path still builds from repo configs; bad override fails with 'config dir not found'.",
+      "tags": [
+        "test-fixture",
+        "scheduler",
+        "limits",
+        "multihost"
+      ],
+      "related_bugs": [],
+      "occurrences": 1,
+      "last_seen": "2026-09-24T13:02:14Z"
     }
   ]
 }

src/tools/deploy-ha.sh

@@ -12,7 +12,8 @@ if [ -z "$BINARY" ] || [ -z "$TOP" ] || [ -z "$PEERS" ]; then
     echo "  peers: comma-separated host:port list (e.g. host1:port1,host2:port2)"
     echo "  TLS cert/key and cluster.bin are generated fresh into <top-dir> every"
     echo "  run (openssl self-signed cert; cluster.bin via xceadmin from"
-    echo "  src/configs/*.yaml) -- no pre-existing data directory needed."
+    echo "  src/configs/*.yaml, or \$XCE_DEPLOY_CONFIG_DIR/*.yaml if set) -- no"
+    echo "  pre-existing data directory needed."
     exit 1
 fi
 
@@ -248,8 +249,13 @@ openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
 
 # Build cluster.bin from src/configs/*.yaml via xceadmin, straight into
 # this deploy's own ctrl-1/conf/ (no intermediate data-dir copy).
+# XCE_DEPLOY_CONFIG_DIR lets a test fixture supply its own config set
+# (e.g. xce-test-multihost drops limits.yaml's limits, bug-1857) without
+# editing the checked-in src/configs/ that every other deploy uses.
+CONFIG_DIR="${XCE_DEPLOY_CONFIG_DIR:-$SCRIPT_DIR/../configs}"
+[ -d "$CONFIG_DIR" ] || { echo "Error: config dir not found: $CONFIG_DIR"; exit 1; }
 CLUSTER_BIN_TMP="$TOP/.cluster.bin.tmp"
-"$XCEADMIN" config build "$SCRIPT_DIR/../configs" -o "$CLUSTER_BIN_TMP"
+"$XCEADMIN" config build "$CONFIG_DIR" -o "$CLUSTER_BIN_TMP"
 TAG=$(sha256sum "$CLUSTER_BIN_TMP" | cut -c1-12)
 mv "$CLUSTER_BIN_TMP" "$TOP/controller/ctrl-1/conf/cluster.bin.$TAG"
 echo "cluster.bin.$TAG" > "$TOP/controller/ctrl-1/conf/current"