diff --git a/.claude/skills/xce-test-ha/SKILL.md b/.claude/skills/xce-test-ha/SKILL.md index 21c18334..97eac085 100644 --- a/.claude/skills/xce-test-ha/SKILL.md +++ b/.claude/skills/xce-test-ha/SKILL.md @@ -266,6 +266,60 @@ invariant is asserted with an immediate `fail()` inside `run_array_chaos_retry_round()`, so a violation aborts the run before this line is ever reached. +### `--worker-capacity-soak` mode: scheduler resource accounting survives a real failover (bug-1841) + +The existing worker-rejoin-resync coverage (sequential/chaotic modes, +`verify_worker_rejoin_resync`) only proves a worker-dispatched job +**survives** a real controller failover — it reaches `DONE` via a genuine +rejoin. It never re-checks the worker's own resource accounting afterward, +which is exactly where bug-1841 lived: `resync_resolve_round()` +(`src/controller/resync.c`) cleared `XCE_WORKER_RECONCILING` on the +controller's own `worker_t` but never told the scheduler, so the +scheduler's copy stayed `RECONCILING` forever once a rejoin's resync round +resolved — permanently freezing that worker's `resInUse` even after every +job on it finished `DONE`, silently starving all *future* dispatch to that +worker. A saturating job reaching `DONE` would **not** have caught this; +only a job submitted *after* recovery actually dispatching proves the +freeze lifted. + +This mode saturates the live worker's real "slots" capacity (live-queried +via `resval -n slots -m `, not assumed — portable across whatever +machine the soak runs on) with real single-slot jobs, then submits one more +"overflow" probe job that must stay `PEND` — self-verifying genuine +saturation rather than guessing at a per-job slot cost. It kills the +primary while the saturating jobs are `RUN`, waits for the worker's real +rejoin and every saturating job to reach `DONE`, then hard-asserts the +*same* probe job — still `PEND` from before the kill — now reaches `RUN`. +That final transition is the actual bug-1841 proof point. + +Requires a real worker+scheduler binary pair (same auto-detection as +sequential/chaotic's own worker-rejoin-resync coverage — build +`worker`/`scheduler` alongside `controller`); **hard-fails**, does not +soft-skip, since this mode is opt-in specifically for this coverage. Single +dedicated round, not a loop — every invariant is a hard `fail()`, matching +`--array-soak`/`--array-chaos-retry`'s "this is a correctness invariant, not +a chaotic/racy outcome class" standard. + +```bash +SKILL_DIR=.claude/skills/xce-test-ha +nix develop ./src --command bash "$SKILL_DIR/run-ha-soak.sh" --worker-capacity-soak ~/tmp/xce-ha-workercap +# tune the post-recovery probe budget / safety cap on saturating job count +XCE_HA_WORKERCAP_RECOVERY_BUDGET=120 XCE_HA_WORKERCAP_MAX_SLOTS=64 \ + nix develop ./src --command bash "$SKILL_DIR/run-ha-soak.sh" --worker-capacity-soak ~/tmp/xce-ha-workercap +``` + +Verdict line (`soak-verdict.sh`'s `verdict_emit_workercapacity`): + +``` +[xce-test-ha] VERDICT mode=worker-capacity-soak slots_capacity=8 saturating_jobs=8 probe_recovery_ms=1250 result=PASS +``` + +Like `--array-soak`, `result` is unconditionally `PASS` here — every hard +invariant is asserted with an immediate `fail()` inside +`run_worker_capacity_soak_round()`, so a violation (most importantly: the +probe never reaching `RUN`, i.e. bug-1841 regressing) aborts the run before +this line is ever reached. + ### `--config-upgrade` mode: live configuration upgrade under chaotic failover (config-upgrade-snapshot task 8.6) Drives a real `OP_upgradeConfig` RPC (the new `upgradeconfig` `xce_test` diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index 04bb9fe4..1d843c1b 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -160,6 +160,36 @@ # hard-failed (fail(), not count-and-tolerate) since "exactly # one record, full range addressable" is a correctness # invariant, not a chaotic/racy outcome class. +# --worker-capacity-soak (bug-1841): one dedicated round proving the +# worker-rejoin-resync feature's SCHEDULER-side resource +# accounting, not just job durability, survives a real +# failover. The existing worker-rejoin-resync coverage +# (sequential/chaotic modes, verify_worker_rejoin_resync) +# only proves a worker-dispatched job reaches DONE via a +# genuine rejoin — it never re-checks the worker's own +# capacity afterward, which is exactly where bug-1841 lived: +# resync_resolve_round() (src/controller/resync.c) cleared +# XCE_WORKER_RECONCILING on the controller's own worker_t +# but never told the scheduler, so the scheduler's copy +# stayed RECONCILING forever once a rejoin's resync round +# resolved — permanently freezing that worker's resInUse +# even after every job on it finished DONE, silently +# starving all FUTURE dispatch. Saturates the live worker's +# REAL "slots" capacity (live-queried via `resval`, not +# assumed) with real single-slot jobs, self-verifies genuine +# saturation with an overflow probe job that must stay PEND, +# kills the primary while those jobs are RUN, waits for the +# worker's real rejoin + every saturating job to reach DONE, +# then hard-asserts the SAME probe job — submitted before +# the kill, still PEND — now reaches RUN. Requires a real +# worker+scheduler binary pair (same auto-detection as +# sequential/chaotic's own worker-rejoin-resync coverage); +# hard-fails, does not soft-skip, since this mode is opt-in +# specifically for this coverage. Single dedicated round, +# not a loop; hard-fails (fail(), not count-and-tolerate) — +# this is bug-1841's own core correctness claim, not a +# chaotic/racy outcome class, same standard --array-soak/ +# --array-chaos-retry hold themselves to. # --selftest run the verdict-parser self-test (soak-verdict-selftest.sh) # and exit — no cluster, no binaries needed. # @@ -273,6 +303,25 @@ # --wal-segment-bytes did not produce the # expected rotation at this job volume # (default 20) +# +# Worker-capacity-soak env overrides (bug-1841): +# XCE_HA_WORKERCAP_MAX_SLOTS safety cap on both how many jobs a +# --worker-capacity-soak round will over-submit +# (2x resval's reported capacity, clamped to +# this) and on the raw resval reading itself — +# fails loud instead of silently running a +# multi-hour soak if `resval` reports an +# implausible capacity (default 128) +# XCE_HA_WORKERCAP_SETTLE seconds to wait, after over-submitting, for +# dispatch to stabilize before snapshotting +# which jobs are RUN vs PEND — the empirical +# saturation point this round proves recovery +# against (default 30) +# XCE_HA_WORKERCAP_RECOVERY_BUDGET seconds to wait, after every +# saturating job reaches DONE via rejoin, for +# the pre-kill overflow probe job to reach RUN +# — the round's actual bug-1841 proof point +# (default 60) set -u # ── Mode / flag parsing (flags may precede the positional [ROUNDS] [DEPLOY_DIR]) ─ @@ -293,6 +342,7 @@ while [ $# -gt 0 ]; do --factstream) MODE="factstream" ;; --array-soak) MODE="array-soak" ;; --array-chaos-retry) MODE="array-chaos-retry" ;; + --worker-capacity-soak) MODE="worker-capacity-soak" ;; --config-upgrade) MODE="config-upgrade" ;; --config-upgrade-wal-trim) MODE="config-upgrade-wal-trim" ;; --selftest) exec bash "$SELF_DIR/soak-verdict-selftest.sh" ;; @@ -661,6 +711,44 @@ verify_worker_resync_jobs_final() { && log "chaotic: all ${#WORKER_RESYNC_JOBIDS[@]} worker-resync job(s) reconciled to DONE" } +# ── Worker-capacity soak helpers (bug-1841) ────────────────────────────────── +# +# Live-queries WORKER_NAME's total "slots" capacity via the real client wire +# protocol (resval -n slots -m ) — NOT the offline sim-REPL "info +# worker" command bug-1841's original bug report used (that only exists +# inside `xce_scheduler --interactive -s `). resval's printed value +# is the worker's self-reported TOTAL installed capacity (w->host.resvals, +# architecturally never decremented by dispatch — see buglog.json bug-1727), +# which is exactly what run_worker_capacity_soak_round below needs: how many +# single-slot jobs it takes to fully occupy the worker, on WHATEVER machine +# this soak happens to run on. Echoes the integer capacity on success. +worker_slot_capacity() { + local out val + out="$(printf 'set masters %s\nset myname soak\nclient\nwait-primary %s\nresval -n slots -m %s\nquit\n' \ + "$MASTERS" "$WAITMS" "$WORKER_NAME" | xtest)" + val="$(printf '%s\n' "$out" | sed -n 's/^value: \([0-9]\{1,\}\)\.[0-9]*$/\1/p' | head -1)" + [ -n "${val:-}" ] && [ "$val" -gt 0 ] 2>/dev/null || return 1 + echo "$val" +} + +# Same jobsub call as submit_worker_job() above, but returns the jobid +# immediately after a successful ack WITHOUT waiting for RUN — used for +# run_worker_capacity_soak_round's overflow probe, which specifically needs +# to observe the job stay PEND while the worker is saturated (a RUN-waiting +# helper would just time out and fail() on exactly the state this scenario +# wants to assert). +submit_worker_job_no_wait() { + local sleep_secs="$1" out jid + out="$(printf 'set masters %s\nset myname soak\nclient\nwait-primary %s\njobsub -u root -q normal -c "sleep %s"\nquit\n' \ + "$MASTERS" "$WAITMS" "$sleep_secs" | xtest)" + jid="$(printf '%s\n' "$out" | sed -n 's/^jobsubmit ok: JobId: \(.*\)$/\1/p')" + if [ -z "$jid" ]; then + log "worker-capacity-soak: overflow probe jobsub itself failed -- raw output: $(printf '%s' "$out" | tr '\n' ' ')" >&2 + return 1 + fi + echo "$jid" +} + # ── Preconditions ──────────────────────────────────────────────────────────── [ -x "$BIN" ] || { echo "[xce-test-ha] FAIL: controller binary missing: $BIN (run /xce-build controller)" >&2; exit 2; } # xce-harness-component task 3.1 decoupled xce_test from the controller build -- @@ -2383,6 +2471,29 @@ poll_addressable_elapsed() { done } +# Poll jobinfo_of every 1s (up to $2 seconds, default 60) until job $1 +# reaches state RUN. Echoes elapsed milliseconds (epoch_ms above) from this +# function's own start to first RUN — run_worker_capacity_soak_round's +# actual bug-1841 proof-point measurement. Mirrors poll_addressable_elapsed's +# own $SECONDS-budget / single-$EPOCHREALTIME-pair shape (this project's own +# logged lesson, .wolf/cerebrum.md 2026-07-07: don't mix sub-second +# timestamps into per-iteration deadline arithmetic). Returns 1 (nothing +# echoed) on timeout. +poll_until_run_elapsed() { + local id="$1" budget="${2:-60}" start_ms st + SECONDS=0 + start_ms="$(epoch_ms)" + while :; do + st="$(jobinfo_of "$id" 2000 | sed -n 's/^state: //p')" + [ "$st" = "RUN" ] && { + echo "$(($(epoch_ms) - start_ms))" + return 0 + } + [ "$SECONDS" -ge "$budget" ] && return 1 + sleep 1 + done +} + # Verify each element index in "$@" (after base string $1) is individually # addressable via jobinfo. Logs + returns 1 on the first miss. verify_array_elements() { @@ -2670,6 +2781,160 @@ run_array_chaos_retry_round() { ARRAYRETRY_LAST_BASE="$base" } +# ── Worker-capacity soak: bug-1841 regression (scheduler-side resource +# accounting for a worker, not just job durability, after a real +# failover) ────────────────────────────────────────────────────────────── +# +# See this file's own header comment (--worker-capacity-soak) for the full +# rationale. Summary: the existing worker-rejoin-resync coverage +# (verify_worker_rejoin_resync above) proves a worker-dispatched job +# SURVIVES a real controller failover — it never re-checks the worker's own +# resource accounting afterward, which is exactly where bug-1841 lived. +# +# Saturation is SELF-VERIFYING, not assumed: submits exactly $cap (the +# worker's own live-queried "slots" capacity, worker_slot_capacity above) +# single-slot jobs, all of which must reach RUN, then submits one more +# single-slot "overflow" probe that must stay PEND — proving the worker is +# genuinely fully occupied, not under- or over-subscribed by a wrong +# per-job slot-cost guess. That SAME probe job is re-checked after the +# failover: once every saturating job reaches DONE via a genuine rejoin +# (mirrors verify_worker_rejoin_resync's own proof, batched like +# verify_worker_resync_jobs_final's chaotic-mode sweep), the probe must +# reach RUN within budget — the actual bug-1841 proof point. Every +# invariant is a hard fail(), matching --array-soak/--array-chaos-retry's +# "this is a correctness invariant, not a chaotic/racy outcome class" +# standard. +# +# Sets WORKERCAP_SLOTS / WORKERCAP_NSATURATED / WORKERCAP_RECOVERY_MS for +# the caller's verdict_emit_workercapacity() call. +run_worker_capacity_soak_round() { + local cap total i jid pre_count prev_term newp st probe_jid pid recovery_ms + local sat_sleep settle_secs done_budget + local DOWN="" + local max_slots="${XCE_HA_WORKERCAP_MAX_SLOTS:-128}" + local recovery_budget="${XCE_HA_WORKERCAP_RECOVERY_BUDGET:-60}" + local submitted=() + local saturating=() + local pending=() + local stuck=() + + [ "$WORKER_ENABLED" -eq 1 ] \ + || fail "worker-capacity-soak: requires a real worker+scheduler -- run /xce-build worker and /xce-build scheduler alongside /xce-build controller" + + cap="$(worker_slot_capacity)" \ + || fail "worker-capacity-soak: could not query $WORKER_NAME's slot capacity via resval -n slots -m $WORKER_NAME" + [ "$cap" -le "$max_slots" ] \ + || fail "worker-capacity-soak: $WORKER_NAME reports an implausible slot capacity ($cap > max $max_slots) -- refusing to submit that many jobs; check resval output or raise XCE_HA_WORKERCAP_MAX_SLOTS" + + # bug-1841 soak-design lesson #1 (found live, first draft): submitting + # exactly $cap jobs SEQUENTIALLY via submit_worker_job (which blocks on + # each one's own RUN confirmation before the next submit even fires) + # took ~6s/job -- 22 jobs took ~127s wall-clock. Against a short + # per-job simulated runtime, the FIRST saturating jobs had already + # finished and freed their slots before the LAST ones were even + # submitted. Fixed by decoupling submit from confirmation (fire every + # jobsub rapidly via submit_worker_job_no_wait first). + # + # bug-1841 soak-design lesson #2 (found live, second draft): resval's + # reported "slots" capacity ($cap) is NOT necessarily the worker's real + # concurrent-dispatch ceiling -- this deployment's own limits.yaml + # (limit1/ug1_normal_slots and friends) or ordinary picker/round timing + # can bind well below the raw resource value (empirically: cap=22 + # reported, only 15 concurrently dispatched). Assuming "exactly $cap + # jobs must all reach RUN" made the round fail on an environment + # artifact that has nothing to do with bug-1841. Fixed by NOT assuming + # the ceiling at all: over-submit generously (2x $cap), let dispatch + # settle, then EMPIRICALLY classify whichever jobs are RUN as the + # "saturating" set and whichever are still PEND as proof of genuine + # saturation (one of them becomes the recovery probe) -- whatever the + # true effective ceiling turns out to be, this round only needs SOME + # jobs to be RUN and AT LEAST ONE to be genuinely PEND. + total=$((cap * 2)) + if [ "$total" -gt "$max_slots" ]; then + total="$max_slots" + fi + settle_secs="${XCE_HA_WORKERCAP_SETTLE:-30}" + sat_sleep=$((total * 3 + 150)) + done_budget=$((sat_sleep + 90)) + log "worker-capacity-soak: $WORKER_NAME reports $cap slot(s) -- over-submitting $total single-slot job(s) to guarantee genuine saturation regardless of the true effective concurrency ceiling" + + for i in $(seq 1 "$total"); do + jid="$(submit_worker_job_no_wait "$sat_sleep")" \ + || fail "worker-capacity-soak: job $i/$total submission failed" + submitted+=("$jid") + done + log "worker-capacity-soak: $total job(s) submitted, settling ${settle_secs}s for dispatch to stabilize" + sleep "$settle_secs" + + for jid in "${submitted[@]}"; do + st="$(jobinfo_of "$jid" 2000 | sed -n 's/^state: //p')" + case "$st" in + RUN) saturating+=("$jid") ;; + PEND) pending+=("$jid") ;; + *) fail "worker-capacity-soak: job $jid in unexpected state '${st:-unknown}' after settling (expected RUN or PEND)" ;; + esac + done + [ "${#saturating[@]}" -gt 0 ] \ + || fail "worker-capacity-soak: zero of $total jobs reached RUN -- $WORKER_NAME never dispatched anything, cannot prove recovery" + [ "${#pending[@]}" -gt 0 ] \ + || fail "worker-capacity-soak: all $total jobs reached RUN -- $WORKER_NAME was not genuinely saturated even at 2x its reported capacity, this round proves nothing (raise XCE_HA_WORKERCAP_MAX_SLOTS to over-submit further)" + probe_jid="${pending[0]}" + log "worker-capacity-soak: settled at ${#saturating[@]} RUN / ${#pending[@]} PEND (resval reported $cap) -- genuinely saturated; using $probe_jid as the recovery probe" + + pre_count="$(worker_log_count "rejoin after transparent reconnect succeeded")" + prev_term="$(latest_primary_term)"; prev_term="${prev_term:-0}" + + pid="$(pgrep -f -- "$(node_pattern "$PRIMARY")")" + [ -n "$pid" ] || fail "worker-capacity-soak: could not find pid for $PRIMARY" + log "worker-capacity-soak: killing primary $PRIMARY (term=$prev_term)" + kill -9 $pid 2>/dev/null + DOWN="$PRIMARY" + sleep 2 + + newp="$(wait_for_new_primary "$prev_term")" + [ -n "$newp" ] || fail "worker-capacity-soak: no new primary elected after kill" + log "worker-capacity-soak: new primary elected: $newp (term=$(latest_primary_term))" + record_primary "$newp" + PRIMARY="$(discover_primary)" + [ -n "$PRIMARY" ] || fail "worker-capacity-soak: new primary $newp elected but client cannot reach it" + restart_scheduler_for_new_primary + verify_no_loss || fail "worker-capacity-soak: a committed job was LOST after failover" + + for i in $(seq 1 30); do + [ "$(worker_log_count "rejoin after transparent reconnect succeeded")" -gt "$pre_count" ] && break + sleep 1 + done + [ "$(worker_log_count "rejoin after transparent reconnect succeeded")" -gt "$pre_count" ] \ + || fail "worker-capacity-soak: worker never logged a fresh rejoin after the failover" + + stuck=() + for jid in "${saturating[@]}"; do + st="" + for i in $(seq 1 "$done_budget"); do + st="$(jobinfo_of "$jid" 2000 | sed -n 's/^state: //p')" + [ "$st" = "DONE" ] && break + sleep 1 + done + [ "$st" = "DONE" ] || stuck+=("$jid(last=${st:-unknown})") + done + [ "${#stuck[@]}" -eq 0 ] \ + || fail "worker-capacity-soak: ${#stuck[@]}/${#saturating[@]} saturating job(s) never reached DONE after rejoin: ${stuck[*]}" + log "worker-capacity-soak: all ${#saturating[@]} saturating job(s) recovered to DONE via genuine rejoin-reconciliation" + + # THE proof: with every saturating job DONE, the worker is genuinely + # idle again -- the probe (still PEND from before the kill) must now be + # dispatched. bug-1841 present => the scheduler's resInUse for this + # worker stays frozen at the saturating jobs' old charge forever, and + # this loop times out with the probe stuck PEND. + recovery_ms="$(poll_until_run_elapsed "$probe_jid" "$recovery_budget")" \ + || fail "worker-capacity-soak: overflow probe $probe_jid never reached RUN within ${recovery_budget}s of every saturating job going DONE -- the scheduler's resource accounting for $WORKER_NAME did not recover after the failover (bug-1841 regression)" + log "worker-capacity-soak: overflow probe $probe_jid reached RUN ${recovery_ms}ms after recovery started -- $WORKER_NAME's capacity correctly unfroze" + + WORKERCAP_SLOTS="$cap" + WORKERCAP_NSATURATED="${#saturating[@]}" + WORKERCAP_RECOVERY_MS="$recovery_ms" +} + # Drive a SPECIFIC node to primary — used to exercise leadership on the # lowest-priority node (ctrl-3), which the natural single-failover loop NEVER # elects: the election timeout is priority-staggered (my_priority * step_ms, p0 @@ -2762,12 +3027,12 @@ PRIMARY="$(discover_primary)" record_primary "$PRIMARY" log "primary is $PRIMARY" -# worker-rejoin-resync live coverage: only sequential/chaotic drive it (see -# this file's own header comment) -- the other dedicated single-round modes -# have their own separate, already-established scenarios and env knobs; -# starting an unrelated worker+scheduler alongside them would only add -# noise, not coverage. -if [ "$MODE" = "sequential" ] || [ "$MODE" = "chaotic" ]; then +# worker-rejoin-resync live coverage: only sequential/chaotic/worker- +# capacity-soak drive it (see this file's own header comment) -- the other +# dedicated single-round modes have their own separate, already-established +# scenarios and env knobs; starting an unrelated worker+scheduler alongside +# them would only add noise, not coverage. +if [ "$MODE" = "sequential" ] || [ "$MODE" = "chaotic" ] || [ "$MODE" = "worker-capacity-soak" ]; then start_worker_and_scheduler fi @@ -2913,6 +3178,24 @@ if [ "$MODE" = "array-chaos-retry" ]; then exit 0 fi +# ── Worker-capacity soak mode: bug-1841 ────────────────────────────────────── +# Opt-in (--worker-capacity-soak / XCE_HA_MODE=worker-capacity-soak). See +# run_worker_capacity_soak_round's own header comment and this file's own +# top-of-file header comment for the full rationale. One dedicated round, +# not a loop; hard-fails via fail() inside the round function, like +# --array-soak/--array-chaos-retry, since "the scheduler's resource +# accounting for a worker recovers after a real failover" is bug-1841's own +# core correctness claim, not a chaotic/racy outcome class. +if [ "$MODE" = "worker-capacity-soak" ]; then + log "MODE=worker-capacity-soak — saturate a real worker's capacity, fail over the primary mid-run, verify the scheduler's resource accounting for it recovers (bug-1841)" + run_worker_capacity_soak_round + verdict_emit_workercapacity "$WORKERCAP_SLOTS" "$WORKERCAP_NSATURATED" "$WORKERCAP_RECOVERY_MS" + log "PASS (worker-capacity-soak): $WORKERCAP_NSATURATED/$WORKERCAP_SLOTS slot(s) saturated, all recovered to DONE via genuine rejoin-reconciliation, overflow probe reached RUN ${WORKERCAP_RECOVERY_MS}ms after recovery -- scheduler resource accounting for the worker correctly unfroze after failover" + cleanup + trap - EXIT INT TERM + exit 0 +fi + # ── Config-upgrade mode: config-upgrade-snapshot task 8.6 ──────────────────── # Opt-in (--config-upgrade / XCE_HA_MODE=config-upgrade). See # run_config_upgrade_round's own header comment for the full mechanism trace. diff --git a/.claude/skills/xce-test-ha/soak-verdict-selftest.sh b/.claude/skills/xce-test-ha/soak-verdict-selftest.sh index 800ce3ff..4fd1dff2 100644 --- a/.claude/skills/xce-test-ha/soak-verdict-selftest.sh +++ b/.claude/skills/xce-test-ha/soak-verdict-selftest.sh @@ -201,6 +201,19 @@ check "arraysoak verdict line passes through the replay bound verdict" \ check "arraysoak verdict result is always PASS (real gate is upstream fail())" \ "PASS" "$(printf '%s' "$v" | grep -oE 'result=[A-Z]+' | cut -d= -f2)" +# verdict_emit_workercapacity (bug-1841): reports the measurements passed +# in. result is unconditionally PASS (the real gate is the fail() in +# run_worker_capacity_soak_round, upstream of this reporting-only function). +v="$(verdict_emit_workercapacity 8 8 1250)" +check "workercapacity verdict line reports slots_capacity" \ + "slots_capacity=8" "$(printf '%s' "$v" | grep -oE 'slots_capacity=[0-9]+')" +check "workercapacity verdict line reports saturating_jobs" \ + "saturating_jobs=8" "$(printf '%s' "$v" | grep -oE 'saturating_jobs=[0-9]+')" +check "workercapacity verdict line reports probe_recovery_ms" \ + "probe_recovery_ms=1250" "$(printf '%s' "$v" | grep -oE 'probe_recovery_ms=[0-9]+')" +check "workercapacity verdict result is always PASS (real gate is upstream fail())" \ + "PASS" "$(printf '%s' "$v" | grep -oE 'result=[A-Z]+' | cut -d= -f2)" + # ── Verdict ────────────────────────────────────────────────────────────────── echo if [ "$fails" -eq 0 ]; then diff --git a/.claude/skills/xce-test-ha/soak-verdict.sh b/.claude/skills/xce-test-ha/soak-verdict.sh index 838d3096..7b27d379 100644 --- a/.claude/skills/xce-test-ha/soak-verdict.sh +++ b/.claude/skills/xce-test-ha/soak-verdict.sh @@ -203,3 +203,22 @@ verdict_emit_arrayretry() { "double_submit=0 silent_loss=0 partial_duplication=0 result=PASS" return 0 } + +# Emit the machine-parseable verdict line for the worker-capacity-soak +# (bug-1841) scenario. Like verdict_emit_arraysoak/verdict_emit_arrayretry, +# this is NOT a count-and-classify verdict: every hard invariant (the +# overflow probe genuinely PEND before the kill -- proving real saturation, +# not an under-subscribed guess -- every saturating job recovering to DONE +# via a genuine post-failover rejoin, and that SAME probe job reaching RUN +# afterward -- proving the scheduler's resource accounting for the worker +# actually unfroze) is asserted with an immediate fail() inside +# run_worker_capacity_soak_round() (run-ha-soak.sh) -- a violation aborts +# the run before this line is ever reached, so result is unconditionally +# PASS here. This function's job is to report the measurements, not gate +# pass/fail itself. Args: slots_capacity n_saturated probe_recovery_ms +verdict_emit_workercapacity() { + local cap="$1" n_saturated="$2" recovery_ms="$3" + echo "[xce-test-ha] VERDICT mode=worker-capacity-soak slots_capacity=$cap" \ + "saturating_jobs=$n_saturated probe_recovery_ms=$recovery_ms result=PASS" + return 0 +}