From 24dafa712fdf75cd97dfa73689e0e167d4d3d3b3 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 1 Oct 2026 13:27:12 +0000 Subject: [PATCH] test(job-array-one-based-index): HA soak and multihost skills check elements 1..N (task 9.2) The HA array soak checks small-array elements 1..N, large-array first, middle and last as 1, N/2 and N, status/cancel/completion on 2, N/2+1 and N-1, and the chaos-retry round over seq 1 N; the multihost verification drives element [1]. Comments now say that after jobsub -a N the success line and $jobid name the array identity. Historical live-run quotes that show [0] are kept and annotated as predating the change. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/xce-test-ha/SKILL.md | 7 +- .claude/skills/xce-test-ha/run-ha-soak.sh | 75 ++++++++++--------- .claude/skills/xce-test-multihost/SKILL.md | 39 ++++++---- .../run-multihost-verify.sh | 49 ++++++------ .../specs/job-array-one-based-index/tasks.md | 2 +- 5 files changed, 95 insertions(+), 77 deletions(-) diff --git a/.claude/skills/xce-test-ha/SKILL.md b/.claude/skills/xce-test-ha/SKILL.md index 75561361..59a7fbf8 100644 --- a/.claude/skills/xce-test-ha/SKILL.md +++ b/.claude/skills/xce-test-ha/SKILL.md @@ -180,6 +180,10 @@ asserts: - **per-element status query, cancel (`jobkill -s 9`), and completion (`jstart`+`jfinish`)** on three distinct LARGE-array elements spread across low/mid/high indices (not clustered near the start), after a real failover. + Elements are numbered from 1 (`jobsub -a N` creates `[1]` .. + `[N]`; the captured `JobId` is the array identity, which names no + element): the addressability spot-check uses elements 1, N/2 and N, and the + status/cancel/completion checks use 2, N/2+1 and N-1. Like `--force-snapshot`, this is **one dedicated round, not a loop**, and it hard-fails (not count-and-tolerate) on any violated invariant — these are the @@ -242,7 +246,8 @@ Per round, asserts (all `fail()`-gated, not just displayed): 0 — a silent loss; never ≥2 — a double-submit), counted via `waldump`'s exact `JOB_ARRAY_SUBMIT` type name (task 6.1's own addition), the same offline-read technique `--array-soak` uses; and -- **every element of the array's full index range** `[0, N)` is individually +- **every element of the array's full index range** `1..N` (elements are + numbered from 1) is individually addressable afterward (not a sampled subset) — a stronger check than `--array-soak`'s low/mid/high sampling, since this scenario's specific concern is a *partial* duplication across the range, not scale. diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index 5491d2c3..97522d23 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -2556,22 +2556,22 @@ quit" # apples comparison point for this specific claim) # (c) every element individually addressable — status query, cancel, # completion — after a failover (Req 5.2), proven against several -# distinct elements of the LARGE array, not just index 0 +# distinct elements of the LARGE array, not just the first element # -# Per-element addressing needs no new xce_test command: `jobinfo`/ -# `expect-job`/`jobkill`'s `-j`/positional argument already parses the -# bracketed array-id format (utils.c parse_jobid, REGEX_JOBARRAY_ID). The -# base jobid `jobsub -a N` captures into $jobid is array_jobid()-stripped -# (tag cleared before printing, per cmd_jobsub/test.c), so it prints WITHOUT -# brackets (e.g. "12-3-1700000000-32768") — element i's addressable string is -# that exact base string with "[i]" appended: parse_jobid reconstructs -# base_jobid_with_tag + i from "[i]", the identical arithmetic -# restore_job_array() (job.c) itself uses for jobid = base_jobid + i. Traced -# through utils.h's MAKE_JOBID/EXTRACT_ARRAY_* bit-layout macros to confirm -# this is exact, not an approximation. +# Array elements are numbered from 1: `jobsub -a N` creates elements +# 1..N (job-array-one-based-index Req 4.1). Per-element addressing needs no +# new xce_test command: `jobinfo`/`expect-job`/`jobkill`'s `-j`/positional +# argument parses the bracketed element form `[k]` (utils.c +# parse_jobid). After `jobsub -a N`, both the success line +# `jobsubmit ok: JobId: ` and the harness's $jobid hold the ARRAY +# IDENTITY, printed WITHOUT brackets (e.g. "12-3-1700000000-32768") (Req +# 4.5) — it names no element, so `expect-job ` reports +# "not found" with a hint to use `[k]`. Element k's addressable +# string is that exact array identity with "[k]" appended (the same form as +# `$jobid[k]` inside an xce_test script), for k in 1..N. -# Submit ONE array of size $1 via `jobsub -a N`. Echoes the captured base -# jobid string (bracket-less) on success; returns 1 with nothing echoed on +# Submit ONE array of size $1 via `jobsub -a N`. Echoes the captured array +# identity string (bracket-less; elements are [1]..[N]) on success; returns 1 with nothing echoed on # failure. Deliberately does NOT append to ALL_JOBIDS — array elements are # addressed by base+index arithmetic here, not individually tracked the way # submit_jobs() tracks single-job submissions, which also keeps this @@ -2596,8 +2596,8 @@ count_array_wal_records() { printf '%s\n' "$out" | grep -c "JOB_ARRAY_SUBMIT" } -# Per-element jobid string for array element $2 (0-based) of base string $1. -# See this section's header comment for why string concatenation is exact. +# Per-element jobid string for array element $2 (numbered from 1, 1..N) of +# array identity string $1. See this section's header comment. array_element_id() { echo "$1[$2]"; } # `jobinfo -j ` against the live primary. Prints the jobinfo text and @@ -2728,7 +2728,7 @@ run_array_soak_round() { delta=$((rec_after - rec_before)) [ "$delta" -eq 1 ] || fail "array-soak: expected exactly 1 WAL_JOB_ARRAY_SUBMIT record for the $small-element array, observed $delta (base=$base_small)" log "array-soak: CONFIRMED exactly ONE WAL_JOB_ARRAY_SUBMIT record for a $small-element array (base=$base_small)" - verify_array_elements "$base_small" 0 $((small - 1)) \ + verify_array_elements "$base_small" 1 "$small" \ || fail "array-soak: small array elements not addressable immediately after commit (pre-failover)" prev_term="$(latest_primary_term)"; prev_term="${prev_term:-0}" @@ -2737,10 +2737,10 @@ run_array_soak_round() { DOWN="$PRIMARY" newp="$(wait_for_new_primary "$prev_term")" || fail "array-soak: no new primary elected after small-array kill" record_primary "$newp" - t_small="$(poll_addressable_elapsed "$(array_element_id "$base_small" 0)" 90)" \ - || fail "array-soak: small array element 0 never became addressable on new primary $newp within 90s" + t_small="$(poll_addressable_elapsed "$(array_element_id "$base_small" 1)" 90)" \ + || fail "array-soak: small array element 1 never became addressable on new primary $newp within 90s" log "array-soak: small-array (N=$small) kill-to-addressable = ${t_small}ms (new primary=$newp)" - verify_array_elements "$base_small" 0 $((small - 1)) \ + verify_array_elements "$base_small" 1 "$small" \ || fail "array-soak: small array elements missing on new primary after failover" log "array-soak: all $small small-array elements confirmed addressable on new primary" @@ -2758,8 +2758,8 @@ run_array_soak_round() { [ "$delta" -eq 1 ] || fail "array-soak: expected exactly 1 WAL_JOB_ARRAY_SUBMIT record for the $large-element array, observed $delta (base=$base_large)" log "array-soak: CONFIRMED exactly ONE WAL_JOB_ARRAY_SUBMIT record for a $large-element array (base=$base_large) — same ONE record as the $small-element array above; record count is O(1) in array size" - mid=$((large / 2)); last=$((large - 1)) - verify_array_elements "$base_large" 0 "$mid" "$last" \ + mid=$((large / 2)); last=$large + verify_array_elements "$base_large" 1 "$mid" "$last" \ || fail "array-soak: large array elements not addressable immediately after commit (pre-failover)" prev_term="$(latest_primary_term)"; prev_term="${prev_term:-0}" @@ -2768,8 +2768,8 @@ run_array_soak_round() { DOWN="$PRIMARY" newp="$(wait_for_new_primary "$prev_term")" || fail "array-soak: no new primary elected after large-array kill" record_primary "$newp" - t_large="$(poll_addressable_elapsed "$(array_element_id "$base_large" 0)" 90)" \ - || fail "array-soak: large array element 0 never became addressable on new primary $newp within 90s" + t_large="$(poll_addressable_elapsed "$(array_element_id "$base_large" 1)" 90)" \ + || fail "array-soak: large array element 1 never became addressable on new primary $newp within 90s" log "array-soak: large-array (N=$large) kill-to-addressable = ${t_large}ms (new primary=$newp)" # Req 4.3 HARD assertion — replay time must be governed by record size, @@ -2795,19 +2795,19 @@ run_array_soak_round() { "(ratio_max=${replay_ratio_max} floor=${replay_floor_ms}ms margin=${replay_margin_ms}ms)" \ "— replay time is NOT governed by record size alone (Req 4.3 violated); possible O(N) regression in array apply/replay" - verify_array_elements "$base_large" 0 "$mid" "$last" \ + verify_array_elements "$base_large" 1 "$mid" "$last" \ || fail "array-soak: large array elements missing on new primary after failover" - log "array-soak: spot-checked elements 0, $mid, $last of the $large-element array all addressable on new primary $newp" + log "array-soak: spot-checked elements 1, $mid, $last of the $large-element array all addressable on new primary $newp" # Per-element status/cancel/completion (Req 5.2) — three DISTINCT, # untouched large-array elements spread across low/mid/high indices - # (like the addressability spot-check above uses 0/mid/last), not + # (like the addressability spot-check above uses 1/mid/last), not # clustered at the start — a bug affecting only high-index elements # (e.g. an off-by-one in a chunking/offset calculation that only bites - # past some element count) would NOT be caught by indices 1/2/3 alone. + # past some element count) would NOT be caught by indices 2/3/4 alone. local idx_status idx_cancel idx_complete local id_status id_cancel id_complete out_cancel out_complete - idx_status=1 # near start (distinct from idx 0's read-only checks) + idx_status=2 # near start (distinct from idx 1's read-only checks) idx_cancel=$((mid + 1)) # near middle (distinct from the read-only mid check) idx_complete=$((last - 1)) # near end (distinct from the read-only last check) id_status="$(array_element_id "$base_large" "$idx_status")" @@ -2880,12 +2880,13 @@ run_array_soak_round() { # call). So every retry attempt inside ONE `jobsub` command targets the # exact same base jobid, by construction, without this harness doing # anything special to force it. -# - Server-side, handle_jobsubmit() (job.c:1326) checks jobid_range_exists() -# then inflight_range_try_add() BEFORE branching into the array append -# path (job.c:1388) — both checks are RANGE-shaped (the full -# [base_jobid, base_jobid+array_size) interval, task 2.2's own doc -# comment), so the array's full index range is dedup-checked as one -# unit, not per-element. +# - Server-side, handle_jobsubmit() (job.c) registers the submission with +# inflight_range_try_add() and then, for an array, checks +# array_identity_exists() BEFORE appending the array record — both are +# RANGE-shaped (the in-flight entry covers the array's whole element +# range 1..N under its array identity; array_identity_exists() scans the +# array's whole block of element keys), so the array's full index range +# is dedup-checked as one unit, not per-element. # Two legitimate outcomes, both of which must resolve to exactly one # committed record: # (a) the original attempt's WAL_JOB_ARRAY_SUBMIT already reached majority @@ -2972,8 +2973,8 @@ run_array_chaos_retry_round() { fi log "array-chaos-retry round $round: CONFIRMED exactly ONE committed WAL_JOB_ARRAY_SUBMIT record for base=$base (delta=1) — retry deduped correctly" - verify_array_elements "$base" $(seq 0 $((n - 1))) \ - || fail "array-chaos-retry round $round: not every element of base=$base's FULL index range [0,$n) is addressable after the chaotic kill-and-retry — partial duplication/loss" + verify_array_elements "$base" $(seq 1 "$n") \ + || fail "array-chaos-retry round $round: not every element of base=$base's FULL index range [1,$n] is addressable after the chaotic kill-and-retry — partial duplication/loss" log "array-chaos-retry round $round: all $n elements of base=$base's FULL index range individually addressable — no partial duplication" walcheck_count "0" diff --git a/.claude/skills/xce-test-multihost/SKILL.md b/.claude/skills/xce-test-multihost/SKILL.md index d6b4c234..25d1e365 100644 --- a/.claude/skills/xce-test-multihost/SKILL.md +++ b/.claude/skills/xce-test-multihost/SKILL.md @@ -17,7 +17,8 @@ fixture is up (`HARNESS_READY`), it (1) submits ONE real multi-task job via plain `jobsub`, polls `jobinfo` for `RUN` then `DONE`, confirms BOTH named workers' own logs independently show them starting their own share of the job's tasks; then (2) submits a 2-element array job via `jobsub -a 2` -requesting the SAME per-element task count, drives element `[0]` through +requesting the SAME per-element task count, drives element `[1]` (elements +are numbered from 1) through the identical `RUN`-then-`DONE`/both-hosts-started proof, and confirms that element's own `jobinfo` `Slots:` line matches the plain job's exactly. A single combined PASS/FAIL verdict reflects both scenarios' real outcome -- @@ -114,11 +115,13 @@ Traces to `.kiro/specs/resreq-maxtasks-wiring/design.md`'s element of an array job** -- immediately after the plain-job scenario above passes, submit a 2-element array job via `jobsub -a 2` requesting the SAME per-element task count (`-n ,`, reusing the same - `nproc`-derived sizing), then drive element `[0]` (addressed as - `[0]`, the same bracketed-string convention `xce-test-ha`'s - `run-ha-soak.sh` array-soak already established -- `jobinfo`'s existing - argument parser reconstructs `base+i` from it, no new `xce_test` command - needed) through the identical `RUN`-then-`DONE` / both-hosts-started + `nproc`-derived sizing), then drive element `[1]` -- the first element; + `jobsub -a N` numbers elements 1..N -- (addressed as `[1]`, + where `` is the `jobsubmit ok: JobId:` value, i.e. the array + identity, which names no element; the same bracketed-string convention + `xce-test-ha`'s `run-ha-soak.sh` array-soak uses and `$jobid[k]` in an + `xce_test` script -- `jobinfo`'s existing argument parser resolves it, no + new `xce_test` command needed) through the identical `RUN`-then-`DONE` / both-hosts-started proof. Confirmed LIVE in `src/scheduler/src/handler.c`'s `sched_on_job_array()`/`make_array_element()` that an array element is NOT a special-cased ingest path: each element gets its own independent @@ -135,6 +138,8 @@ Traces to `.kiro/specs/resreq-maxtasks-wiring/design.md`'s ``` [xce-test-multihost] HARNESS_READY controller=up scheduler=up workers=2/2 top=/home/claude/tmp/xce-test-multihost ... + # (recorded before job-array-one-based-index, when elements were numbered + # from 0; a current run reports element [1]) [xce-test-multihost] PASS: plain job 0-13-1788686453-901606 (23 tasks, sleep 5s) submitted via plain jobsub AND array-job element 0-4-1788686458-524288[0] (base=0-4-1788686458-524288, 2 elements, 23 tasks, sleep 5s) submitted via jobsub -a 2 both reached RUN then DONE; mh-worker-1 and mh-worker-2 each independently logged starting their own share of both jobs' tasks -- the scheduler's own decision genuinely spans both hosts for BOTH the plain-job and array-job cases, and the array element's task-count wiring (Slots: 23,23) matches the plain job's exactly ``` (Verbatim from a real local run.) Any structural failure (a role never @@ -184,7 +189,8 @@ bash .claude/skills/xce-test-multihost/run-multihost-verify.sh [--keep-running] `$XCE_MULTIHOST_RUN_TIMEOUT` (default `30`), `$XCE_MULTIHOST_DONE_TIMEOUT` (default `$XCE_MULTIHOST_JOB_SLEEP + 30`), `$XCE_MULTIHOST_ARRAY_SIZE` (default `2` -- task 4.3's array-job scenario: element count for - `jobsub -a N`; only element `[0]` is driven through the full proof + `jobsub -a N`; only element `[1]` (the first, elements numbered 1..N) is + driven through the full proof regardless of this value). Must be invoked under an environment with `nix` on `PATH` -- the script @@ -280,10 +286,10 @@ signal, but corroborates the two above when inspecting a log by hand.) `verify_task_started_on_host()`'s jobid is passed through `ere_escape()` before being embedded in the `grep -E` alternation above (task 4.3): an -array-element jobid string like `[0]` contains `[`/`]`, which `grep +array-element jobid string like `[1]` contains `[`/`]`, which `grep -E` would otherwise parse as a one-character bracket-expression class -(matching a literal `0`) rather than the literal 3-character substring -`[0]` -- silently breaking the match for every array element. Reproduced +(matching a literal `1`) rather than the literal 3-character substring +`[1]` -- silently breaking the match for every array element. Reproduced this exact false negative live before adding the fix, then confirmed `ere_escape()` corrects it; a no-op for a plain (non-array) jobid, which contains no ERE metacharacters. @@ -294,10 +300,11 @@ After the plain-job scenario passes, `run_multihost_array_job_scenario()` submits a `$ARRAY_SIZE`-element (default `2`) array job via `submit_multihost_array_job()` (`jobsub -a $ARRAY_SIZE`, composed with the SAME `-n ,` flag the plain job used), then drives element -`[0]` -- addressed via `array_element_id()`'s `"[0]"` string, the -identical convention `xce-test-ha`'s `run-ha-soak.sh` array-soak already -established (`jobinfo`'s existing `parse_jobid`/`REGEX_JOBARRAY_ID` -reconstructs `base+i` from it; no new `xce_test` command needed) -- through +`[1]` (the first element; elements are numbered 1..N) -- addressed via +`array_element_id()`'s `"[1]"` string, the identical convention +`xce-test-ha`'s `run-ha-soak.sh` array-soak uses (`jobinfo`'s existing +`parse_jobid` resolves the `[k]` element form; no new `xce_test` +command needed) -- through the SAME `RUN`-then-`DONE`/both-hosts-started proof `run_multihost_job_scenario()` uses for the plain job. @@ -317,7 +324,9 @@ also checks it empirically, by comparing the array element's own `jobinfo` `Slots:` line (`job->submit->mintasks/maxtasks`) byte-for-byte against the plain job's, failing the run if they ever differ. -Live evidence from a real run (`base=0-4-1788686458-524288`, `ntasks=23`): +Live evidence from a real run (`base=0-4-1788686458-524288`, `ntasks=23`; +recorded before job-array-one-based-index, when the first element was `[0]` -- +a current run shows `[1]` in the same two lines): `xcecd-mh-worker-1.log` logged `"Job 0-4-1788686458-524288[0] ntasks is: 12"` (the startjob path) and `xcecd-mh-worker-2.log` logged `"run task: jobid = 0-4-1788686458-524288[0]"` (the relayed runtasks path) -- the exact diff --git a/.claude/skills/xce-test-multihost/run-multihost-verify.sh b/.claude/skills/xce-test-multihost/run-multihost-verify.sh index 7422ec7f..4b30db26 100755 --- a/.claude/skills/xce-test-multihost/run-multihost-verify.sh +++ b/.claude/skills/xce-test-multihost/run-multihost-verify.sh @@ -30,7 +30,8 @@ # harness-ready. # # Task 4.3 (THIS revision) adds a SECOND scenario on top of 4.2's: the same -# proof, submitted as element [0] of a 2-element array job via `jobsub -a N` +# proof, submitted as element [1] (elements are numbered from 1) of a +# 2-element array job via `jobsub -a N` # instead of a plain single-job submission. Confirmed LIVE via # src/scheduler/src/handler.c's sched_on_job_array()/make_array_element() # that an array element is NOT a special-cased or shortcut ingest path -- @@ -47,8 +48,8 @@ # empirically by comparing jobinfo's own `Slots:` line between the plain job # and the array element. Also fixed, as part of adding this: verify_task_ # started_on_host()'s grep pattern now escapes the jobid before embedding it -# in an ERE alternation -- an array element's bracketed id (e.g. "[0]") -# would otherwise have its "[0]" misparsed by `grep -E` as a one-character +# in an ERE alternation -- an array element's bracketed id (e.g. "[1]") +# would otherwise have its "[1]" misparsed by `grep -E` as a one-character # bracket-expression class instead of a literal 3-character substring, # silently breaking the match (confirmed live: reproduced the exact false # negative, then confirmed ere_escape() fixes it, before wiring it in below). @@ -182,7 +183,8 @@ RUN_TIMEOUT_SECS="${XCE_MULTIHOST_RUN_TIMEOUT:-30}" DONE_TIMEOUT_SECS="${XCE_MULTIHOST_DONE_TIMEOUT:-$((JOB_SLEEP_SECS + 30))}" # Task 4.3's array-job scenario: element count for `jobsub -a N`. 2 is enough # to prove the array-submission path (Requirement 5.5 only requires "at -# least once"); only element [0] is verified end-to-end below (design.md's +# least once"); only element [1] (the first; elements are numbered 1..N) +# is verified end-to-end below (design.md's # xce-test-multihost Implementation Notes: "a multi-task job submitted as # part of an array job" -- singular, not "every element of"). ARRAY_SIZE="${XCE_MULTIHOST_ARRAY_SIZE:-2}" @@ -624,9 +626,10 @@ submit_multihost_job() { # element genuinely receives the SAME per-element task-count wiring a # standalone job would (Requirement 5.5), by construction of the production # code this test drives, not merely by this script's own assumption. -# Echoes the captured BASE jobid (bracket-less -- cmd_jobsub prints -# fmt_jobid(array_jobid(jobid)), the array-tag-stripped form, same as -# run-ha-soak.sh's own submit_array()) on success; returns 1 (echoing +# Echoes the captured ARRAY IDENTITY (bracket-less -- after `jobsub -a N` +# both the `jobsubmit ok: JobId:` line and $jobid hold the array identity, +# which names no element; same as run-ha-soak.sh's own submit_array()) on +# success; returns 1 (echoing # nothing, diagnostic on stderr) on a rejected submit. submit_multihost_array_job() { local ntasks="$1" sleep_secs="$2" arsize="$3" out base @@ -641,13 +644,12 @@ submit_multihost_array_job() { return 0 } -# Per-element addressable jobid string for array element $2 (0-based) of -# base jobid $1 -- IDENTICAL string-concatenation convention to -# run-ha-soak.sh's own array_element_id(): no new xce_test command needed, -# since `jobinfo -j`/`-a`'s existing argument parser (utils.c parse_jobid, -# REGEX_JOBARRAY_ID) already reconstructs base_jobid+i from the bracketed -# "[i]" string form -- the identical arithmetic restore_job_array() -# (controller/job.c) itself uses. jobinfo_of()/wait_for_job_state()/ +# Per-element addressable jobid string for array element $2 (numbered from +# 1, i.e. 1..N) of array identity $1 -- IDENTICAL string-concatenation +# convention to run-ha-soak.sh's own array_element_id() and to `$jobid[k]` +# in an xce_test script: no new xce_test command needed, since `jobinfo +# -j`/`-a`'s existing argument parser (utils.c parse_jobid) resolves the +# bracketed "[k]" element form. jobinfo_of()/wait_for_job_state()/ # verify_task_started_on_host() below all work UNCHANGED against this # string: none of them bash-glob-expand it (always interpolated via printf # %s into an xce_test session line, or -- verify_task_started_on_host() @@ -674,10 +676,10 @@ wait_for_job_state() { # Escape ERE metacharacters in $1 so it can be embedded in a `grep -E` # pattern as a LITERAL substring match (task 4.3). Needed because an -# array-element jobid (e.g. "12-3-1700000000-32768[0]") contains `[`/`]` -- -# without escaping, grep -E parses "[0]" as a one-character bracket- -# expression class matching a literal '0', NOT the literal 3-character -# substring "[0]", so verify_task_started_on_host()'s jobid-tagged log-line +# array-element jobid (e.g. "12-3-1700000000-32768[1]") contains `[`/`]` -- +# without escaping, grep -E parses "[1]" as a one-character bracket- +# expression class matching a literal '1', NOT the literal 3-character +# substring "[1]", so verify_task_started_on_host()'s jobid-tagged log-line # match would silently and always miss for every array element (reproduced # and confirmed live before adding this fix). A plain (non-array) jobid # contains no ERE metacharacters today (fmt_jobid's format is digits and @@ -723,7 +725,7 @@ ere_escape() { # it is expected to already be present by the time wait_for_job_state() # above observes RUN -- this wait is a safety margin, not the primary gate). # $jid is generic -- works UNCHANGED for either a plain jobid (task 4.2) or a -# bracketed array-element jobid string like "[0]" (task 4.3): passed +# bracketed array-element jobid string like "[1]" (task 4.3): passed # through ere_escape() above before being embedded in the grep -E pattern, # so the alternation matches it as a literal substring either way. verify_task_started_on_host() { @@ -769,14 +771,15 @@ run_multihost_job_scenario() { } # Orchestrates the task 4.3 scenario: the SAME job-outcome proof as -# run_multihost_job_scenario() above, but submitted as element [0] of a +# run_multihost_job_scenario() above, but submitted as element [1] of a # $ARRAY_SIZE-element array job via `jobsub -a N` (Requirement 5.5). Run # AFTER the plain-job scenario has already passed (main(), below), and # deliberately reuses this scenario's own compute_task_count() sizing so the # array element requests the IDENTICAL per-element task count the plain job # did -- this is what makes the closing jobinfo `Slots:` comparison below a # genuine like-for-like check, not just two arbitrary numbers. Only element -# [0] is driven through RUN/task-start/DONE (design.md's Implementation +# [1] (the first; elements are numbered 1..N) is driven through +# RUN/task-start/DONE (design.md's Implementation # Notes: "a multi-task job submitted as part of an array job" -- "at least # once" per Requirement 5.5's own wording, not "every element"). Hard-fails # (via the shared fail() helper) on the first violated step, same discipline @@ -789,8 +792,8 @@ run_multihost_array_job_scenario() { base="$(submit_multihost_array_job "$ntasks" "$JOB_SLEEP_SECS" "$ARRAY_SIZE")" \ || fail "array-job submission failed -- see the diagnostic line just above" - eid="$(array_element_id "$base" 0)" - log "submitted array job base=$base -- verifying element $eid (index 0 of $ARRAY_SIZE) end to end; polling for RUN (timeout ${RUN_TIMEOUT_SECS}s)" + eid="$(array_element_id "$base" 1)" + log "submitted array job base=$base -- verifying element $eid (index 1 of 1..$ARRAY_SIZE) end to end; polling for RUN (timeout ${RUN_TIMEOUT_SECS}s)" wait_for_job_state "$eid" "RUN" "$RUN_TIMEOUT_SECS" >/dev/null \ || fail "array element $eid never reached RUN within ${RUN_TIMEOUT_SECS}s -- the scheduler's own allocation decision never dispatched it" diff --git a/.kiro/specs/job-array-one-based-index/tasks.md b/.kiro/specs/job-array-one-based-index/tasks.md index a27db1e0..74a2edf6 100644 --- a/.kiro/specs/job-array-one-based-index/tasks.md +++ b/.kiro/specs/job-array-one-based-index/tasks.md @@ -212,7 +212,7 @@ - _Requirements: 1.3, 4.2_ - _Depends: 4.1, 6.2_ -- [ ] 9.2 Update test skills for elements numbered from 1 +- [x] 9.2 Update test skills for elements numbered from 1 - Make the HA soak and multihost verification scripts and their skill descriptions verify elements 1..N (first, middle, last) instead of 0..N-1 - Done when both scripts pass a syntax check and every element check in them refers to indices 1..N - _Requirements: 4.1, 4.5_