From a14785108c8077526d3ad916212fbb0b934ba1d7 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Wed, 22 Jul 2026 10:11:08 +0000 Subject: [PATCH] docs(admin-config-tool): commit spec docs and example configs design.md, research.md (new), brief.md and requirements.md (updated during spec authoring: Req 1.6 resolved/superseded by bundling, Req 5.5 added, Req 4.3/4.4 extended for limits.yaml) had been sitting uncommitted since before implementation began -- the entire 18-task implementation was built against these documents without them ever landing in git. src/configs/*.yaml + schema/*.schema.json: the example config directory and bundled JSON schemas most of the test suite reads as "real repo fixtures" (test_config_build.py, test_mapping.py, test_validation.py, etc.) were likewise never committed -- those tests would silently skip (skipif guards) rather than run for anyone else checking out this branch. --- .kiro/specs/admin-config-tool/brief.md | 41 +- .kiro/specs/admin-config-tool/design.md | 544 ++++++++++++++++++ .kiro/specs/admin-config-tool/requirements.md | 58 +- .kiro/specs/admin-config-tool/research.md | 158 +++++ src/configs/hosts.yaml | 39 ++ src/configs/limits.yaml | 49 ++ src/configs/queues.yaml | 94 +++ src/configs/resources.yaml | 199 +++++++ src/configs/scaler.yaml | 13 + src/configs/schema/hosts.schema.json | 101 ++++ src/configs/schema/limits.schema.json | 54 ++ src/configs/schema/queues.schema.json | 120 ++++ src/configs/schema/resources.schema.json | 41 ++ src/configs/schema/scaler.schema.json | 28 + src/configs/schema/users.schema.json | 39 ++ src/configs/schema/xce.schema.json | 35 ++ src/configs/users.yaml | 8 + src/configs/xce.yaml | 6 + 18 files changed, 1574 insertions(+), 53 deletions(-) create mode 100644 .kiro/specs/admin-config-tool/design.md create mode 100644 .kiro/specs/admin-config-tool/research.md create mode 100644 src/configs/hosts.yaml create mode 100644 src/configs/limits.yaml create mode 100644 src/configs/queues.yaml create mode 100644 src/configs/resources.yaml create mode 100644 src/configs/scaler.yaml create mode 100644 src/configs/schema/hosts.schema.json create mode 100644 src/configs/schema/limits.schema.json create mode 100644 src/configs/schema/queues.schema.json create mode 100644 src/configs/schema/resources.schema.json create mode 100644 src/configs/schema/scaler.schema.json create mode 100644 src/configs/schema/users.schema.json create mode 100644 src/configs/schema/xce.schema.json create mode 100644 src/configs/users.yaml create mode 100644 src/configs/xce.yaml diff --git a/.kiro/specs/admin-config-tool/brief.md b/.kiro/specs/admin-config-tool/brief.md index 55842ff7..221f797b 100644 --- a/.kiro/specs/admin-config-tool/brief.md +++ b/.kiro/specs/admin-config-tool/brief.md @@ -4,8 +4,8 @@ XCE operators have no tool to build and deploy cluster configuration. Currently: -- **Config authoring**: YAML config files (`configs/hosts.yaml`, `queues.yaml`, `resources.yaml`, `users.yaml`) exist as documentation examples with JSON schemas for validation (`configs/schema/*.schema.json`), but **no tool compiles them** into the `ClusterConfig` Cap'n Proto binary that the controller expects. -- **Config deployment**: `OP_upgradeConfig` exists on the controller side (`config_repl.c:1536` — receives binary blob, verifies SHA-256, saves, pushes to standbys, awaits majority ACK, signals restart), but **no CLI sends it**. Today: manual scp + restart. +- **Config authoring**: YAML config files (`configs/hosts.yaml`, `queues.yaml`, `resources.yaml`, `users.yaml`, `limits.yaml`, `xce.yaml`, `scaler.yaml`) exist as documentation examples with JSON schemas for validation (`configs/schema/*.schema.json`), but **no tool compiles them** into the `ClusterConfig` Cap'n Proto binary that the controller expects. +- **Config deployment**: `OP_upgradeConfig` exists on the controller side (`config_repl.c:1591` `do_upgrade_config_with_bytes()` — receives binary blob, verifies SHA-256, saves, pushes to standbys, awaits majority ACK, signals restart), but **no CLI sends it**. Today: manual scp + restart. End users use `xcecli` (Go/Cobra → REST → apiserver → CGO → controller) for non-admin operations. There is no admin-facing tool. @@ -15,10 +15,10 @@ End users use `xcecli` (Go/Cobra → REST → apiserver → CGO → controller) | What | Where | Status | |------|-------|--------| -| `ClusterConfig` capnp struct | `src/schema/controller.capnp:710-732` | 21 fields, wire-ready | -| YAML config examples | `configs/{hosts,queues,resources,users}.yaml` | Human-authored, NOT loaded by any code | +| `ClusterConfig` capnp struct | `src/schema/controller.capnp:694-708` | 13 fields (@0-@12), wire-ready — down from 21 after `clusterconfig-schema-split` removed `port`/`candidates`/hash-size/`xstrings` | +| YAML config examples | `configs/{hosts,queues,resources,users,limits,xce,scaler}.yaml` | Human-authored, NOT loaded by any code | | JSON schemas | `configs/schema/*.schema.json` | JSON Schema 2020-12 | -| `OP_upgradeConfig` handler | `config_repl.c:1536` `do_upgrade_config_with_bytes()` | Receives binary, verifies, pushes, awaits majority | +| `OP_upgradeConfig` handler | `config_repl.c:1591` `do_upgrade_config_with_bytes()` | Receives binary, verifies, pushes, awaits majority | | `UpgradeConfigRequest/Reply` | `controller.capnp:792-797` | Defined in schema | | libxce upgrade API | `request.c:1046` has extractors only | No send function | | Controller wire protocol | TCP+TLS, capnp-encoded `Request`/`Reply`, same CA as controller-to-controller | Exists, used by all C daemons | @@ -47,14 +47,13 @@ configs/ ├── queues.yaml # optional ├── resources.yaml # optional ├── users.yaml # optional -└── schema/ # required for validation - ├── hosts.schema.json - ├── queues.schema.json - ├── resources.schema.json - └── users.schema.json +├── limits.yaml # optional +├── xce.yaml # optional — missing = clusterName/heartbeat/etc. left unset +└── scaler.yaml # optional ``` +No `schema/` subdirectory — **RESOLVED (2026-07-21)**: the 7 JSON schemas are bundled with the `xceadmin` package itself (`xcecommon` `package_data`, synced from `src/configs/schema/` at build time), not supplied per `configs_dir`. This mirrors the `controller.capnp` bundling decision and for the same reason: the schemas are the formal spec of what `xcecommon.mapping`'s field-extraction code actually expects, so they need to stay version-locked to the `xceadmin` release doing the mapping, not left for an operator to drift out of sync. `src/configs/schema/*.schema.json` remains in the repo as the source of truth that gets synced — an operator's `configs_dir` never needs its own `schema/` folder, and one present there is not read. -- **config build**: Scan directory → validate present YAML files against corresponding JSON schemas → map to `ClusterConfig` capnp struct → serialize to `cluster.bin` → output SHA-256. Missing files = empty section (not error). Schema directory must exist. +- **config build**: Scan directory → validate present YAML files against `xceadmin`'s bundled JSON schemas → map to `ClusterConfig` capnp struct → serialize to `cluster.bin` → output SHA-256. Missing files = empty section (not error). No schema directory to require — bundled, not operator-supplied. - **config push**: Read `cluster.bin` → connect to controller (default `localhost:9991`) → controller reply includes peer list + primary address → follow redirect if needed → send capnp `Request{OP_upgradeConfig, UpgradeConfigRequest{sha256, bytes}}` → receive `Reply` → display `{versionTag, ackedCount, error}`. `configGeneration` is fully controller-owned: the primary assigns `new_gen = current_gen + 1` on receipt and returns it as `versionTag`; xceadmin never reads, sets, or overrides it — no `--generation` flag. - **Shared package**: `xcecommon` Python package with capnp schema handling, wire protocol, and config models — reusable by future Python-based tools @@ -148,6 +147,10 @@ Decided 2026-07-20. Governs `ClusterConfig`/`cluster.bin` field classification n | `candidates` | passes (adding a standby doesn't require other nodes' config to change) | passes (routine op) | OUT | | `jobmgr`/`workermgr`/`qmgr`/`resmgr` hash size | passes (perf tuning, local effect only) | — | OUT | | `xstrings` hash size | passes (perf tuning, local effect only) | — | OUT | +| `limits` | fails (quotas must be enforced identically on every controller) | fails (rarely changes) | IN | +| `fetchJobInterval` | passes (per-controller scheduler-loop tuning, no cross-node dependency) | — | IN (operational-uniformity override, bundled into `xce.yaml` with the other cluster-identity/liveness fields rather than split into a separate file) | +| `netioTimeout` | passes (per-controller network tuning, no cross-node dependency) | — | IN (same override, same file — see `fetchJobInterval` row) | +| `hproviders` (scaler providers) | fails (a queue's scaler reference must resolve identically after failover) | fails (rarely changes) | IN | | `configGeneration` | N/A — not operator input; controller-assigned on every push | N/A | OUT (different reason: not a config value at all, it's protocol state) | When a new field is proposed for `ClusterConfig` in the future, run it through rules 1–2 before adding it to xceadmin's YAML→capnp mapping; document the verdict in this table. @@ -157,7 +160,6 @@ When a new field is proposed for `ClusterConfig` in the future, run it through r - **In**: - `xceadmin config build`: YAML parsing, JSON Schema validation, capnp ClusterConfig serialization, cluster.bin output with SHA-256. `configGeneration`, `port`, and the per-instance perf-tuning fields (manager hash-table sizes, `xstrings`) are left unset — not xceadmin's to set. - `xceadmin config push`: single controller address (default `localhost:9991`), auto primary detection via peer-list-from-reply, TLS+capnp wire protocol, capnp UpgradeConfigRequest/Reply, result display. `configGeneration` assignment is entirely controller-side (primary assigns `current + 1`, returns it as `versionTag`) — no client probe, increment, or override flag. - - `xceadmin config diff`: (nice-to-have) decode two cluster.bin files, human-readable field-by-field diff. Pre-push safety: "what will this config change actually do?" - `xceadmin config push --dry-run`: (nice-to-have) validate controller will accept the config without committing it - `xcecommon` shared Python package: capnp schema loading, wire protocol (TLS+capnp framing), config model types - Build system integration: pip-based install (editable for dev, pinned for release), nix dev shell packages, xce-build.sh component target @@ -440,10 +442,9 @@ tests/ │ ├── minimal/ # hosts.yaml + queues.yaml only │ │ ├── configs/ │ │ │ ├── hosts.yaml -│ │ │ ├── queues.yaml -│ │ │ └── schema/ +│ │ │ └── queues.yaml # no schema/ — validated against xceadmin's bundled schemas │ │ └── expected.bin # pre-built golden output -│ ├── full/ # all 4 YAML files +│ ├── full/ # all 7 YAML files │ └── invalid/ # schema violations → expect error ├── test_config_build.py # pytest: build → compare binary, verify SHA-256 └── test_schema_validation.py # pytest: each schema violation case @@ -479,7 +480,7 @@ Issues identified during 2026-07-07 review — need resolution before or during | # | Question | Why it matters | Candidates | |---|----------|---------------|------------| -| 1 | **YAML→capnp mapping incomplete** | `ClusterConfig` has 21 fields; 4 YAML files cover ~10. Of the remaining 11, all now classified per the Configuration Inclusion Rules (see below): **excluded** — `port`, `jobmgr`/`workermgr`/`qmgr`/`resmgr` hash configs, `xstrings` (Req 1.8) and `candidates` (Req 1.10, see row 3) — never sourced by xceadmin, left unset in `cluster.bin`. `configGeneration` **excluded, different reason** — controller-assigned, not operator input (row 2). **Included** — `clusterName`, `heartbeatInterval`/`inactiveHeartbeatCount` (controller→worker liveness — NOT the HA/Raft election heartbeat, which has no `ClusterConfig` field at all, see `config-design.md` Rule 0) (Req 1.11) — must stay consistent cluster-wide; still need a YAML source | For the 3 still-included-but-unsourced fields: (a) xceadmin fills defaults from controller's `get_default_conf()` values, (b) add a `cluster.yaml` for system-level fields, (c) accept CLI flags `--cluster-name` | +| 1 | **YAML→capnp mapping incomplete** | **RESOLVED — all 13 fields accounted for.** `ClusterConfig` is now 13 fields (@0-@12, post-`clusterconfig-schema-split`), covered by 7 YAML files: `resources`→`resources.yaml`, `queues`→`queues.yaml`, `ugroups`→`users.yaml`, `hgroups`+`hostConfigs`→`hosts.yaml`, `limits`→`limits.yaml` (6 fields, 5 files). `clusterName`/`heartbeatInterval`/`inactiveHeartbeatCount`/`fetchJobInterval`/`netioTimeout`→`xce.yaml` (5 fields, 1 file, Req 1.11). `hproviders`→`scaler.yaml` (1 field, 1 file, Req 1.12). `configGeneration` excluded, controller-assigned (1 field, row 2). 6+5+1+1 = 13. | Closed — no candidates remain | | 2 | **configGeneration** | Monotonic counter. **RESOLVED**: fully controller-owned. Controller auto-increments `new_gen = current_gen + 1` on every `OP_upgradeConfig` receipt (already implemented, `config_repl.c`) and returns it as `versionTag`. xceadmin never reads, sets, or overrides it — `config build` leaves the field unset, `config push` sends the binary as-is, no `--generation` flag. Same generation as current = no-op is a controller-internal detail, not something xceadmin needs to detect. | ✅ Decided: controller-only, no client involvement | | 3 | **candidates field** | Controller peer list (`List(MasterCandidate)`). **RESOLVED — excluded from configuration** (Req 1.10): standby add/remove changes independently and more frequently than the rest of cluster config, so it fails both Configuration Inclusion Rules below. xceadmin never sources it from YAML/CLI, `cluster.bin` leaves it unset. Open sub-question: how peer membership IS sourced/updated instead (today: CLI args to `deploy-ha.sh`) — not a `cluster.bin` field going forward | (a) keep as `deploy-ha.sh` CLI args (status quo, out of xceadmin scope), (b) separate `xceadmin peer add/remove` command (future), (c) controller-side dynamic membership (future, own spec) | | 4 | **YAML key ≠ capnp field name** | YAML `groupname` → capnp `GroupConfig.name`, YAML `exclude_members` → capnp `exmembers`. Every field needs explicit mapping definition | Write a field-mapping table in design.md; auto-generate from capnp schema annotations if possible | @@ -491,7 +492,7 @@ Issues identified during 2026-07-07 review — need resolution before or during |---|----------|---------------|------------| | 6 | **Wire framing format** | Controller expects 8-byte uint64 size prefix (host byte order) + capnp payload. ~5 lines of Python but wrong byte order = silent reject | Document in design.md; mock controller test catches framing errors | | 7 | **TLS cert discovery** | ✅ Decided: reuse server-side cert bundle. xceadmin loads `xce.crt`/`xce.key` from `ha/data/` by default (same files `deploy-ha.sh` copies for controller-to-controller TLS), overridable via `--cert-dir`. No separate CA flag, no client cert issuance — same trust model as `xce_test`. See Requirement 2.3/2.4. | -| 8 | **capnp schema compilation** | `capnp compile -o python controller.capnp` → `controller_capnp.py`. When and where? | (a) build time in xce-build.sh (committed output), (b) runtime in xceadmin startup (needs capnp binary), (c) nix derivation that produces the .py file | +| 8 | **capnp schema compilation** | ✅ **RESOLVED (design phase)**: no `capnp compile -o python` codegen step exists — `pycapnp` loads `controller.capnp` via runtime dynamic reflection (`capnp.load()`, `DynamicStruct`), not static generated code. The only build-time step is a plain file sync (not compilation): `src/schema/{controller,c}.capnp` and `src/configs/schema/*.schema.json` are copied into `xcecommon`'s `package_data` by `xce-build.sh`'s `xceadmin)` case so both schema types survive `pip install .` (non-editable), resolved at runtime via `importlib.resources`. See `design.md` `xcecommon.schema`. | | 9 | **Python package layout** | Where does code live? Affects pip install, import paths, xce-build.sh | (a) `src/xceadmin/` alongside C code, (b) `tools/xceadmin/` with deploy-ha.sh, (c) top-level `xceadmin/` | | 10 | **existing cluster.bin provenance** | `ha/data/cluster.bin` is 8704 bytes, SHA 4986624f..., built June 21 by unknown method. Can't validate xceadmin output against it | Regenerate from known YAML inputs; use as golden-file test once xceadmin produces identical output for same inputs | @@ -528,7 +529,7 @@ Git manages the **source**. These parts need implementation: | # | Capability | What it does | Owned by | |---|-----------|-------------|----------| -| R1 | **Binary diff** | `xceadmin config diff cluster-v1.bin cluster-v2.bin` — human-readable diff of two built config binaries (decode both → compare field-by-field → pretty-print changes). Useful for pre-push review: "what will this config change actually do?" | admin-config-tool | +| R1 | **Binary diff** | `xceadmin config diff cluster-v1.bin cluster-v2.bin` — human-readable diff of two built config binaries (decode both → compare field-by-field → pretty-print changes). Useful for pre-push review: "what will this config change actually do?" | deferred (2026-07-21) — no spec owner yet | | R2 | **Dry-run push** | `xceadmin config push --dry-run cluster.bin` — validate controller will accept the config (SHA check, version check) without committing. Safety net before real push | admin-config-tool | | R3 | **Config change snapshot** | After config upgrade, take an entity-bearing snapshot of the new configuration state. This anchors the config change in the snapshot chain — without it, rollback has no durable record of what the cluster looked like before/after the change. Controller-side: insert snapshot step into the `OP_upgradeConfig` pipeline (between step 5 "majority ACK" and step 6 "commit WAL_CONFIG_MARKER" in the current handler). | config-upgrade-snapshot | | R4 | **Controller version history** | Controller tracks last N config versions (tag + SHA + timestamp). `xceadmin config history` queries it. Needed for: "what config is the cluster running?" and "what was running before the last push?" Depends on R3 (snapshot chain provides durable version records). | config-upgrade-snapshot | @@ -594,7 +595,7 @@ xceadmin config status |-----------|-----------|-----| | YAML source in git | ✅ (documentation) | Free — already how configs/ works | | Deterministic build | ✅ (core) | Same YAML → same binary (generation not set — controller-owned) | -| Binary diff (R1) | ✅ (nice-to-have) | Pure Python, offline, high operator value | +| Binary diff (R1) | ❌ deferred (2026-07-21) | Descoped from admin-config-tool — pure Python/offline, no controller dependency, but no operator has asked for it yet; revisit as follow-up once `config push` sees real-world usage | | Dry-run push (R2) | ✅ (nice-to-have) | Needs `ProbeConfigTag` RPC (already exists in controller) | | Config change snapshot (R3) | ❌ deferred | Controller-side — insert entity-bearing snapshot into OP_upgradeConfig pipeline; blocked on entity-bearing-snapshots | | Version history (R4) | ❌ deferred | Controller-side — depends on R3 snapshot chain | @@ -609,4 +610,4 @@ xceadmin config status - **Adjacent**: `ha-term-safety` (the WAL_CONFIG_MARKER this tool's upload commits anchors on the Raft log via ha-term-safety's term-index infrastructure) - **Upstream blocker**: `entity-bearing-snapshots` — must land before R3 (config change snapshot) and R4 (version history) can be implemented - **Blocks**: nothing — xceadmin ships independently; the current controller pipeline (SHA verify → decode → gen++ → save → replicate → WAL marker → restart) already works without the snapshot step -- **Future**: configuration lifecycle (R1–R8 above) — split across admin-config-tool (R1, R2, R8) and config-upgrade-snapshot (R3–R7) +- **Future**: configuration lifecycle (R1–R8 above) — split across admin-config-tool (R2, R8) and config-upgrade-snapshot (R3–R7); R1 (binary diff) deferred, no spec owner yet diff --git a/.kiro/specs/admin-config-tool/design.md b/.kiro/specs/admin-config-tool/design.md new file mode 100644 index 00000000..9be4bee7 --- /dev/null +++ b/.kiro/specs/admin-config-tool/design.md @@ -0,0 +1,544 @@ +# Technical Design: admin-config-tool + +## Overview + +`xceadmin` is a standalone Python 3.10+ CLI that compiles operator-authored YAML into the `ClusterConfig` Cap'n Proto binary the XCE controller expects (`config build`), and delivers that binary to a running controller cluster over the same TLS+Cap'n Proto wire protocol controllers already speak to each other (`config push`). It is the first Python component in an otherwise C/Go codebase and the first external (non-daemon) client of the controller's raw wire protocol. + +**Users**: XCE cluster operators managing cluster topology, queues, resources, users, limits, autoscaler providers, and cluster-identity settings through git-managed YAML rather than hand-crafted binaries or manual `scp`. + +**Impact**: Today `cluster.bin` is a hand-built, provenance-free artifact and `OP_upgradeConfig` has no external caller. This spec adds the missing YAML→binary compiler and the missing binary→controller uploader, closing both gaps client-side. + +### Goals +- Deterministic YAML → `cluster.bin` compilation covering all 13 `ClusterConfig` fields (9.1–9.3, 1.1–1.12). +- TLS+capnp delivery of `cluster.bin` to a controller, following redirects to the primary (2.1–2.7, 3.1–3.4). +- Fail-fast validation with actionable errors before anything reaches a running cluster (4.1–4.5, 7.1–7.6). +- A reusable `xcecommon` package so a future Python `xcecli` does not re-derive capnp/wire-protocol code (5.1–5.4). +- Toolchain integration: `xce-build`, `xce-deploy-ha`, nix dev shell (8.1–8.6). + +### Non-Goals +- Controller-side replication, snapshot, restart, rollback, version history (owned by `config-upgrade-snapshot`). +- Fixing the controller-side `OP_upgradeConfig` dispatch defect discovered during this design's discovery phase (see Risks) — that is a controller bug, not an xceadmin gap. +- `config diff` — descoped to roadmap follow-up during requirements review (2026-07-21). +- Peer/candidate membership management, per-instance perf tuning (`port`, hash-table sizes, `xstrings`) — physically absent from `ClusterConfig` after `clusterconfig-schema-split`. + +## Boundary Commitments + +### This Spec Owns +- The `xceadmin` CLI (`config build`, `config push`, `config push --dry-run`). +- The `xcecommon` package: capnp schema loading, YAML→capnp field mapping, JSON Schema + business-rule validation, TLS+framing wire client. Both schema types (`controller.capnp` and the 7 JSON schemas) are packaged as bundled copies (`package_data`), synced from `src/schema/` and `src/configs/schema/` at build time — not read from an operator-supplied path at runtime (2026-07-21 revision; superseded Requirement 1.6, see Requirement 5.5). +- Build/toolchain integration: `xce-build.sh` dispatch (not meson — see File Structure Plan Design Review correction), nix dev-shell Python packages, `xce-deploy-ha` `--build-config` flag. +- Test fixtures and golden files under `src/xceadmin/tests/`. + +### Out of Boundary +- **Controller-side `OP_upgradeConfig` dispatch** — `ha_dispatch_gate()` excludes `OPCode_upgradeConfig` from the standby allow-list (`ha.c:3926-3951`), and `cluster.c:667-679` calls `handle_upgrade_config()` only when `ha_is_primary()` is **false**. Combined, no controller role reaches `do_upgrade_config_with_bytes()` via the network today — it is exercised only by unit tests calling it directly. This is a controller defect, not an xceadmin defect; xceadmin implements the client side of the documented contract (`config_repl.h:347-349` — "handle an OP_upgradeConfig request from an admin tool") and cannot itself fix the gate. Flagged as a blocking risk below. +- **Redirect wiring for `OP_upgradeConfig`** — `fill_redirect_reply()` (`join.c:27-31`) is only invoked from the `OPCode_join` path (`join.c:277`). `RequestReply.redirectHost`/`redirectPort` are never populated for an `upgradeConfig` reply today. xceadmin implements generic, opcode-agnostic redirect-following per the `RequestReply` contract (forward-compatible), but it will not fire for `config push` until the controller is extended. +- **`ProbeConfigTag` server-side handling** — `xce_probe_config_tag()` (`config_repl.c:1829`) has no server-side dispatch case anywhere in `cluster.c` and no production caller; it is dormant plumbing. xceadmin's dry-run client implements the wire contract but has nothing live to call. +- Entity-bearing snapshots, config rollback, version history, config activation status (owned by `config-upgrade-snapshot`, blocked upstream on `entity-bearing-snapshots`). +- TLS mutual authentication / hostname verification — none exists anywhere in the codebase (`SSL_VERIFY_NONE` is set explicitly for the controller's own outbound client context, `myapp.c:109`; no `X509_check_host` call exists). xceadmin matches this existing posture; it neither introduces nor removes an authentication guarantee. + +### Allowed Dependencies +- `src/schema/controller.capnp` and `src/schema/c.capnp` — schema source of truth, read-only. `xceadmin` packages a synced *copy* (see File Structure Plan / `xcecommon.schema`) rather than referencing this path directly at runtime — `src/schema/` itself stays owned by the C build, not by this spec. +- `src/configs/*.yaml` — the 7 example configs already authored (this session) as operator-facing documentation; xceadmin's test fixtures are separate, purpose-built golden inputs under `src/xceadmin/tests/fixtures/`, not these files. `src/configs/schema/*.schema.json` is different in kind from the YAML examples: it's the source of truth `xcecommon` packages a synced copy of (2026-07-21 revision — see `xcecommon.validation`), the same relationship `src/schema/controller.capnp` already has. +- `xce.crt`/`xce.key` under `ha/data/` (or `--cert-dir`) — same cert bundle the controller cluster already uses. +- Nix `python3Packages`: `pycapnp`, `pyyaml`, `jsonschema`, `click`, `pytest`, `ruff` (net-new additions to `src/nixSupport/xce.nix`). +- `src/tools/xce-build.sh`'s case-statement dispatch pattern (not meson) as the build-integration point — see Design Review correction below. + +### Revalidation Triggers +- `controller.capnp` `ClusterConfig` field renumbering/addition/removal (already happened once this session via `clusterconfig-schema-split`) — invalidates the field-mapping table in Data Models. +- A fix to the `cluster.c`/`ha.c` dispatch-gate contradiction — flips `config push` E2E testing from "mock controller only" to "real cluster required," per Testing Strategy. +- `fill_redirect_reply()` extended to `OPCode_upgradeConfig` — unlocks live auto-primary-discovery testing for config push (currently only testable against `OPCode_join`-shaped mocks). +- A server-side handler added for `OPCode_probeConfigTag` — unlocks `config push --dry-run` E2E testing. + +## Architecture + +### Existing Architecture Analysis +XCE's daemon-to-daemon wire protocol is hand-rolled: `iomgr`/`vtaskmgr` (epoll loop + thread-pool dispatch) drive a raw-`SSL`-over-TCP channel framed as `uint64_t` size prefix (native/host byte order, **not** network byte order — confirmed at `netio.c:190-199`, `request.c:863-938`, `config_repl.c:1925-1968`, no byte-swap anywhere in the path) followed by that many bytes of a Cap'n Proto `Request`/`Reply` message. There is no Cap'n Proto RPC layer (no vat network) — every exchange is one framed message out, one framed message back. `src/lib/request.c` already contains a minimal one-shot raw-SSL helper pattern (`xce_probe_config_tag`-adjacent code, `request.c:863-938`) that this design mirrors from the Python side. + +No precedent exists for a non-daemon client of this protocol: `src/tools/` has no socket/capnp client, `xce_test` is a WAL/snapshot file-inspection CLI with no network code, and `xcecli` (Go) speaks HTTP/JSON to `apiserver`, never this raw protocol. `xceadmin` is the first. + +### Architecture Pattern & Boundary Map + +**Selected pattern**: layered CLI — thin `click` entry point over a reusable library (`xcecommon`), matching the project's existing `controllers → services → store/backend` inward-dependency convention (`structure.md`) adapted for a one-shot tool (no persistent store; "runtime" is a single TLS exchange or a single file write). + +```mermaid +graph TB + CLI[xceadmin cli] --> Build[xceadmin build] + CLI --> Push[xceadmin push] + Build --> Validation[xcecommon validation] + Build --> Mapping[xcecommon mapping] + Push --> Wire[xcecommon wire] + Push --> Discovery[xceadmin discovery] + Discovery --> Wire + Mapping --> Schema[xcecommon schema] + Wire --> Schema + Validation --> YamlFiles[YAML configs] + Mapping --> ClusterBin[cluster bin file] + Wire --> Controller[XCE controller] +``` + +**Architecture Integration**: +- Selected pattern: layered CLI over a shared library — the library (`xcecommon`) has zero CLI/`click` dependency so a future Python `xcecli` (brief.md Future Roadmap) can import it directly. +- Domain boundaries: `xcecommon` owns protocol/schema/mapping concerns; `xceadmin` owns orchestration and CLI ergonomics (exit codes, stderr/stdout conventions, flag parsing). Neither owns controller-side behavior. +- Existing patterns preserved: framing format, TLS cert bundle reuse, opcode/union shapes are consumed exactly as the C side defines them — no protocol reinterpretation. +- New components rationale: `xcecommon` did not exist (first Python in the repo); every module below is net-new. +- Steering compliance: Cap'n Proto for wire format (`structure.md` — "Cap'n Proto for wire formats"), component lives under `src//` (`structure.md` — "Organization Philosophy"), fail-fast input validation (`security.md` — "Validate at the API boundary"). + +**Dependency Direction**: `xcecommon.schema` → `{xcecommon.mapping, xcecommon.wire}` → `xcecommon.validation` (independent, no inbound dependency from schema/mapping/wire) → `{xceadmin.build, xceadmin.discovery}` → `xceadmin.push` → `xceadmin.cli`. Each layer imports only from layers to its left; `xcecommon.*` never imports anything from `xceadmin.*` — this is what keeps `xcecommon` importable by a future Python `xcecli` without dragging in `click` or CLI-specific exit-code handling. + +**Parallel-safe implementation boundaries**: within `xcecommon`, `validation.py` has no dependency on `schema.py`/`mapping.py`/`wire.py` and can be implemented independently of them; `mapping.py` and `wire.py` both depend only on `schema.py`, not on each other, so they can be implemented in parallel once `schema.py` is stable. Within `xceadmin`, `build.py` (depends on `xcecommon.{validation,mapping}`) and `discovery.py`+`push.py` (depend on `xcecommon.wire`) touch disjoint `xcecommon` modules and can be implemented in parallel once the underlying `xcecommon` layer is stable. + +### Technology Stack + +| Layer | Choice / Version | Role in Feature | Notes | +|-------|------------------|------------------|-------| +| CLI | `click` (nixpkgs `python3Packages.click`) | `config build`/`config push`/`config push --dry-run` subcommand groups | Grouped-command idiom: `@click.group()` → `@cli.group()` → `@config.command()` | +| Capnp | `pycapnp` ~2.2.x (nixpkgs) | Load `controller.capnp`, build/encode `ClusterConfig`, decode `Request`/`Reply` | Dynamic-reflection (`DynamicStruct`) API; `$C.mapname(...)` annotations are declared-and-inert, not a parse hazard (verified against `capnp.pyx` source) | +| YAML | `pyyaml` (nixpkgs) | Parse the 7 config files | — | +| Schema validation | `jsonschema` (nixpkgs), Draft 2020-12 | Validate each YAML against `xceadmin`'s bundled JSON schemas (`package_data`, same mechanism as `xcecommon.schema`'s `controller.capnp`) | Source of truth is `src/configs/schema/*.schema.json`; synced into the package at build time, never read from `configs_dir` at runtime (2026-07-21 revision — see `xcecommon.validation`) | +| Transport | Python stdlib `ssl` + `socket` | TLS-wrapped TCP to controller | `ssl.SSLContext` with `check_hostname=False`, `verify_mode=ssl.CERT_NONE` — matches the controller's own outbound posture (`myapp.c:109`), not a weakening | +| Test framework | `pytest` (nixpkgs) | Golden-file + mock-controller tests | Plain byte-equality fixture comparison (see Testing Strategy — `pytest-regressions` considered, not adopted, to avoid an unverified new nix package) | +| Packaging | `pyproject.toml` + `pip install -e .` (dev) / `pip install .` (release) | Editable vs. pinned install | Runs inside nix dev shell so `pycapnp`'s C-extension build links nix-provided `capnproto` | + +## File Structure Plan + +### Directory Structure +``` +src/xceadmin/ +├── pyproject.toml # package metadata; entry point xceadmin = xceadmin.cli:main +│ # NOT wired into meson — see Design Review correction below +├── src/ +│ ├── xcecommon/ # shared library — no click/CLI dependency, reusable by future Python xcecli +│ │ ├── __init__.py +│ │ ├── errors.py # XceAdminError base + all subclasses (single CLI dispatch surface) +│ │ ├── schema_files/ # packaged copies, synced from src/ at build (not hand-edited) +│ │ │ ├── controller.capnp # <- src/schema/controller.capnp +│ │ │ ├── c.capnp # <- src/schema/c.capnp +│ │ │ └── json/ # <- src/configs/schema/*.schema.json (7 files) +│ │ ├── schema.py # capnp.load(controller.capnp) wrapper + typed accessors; resolves schema_files/ via importlib.resources +│ │ ├── mapping.py # declarative YAML-section -> capnp-field table + 2 mapper shapes (list, scalar) +│ │ ├── validation.py # JSON Schema validation (bundled schema_files/json/), reserved-name / dup / undefined-ref checks +│ │ └── wire.py # TLS connect, 8-byte native-order framing, generic send_request() +│ └── xceadmin/ +│ ├── __init__.py +│ ├── cli.py # click group: config build / config push / config push --dry-run +│ ├── build.py # config build orchestration: scan dir -> validate -> map -> serialize -> sha256 +│ ├── push.py # config push orchestration: connect -> discover primary -> send -> display +│ └── discovery.py # generic redirect-following (Req 3) — opcode-agnostic +└── tests/ + ├── fixtures/ + │ ├── minimal/configs/{hosts,queues}.yaml + expected.bin # no schema/ — validated against xceadmin's bundled schemas + │ ├── full/configs/{hosts,queues,resources,users,limits,xce,scaler}.yaml + expected.bin + │ └── invalid/configs/... # one fixture dir per validation-error case (4.1, 4.2, 4.3, 4.4, 4.5) + ├── test_schema_bundling.py # default_schema_path()/default_schema_dir() resolve under the active install (5.5) — only meaningful under `xce-build xceadmin release`, see Testing Strategy + ├── test_mapping.py # pure unit tests: map_list_section/map_scalar_section against in-memory dicts, no file I/O (1.1, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 5.3, 9.1, 9.2, 9.3) — gap closed during task generation, was in Testing Strategy's "Unit — mapping" bullet but missing from this file list + ├── test_config_build.py # golden-file build tests (1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 9.1, 9.2, 9.3) + ├── test_schema_validation.py # per-error-case validation tests (4.1, 4.2, 4.3, 4.4, 4.5) + ├── test_wire_framing.py # framing pack/unpack against known byte sequences (2.1) + └── test_config_push_mock.py # mock TLS server, opcode + redirect + error-reply cases (2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.1, 3.2, 3.3, 3.4, 7.1, 7.2, 7.3) +``` + +`xcecommon` and `xceadmin` are separate top-level packages under one `src/xceadmin/src/` tree, installed together via one `pyproject.toml` for now (two console-ready packages, one distributable unit) — splitting into two distributions is deferred until a second consumer (the future Python `xcecli`) actually exists (Simplification: no speculative packaging split). + +### Modified Files +- `src/tools/xce-build.sh` — new `xceadmin)` case: sync `src/schema/{controller,c}.capnp` → `src/xceadmin/src/xcecommon/schema_files/` and `src/configs/schema/*.schema.json` → `src/xceadmin/src/xcecommon/schema_files/json/` (`cp`, source of truth stays `src/schema/` and `src/configs/schema/` respectively — 2026-07-21 revision extends the sync to JSON schemas, Requirement 5.5), then `pip install -e .`/`pip install .` by profile, then `pytest` (8.1, 8.2). The sync step runs on every build so a `pip install .` (non-editable, Req 8.2) always packages the current schemas as `package_data` — without it, `load_schema()`/`xcecommon.validation`'s default resolution would silently package stale or missing copies. **`pytest` runs after whichever install just happened, not before** — this ordering is what makes the `release` profile's test run (8.2) the only one that actually exercises `default_schema_path()`/`default_schema_dir()` resolving under a real non-editable `site-packages` layout; the default (dev/editable) profile's test run (8.1) never would, since editable installs keep resolving into the checked-out `schema_files/` copy either way. **Design Review correction**: `xceadmin` is dispatched directly by this case statement and is deliberately **not** wired into `src/meson.build`/`src/meson.options`/a `src/xceadmin/meson.build` — `xcecli`'s `custom_target()` pattern fits a single static-binary output; `pip install -e .` mutates a venv/site-packages tree with no single trackable output artifact, so forcing it through meson's build graph would be a dead end. `xceadmin` sits orthogonal to the meson component option entirely, matching what `brief.md`'s harness sketch already specified before this design pass over-engineered a meson integration. +- `src/nixSupport/xce.nix` — add `python3`, `python3Packages.{pycapnp,pyyaml,jsonschema,click,pytest,ruff}` to `nativeBuildInputs` (8.6). +- `.claude/skills/xce-build/SKILL.md` — document the `xceadmin` component and its pip-based (non-meson-profile) build path (8.1, 8.2). +- `src/tools/deploy-ha.sh` — **(gap closed during task generation — the actual runtime behavior for 8.3-8.5 belongs here, not just in documentation)** add a `--build-config` flag; before starting the cluster, if `--build-config` is passed or `ha/data/cluster.bin` is missing, run `xceadmin config build configs/ -o ha/data/cluster.bin` (8.3, 8.4); if `cluster.bin` already exists and `--build-config` is not passed, use it as-is, unchanged (8.5). +- `.claude/skills/xce-deploy-ha/SKILL.md` — document the `--build-config` flag and the new precondition step added to `deploy-ha.sh` above (8.3, 8.4, 8.5). +- `.claude/settings.local.json` / `.claude/settings.json` — permission allowlist and ruff format/check hooks for `*.py` (full JSON already drafted in `brief.md` Harness Enhancement section; applied verbatim here, not re-derived). + +## System Flows + +### Config Build +```mermaid +flowchart TD + Start[xceadmin config build dir] --> ScanFiles[scan 7 known yaml filenames] + ScanFiles --> PerFile{file present} + PerFile -- no --> EmptySection[treat section as empty] + PerFile -- yes --> Validate[validate against bundled jsonschema] + Validate -- fail --> ErrValidation[report file field constraint exit nonzero] + Validate -- pass --> BusinessRules[reserved name dup check undefined ref warn] + BusinessRules --> Map[map to ClusterConfig fields] + EmptySection --> Map + Map --> Encode[pycapnp encode] + Encode --> Write[write cluster bin and sha256 to stderr] +``` + +Gating conditions: a missing individual YAML file is not an error (1.5, empty section) — there is no "missing schema" error mode at all (1.6, superseded 2026-07-21: schemas are bundled with `xceadmin`, not read from `configs_dir`). Schema validation failures are hard errors (4.1); business-rule warnings (undefined references) do not stop the build (4.4). + +### Config Push (including auto-discovery) +```mermaid +sequenceDiagram + participant Op as Operator + participant X as xceadmin push + participant C1 as Controller node1 + participant C2 as Controller node primary + + Op->>X: config push cluster.bin + X->>C1: TLS connect, Request OP_upgradeConfig + C1-->>X: Reply + alt redirectHost set + X->>C2: TLS connect, resend Request + C2-->>X: Reply UpgradeConfigReply + end + X->>Op: versionTag ackedCount error, exit code +``` + +At most 3 redirect hops (3.2); each hop is a fresh TLS connection with the same 30s timeout (3.4, 7.6). Because `fill_redirect_reply()` does not currently fire for `OPCode_upgradeConfig` (see Boundary Commitments), the `alt` branch above is exercised only in mock-controller tests until the controller gap is closed. + +## Requirements Traceability + +| Requirement | Summary | Components | Interfaces | Flows | +|---|---|---|---|---| +| 1 | Config Build | `xceadmin.build`, `xcecommon.{validation,mapping,schema}` | `build_cluster_config()`, `map_section()` | Config Build | +| 2 | Config Push | `xceadmin.push`, `xcecommon.wire` | `send_request()`, `push_config()` | Config Push | +| 3 | Auto Primary Discovery | `xceadmin.discovery` | `follow_redirects()` | Config Push | +| 4 | Validation | `xcecommon.validation` | `validate_yaml()`, `check_business_rules()` | Config Build | +| 5 | xcecommon Shared Package | `xcecommon.{schema,wire,mapping,validation}` | package-level exports, `default_schema_path()`, `default_schema_dir()` | — | +| 6 | Dry-Run Push (Nice-to-Have) | `xceadmin.push`, `xcecommon.wire` | `probe_config()` | Config Push (dry-run variant, not diagrammed — mirrors push minus commit) | +| 7 | Error Handling & Resilience | `xceadmin.cli`, `xcecommon.wire` | `click.ClickException` mapping, socket timeout handling | Config Push | +| 8 | Skill & Harness Integration | build/nix/skill files (File Structure Plan — Modified Files) | — | — | +| 9 | Deterministic Builds | `xcecommon.mapping`, `xceadmin.build` | `build_cluster_config()` (no wall-clock/host/user inputs) | Config Build | + +## Components and Interfaces + +| Component | Domain/Layer | Intent | Req Coverage | Key Dependencies (P0/P1) | Contracts | +|---|---|---|---|---|---| +| `xcecommon.schema` | Library | Load `controller.capnp`, expose typed roots | 5.1, 5.5, 7.4 | `controller.capnp` (P0) | Service | +| `xcecommon.mapping` | Library | YAML section dict → capnp field values | 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 5.3, 9.1, 9.2, 9.3 | `xcecommon.schema` (P0) | Service | +| `xcecommon.validation` | Library | JSON Schema + business-rule checks | 4.1, 4.2, 4.3, 4.4, 4.5, 5.5 | `xcecommon/schema_files/json/*.schema.json` (P0, bundled) | Service | +| `xcecommon.wire` | Library | TLS connect, framing, generic request/reply | 2.1, 2.3, 2.4, 5.2, 6.1, 7.1, 7.3, 7.6 | `xce.crt`/`xce.key` (P0), `xcecommon.schema` (P0) | Service | +| `xceadmin.build` | CLI orchestration | `config build` command | 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 9.1, 9.2, 9.3 | `xcecommon.{validation,mapping}` (P0) | Service | +| `xceadmin.push` | CLI orchestration | `config push` / `--dry-run` | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 6.1, 6.2, 6.3, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6 | `xcecommon.wire` (P0), `xceadmin.discovery` (P0) | Service | +| `xceadmin.discovery` | CLI orchestration | Redirect-following | 3.1, 3.2, 3.3, 3.4 | `xcecommon.wire` (P0) | Service | +| `xceadmin.cli` | CLI entry | Argument parsing, exit codes | 1.2, 1.3, 2.2, 2.4, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6, 8.1, 8.2 | all `xceadmin.*` (P0) | Service | + +Only components introducing a new boundary get a full block below; `xceadmin.cli` is presentation-only (click wiring) and is covered by the summary row plus its Implementation Note. + +### Library + +#### xcecommon.schema + +| Field | Detail | +|-------|--------| +| Intent | Load and cache the compiled `controller.capnp` schema; expose the `ClusterConfig`, `Request`, `Reply`, `UpgradeConfigRequest/Reply`, `ProbeConfigTagRequest/Reply` types. | +| Requirements | 5.1, 5.5, 7.4 | + +**Responsibilities & Constraints** +- Single load point — `capnp.load(path, imports=[schema_dir])` called once per process; other modules import the returned module object, never re-load. +- Resolves `c.capnp`'s sibling-directory import via the `imports` argument, pointed at wherever `default_schema_path()` resolved the copy to. +- Read-only: never mutates or re-serializes the schema itself. +- **Default path resolution (Design Review correction, 2026-07-21 — a hosted-discussion follow-up caught that the first draft never specified this, which meant `pip install .` / Req 8.2 was unimplementable as designed)**: `default_schema_path()` uses `importlib.resources.files("xcecommon") / "schema_files" / "controller.capnp"` — this resolves correctly for both `pip install -e .` (Req 8.1, still points into the checked-out repo's `schema_files/` copy) and `pip install .` (Req 8.2, resolves into whatever `site-packages` copy `package_data` installed) identically, unlike a `__file__`-relative `../../../schema/` walk, which only survives editable installs. `--schema-path ` (CLI) / `XCEADMIN_SCHEMA_PATH` (env var) override the default for troubleshooting, per 7.4's "suggest checking the installation." + +**Dependencies** +- External: `pycapnp` (P0) — schema compilation and `DynamicStruct` message API. +- External: `xcecommon/schema_files/{controller,c}.capnp` (P0) — packaged copy, synced from `src/schema/` at build time (File Structure Plan → Modified Files); failure path is 7.4. + +**Contracts**: Service [x] / API [ ] / Event [ ] / Batch [ ] / State [ ] + +##### Service Interface +```python +def default_schema_path() -> str: + """importlib.resources.files('xcecommon') / 'schema_files' / 'controller.capnp', + overridable via --schema-path / XCEADMIN_SCHEMA_PATH.""" + +def load_schema(schema_path: str | None = None) -> "capnp.Capnp": + """Load controller.capnp (default_schema_path() if schema_path is None); + raise SchemaNotFoundError if the resolved path or its c.capnp import is missing.""" + +class SchemaNotFoundError(XceAdminError): + path: str +``` +- Preconditions: `schema_path` (explicit or resolved) points at `controller.capnp`; `c.capnp` sits alongside it. +- Postconditions: returned module exposes `.ClusterConfig`, `.Request`, `.Reply`, `.UpgradeConfigRequest`, `.UpgradeConfigReply`, `.ProbeConfigTagRequest`, `.ProbeConfigTagReply`. +- Invariants: no network or filesystem writes. + +#### xcecommon.mapping + +| Field | Detail | +|-------|--------| +| Intent | Convert validated YAML section dicts into `ClusterConfig` capnp field values, and back (field metadata reused by any future `config diff`). | +| Requirements | 1.1, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 5.3, 9.1, 9.2, 9.3 | + +**Responsibilities & Constraints** +- One declarative table (`SECTION_MAP`) drives all 7 sections — each entry states: YAML filename, top-level YAML key, target capnp field name (`@N`), and mapper kind (`list` or `scalar`). Two mapper functions (`map_list_section`, `map_scalar_section`) implement every section — no per-file bespoke mapper (Generalization, see `research.md`). +- Two mapper shapes only: **list** sections (`resources`, `queues`, `ugroups`, `hgroups`, `hostConfigs`, `hproviders`, `limits` — YAML top key is an array, mapped to `List(Struct)`) and **scalar** section (`xce.yaml` — flat key/value, mapped to 5 top-level `ClusterConfig` scalar fields). +- Enforces 9.1, 9.2, 9.3: no timestamp/hostname/username is ever read into a mapped value; list-section ordering follows YAML source order exactly, not sorted or hashed (order stability is required for byte-identical output). +- `configGeneration` (`@12`) is never touched by this module (1.9) — capnp leaves it at its default (0) unconditionally. +- `candidates`, `port`, hash-table-size fields, `xstrings` do not exist in the schema (post `clusterconfig-schema-split`) — there is nothing for this module to exclude; 1.8/1.10 are satisfied structurally, not by runtime logic. + +**Dependencies** +- Inbound: `xceadmin.build` (P0) — sole caller. +- Outbound: `xcecommon.schema` (P0) — target field types. + +**Contracts**: Service [x] + +##### Service Interface +```python +from typing import Literal, TypedDict + +class SectionSpec(TypedDict): + yaml_file: str # e.g. "queues.yaml" + yaml_key: str # e.g. "queues" + capnp_field: str # e.g. "queues" + kind: Literal["list", "scalar"] + +SECTION_MAP: list[SectionSpec] # 7 entries, one per YAML file + +def map_list_section(spec: SectionSpec, items: list[dict], target: "capnp._DynamicStructBuilder") -> None: + """Populate target. as a capnp List sized to len(items).""" + +def map_scalar_section(spec: SectionSpec, values: dict, target: "capnp._DynamicStructBuilder") -> None: + """Set each scalar capnp field on target directly from values.""" +``` +- Preconditions: `items`/`values` already passed `xcecommon.validation`. +- Postconditions: `target` (a `ClusterConfig` builder) has exactly the fields for the given section set; no other field is touched. +- Invariants: idempotent given the same input dict — calling twice with identical input produces byte-identical serialized output (9.1). + +**Full field-mapping table** (Data Models has the authoritative field-by-field version — see Data Contracts & Integration). + +#### xcecommon.validation + +| Field | Detail | +|-------|--------| +| Intent | JSON Schema validation plus business rules (reserved names, duplicates, undefined references) not expressible in JSON Schema alone. | +| Requirements | 4.1, 4.2, 4.3, 4.4, 4.5, 5.5 | + +**Responsibilities & Constraints** +- **Bundled schemas (2026-07-21 revision, Requirement 5.5, supersedes Requirement 1.6)**: `default_schema_dir()` uses `importlib.resources.files("xcecommon") / "schema_files" / "json"` — same `importlib.resources` mechanism as `xcecommon.schema.default_schema_path()`, correct for both `pip install -e .` and `pip install .`. `configs_dir` is never consulted for a `schema/` subdirectory; there is no "schema not found in configs_dir" error path anymore. No CLI override exists for this one (unlike `--schema-path` for the capnp schema) — Simplification: no requirement or use case calls for operator-swappable *validation* schemas, only for pointing at a different `controller.capnp` during cross-version troubleshooting. +- `validate_yaml(file, schema_name)` — `schema_name` is the YAML file's base name with `.yaml` stripped (e.g. `queues.yaml` → `"queues"` → `default_schema_dir() / "queues.schema.json"`), matching `SECTION_MAP`'s `yaml_file` entries and the naming convention the 7 files under `src/configs/schema/` already follow. `jsonschema.Draft202012Validator`; on failure raises with file name, JSON-pointer field path, and violated constraint (4.1). +- `check_reserved_names(data)` — rejects `"lost_and_found"` for any queue/host name (4.2), independent of schema validity. +- `check_duplicates(all_sections)` — per-file/per-section uniqueness only; a limit name may collide with a queue name (4.3, scoped intentionally). Raises like `check_reserved_names`, not return-and-let-caller-decide — 4.3, like 4.2, is an unconditional hard-fail per requirements.md, so the two share the same raise-on-violation contract (Design Review correction: an earlier draft had `check_duplicates` return messages for the caller to judge, which was an unjustified control-flow inconsistency between two checks with identical fail-semantics). +- `check_undefined_refs(all_sections)` — cross-file warnings only (queue→host-group, limit→group/`resname`); never blocks the build (4.4). `resources.yaml` is treated as the exhaustive resource catalog — every `resname` elsewhere, including the built-in/dynamic names (`slots`, `mem`, the 8 `add_hostattrs_resvals()` attributes), must resolve against it or this check warns (4.5). + +**Dependencies** +- Inbound: `xceadmin.build` (P0). +- External: `jsonschema` (P0), `xcecommon/schema_files/json/*.schema.json` (P0, bundled — synced from `src/configs/schema/` at build time, File Structure Plan → Modified Files). + +**Contracts**: Service [x] + +##### Service Interface +```python +class ValidationError(XceAdminError): + file: str + json_pointer: str + constraint: str + +class ReservedNameError(XceAdminError): ... +class DuplicateEntryError(XceAdminError): ... + +def default_schema_dir() -> str: + """importlib.resources.files('xcecommon') / 'schema_files' / 'json'.""" + +def validate_yaml(file_path: str, schema_name: str) -> dict: ... # raises ValidationError; schema_name resolved under default_schema_dir() +def check_reserved_names(sections: dict) -> None: ... # raises ReservedNameError +def check_duplicates(sections: dict) -> None: ... # raises DuplicateEntryError +def check_undefined_refs(sections: dict) -> list[str]: # returns warning messages, never raises — the one intentionally non-fatal check (4.4) +``` + +All raising exceptions here subclass `XceAdminError` (defined in `xcecommon.errors`, see `xceadmin.cli` Implementation Note) so the CLI boundary has one dispatch surface instead of catching unrelated types per module. + +#### xcecommon.wire + +| Field | Detail | +|-------|--------| +| Intent | TLS connection, the 8-byte native-order framing, and a generic single-request/single-reply exchange — opcode-agnostic so it also serves `ProbeConfigTag`'s direct-struct variant. | +| Requirements | 2.1, 2.3, 2.4, 5.2, 6.1, 7.1, 7.3, 7.6 | + +**Responsibilities & Constraints** +- `connect(host, port, cert_dir, timeout=30)` — `socket.create_connection` wrapped by `ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)` with `check_hostname=False`, `verify_mode=ssl.CERT_NONE`, loading `xce.crt`/`xce.key` from `cert_dir` (2.3, 2.4, 7.1, 7.6). This mirrors the controller's own outbound TLS posture exactly (`myapp.c:103-109`) — confirmed no CN/peer-cert check exists anywhere in the codebase to diverge from. +- `send_frame(sock, payload: bytes)` / `recv_frame(sock) -> bytes` — `struct.pack("=Q", len(payload))` / `struct.unpack("=Q", ...)`, **native byte order**, matching `netio.c`'s raw-memory-copy framing exactly (verified: no `htole64`/`htobe64` anywhere in the C framing path). +- `send_request(sock, request: "Request") -> "Reply"` — generic: encode via `.to_bytes()`, frame, send, receive frame, decode via `Reply.from_bytes()`. Used for `OP_upgradeConfig`; the same primitives (not the same function, since `ProbeConfigTagRequest`/`Reply` are direct structs, not `Request`/`Reply`-union-wrapped) serve `config push --dry-run`'s `ProbeConfigTag` client. +- Never passes an `SSLSocket` or its `fileno()` into a pycapnp read/write call — decode/encode happens in-memory (`from_bytes`/`to_bytes`) against plaintext buffers assembled by `recv_frame`/`send_frame`, so TLS is never bypassed (verified gotcha from discovery: pycapnp's `.write()` calls `fileno()` and writes to the raw fd, which would skip encryption entirely on a wrapped socket). + +**Dependencies** +- Inbound: `xceadmin.push`, `xceadmin.discovery` (P0). +- External: Python stdlib `ssl`, `socket`, `struct` (P0); `xcecommon.schema` (P0). + +**Contracts**: Service [x] + +##### Service Interface +```python +import socket, ssl + +def connect(host: str, port: int, cert_dir: str, timeout_s: float = 30.0) -> ssl.SSLSocket: ... +def send_frame(sock: ssl.SSLSocket, payload: bytes) -> None: ... +def recv_frame(sock: ssl.SSLSocket) -> bytes: ... +def send_request(sock: ssl.SSLSocket, request_bytes: bytes) -> bytes: + """Frame+send request_bytes, receive+unframe one reply. Raises + ConnectionTimeoutError / ConnectionRefusedError on failure.""" + +class ConnectionTimeoutError(XceAdminError): host: str; port: int +class ReplyDecodeError(XceAdminError): reason: str # corrupt/truncated capnp bytes, 7.3 +``` +- Preconditions: `cert_dir` contains `xce.crt`+`xce.key`. +- Postconditions: exactly one request sent, exactly one reply returned, per call — no retry, no connection pooling (one-shot CLI, not a daemon; Simplification). +- Invariants: socket closed on every exit path, including exceptions and `SIGINT` (7.5). + +### CLI Orchestration + +#### xceadmin.build + +| Field | Detail | +|-------|--------| +| Intent | `config build ` command: scan → validate → map → serialize → write → print SHA-256. | +| Requirements | 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.9, 8.3, 8.4 | + +**Responsibilities & Constraints** +- Scans for the 7 known filenames only (`SECTION_MAP` keys) — not an open-ended directory walk (1.1). +- `-o ` / `-o -` (stdout) output targets (1.2, 1.3); SHA-256 always to stderr, regardless of `-o` target (1.4). +- No `schema/` subdirectory check — schemas are bundled (1.6, superseded); missing individual YAML file → empty section, continue (1.5). +- Unrecognized-but-schema-valid fields → warning, not an error (1.7). + +**Dependencies** +- Outbound: `xcecommon.validation` (P0), `xcecommon.mapping` (P0), `xcecommon.schema` (P0, transitively). + +**Contracts**: Service [x] + +##### Service Interface +```python +def build_cluster_config(configs_dir: str) -> bytes: + """Full pipeline: validate all present sections, map to ClusterConfig, + serialize. Raises ValidationError / ReservedNameError / DuplicateEntryError + on hard failures (all subclass XceAdminError).""" + +def write_output(data: bytes, output_path: str) -> str: + """Writes data to output_path ('-' => stdout). Returns hex SHA-256.""" +``` + +#### xceadmin.push + +| Field | Detail | +|-------|--------| +| Intent | `config push ` (commit) and `config push --dry-run ` (probe): connect, discover primary, send, display result. Two distinct functions, not one boolean-flagged function — the two wire messages (`Request{upgradeConfig}` vs. direct-struct `ProbeConfigTagRequest`) and their replies (`UpgradeConfigReply{versionTag, ackedCount, error}` vs. `ProbeConfigTagReply{configTag}`) share no fields, so a shared return type would misrepresent one of the two (Design Review correction — an earlier draft's single `push_config(..., dry_run: bool) -> PushResult` implied `acked_count` exists on a dry-run probe reply, which it does not). | +| Requirements | 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 6.1, 6.2, 6.3, 7.1, 7.2, 7.3, 7.4, 7.5, 7.6 | + +**Responsibilities & Constraints** +- Builds the `Request{opcode: upgradeConfig, upgradeconfigrequestmsg: {sha256, bytes}}` message exactly per `controller.capnp:628-654,768-771` (2.1). +- Delegates connection + redirect-following to `xceadmin.discovery` before sending (3.1, 3.2, 3.3, 3.4). +- `--dry-run` sends `ProbeConfigTagRequest` instead (direct struct, no `Request` wrapper — see `xcecommon.wire`), reports acceptance without committing (6.1, 6.2, 6.3). Per Boundary Commitments, this path has no live server counterpart today; it is implemented against the documented contract and exercised only by the mock-controller test suite until the controller gains a handler. +- Maps every failure mode to a specific message + non-zero exit per 7.1, 7.2, 7.3, 7.5, 7.6 (connection failure, non-zero `Reply.status`, corrupt binary, SIGINT, timeout). + +**Dependencies** +- Outbound: `xcecommon.wire` (P0), `xceadmin.discovery` (P0). + +**Contracts**: Service [x] + +##### Service Interface +```python +class PushResult: + version_tag: int + acked_count: int + error: str + +class DryRunResult: + current_generation: int # from ProbeConfigTagReply.configTag + would_be_generation: int # current_generation + 1, computed client-side for display only (6.2) + accepted: bool + reason: str # populated when accepted is False (6.3) + +def push_config(cluster_bin_path: str, controller: str, cert_dir: str) -> PushResult: ... +def probe_config(cluster_bin_path: str, controller: str, cert_dir: str) -> DryRunResult: ... +``` +- Postconditions (`push_config`): exit 0 iff `acked_count > 0` and `error == ""` (2.6); otherwise exit non-zero with `error` displayed (2.7). +- Postconditions (`probe_config`): exit 0 and `"OK: controller will accept this config (generation N→M)"` iff `accepted` (6.2); otherwise exit non-zero with `reason` displayed (6.3). + +#### xceadmin.discovery + +| Field | Detail | +|-------|--------| +| Intent | Generic, opcode-agnostic redirect-following against `RequestReply.redirectHost`/`redirectPort`. | +| Requirements | 3.1, 3.2, 3.3, 3.4 | + +**Responsibilities & Constraints** +- Inspects any `Reply` for non-empty `redirectHost`/`redirectPort`; reconnects and resends up to 3 hops (3.1, 3.2), then errors. +- Displays the redirect target when a standby redirects (3.3). +- 30s "no primary elected" timeout is a property of each connection attempt, not the overall discovery loop (3.4, delegated to `xcecommon.wire.connect`). +- Implemented generically against the `RequestReply` schema shape (not `upgradeConfig`-specific) so it is correct today for `OPCode_join`-style redirects and automatically correct for `upgradeConfig` once the controller wires `fill_redirect_reply()` for that opcode too (Simplification — no opcode-specific branching to remove later). + +**Dependencies** +- Outbound: `xcecommon.wire` (P0). + +**Contracts**: Service [x] + +##### Service Interface +```python +def follow_redirects(initial_host: str, initial_port: int, cert_dir: str, send_fn) -> tuple[str, int, bytes]: + """send_fn(host, port) -> reply_bytes. Returns (final_host, final_port, reply_bytes) + after following <=3 redirects. Raises TooManyRedirectsError / NoPrimaryError.""" + +class TooManyRedirectsError(XceAdminError): hops: int +class NoPrimaryError(XceAdminError): host: str; port: int +``` + +### xceadmin.cli — Implementation Note +Thin `click` layer: `@click.group() cli()` → `@cli.group() config()` → `@config.command() build(...)`/`push(...)`. + +**Shared exception contract** (Design Review correction — an earlier draft had 7 unrelated exception types with no unifying dispatch mechanism): `xcecommon.errors` defines one base, `class XceAdminError(Exception): message: str`, and every raising exception across `xcecommon`/`xceadmin` (`SchemaNotFoundError`, `ValidationError`, `ReservedNameError`, `DuplicateEntryError`, `ConnectionTimeoutError`, `ReplyDecodeError`, `TooManyRedirectsError`, `NoPrimaryError`) subclasses it. `xceadmin.cli` has exactly one `except XceAdminError as e: raise click.ClickException(e.message)` — a single dispatch surface, per-type message formatting happens in each exception's `__str__`/`message` construction at raise time, not in the CLI layer (7.1, 7.2, 7.3, 7.4). A `SIGINT` handler closes any open socket before re-raising as a "cancelled" message (7.5). No business logic lives here — covered by the summary row. + +## Data Models + +### Domain Model +No persistent entities or aggregates — `xceadmin` is a stateless, one-shot transform (YAML → bytes) and a stateless one-shot RPC client (bytes → controller → result). The only domain concept is the **`ClusterConfig` field set**, whose provenance (which YAML file owns which field) is the core invariant this design enforces. + +### Data Contracts & Integration + +**YAML section → `ClusterConfig` field mapping** (all 13 fields, `controller.capnp:694-708`): + +| capnp field (`@N`) | YAML source | Kind | Req | +|---|---|---|---| +| `clusterName` (@0), `heartbeatInterval` (@1), `inactiveHeartbeatCount` (@2), `fetchJobInterval` (@3), `netioTimeout` (@4) | `xce.yaml` (flat keys `cluster_name`, `heartbeat_interval`, `inactive_heartbeat_count`, `fetch_job_interval`, `netio_timeout`) | scalar | 1.11 | +| `resources` (@5) | `resources.yaml` → `resources[]` | list | 1.1 | +| `queues` (@6) | `queues.yaml` → `queues[]` | list | 1.1 | +| `ugroups` (@7) | `users.yaml` → `usergroups[]` | list | 1.1 | +| `hgroups` (@8) | `hosts.yaml` → `hostgroups[]` | list | 1.1 | +| `hostConfigs` (@9) | `hosts.yaml` → `hosts[]` | list | 1.1 | +| `hproviders` (@10) | `scaler.yaml` → `scalers[]` | list | 1.12 | +| `limits` (@11) | `limits.yaml` → `limits[]` | list | 1.1 | +| `configGeneration` (@12) | none — controller-assigned | excluded | 1.9 | + +`hosts.yaml` is the one file feeding two capnp fields (`hgroups` + `hostConfigs`) — `xcecommon.mapping`'s `SECTION_MAP` has two entries pointing at the same `yaml_file` with different `yaml_key`/`capnp_field` pairs; this is the only many-to-one case in the table and is called out explicitly here so it is not mistaken for a bug during implementation. + +Per-field YAML-key-to-capnp-field-name renames (e.g., `groupname`→`GroupConfig.name`, `exclude_members`→`exmembers`) are enumerated in `research.md` (too granular for this table) and enforced by `xcecommon.mapping`'s per-section spec, not restated here. + +### Logical Data Model +`cluster.bin` has no indexing/sharding concerns (single-file capnp blob, read once at controller startup/config-push time). Byte-for-byte determinism (9.1, 9.2, 9.3) is the only "consistency" property that matters, guaranteed by: (a) `SECTION_MAP` iterating YAML list order verbatim, never re-sorting; (b) no wall-clock/host/user value ever entering a mapped field; (c) capnp's own canonical single-segment encoding for a message built once and never mutated after `.to_bytes()`. + +## Error Handling + +### Error Strategy +Every `xcecommon`/`xceadmin` internal exception is caught once, at the `xceadmin.cli` boundary, and re-raised as `click.ClickException(message)` — giving a single place that guarantees "clear, actionable error, non-zero exit" (7.1, 7.2, 7.3, 7.4) without scattering `sys.exit()` calls through the library layer (library code raises; only the CLI layer exits). + +### Error Categories and Responses +| Category | Trigger | Response | Req | +|---|---|---|---| +| Connection | TLS connect fails | `"cannot connect to : "`, exit non-zero | 7.1 | +| Protocol | `Reply.status != 0` | status code + reply error text, exit non-zero | 7.2 | +| Decode | corrupt/truncated capnp bytes | `"invalid cluster.bin: "`, exit non-zero | 7.3 | +| Environment | schema file missing at expected path | missing path + installation hint | 7.4 | +| Interrupt | `SIGINT` during push | close TLS socket, `"cancelled"`, graceful exit | 7.5 | +| Timeout | any network op > 30s | `"timeout connecting to "` | 7.6 | + +### Monitoring +Out of scope — `xceadmin` is a one-shot CLI with stderr/exit-code as its only observability surface; no logging framework, no metrics endpoint (Simplification — a persistent log/metrics story belongs to a daemon, not a CLI invoked from a shell or a skill). + +## Testing Strategy + +- **Unit — schema bundling**: `default_schema_path()`/`default_schema_dir()` resolve to a real, non-empty, parseable `controller.capnp`/JSON schema set under whatever install is currently active — trivial in isolation, but only meaningful when it actually runs against a **non-editable** install; every other test in this list runs fine against an editable install even if `package_data` packaging is broken, since editable installs keep resolving into the checked-out source tree regardless. This is what `/xce-build xceadmin release` (8.2) validates that `/xce-build xceadmin` (8.1) cannot — a gap discovered post-design-approval (see `research.md`) that the schema-bundling fix itself would otherwise ship untested. +- **Unit — mapping**: `SECTION_MAP` list-section and scalar-section mappers each produce the expected capnp field values from a minimal in-memory dict (no file I/O) — one test per `kind`, plus the `hosts.yaml`-feeds-two-fields case explicitly. +- **Unit — wire framing**: `send_frame`/`recv_frame` round-trip against a fixed byte sequence, and against a loopback `socket.socketpair()` — verifies native-order packing matches `struct.pack("=Q", ...)` semantics without needing a real controller. +- **Integration — config build (golden-file)**: `minimal`/`full`/`invalid` fixture directories (File Structure Plan) — `full` now covers all 7 YAML files including the newly-added `xce.yaml`/`scaler.yaml`; byte-compares `build_cluster_config()` output against a committed `expected.bin` and its SHA-256 (1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 9.1, 9.2, 9.3). Plain `assert produced == expected_bytes` fixture comparison — `pytest-regressions` was evaluated (see `research.md`) but not adopted, since this project's nix-pinned dependency list would need independent verification and the byte-equality need here is simple enough not to warrant it (Simplification). +- **Integration — validation**: one test per `invalid/` fixture case — schema violation, reserved name, duplicate name (including a limit/queue name collision, proving 4.3's intentionally-narrow scope), undefined reference (warn, not fail). +- **Integration — config push (mock controller)**: a Python TLS server speaking just enough of the wire protocol (frame in, decode `Request` or direct-struct `ProbeConfigTagRequest`, encode a canned `Reply`/`ProbeConfigTagReply`) to exercise `push_config()` (2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 3.1, 3.2, 3.3 redirect, 7.1, 7.2, 7.3) and `probe_config()` (6.1, 6.2, 6.3 — accepted and rejected probe replies) against the same mock, since both share `xcecommon.wire`'s framing/send-receive primitives. **This is the only realistic way to test config push today** — not merely "preferred for the dev loop" as `brief.md` originally framed it, but load-bearing: the controller-side dispatch-gate defect found during discovery (Boundary Commitments) means a real 3-node cluster currently rejects `OP_upgradeConfig` from every role, so live-cluster E2E (`brief.md`'s "Option B") is **blocked**, not merely deferred, until that controller defect is fixed (Revalidation Trigger). +- **E2E (deferred)**: real-cluster `config push` round-trip via `xce-test-ha` — tracked in `brief.md`, explicitly gated on both the controller dispatch fix and `config-upgrade-snapshot` landing. + +## Security Considerations + +- **Transport is encryption-only, not authenticated** — no peer-certificate validation exists anywhere in XCE's controller-to-controller TLS today (`SSL_VERIFY_NONE`, no `X509_check_host`); `xceadmin` matches this exactly rather than silently adding (and thereby implying) a stronger guarantee the rest of the system doesn't have. An attacker with network access to the controller port can MITM either direction. This is a pre-existing, project-wide posture (`product.md` — "trust OS boundary"), not something this spec introduces or should unilaterally fix. +- **Input validation at the boundary** — every YAML file is JSON-Schema-validated before any byte of it reaches capnp encoding (4.1), consistent with `security.md`'s "validate at the API boundary, never trust external data." +- **No secrets in xceadmin** — no JWT, no RBAC, no credential storage; trust model is "same TLS cert bundle as the controller cluster," matching `xce_test`'s existing zero-config posture. Nothing new to rotate or leak. +- **Reply decoding must not crash on malformed input** — `xcecommon.wire.send_request`'s `Reply.from_bytes()` call is wrapped to convert a pycapnp decode exception into `7.3`'s "invalid" error path rather than an unhandled traceback, even though the reply source (a controller within the operator's own cluster) is semi-trusted. + +## Supporting References +- Full per-field YAML-key ↔ capnp-field-name rename table, pycapnp API usage notes, and the two controller-side defect write-ups: `research.md`. +- Nix package list, harness hook JSON (ruff format/check, YAML schema guard), and settings allowlist: `brief.md` Harness Enhancement section (unchanged, referenced not restated here). diff --git a/.kiro/specs/admin-config-tool/requirements.md b/.kiro/specs/admin-config-tool/requirements.md index 0914dddc..27e06d58 100644 --- a/.kiro/specs/admin-config-tool/requirements.md +++ b/.kiro/specs/admin-config-tool/requirements.md @@ -2,7 +2,7 @@ ## Introduction -XCE operators need a tool to build and deploy cluster configuration. Currently, YAML config files (`configs/{hosts,queues,resources,users}.yaml`) exist with JSON schemas for validation, but no tool compiles them into the `ClusterConfig` Cap'n Proto binary the controller expects. The controller-side `OP_upgradeConfig` handler exists (`config_repl.c` — receives binary, verifies SHA-256, pushes to standbys, awaits majority ACK), but no CLI sends it. +XCE operators need a tool to build and deploy cluster configuration. Currently, YAML config files (`configs/{hosts,queues,resources,users,limits,xce,scaler}.yaml`) exist with JSON schemas for validation, but no tool compiles them into the `ClusterConfig` Cap'n Proto binary the controller expects. The controller-side `OP_upgradeConfig` handler exists (`config_repl.c` — receives binary, verifies SHA-256, pushes to standbys, awaits majority ACK), but no CLI sends it. `xceadmin` is a Python CLI that fills this gap: `config build` compiles YAML into `cluster.bin`, `config push` sends it to a controller over TLS+capnp. @@ -19,7 +19,6 @@ Config source of truth is git-managed YAML. `cluster.bin` is a derived artifact - `xcecommon` shared Python package (capnp schema, wire protocol, config models) - Integration with xce-build and xce-deploy-ha skills - Nix dev shell: Python package dependencies - - `config diff` — human-readable diff of two built config binaries (nice-to-have) - `config push --dry-run` — validate without committing (nice-to-have) - **Out of scope**: @@ -30,6 +29,7 @@ Config source of truth is git-managed YAML. `cluster.bin` is a derived artifact - `configGeneration` management — fully controller-owned (see Adjacent Expectations below); `xceadmin` never reads, increments, or overrides it. - xcecli Python rewrite (future, separate spec) - Controller config version history, rollback safety validation, config activation status (owned by `config-upgrade-snapshot`, see Future Roadmap in brief.md) + - `config diff` — deferred to roadmap follow-up (2026-07-21): no operator has asked for it yet; revisit once `config push` has real-world usage (see brief.md R1) - **Adjacent expectations**: - **Controller `OP_upgradeConfig` handler** (`config_repl.c`) exists and is operational. On receiving `UpgradeConfigRequest{sha256, bytes}`: @@ -40,17 +40,17 @@ Config source of truth is git-managed YAML. `cluster.bin` is a derived artifact 5. Commit `WAL_CONFIG_MARKER`, signal controller restart 6. Return `UpgradeConfigReply{versionTag, ackedCount, error}` - **Controller redirect**: Standby returns `redirectHost`/`redirectPort` in the `RequestReply` union — xceadmin follows the redirect to reach the primary. - - **Per-instance field preservation** — RESOLVED structurally, not by controller behavior: `port`, `candidates`, the hash-table-size fields, and `xstrings` are being removed from the `ClusterConfig` capnp struct entirely, owned by the new `clusterconfig-schema-split` spec. Once that lands, there is no zero-default-clobber risk to preserve against — these fields simply won't exist in `cluster.bin`. - - **ProbeConfigTag RPC** (`config_repl.c:1774`): Separate direct-capnp RPC (not in `Request` union) for reading current config tag. Used by `config push --dry-run` to validate before committing. + - **Per-instance field preservation** — RESOLVED structurally, not by controller behavior: `port`, `candidates`, the hash-table-size fields, and `xstrings` have been removed from the `ClusterConfig` capnp struct entirely (`clusterconfig-schema-split` spec, landed commit `519e11c`). `ClusterConfig` is now 13 fields (@0–@12, `controller.capnp:694-708`); these fields simply do not exist in `cluster.bin` — there is no zero-default-clobber risk to guard against. Req 1.8/1.9/1.10 remain as defense-in-depth documentation (if any of these fields is ever reintroduced, xceadmin must still not source it), not as active field-suppression logic. + - **ProbeConfigTag RPC** (`xce_probe_config_tag()`, `config_repl.c:1829`): Separate direct-capnp RPC (not in `Request` union) for reading current config tag. Used by `config push --dry-run` to validate before committing. - `controller.capnp` schema is the single source of truth for `ClusterConfig`, `Request`/`Reply` unions, and all OP codes. - `config-upgrade-snapshot` spec will extend the controller-side pipeline with: entity-bearing snapshots after config change, rollback safety validation, version history tracking, config activation status. - **Blocked controller-side gaps** (NOT owned by admin-config-tool): - Entity-bearing snapshot after config upgrade (blocked on `entity-bearing-snapshots` spec) - Config rollback safety (e.g., reject rollback while jobs reference deleted queues) - Config version history query (`xceadmin config history`) - - **Dependency on `clusterconfig-schema-split`** — split by subcommand: - - `config build`: no dependency. Req 1.8/1.10 already exclude these fields from YAML/CLI regardless of schema state — safe to build in parallel, ship anytime. - - `config push`: **soft-blocked in practice**. Until `clusterconfig-schema-split` lands (fields actually removed from the schema) or someone independently verifies `config_repl.c`'s `do_upgrade_config_with_bytes` preserves per-instance fields on a zero-valued push, running `xceadmin config push` against a real cluster risks silently wiping `candidates` (peer list — HA/Raft membership), `port`, and perf-tuning on every node it touches. That's not a hypothetical: nobody has checked the controller does the right thing here today. Recommend `clusterconfig-schema-split` (or equivalent verification) land before `config push` is exercised against anything but a disposable test cluster; `config build` and the rest of the CLI can proceed in parallel. + - **Dependency on `clusterconfig-schema-split`** — **RESOLVED, landed** (commit `519e11c`, phase=`implementation-complete`). Both subcommands are now unblocked: + - `config build`: no dependency, unaffected either way. + - `config push`: previously soft-blocked over the risk of `xceadmin` silently wiping `candidates`/`port`/perf-tuning on a zero-valued push. That risk no longer exists — those fields are physically absent from the `ClusterConfig` schema, so there is nothing for a push to clobber. `config push` may be exercised against a real cluster without this precondition. ## Requirements @@ -60,19 +60,20 @@ Config source of truth is git-managed YAML. `cluster.bin` is a derived artifact #### Acceptance Criteria -1. When the operator runs `xceadmin config build `, the system shall scan the directory for YAML config files (`hosts.yaml`, `queues.yaml`, `resources.yaml`, `users.yaml`), validate each against its corresponding JSON schema from the `schema/` subdirectory, map the validated data to the `ClusterConfig` capnp struct, serialize it to binary, and write the result to `cluster.bin` in the current working directory. +1. When the operator runs `xceadmin config build `, the system shall scan the directory for YAML config files (`hosts.yaml`, `queues.yaml`, `resources.yaml`, `users.yaml`, `limits.yaml`, `xce.yaml`, `scaler.yaml`), validate each against its corresponding JSON schema bundled with `xceadmin`, map the validated data to the `ClusterConfig` capnp struct, serialize it to binary, and write the result to `cluster.bin` in the current working directory. 2. When the operator specifies `-o `, the system shall write the output to the given path instead of the default `cluster.bin`. 3. When the operator specifies `-o -`, the system shall write the binary output to stdout. 4. After writing output, the system shall print the SHA-256 hash of the resulting binary to stderr. 5. Where a YAML file is missing from the configs directory, the system shall treat the corresponding section as empty and continue building (missing files are not errors). -6. Where the `schema/` subdirectory is missing from the configs directory, the system shall report an error and exit non-zero. +6. **RESOLVED, superseded (2026-07-21)**: JSON Schema validation is performed against schemas bundled with the `xceadmin` package (see Requirement 5.5), not a `schema/` subdirectory the operator supplies. `configs_dir` therefore never needs its own `schema/` folder, and there is no "missing schema" error mode — the schemas travel with the tool, version-locked to the same release as the `ClusterConfig` field mapping they validate against (`xcecommon.mapping`), the same rationale already applied to `controller.capnp` (see Requirement 5.5, brief.md). An operator-supplied `schema/` subdirectory, if present, is ignored — not read, not merged, not treated as an override. 7. Where the YAML content passes JSON Schema validation but maps to fields not present in the `ClusterConfig` capnp struct, the system shall warn the operator about unrecognized fields. 8. The system shall not source `port`, the `jobmgr`/`workermgr`/`qmgr`/`resmgr` hash-table-size fields, or `xstrings` (xstring hash size) from YAML or CLI input under any circumstance. These are per-controller-instance performance-tuning parameters, not cluster-shared configuration, and the built `cluster.bin` shall leave them unset (capnp default). 9. The system shall not source, read, or write `configGeneration`. The built `cluster.bin` shall leave it unset (capnp default) — the primary controller is the sole authority for assigning it (see Requirement 2 note). 10. The system shall not source `candidates` (controller peer/standby list) from YAML or CLI input. Standby membership changes independently of, and more frequently than, the rest of cluster configuration, so it shall not require a `config build`/`config push` cycle; the built `cluster.bin` shall leave `candidates` unset (capnp default). Peer membership sourcing mechanism is out of scope for this spec (open question — see brief.md Open Design Question #3). -11. `clusterName` and the `heartbeatInterval`/`inactiveHeartbeatCount` fields (controller→worker liveness — not the HA/Raft election heartbeat, which has no `ClusterConfig` field, see `.kiro/steering/config-design.md` Rule 0) SHALL be sourced from YAML and included in `cluster.bin` — these must stay consistent across all controller instances (unlike Criteria 8–10, which are intentionally excluded). Source location (existing YAML file vs. new `cluster.yaml`) is a design-phase decision. +11. `clusterName`, `heartbeatInterval`/`inactiveHeartbeatCount` (controller→worker liveness — not the HA/Raft election heartbeat, which has no `ClusterConfig` field, see `.kiro/steering/config-design.md` Rule 0), `fetchJobInterval` (scheduler job-fetch pass interval), and `netioTimeout` (network I/O timeout) SHALL be sourced from a new `xce.yaml` file and included in `cluster.bin` — these must stay consistent across all controller instances (unlike Criteria 8–10, which are intentionally excluded). **RESOLVED**: source is `xce.yaml`, a flat (non-list) YAML file holding these 5 scalar `ClusterConfig` fields. `cluster_name` is required (no sensible default). `heartbeat_interval` and `fetch_job_interval` are optional — both have controller-side defaults (`fill_default_cluster_config()`, `myconf.c:72-76`: 5 and 10 respectively). `inactive_heartbeat_count` is required and has **no** controller-side default — `cluster.c:1622` computes the worker-dead timeout as `heartbeat_interval * inactive_heartbeat_count`, so an unset (0) value would mark workers dead immediately; xceadmin must not let this ship unset. `netio_timeout` is optional; note it currently has no reader anywhere in the codebase outside the struct declaration itself (`clusterconfig.h:12`) — the field is wired on the wire/schema but not yet consumed by any controller logic. That's a controller-side gap, out of scope for this spec, but xceadmin should still source it from YAML if present rather than silently drop it. +12. `hproviders` (`List(ScalerConfig)` — autoscaler provider definitions: `name`, `type`, `conf_path`, `script_path`, `script_option`) SHALL be sourced from a new `scaler.yaml` file and included in `cluster.bin`. Scaler definitions must be identical across controller instances (a queue configured against a scaler must resolve it the same way after failover) and change rarely — both Configuration Inclusion Rules fail, so this field is included per the Default rule. -Criteria 8–11 apply the **Configuration Inclusion Rules** (brief.md): a field is excluded from `cluster.bin` if it can change independently of the rest of cluster config, OR if it's expected to change frequently; otherwise it's included. Any future `ClusterConfig` field proposed for xceadmin's YAML mapping must be classified against these two rules first. +Criteria 8–12 apply the **Configuration Inclusion Rules** (brief.md): a field is excluded from `cluster.bin` if it can change independently of the rest of cluster config, OR if it's expected to change frequently; otherwise it's included. Any future `ClusterConfig` field proposed for xceadmin's YAML mapping must be classified against these two rules first. ### Requirement 2: Config Push @@ -107,8 +108,9 @@ Criteria 8–11 apply the **Configuration Inclusion Rules** (brief.md): a field 1. When a YAML file fails JSON Schema validation, the system shall report the specific validation error with file name, line/field path, and the constraint violated, then exit non-zero. 2. When a YAML file uses the reserved name `"lost_and_found"` for a queue or host, the system shall reject it with a clear error message explaining the reservation, even if it passes schema validation. -3. When a YAML file contains duplicate entries (duplicate queue names, host names, resource names, group names), the system shall report the duplicates and exit non-zero. -4. When a YAML file references an undefined entity (e.g., a queue references a host group that is not defined), the system shall warn the operator but allow the build to proceed (the controller performs final referential integrity checks at runtime). +3. When a YAML file contains duplicate entries (duplicate queue names, host names, resource names, group names, limit names — each checked within its own file/section; a limit name is not required to differ from a queue, host, or group name), the system shall report the duplicates and exit non-zero. +4. When a YAML file references an undefined entity (e.g., a queue references a host group that is not defined, or a limit references a user/queue/project/host group or a `resname` that is not defined in `resources.yaml`), the system shall warn the operator but allow the build to proceed (the controller performs final referential integrity checks at runtime). +5. `resources.yaml` is the exhaustive resource catalog: every `resname` referenced elsewhere (e.g., `limits.yaml`, `queues.yaml` `sched_stop`/`res_req`/`gpu_req`, `hosts.yaml` `sched_stop`) — including built-in/dynamic resources such as `slots`, `mem`, and the 8 per-host attributes the worker registers automatically (`type`, `model`, `cpuf`, `maxcpus`, `maxmem`, `maxswap`, `maxtmp`, `ndisks` — see `add_hostattrs_resvals()`, `controller/worker.c`) — shall be declared as an entry in `resources.yaml`. The system shall not treat any resource name as an implicit built-in exempt from this catalog; Criterion 4's undefined-entity warning applies equally to built-in resource names. ### Requirement 5: xcecommon Shared Package @@ -120,18 +122,9 @@ Criteria 8–11 apply the **Configuration Inclusion Rules** (brief.md): a field 2. The `xcecommon` package shall provide a wire protocol module that handles TLS connection, capnp message framing (8-byte uint64 size prefix in host byte order), send, and receive. 3. The `xcecommon` package shall provide YAML→capnp mapping functions that convert validated YAML data to `ClusterConfig` capnp struct fields. 4. Where the `xcecommon` package is installed for development, it shall be importable under the name `xcecommon` from any tool within the project. +5. **(2026-07-21)** The `xcecommon` package shall bundle its own copy of `controller.capnp`/`c.capnp` and the 7 JSON Schema files (one per `config build` input file), synced from `src/schema/` and `src/configs/schema/` respectively at build time (see Requirement 8.1/8.2) and packaged so they are available regardless of install mode (`pip install -e .` or `pip install .`). Neither schema is read from a path outside the installed package at runtime; an operator-supplied `configs_dir/schema/` directory, if present, is not consulted (superseded Requirement 1.6). -### Requirement 6: Config Diff (Nice-to-Have) - -**Objective:** As an XCE operator, I want to see what changed between two config binaries, so that I can review changes before pushing. - -#### Acceptance Criteria - -1. When the operator runs `xceadmin config diff `, the system shall decode both binaries, compare them field-by-field, and display a human-readable summary of additions, removals, and modifications. -2. The diff output shall group changes by section (queues, resources, hosts, users, limits, scalers) and show the specific fields that differ. -3. When the two binaries are identical, the system shall report "no differences" and exit 0. - -### Requirement 7: Dry-Run Push (Nice-to-Have) +### Requirement 6: Dry-Run Push (Nice-to-Have) **Objective:** As an XCE operator, I want to validate that the controller will accept a config before committing to the push. @@ -141,7 +134,7 @@ Criteria 8–11 apply the **Configuration Inclusion Rules** (brief.md): a field 2. When dry-run passes, the system shall report "OK: controller will accept this config (generation N→M)" and exit 0. 3. When dry-run fails, the system shall report the specific rejection reason and exit non-zero. -### Requirement 8: Error Handling & Resilience +### Requirement 7: Error Handling & Resilience **Objective:** As an XCE operator, I want clear, actionable error messages for all failure modes, so that I can diagnose and fix configuration issues without reading controller logs. @@ -154,20 +147,20 @@ Criteria 8–11 apply the **Configuration Inclusion Rules** (brief.md): a field 5. If the operator interrupts the push (SIGINT), the system shall clean up the TLS connection and exit gracefully with a "cancelled" message. 6. The system shall time out after 30 seconds on any network operation and report "timeout connecting to ". -### Requirement 9: Skill & Harness Integration +### Requirement 8: Skill & Harness Integration **Objective:** As a developer, I want xceadmin integrated into the existing development toolchain, so that building, deploying, and testing XCE clusters uses consistent tooling. #### Acceptance Criteria 1. When the developer runs `/xce-build xceadmin`, the system shall make the `xceadmin` CLI available on PATH and run the full test suite, reporting pass/fail. -2. When the developer runs `/xce-build xceadmin release`, the system shall install `xceadmin` for production use (pinned, non-editable). +2. When the developer runs `/xce-build xceadmin release`, the system shall install `xceadmin` for production use (pinned, non-editable) **and then run the full test suite against that non-editable install**, reporting pass/fail — this is the only path that exercises the packaged (`pip install .`) resolution of the bundled `controller.capnp`/JSON schemas (Requirement 5.5); an editable-install-only test run would never catch a regression there. 3. When the developer runs `/xce-deploy-ha` with the `--build-config` flag, the system shall invoke `xceadmin config build configs/ -o ha/data/cluster.bin` as a precondition before deploying the cluster. 4. When `cluster.bin` is missing from `ha/data/` during deploy, `/xce-deploy-ha` shall run `xceadmin config build` automatically (same as `--build-config`). 5. When `cluster.bin` already exists and `--build-config` is not passed, `/xce-deploy-ha` shall use the existing binary as-is (backward compatible). 6. The nix dev shell (`nix develop ./src`) shall include all Python dependencies required by xceadmin after the nix environment is updated (see Nix Environment section in brief.md). -### Requirement 10: Deterministic Builds +### Requirement 9: Deterministic Builds **Objective:** As an XCE operator, I want config builds to be deterministic, so that I can reproduce any historical config binary from its YAML source. @@ -186,8 +179,7 @@ Criteria 8–11 apply the **Configuration Inclusion Rules** (brief.md): a field | 3 | Auto Primary Discovery | **Core** | Implementation | | 4 | Validation | **Core** | Implementation | | 5 | xcecommon Package | **Core** | Implementation | -| 6 | Config Diff | Nice-to-Have | Follow-up | -| 7 | Dry-Run Push | Nice-to-Have | Follow-up | -| 8 | Error Handling | **Core** | Implementation | -| 9 | Skill & Harness | **Core** | Implementation | -| 10 | Deterministic Builds | **Core** | Implementation | +| 6 | Dry-Run Push | Nice-to-Have | Follow-up | +| 7 | Error Handling | **Core** | Implementation | +| 8 | Skill & Harness | **Core** | Implementation | +| 9 | Deterministic Builds | **Core** | Implementation | diff --git a/.kiro/specs/admin-config-tool/research.md b/.kiro/specs/admin-config-tool/research.md new file mode 100644 index 00000000..5690dfe4 --- /dev/null +++ b/.kiro/specs/admin-config-tool/research.md @@ -0,0 +1,158 @@ +# Research & Design Decisions: admin-config-tool + +## Summary +- **Feature**: `admin-config-tool` +- **Discovery Scope**: Complex Integration — first Python component in a C/Go codebase, first external client of the controller's raw TLS wire protocol, no in-repo precedent for either. +- **Key Findings**: + - The controller's `OP_upgradeConfig` network dispatch path is currently broken for every role (standby rejected by `ha_dispatch_gate`, primary rejected by an inverted `ha_is_primary()` check in `cluster.c`) — a controller-side defect discovered during this design's discovery phase, out of this spec's boundary but blocking real E2E validation of `config push`. + - Redirect (`redirectHost`/`redirectPort`) is wired only for `OPCode_join`, never `OPCode_upgradeConfig` — auto-primary-discovery (Req 3) has no live data to act on for config push until the controller is extended. + - `pycapnp` handles `controller.capnp`'s `$C.mapname(...)` annotations without issue (declared, inert metadata under dynamic reflection) — no custom-annotation workaround needed. + - `src/frontend/xcecli/meson.build`'s `go_build`-wrapper + `custom_target()` pattern is a directly reusable template for `src/xceadmin/meson.build`, resolving brief.md's open question on Python package location/build integration. + +## Research Log + +### TLS setup and certificate validation +- **Context**: `config push` needs to establish the same trust model the controller cluster already uses (brief.md Req 2.3-2.4 assumption). +- **Sources Consulted**: `src/lib/netio.c:1208-1243` (`create_ssl_context`), `src/controller/myapp.c:94-109`, `src/controller/main.c:374-375,90`, `src/controller/join.c:13-16`. +- **Findings**: + - `create_ssl_context(cert_file, key_file)` builds a client context (`TLS_client_method()`) when no cert/key is given, a server context otherwise. `myconf->certfile`/`keyfile` resolve to `/xce.crt`/`xce.key`. + - The controller's own outbound client context explicitly sets `SSL_CTX_set_verify(client_ssl_ctx, SSL_VERIFY_NONE, NULL)` (`myapp.c:109`). + - No `SSL_CTX_set_verify(..., SSL_VERIFY_PEER, ...)`, `X509_check_host`, or `SSL_get_peer_certificate` call exists anywhere in the codebase. `validate_host()` (`join.c:13-16`) is an unrelated stub (`return 1;`) used only in the cluster-join path, not TLS. + - No minimum TLS version or cipher-suite pinning exists (`SSL_CTX_set_min_proto_version`/`set_ciphersuites` not found). +- **Implications**: `xceadmin` should connect with `ssl.CERT_NONE`/`check_hostname=False` to match current behavior exactly — this is encryption-only transport security throughout XCE today, not a weakening introduced by this spec. Captured in design.md Security Considerations. + +### Wire framing byte order +- **Context**: brief.md asserted "8-byte uint64 size prefix in host byte order" — needed independent confirmation before committing a Python `struct.pack` format string. +- **Sources Consulted**: `src/include/netio.h:30`, `src/lib/netio.c:111-199`, `src/lib/request.c:863-938`, `src/controller/config_repl.c:1925-1968`. +- **Findings**: `io_packet_t.sz` (`uint64_t`) is written/read as raw memory (`&pkt->sz`, `sizeof(...)`) directly into `SSL_read`/`SSL_write` — no `htole64`/`htobe64`/`ntohll` anywhere in the path. `request.c:863-864` states this explicitly in a comment; `config_repl.c`'s `xce_probe_config_tag` send/recv code (`:1925-1968`) follows the identical pattern. +- **Implications**: Python must use `struct.pack("=Q", size)` (native byte order), not network byte order (`">Q"`). Documented in design.md `xcecommon.wire`. + +### OP_upgradeConfig dispatch path +- **Context**: Verifying the request would actually reach `do_upgrade_config_with_bytes()` on a real cluster before designing around it as a working dependency. +- **Sources Consulted**: `src/controller/ha.c:3794-3960` (`ha_dispatch_gate`), `src/controller/cluster.c:238-682` (`myreq_handler`), `src/controller/config_repl.c:1591-1810`, `src/controller/config_repl.h:347-349`, `src/controller/testsuite/test_config_repl.c:2042`, `test_commit_barrier_rollout.c:1484,1531`. +- **Findings**: + - `ha_dispatch_gate()`'s `CTRL_STANDBY` allow-list (`ha.c:3926-3951`) does not include `OPCode_upgradeConfig` (only `upgradeConfigAck` is listed) — standbys are rejected with `-XCE_ERR_STANDBY` before reaching the opcode switch. + - For `CTRL_PRIMARY` with quorum, the gate falls through (`ha.c:3880`) — primaries pass the gate. + - But `cluster.c:667-679`'s in-switch handler does `if (ha_is_primary(&c->ha)) { /* reject, -EAGAIN */ } else { handle_upgrade_config(...) }` — i.e., it only calls the real handler for a **non-primary**, the opposite of what passed the gate. + - Net: standby → rejected at the gate; primary → passes the gate, rejected by the inverted in-switch check. No role reaches `handle_upgrade_config()`/`do_upgrade_config_with_bytes()` via the network today. That function is exercised only by unit tests calling it directly. + - `config_repl.h:347-349`'s doc comment ("handle an OP_upgradeConfig request from an admin tool") and the handler's actual logic (WAL marker commit, `push_config_to_standbys`, restart signal) both imply it should run on the **primary** — the `cluster.c:671` guard looks inverted/stale relative to both the dispatch gate and the handler's own semantics. +- **Implications**: This is a controller-side defect, not an xceadmin gap — out of this spec's boundary (see design.md Boundary Commitments), but it makes real-cluster E2E testing of `config push` currently impossible. Testing Strategy in design.md treats the mock-controller test suite as load-bearing, not merely a dev-loop convenience, until this is fixed. Recommend logging this as a bug/blocking item outside this spec's implementation tasks. + +### Redirect wiring scope +- **Context**: Requirement 3 (Auto Primary Discovery) assumes `redirectHost`/`redirectPort` get populated on a standby's reply to any request, including `upgradeConfig`. +- **Sources Consulted**: `src/schema/controller.capnp:573-595` (`RequestReply`), `src/controller/join.c:27-31,64,277`. +- **Findings**: `fill_redirect_reply()` is defined at `join.c:27-31` and its only production call site is `join.c:277`, inside the `OPCode_join` handling path. No other opcode handler calls it. +- **Implications**: `xceadmin.discovery`'s redirect-following is implemented generically against the `RequestReply` schema shape (correct for any opcode that does populate these fields, e.g. `join`), but will not actually fire for `config push` until the controller extends `fill_redirect_reply()`'s call sites to include `upgradeConfig`. Documented as an Out-of-Boundary item and Revalidation Trigger in design.md. + +### ProbeConfigTag liveness +- **Context**: `config push --dry-run` (Requirement 6) was planned around reusing `ProbeConfigTag`. +- **Sources Consulted**: `src/controller/config_repl.c:1811-1998` (`xce_probe_config_tag` + doc comment), `src/schema/controller.capnp:821-827`, `ha.c:7409`, `ha.c:3828,3860,3889,3911,3943` (dispatch-gate allow-lists). +- **Findings**: `xce_probe_config_tag()` is a direct-struct RPC (no `Request`/`Reply` wrapper) with only test callers (`test_ha_bootstrap.c:259` and a mock in `test_ha_bootstrap_support.c`). No server-side dispatch case for `OPCode_probeConfigTag` exists anywhere in `cluster.c`, despite `ha_dispatch_gate` listing the opcode as allowed for every role — the allow-list entry is moot because nothing answers it. A stale doc comment at `ha.c:7409` claims it's "already called in myapp_run before ha_start_election_async"; no such call exists in `myapp.c` or anywhere in production code. +- **Implications**: `config push --dry-run`'s wire client can and should still be implemented against the documented contract (Requirement 6 remains valid future-facing design), but it has no live counterpart to call today. Reflected in design.md as an Out-of-Boundary item; Requirement 6 stays Nice-to-Have/Follow-up as already scoped in requirements.md. + +### No existing external-client precedent +- **Sources Consulted**: `src/tools/` (`xce-build.sh`, `deploy-ha.sh`, `go_build`, `appimage/entry.sh`), `src/controller/testsuite/test.c` (built as `xce_test`), `src/frontend/xcecli/common/request/request.go`, repo-wide `find . -iname "*.py"`. +- **Findings**: `src/tools/` has no socket/capnp client. `xce_test` is a WAL/snapshot file-inspection CLI with zero network code. `xcecli` (Go) speaks HTTP/JSON to `apiserver`, never the raw wire protocol. No Python file exists anywhere in the repository. +- **Implications**: `xceadmin` is the first client of this wire protocol from outside a C daemon — there is no reference implementation to diff against; correctness rests entirely on independently re-deriving the framing/union contract from the C source (done above), not on copying an existing pattern. + +### Build tooling precedent for a non-C, non-Meson-native component +- **Sources Consulted**: `src/tools/xce-build.sh:52,63-131`, `.claude/skills/xce-build/SKILL.md`, `.claude/skills/xce-deploy-ha/SKILL.md`, `src/nixSupport/xce.nix:18-70`, `src/frontend/xcecli/meson.build`, `src/meson.build:34-35,84-86`, `src/meson.options:3-6`. +- **Findings**: `xce-build.sh` dispatches on component name via a `case` statement; `apiserver`/`xcecli` branches additionally shell out to `make test`/`go test`. `xcecli`'s `meson.build` wraps `go build` via a `custom_target()` calling a generic `tools/go_build` script, gated by the `component` meson option (`combo`: `all|xcecore|xcecli`) and conditionally included via `if component != 'xcecore': subdir('frontend/xcecli')`. `xce.nix`'s `nativeBuildInputs` has zero Python packages today — confirmed genuinely absent, not just undocumented. +- **Implications**: `src/xceadmin/meson.build` should mirror `xcecli`'s pattern exactly (a new `tools/py_build` wrapper + `custom_target()` + a `component` choice or unconditional `subdir('xceadmin')`), resolving brief.md's open question #9 (Python package layout) in favor of `src/xceadmin/`, consistent with `structure.md`'s "each major subsystem lives under `src//`" convention. + +### pycapnp schema loading and message encoding +- **Context**: Confirm `controller.capnp`'s C-specific annotations (`$C.mapname(...)`) don't break pycapnp's schema compilation, and confirm the right API surface for non-RPC encode/decode. +- **Sources Consulted**: [pycapnp GitHub/README](https://github.com/capnproto/pycapnp), [pycapnp CHANGELOG](https://github.com/capnproto/pycapnp/blob/master/CHANGELOG.md), [Quickstart (2.2.4 docs)](https://capnproto.github.io/pycapnp/quickstart.html), [capnp.pyx source](https://github.com/capnproto/pycapnp/blob/master/capnp/lib/capnp.pyx). +- **Findings**: `$C.mapname(...)` is a properly declared annotation (via `c.capnp`'s `annotation mapname @0x...`), not an unknown construct — pycapnp's `DynamicStruct` reflection parses it as inert metadata; the only real failure mode is an unresolved `/c.capnp` import path, handled via `capnp.load(path, imports=[schema_dir])`. `msg.write(f)` calls `f.fileno()` and writes to the raw fd, bypassing any Python-level wrapper — safe for `open(path, 'wb')`, unsafe for a socket. `msg.to_bytes()`/`Struct.from_bytes()` are pure in-memory and the correct choice for the socket path. +- **Implications**: No custom annotation-stripping preprocessing needed before schema load. `xcecommon.wire` must use `from_bytes()`/`to_bytes()` exclusively (never `.write()`/`.read()` on a live socket) — captured as an explicit constraint in design.md. + +### pycapnp + Python ssl module composition +- **Findings**: pycapnp's async/KJ-loop machinery (`AsyncIoStream`, `capnp.run()`) is scoped to its RPC/vat-network layer (`TwoPartyClient/Server`) and does not intrude on plain synchronous `from_bytes()`/`to_bytes()` usage. `ssl.SSLContext().wrap_socket()` gives a normal `SSLSocket` whose `.recv()`/`.sendall()` perform the framing at the plaintext layer; pycapnp never touches the socket object directly in this design. +- **Implications**: Confirms the `xcecommon.wire` design (manual framing over `ssl.SSLSocket`, in-memory capnp encode/decode) has no hidden event-loop conflict. + +### click and pytest conventions +- **Findings**: Current stable `click` (8.3.x/8.4.x) idiom for grouped subcommands: `@click.group()` → `@cli.group()` → `@config.command()`, giving `xceadmin config build`/`xceadmin config push` naturally. `pytest-regressions`' `file_regression` fixture (binary-capable, deliberate `--force-regen` updates) is the more actively maintained golden-file library versus `pytest-golden` (YAML-only, no binary support) — evaluated but not adopted (see Design Decisions) in favor of a plain `assert produced == expected_bytes` fixture comparison, since the byte-equality need here doesn't need the extra dependency's ergonomics. + +## Architecture Pattern Evaluation + +| Option | Description | Strengths | Risks / Limitations | Notes | +|--------|-------------|-----------|----------------------|-------| +| Layered CLI over shared library (selected) | Thin `click` CLI (`xceadmin`) over a protocol/schema/mapping library (`xcecommon`) with no CLI dependency | Matches project's inward-dependency convention; `xcecommon` directly reusable by future Python `xcecli` (brief.md Future Roadmap); each layer independently testable | None material for this scope | — | +| Single-file script | One `xceadmin.py` doing everything | Fastest to write initially | Not reusable by future `xcecli`; violates project's "one domain per file" convention (`structure.md`); harder to unit-test wire framing in isolation | Rejected | +| Two separate pip distributions (`xceadmin`, `xcecommon`) from day one | Split packaging immediately | Cleaner dependency graph if `xcecommon` gets a second consumer soon | No second consumer exists yet; premature packaging overhead (Simplification lens) | Deferred, not rejected — revisit when Python `xcecli` work starts | + +## Design Decisions + +### Decision: `src/xceadmin/` as the package location +- **Context**: brief.md left this as Open Design Question #9 (candidates: `src/xceadmin/`, `tools/xceadmin/`, top-level `xceadmin/`). +- **Alternatives Considered**: 1) `tools/xceadmin/` alongside `deploy-ha.sh`. 2) top-level `xceadmin/`. 3) `src/xceadmin/`. +- **Selected Approach**: `src/xceadmin/`, mirroring `src/frontend/`'s existing precedent for a non-C, non-Meson-native language living directly under `src/` with its own build wrapper. +- **Rationale**: `structure.md`'s "Organization Philosophy" states "each major subsystem lives under `src//`" — `xcecli` (Go) already establishes that this applies across languages, not just C. Discovery confirmed a directly reusable `custom_target()` + wrapper-script pattern from `xcecli/meson.build`. +- **Trade-offs**: None material — this is the path of least resistance given an existing, working precedent. +- **Follow-up**: Confirm during implementation whether `component` meson option needs a new `xceadmin` choice or can use unconditional `subdir()` inclusion (both patterns exist for `xcecli` depending on option value — see `src/meson.build:84-86`). + +### Decision: plain byte-equality golden-file tests, not `pytest-regressions` +- **Context**: External research surfaced `pytest-regressions` as the more idiomatic 2025/2026-era binary golden-file library versus brief.md's unspecified "compare binary, verify SHA-256." +- **Alternatives Considered**: 1) `pytest-regressions` (`file_regression.check(..., binary=True)`). 2) `pytest-golden` (YAML-only, ruled out — no binary support). 3) Plain `assert produced_bytes == open(expected_path, 'rb').read()` in a parametrized fixture. +- **Selected Approach**: Option 3 — plain assertion. +- **Rationale**: `pytest-regressions` is not yet in this project's nix-pinned dependency list (`brief.md`'s Nix Environment section) and would need independent nixpkgs-availability verification before adoption; the actual requirement (9.1-9.3, byte-for-byte determinism against a committed fixture) doesn't need `file_regression`'s deliberate-update ergonomics for a small, stable fixture set (Simplification lens). +- **Trade-offs**: Loses `--force-regen`'s one-flag golden-file update convenience; gains zero new dependencies to verify/pin. +- **Follow-up**: Revisit if the fixture set grows large enough that manual golden-file regeneration becomes a recurring pain point. + +### Decision: generic (opcode-agnostic) redirect-following and wire exchange +- **Context**: Discovery found `fill_redirect_reply()` only wired for `OPCode_join`, and two distinct message shapes (`Request`/`Reply`-wrapped vs. `ProbeConfigTagRequest`/`Reply` direct-struct) both need send/receive. +- **Alternatives Considered**: 1) An `upgradeConfig`-specific redirect function, built now and generalized later if `dry-run`/other opcodes need it. 2) One generic `follow_redirects()` against the `RequestReply` shape, and one generic `send_request()`/framing pair reused by both message shapes. +- **Selected Approach**: Option 2, generic from the start. +- **Rationale**: Generalization lens — both consumers (upgradeConfig today, join-shaped mocks in tests, potentially dry-run's direct-struct variant) are instances of the same "read `RequestReply`-shaped fields, frame-and-exchange bytes" problem; building the opcode-specific version first and generalizing later would mean deleting code, not adding it (the redirect logic and framing logic are trivial regardless — no meaningful cost to being generic from day one). +- **Trade-offs**: None material. +- **Follow-up**: None. + +### Decision: `push_config()`/`probe_config()` split into two functions, not one boolean-flagged function (Design Review correction, 2026-07-21, second pass) +- **Context**: A second `/kiro-validate-design` pass found the first draft's `push_config(..., dry_run: bool) -> PushResult` implied a dry-run reply has `acked_count`, which `ProbeConfigTagReply` (only a `configTag` field) does not have — the shared return type would misrepresent one of the two call shapes. +- **Alternatives Considered**: 1) Keep one function, make `PushResult` fields `Optional` for the dry-run case. 2) Two functions, two result types. +- **Selected Approach**: Option 2 — `push_config() -> PushResult` and `probe_config() -> DryRunResult`. +- **Rationale**: The two wire messages and replies share no fields; a boolean flag silently switching which fields are populated is a Type Safety anti-pattern this project's `design-principles.md` implicitly warns against ("discriminated unions for error handling" — same spirit applies to result shapes). +- **Trade-offs**: None material — strictly a clarity improvement over the first draft. +- **Follow-up**: None. + +### Decision: schema files are packaged as `package_data`, not referenced from `src/schema/` at runtime +- **Context**: User question after design approval — does `pip install .` (Req 8.2, non-editable) actually work? It doesn't as originally drafted: non-editable installs copy files into `site-packages`, severing any fixed relative path back to `src/schema/controller.capnp`. A `__file__`-relative walk (e.g. `Path(__file__).parents[3] / "schema"`) only survives editable installs, where `__file__` still resolves into the checked-out repo tree. +- **Alternatives Considered**: 1) Require `--schema-path` on every invocation (no default) — rejected, bad ergonomics for the common case. 2) `__file__`-relative resolution — rejected, breaks for `pip install .`. 3) Package a synced copy of the schema as `package_data`, resolved via `importlib.resources` (packaging-aware, correct for both install modes). +- **Selected Approach**: Option 3. `src/schema/{controller,c}.capnp` synced (`cp`) into `src/xceadmin/src/xcecommon/schema_files/` by `xce-build.sh`'s `xceadmin)` case on every build, packaged via `pyproject.toml` `package_data`, resolved at runtime via `importlib.resources.files("xcecommon") / "schema_files" / "controller.capnp"`. +- **Rationale**: `importlib.resources` is the standard-library-correct way to locate packaged non-code files regardless of install mode (editable, wheel-installed, zipped) — it doesn't care where `site-packages` physically is. `src/schema/` remains the single source of truth (this spec doesn't own it, doesn't hand-edit the copy); the sync step is mechanical and runs every build, so it can't drift stale. +- **Trade-offs**: One more build-step to keep in sync (mitigated: it's a single `cp`, not a generation/compilation step); the packaged copy is momentarily stale between a `controller.capnp` edit and the next `xce-build xceadmin` run — acceptable, matches how any other build artifact in this repo already works (nothing is watched/hot-reloaded). +- **Follow-up**: None. + +## Risks & Mitigations +- **Controller `OP_upgradeConfig` dispatch defect (`cluster.c:667-679` vs `ha.c:3926-3951`)** — no role can process a push today. Mitigation: documented as an explicit Out-of-Boundary/Revalidation-Trigger item in design.md; `config push` implementation and testing proceed against the documented contract via a mock controller; real-cluster E2E is blocked until a separate fix (outside this spec) lands. Recommend raising this as its own tracked defect. +- **Redirect not wired for `upgradeConfig`** — auto-discovery (Requirement 3) is untestable end-to-end against a real cluster for this opcode specifically. Mitigation: generic implementation is correct and forward-compatible; mock-controller tests exercise the logic using synthetic `redirectHost`/`redirectPort` replies. +- **`ProbeConfigTag` has no server handler** — `config push --dry-run` cannot be validated live. Mitigation: already Nice-to-Have/Follow-up in requirements.md; client implemented against the schema contract, tested via mock only. +- **First Python component in the repo** — nix/build/CI integration has no precedent to copy exactly (only an analogous one, `xcecli`'s Go integration). Mitigation: `xcecli`'s `meson.build`/`custom_target()` pattern is close enough structurally to de-risk this; flagged for extra scrutiny during task review. +- **`pycapnp` C-extension build inside nix** — `pip install pycapnp` normally compiles a C extension against `capnproto`; getting this wrong would break `nix develop`. Mitigation: use `python3Packages.pycapnp` from nixpkgs (pre-built against nix-provided `capnproto`), not a bare `pip install pycapnp`, exactly as brief.md's Nix Environment section already specifies. + +### Decision: `xceadmin` is not wired into meson (Design Review correction, 2026-07-21) +- **Context**: `/kiro-validate-design` review flagged that the initial design draft's `src/xceadmin/meson.build` (mirroring `xcecli`'s `custom_target()`) was architecturally unsound — Go's pattern gates on a single static-binary output; `pip install -e .` mutates a venv/site-packages tree with no single trackable output for meson to gate on. +- **Alternatives Considered**: 1) Force it through meson anyway via a stamp-file `custom_target()`. 2) Skip meson entirely, dispatch purely from `xce-build.sh`'s case statement. +- **Selected Approach**: Option 2. +- **Rationale**: `brief.md`'s own harness sketch (`xce-build.sh`'s `xceadmin)` case) already specified direct `pip install`/`pytest` dispatch with no `meson compile` step — the meson integration was an unjustified addition during design drafting, not something the original harness plan called for. +- **Trade-offs**: `xceadmin` doesn't participate in `meson compile -C build ` the way C/Go components do; acceptable since it has no C-extension compile step of its own that meson would need to orchestrate (pycapnp's C extension is pip/nix's concern, not this project's meson graph). +- **Follow-up**: None — resolved at design time, `component` meson option is unchanged. + +### Decision: bundle the 7 JSON Schema files too, not just `controller.capnp` (2026-07-21, post-approval user question) +- **Context**: After design approval, user asked whether the JSON schemas used to validate YAML input should also ship with `xceadmin`, the same way `controller.capnp` does. Initial answer was "no, they're operator-supplied input per Req 1.1/1.6" — but on reconsideration, that framing didn't hold up: the JSON schemas are the formal spec of what `xcecommon.mapping`'s hardcoded field-extraction logic (`groupname`, `exclude_members`, `resname`, …) actually expects, not independent operator content. An operator-managed copy that drifts from the shipped mapping code risks a *silent* failure mode — validation passes on YAML that then maps wrong (or rejects YAML that would map fine) — arguably worse than the capnp case (hard crash) because nothing raises. No requirement or brief text ever described org-custom validation schemas as an actual use case; the `configs_dir/schema/` layout in `brief.md`'s original sketch was inherited unreflectively from `src/configs/`'s example layout, not a deliberate feature. +- **Alternatives Considered**: 1) Keep operator-supplied, `configs_dir/schema/` required (original design). 2) Bundle as `package_data`, same mechanism as `controller.capnp`, `configs_dir/schema/` becomes irrelevant. +- **Selected Approach**: Option 2 — confirmed with the user directly (`AskUserQuestion`, "Yes, bundle" selected over "Keep operator-supplied"). +- **Rationale**: Consistency with the capnp decision's own rationale (version-lock a tool-internal contract to the release that consumes it); removes an entire operator-facing requirement (maintaining a `schema/` folder they didn't author) for zero documented benefit forfeited. +- **Trade-offs**: No mechanism left for an org to enforce custom/stricter YAML validation rules — acceptable, since none was ever a stated requirement; can be revisited as a follow-up feature (e.g. a future `--extra-schema` overlay) if a real need surfaces. +- **Process note**: this reopened an *approved* acceptance criterion (Requirement 1.6) — amended requirements.md directly (added 5.5, rewrote 1.6 as "resolved, superseded") rather than looping back through `/kiro-spec-requirements`, since the user's direct confirmation in this conversation constitutes the approval for this specific amendment. +- **Follow-up**: None. + +## References +- [pycapnp GitHub](https://github.com/capnproto/pycapnp) — README, quickstart usage patterns +- [pycapnp CHANGELOG](https://github.com/capnproto/pycapnp/blob/master/CHANGELOG.md) — confirms synchronous `from_bytes`/`to_bytes`/`write`/`read` remain available through 2.2.x after RPC-layer sync support removal in 2.0.0b1 +- [pycapnp Quickstart (2.2.4 docs)](https://capnproto.github.io/pycapnp/quickstart.html) +- [capnp.pyx source](https://github.com/capnproto/pycapnp/blob/master/capnp/lib/capnp.pyx) — verified `.write()`'s `fileno()` behavior directly from source +- [Click — Commands and Groups](https://click.palletsprojects.com/en/stable/commands-and-groups/) +- [pytest-regressions](https://pytest-regressions.readthedocs.io/en/latest/overview.html) / [GitHub](https://github.com/ESSS/pytest-regressions) — evaluated, not adopted (see Design Decisions) +- [pytest-golden PyPI](https://pypi.org/project/pytest-golden/) — ruled out, YAML-only +- `src/schema/controller.capnp`, `src/schema/c.capnp`, `src/lib/netio.c`, `src/lib/request.c`, `src/controller/{ha,cluster,config_repl,join,myapp,main}.c`, `src/frontend/xcecli/meson.build`, `src/nixSupport/xce.nix` — primary in-repo sources for all findings above diff --git a/src/configs/hosts.yaml b/src/configs/hosts.yaml new file mode 100644 index 00000000..d9d57167 --- /dev/null +++ b/src/configs/hosts.yaml @@ -0,0 +1,39 @@ +--- +hosts: + - hostname: default + type: "x86_64-linux-gnu" + model: "x86_64-linux-gnu" + max_slot: -1 + description: "default host" + resource: + - name: mg + - name: maxswap + value: 10000 + sched_stop: + - resname: r1m + sched: 3.5 + stop: 4.5 + - resname: r15s + sched: 4.0 + stop: 10.0 + dispatch_window: + - start_time: "9:00" + end_time: "18:00" + - start_time: "21:00" + end_time: "7:00" + + - hostname: worker2 + model: "!" + type: "!" + max_slot: 256 + resource: + - name: gstring + value: "This is a G string" + + +hostgroups: + - groupname: hg1 + members: + - all + exclude_members: [h3] + diff --git a/src/configs/limits.yaml b/src/configs/limits.yaml new file mode 100644 index 00000000..127312fd --- /dev/null +++ b/src/configs/limits.yaml @@ -0,0 +1,49 @@ +--- +limits: + # cap total slots for ug1 across the whole cluster + - name: ug1_slots + users: + members: + - ug1 + resname: slots + value: 10 + + # per-user slot cap inside ug2 (each member gets up to 4, not a shared pool) + - name: ug2_slots_per_user + users: + members: + - ug2 + exclude_members: + - user3 + resname: slots + value: 4 + per_user: true + + # queue-scoped memory cap, percentage of total mem resource + - name: night_mem_pct + queues: + members: + - night + resname: mem + value: 50 + percent: true + + # host-scoped cap, applied per host in hg1 + - name: hg1_slots_per_host + hosts: + members: + - hg1 + resname: slots + value: 20 + per_host: true + + # combined scope: ug1 members on the normal queue + - name: ug1_normal_slots + users: + members: + - ug1 + queues: + members: + - normal + resname: slots + value: 8 diff --git a/src/configs/queues.yaml b/src/configs/queues.yaml new file mode 100644 index 00000000..9bafd74c --- /dev/null +++ b/src/configs/queues.yaml @@ -0,0 +1,94 @@ +--- +queues: + - name: xce + description: "xce queue" + priority: 30 + fairshare: + - user: default + share: 100 + hosts: + members: + - all + + - name: normal + description: "normal queue" + priority: 40 + nice: 10 + fairshare: + - user: default + share: 100 + hosts: + members: + - all + + + - name: night + description: "night queue" + priority: 30 + nice: 10 + res_req: "mem>1000 rusage[mem=1000]" + gpu_req: "num=1:mode=shared:gvendor=NVIDIA:gmodel=A100" + fairshare: + - user: ug1 + share: 1000 + - user: others + share: 100 + cpulimit: 100 + memlimit: 1000 + runlimit: 20 + runtimeest: 30 + dispatch_window: + - start_time: "19:00" + end_time: "8:30" + run_window: + - start_time: "5:19:00" + end_time: "1:8:30" + - start_time: "20:00" + end_time: "8:30" + sched_stop: + - resname: tmp + sched: 2000 + stop: 1000 + - resname: res1 + sched: 1.0 + stop: 2.5 + reservation: 100 + preemption: preemptive + backfill: true + exclusive: false + rerunable: true + pre_exec: "/path/to/preexec.sh" + post_exec: "/path/to/postexec.sh" + job_starter: "/path/to/jobstarter.sh" + users: + members: + - ug1 + - ug2 + exclude_members: + - user3 + hosts: + members: + - hgroup1 + - hgroup2 + exclude_members: + - host3 + + - name: short + description: "for short jobs" + priority: 45 + cpulimit: 15 + fairshare: + - user: ug1 + share: 1000 + - user: default + share: 100 + reservation: 10 + preemption: preemptable + backfill: true + users: + members: + - ugroup1 + hosts: + members: + - hgroup1 + diff --git a/src/configs/resources.yaml b/src/configs/resources.yaml new file mode 100644 index 00000000..f12b9f82 --- /dev/null +++ b/src/configs/resources.yaml @@ -0,0 +1,199 @@ +--- +resources: + # built-in: job slot count (consumed by running jobs, decreases as slots fill) + - name: slots + type: numeric + order: decrease + consumable: true + description: "job slot count (built-in)" + + # built-in: dynamic memory load index reported by workers + - name: mem + type: numeric + interval: 60 + consumable: false + order: decrease + description: "available memory in MB (built-in dynamic load index)" + + # built-in host attributes, registered per-worker by + # controller/worker.c add_hostattrs_resvals() + - name: type + type: string + description: "host OS/arch type (built-in host attribute)" + + - name: model + type: string + description: "host hardware model (built-in host attribute)" + + - name: cpuf + type: numeric + description: "CPU speed factor (built-in host attribute)" + + - name: maxcpus + type: numeric + description: "max CPU count (built-in host attribute)" + + - name: maxmem + type: numeric + description: "max memory in MB (built-in host attribute)" + + - name: maxswap + type: numeric + description: "max swap in MB (built-in host attribute)" + + - name: maxtmp + type: numeric + description: "max tmp disk space in MB (built-in host attribute)" + + - name: ndisks + type: numeric + description: "disk count (built-in host attribute)" + + # dynamic hostbased resource, external load index + - name: dynres1 + type: numeric + interval: 60 + order: decrease + consumable: true + release: true + description: "user defined dynamic resource" + resourcemap: + - location: + - default + + + # dynamic share resource, 2 instances + - name: dynres2 + type: numeric + interval: 60 + consumable: false + order: decrease + description: "user defined dynamic resource" + resourcemap: + - location: + - host1 + - host2 + - location: + - host5 + - host6 + + + # static share resource, all host share 100 res. + - name: staticres4 + type: numeric + interval: 0 + consumable: true + order: decrease + resourcemap: + - location: + - all + value: 100 + + # static share resource, host1 and host2 share the 100 res. + - name: staticres1 + type: numeric + interval: 0 + consumable: true + order: decrease + resourcemap: + - value: 100 + location: + - host1 + - host2 + + # static share resource, each of host1 and host2 shares the 100 res. + - name: staticres2 + type: numeric + interval: 0 + consumable: false + order: decrease + resourcemap: + - value: 100 + location: + - host1 + - value: 100 + location: + - host2 + + # static hostbased resource. + - name: staticres3 + type: numeric + interval: 0 + consumable: true + order: decrease + resourcemap: + - value: 100 + location: + - default + + - name: res1 + type: numeric + interval: 0 + consumable: true + order: decrease + resourcemap: + - location: + - all + value: 100 + + - name: sres1 + type: numeric + interval: 0 + consumable: true + order: decrease + + + # boolean resource + - name: bigmem + type: boolean + order: na + resourcemap: + - value: true + location: + - host1 + - value: true + location: + - host2 + - value: true + location: + - host3 + + # dynamic string resource + - name: strres1 + type: string + interval: 60 + consumable: false + order: na + description: "string resources" + resourcemap: + - location: + - default + + # static string resource + - name: strres2 + type: string + description: "string resources" + resourcemap: + - value: v1 + location: + - host1 + - value: v2 + location: + - host2 + - value: v3 + location: + - host3 + - name: host_quota + description: quota on each host + type: numeric + order: decrease + consumable: true + interval: 0 + resourcemap: + - value: 10 + location: + - default + + - name: gstring + description: string resource + type: string diff --git a/src/configs/scaler.yaml b/src/configs/scaler.yaml new file mode 100644 index 00000000..9f904ea1 --- /dev/null +++ b/src/configs/scaler.yaml @@ -0,0 +1,13 @@ +--- +scalers: + # script-based cloud autoscaler provider + - name: aws-asg + type: script + conf_path: /etc/xce/scalers/aws-asg.conf + script_path: /opt/xce/scalers/aws_asg.sh + script_option: "--region us-east-1" + + # minimal provider, no conf file or options + - name: k8s-pool + type: script + script_path: /opt/xce/scalers/k8s_pool.sh diff --git a/src/configs/schema/hosts.schema.json b/src/configs/schema/hosts.schema.json new file mode 100644 index 00000000..44769c0f --- /dev/null +++ b/src/configs/schema/hosts.schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schemas/hosts.schema.json", + "title": "Hosts and Hostgroups", + "type": "object", + "additionalProperties": false, + "properties": { + "hosts": { + "type": "array", + "items": { "$ref": "#/$defs/Host" } + }, + "hostgroups": { + "type": "array", + "items": { "$ref": "#/$defs/HostGroup" } + } + }, + "required": ["hosts"], + "$defs": { + "Host": { + "type": "object", + "additionalProperties": false, + "properties": { + "hostname": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, + "type": { "type": "string" }, + "model": { "type": "string" }, + "max_slot": { "type": "integer", "minimum": -1, "description": "-1 means auto-detect" }, + "dispatch_window": { "type": "array", "items": { "$ref": "#/$defs/TimeWindow" } }, + "sched_stop": { "type": "array", "items": { "$ref": "#/$defs/SchedStop" } }, + "resource": { "type": "array", "items": { "$ref": "#/$defs/HostResource" } }, + "description": { "type": "string", "maxLength": 4096 } + }, + "required": ["hostname"] + }, + "HostResource": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, + "value": { + "description": "Resource value (string or number)", + "oneOf": [ + { "type": "number" }, + { "type": "string" } + ] + } + }, + "required": ["name"] + }, + "TimeWindow": { + "type": "object", + "additionalProperties": false, + "properties": { + "start_time": { + "type": "string", + "description": "[day:]hour[:minute]; day 0-6, hour 0-23, minute 0-59", + "pattern": "^((([0-6]):)?([01]?\\d|2[0-3])(:[0-5]?\\d)?)$" + }, + "end_time": { + "type": "string", + "description": "[day:]hour[:minute]; day 0-6, hour 0-23, minute 0-59", + "pattern": "^((([0-6]):)?([01]?\\d|2[0-3])(:[0-5]?\\d)?)$" + } + }, + "required": ["start_time", "end_time"] + }, + "SchedStop": { + "type": "object", + "additionalProperties": false, + "properties": { + "resname": { "type": "string" }, + "sched": { "type": "number" }, + "stop": { "type": "number" } + }, + "required": ["resname"], + "anyOf": [ + { "required": ["sched"] }, + { "required": ["stop"] } + ] + }, + "HostGroup": { + "type": "object", + "additionalProperties": false, + "properties": { + "groupname": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, + "members": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^(?:all|!|[A-Za-z0-9._*\\[\\]-]+)$" + } + }, + "exclude_members": { + "type": "array", + "items": { "type": "string", "pattern": "^(?:all|!|[A-Za-z0-9._*\\[\\]-]+)$" } + } + }, + "required": ["groupname", "members"] + } + } +} \ No newline at end of file diff --git a/src/configs/schema/limits.schema.json b/src/configs/schema/limits.schema.json new file mode 100644 index 00000000..2ca5721d --- /dev/null +++ b/src/configs/schema/limits.schema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schemas/limits.schema.json", + "title": "Limits", + "type": "object", + "additionalProperties": false, + "properties": { + "limits": { + "type": "array", + "items": { "$ref": "#/$defs/Limit" } + } + }, + "required": ["limits"], + "$defs": { + "Membership": { + "type": "object", + "additionalProperties": false, + "properties": { + "members": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^(?:all|[A-Za-z0-9._*\\[\\]-]+)$" + } + }, + "exclude_members": { + "type": "array", + "items": { "type": "string", "pattern": "^[A-Za-z0-9._*\\[\\]-]+$" } + } + }, + "required": ["members"] + }, + "Limit": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, + "users": { "$ref": "#/$defs/Membership" }, + "queues": { "$ref": "#/$defs/Membership" }, + "projects": { "$ref": "#/$defs/Membership" }, + "hosts": { "$ref": "#/$defs/Membership" }, + "resname": { "type": "string" }, + "value": { "type": "number" }, + "percent": { "type": "boolean" }, + "per_user": { "type": "boolean" }, + "per_queue": { "type": "boolean" }, + "per_project": { "type": "boolean" }, + "per_host": { "type": "boolean" } + }, + "required": ["name", "resname", "value"] + } + } +} diff --git a/src/configs/schema/queues.schema.json b/src/configs/schema/queues.schema.json new file mode 100644 index 00000000..470dd681 --- /dev/null +++ b/src/configs/schema/queues.schema.json @@ -0,0 +1,120 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schemas/queues.schema.json", + "title": "Queues", + "type": "object", + "additionalProperties": false, + "properties": { + "queues": { + "type": "array", + "items": { "$ref": "#/$defs/Queue" } + } + }, + "required": ["queues"], + "$defs": { + "TimeWindow": { + "type": "object", + "additionalProperties": false, + "properties": { + "start_time": { + "type": "string", + "description": "[day:]HH[:MM] (day 0-6 optional, minute optional)", + "pattern": "^((([0-6]):)?([01]?\\d|2[0-3])(:[0-5]?\\d)?)$" + }, + "end_time": { + "type": "string", + "description": "[day:]HH[:MM] (day 0-6 optional, minute optional)", + "pattern": "^((([0-6]):)?([01]?\\d|2[0-3])(:[0-5]?\\d)?)$" + } + }, + "required": ["start_time", "end_time"] + }, + "SchedStop": { + "type": "object", + "additionalProperties": false, + "properties": { + "resname": { "type": "string" }, + "sched": { "type": "number" }, + "stop": { "type": "number" } + }, + "required": ["resname"], + "anyOf": [ + { "required": ["sched"] }, + { "required": ["stop"] } + ] + }, + "FairShare": { + "type": "object", + "additionalProperties": false, + "properties": { + "user": { "type": "string" }, + "share": { "type": "integer", "minimum": 0 } + }, + "required": ["user", "share"] + }, + "Membership": { + "type": "object", + "additionalProperties": false, + "properties": { + "members": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", "pattern": "^(?:all|[A-Za-z0-9._*\\[\\]-]+)$" + } + }, + "exclude_members": { + "type": "array", + "items": { "type": "string", "pattern": "^[A-Za-z0-9._*\\[\\]-]+$" } + } + }, + "required": ["members"] + }, + "Queue": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, + "description": { "type": "string" }, + "priority": { "type": "integer" }, + "nice": { "type": "integer" }, + "res_req": { "type": "string" }, + "gpu_req": { "type": "string" }, + "fairshare": { + "type": "array", + "items": { "$ref": "#/$defs/FairShare" } + }, + "cpulimit": { "type": "number" }, + "memlimit": { "type": "number" }, + "runlimit": { "type": "number" }, + "runtimeest": { "type": "number" }, + "dispatch_window": { + "type": "array", + "items": { "$ref": "#/$defs/TimeWindow" } + }, + "run_window": { + "type": "array", + "items": { "$ref": "#/$defs/TimeWindow" } + }, + "sched_stop": { + "type": "array", + "items": { "$ref": "#/$defs/SchedStop" } + }, + "reservation": { "type": "integer", "minimum": 0 }, + "preemption": { + "type": "string", + "enum": ["preemptive", "preemptable"] + }, + "backfill": { "type": "boolean" }, + "exclusive": { "type": "boolean" }, + "rerunable": { "type": "boolean" }, + "pre_exec": { "type": "string" }, + "post_exec": { "type": "string" }, + "job_starter": { "type": "string" }, + "users": { "$ref": "#/$defs/Membership" }, + "hosts": { "$ref": "#/$defs/Membership" } + }, + "required": ["name", "priority"] + } + } +} \ No newline at end of file diff --git a/src/configs/schema/resources.schema.json b/src/configs/schema/resources.schema.json new file mode 100644 index 00000000..b11620a4 --- /dev/null +++ b/src/configs/schema/resources.schema.json @@ -0,0 +1,41 @@ +{ + "type": "object", + "properties": { + "resources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "type": { "type": "string", "enum": ["numeric", "boolean", "string"] }, + "interval": { "type": "integer" }, + "order": { "type": "string", "enum": ["increase", "decrease", "na"] }, + "consumable": { "type": "boolean" }, + "release": { "type": "boolean" }, + "description": { "type": "string" }, + "resourcemap": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "oneOf": [ + { "type": "number" }, + { "type": "string" }, + { "type": "boolean" } + ] + }, + "location": { + "type": "array", + "items": { "type": "string" } + } + } + } + } + }, + "required": ["name", "type"] + } + } + }, + "required": ["resources"] +} diff --git a/src/configs/schema/scaler.schema.json b/src/configs/schema/scaler.schema.json new file mode 100644 index 00000000..7fc51750 --- /dev/null +++ b/src/configs/schema/scaler.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schemas/scaler.schema.json", + "title": "Scalers", + "type": "object", + "additionalProperties": false, + "properties": { + "scalers": { + "type": "array", + "items": { "$ref": "#/$defs/Scaler" } + } + }, + "required": ["scalers"], + "$defs": { + "Scaler": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { "type": "string", "pattern": "^[A-Za-z0-9._-]+$" }, + "type": { "type": "string", "description": "provider type, e.g. \"script\"" }, + "conf_path": { "type": "string" }, + "script_path": { "type": "string" }, + "script_option": { "type": "string" } + }, + "required": ["name", "type", "script_path"] + } + } +} diff --git a/src/configs/schema/users.schema.json b/src/configs/schema/users.schema.json new file mode 100644 index 00000000..b1fbaf49 --- /dev/null +++ b/src/configs/schema/users.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schemas/users.schema.json", + "title": "Usergroups", + "type": "object", + "additionalProperties": false, + "properties": { + "usergroups": { + "type": "array", + "items": { "$ref": "#/$defs/UserGroup" } + } + }, + "required": ["usergroups"], + "$defs": { + "UserGroup": { + "type": "object", + "additionalProperties": false, + "properties": { + "groupname": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]+$" + }, + "members": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^(?:all|!|[A-Za-z0-9._*\\[\\]-]+)$" + } + }, + "exclude_members": { + "type": "array", + "items": { "type": "string", "pattern": "^(?:all|!|[A-Za-z0-9._*\\[\\]-]+)$" } + } + }, + "required": ["groupname", "members"] + } + } +} \ No newline at end of file diff --git a/src/configs/schema/xce.schema.json b/src/configs/schema/xce.schema.json new file mode 100644 index 00000000..f799b200 --- /dev/null +++ b/src/configs/schema/xce.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/schemas/xce.schema.json", + "title": "XCE Cluster Settings", + "type": "object", + "additionalProperties": false, + "properties": { + "cluster_name": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]+$", + "description": "cluster identity — must be identical on every controller instance" + }, + "heartbeat_interval": { + "type": "integer", + "minimum": 1, + "description": "seconds between controller->worker liveness heartbeats" + }, + "inactive_heartbeat_count": { + "type": "integer", + "minimum": 1, + "description": "missed heartbeats before a worker is marked unavailable" + }, + "fetch_job_interval": { + "type": "integer", + "minimum": 0, + "description": "seconds between scheduler job-fetch passes" + }, + "netio_timeout": { + "type": "integer", + "minimum": 0, + "description": "network I/O timeout in seconds" + } + }, + "required": ["cluster_name", "heartbeat_interval", "inactive_heartbeat_count"] +} diff --git a/src/configs/users.yaml b/src/configs/users.yaml new file mode 100644 index 00000000..b8841ee2 --- /dev/null +++ b/src/configs/users.yaml @@ -0,0 +1,8 @@ +--- +#define usergroup share +usergroups: + - groupname: ug1 + members: + - user1 + - user2 + diff --git a/src/configs/xce.yaml b/src/configs/xce.yaml new file mode 100644 index 00000000..05a0fef2 --- /dev/null +++ b/src/configs/xce.yaml @@ -0,0 +1,6 @@ +--- +cluster_name: eda1 +heartbeat_interval: 30 +inactive_heartbeat_count: 3 +fetch_job_interval: 10 +netio_timeout: 30