fix(admin-config-tool): resolve xceadmin config build warnings

Cross-reference warnings ("references undefined entry") were genuine
gaps in the example configs, not validator bugs: src/configs/hosts.yaml
had a typo'd resource ref (mg -> mem) and referenced a nonexistent
host3/hostgroup pair; queues.yaml referenced hgroup1/hgroup2/ugroup1
instead of the real hg1/ug1 names; resources.yaml was missing the
r1m/r15s/tmp built-in worker load-index resources
(worker/src/resource_sync.c's init_builtin_load_indices()). Added the
missing host3 host, hg2 hostgroup, ug2 usergroup, and three resource
entries; fixed the typo'd refs.

"schema-valid but has no ClusterConfig destination" warnings for
resources.yaml's type/order/consumable/release fields were a real gap
in xcecommon/mapping.py's _build_resource, not an inherent limitation
-- the encoding (XCE_VAL_*/xce_order_t/XCE_RESF_CONSUMABLE/
XCE_RESF_RELEASE) is documented in include/xce/types.h and actively
read by the scheduler; the old docstring's claim that it was
undocumented was wrong. Only resourcemap remains genuinely unmapped
(its capnp destination fields are commented out).

Updated test_mapping.py/test_config_build.py for the new mapping
behavior and regenerated the full golden fixture's expected.bin
(legitimate representation-drift from the mapping fix, not a bug).
This commit is contained in:
Rongsong Shen 2026-07-23 09:10:14 +00:00
parent 4ec401672b
commit 1705495a5e
9 changed files with 139 additions and 32 deletions

View file

@ -32,7 +32,11 @@ from xcecommon import mapping, validation
#: allows. Keyed by SECTION_MAP's ``capnp_field`` (not ``yaml_file``,
#: since ``hosts.yaml`` feeds two fields with two different item shapes).
_UNMAPPED_BUT_VALID_ITEM_KEYS: dict[str, frozenset[str]] = {
"resources": frozenset({"type", "order", "consumable", "release", "resourcemap"}),
# type/order/consumable/release ARE mapped (mapping.py's
# _RESOURCE_VALTYPE/_RESOURCE_ORDTYPE/flag constants) -- only
# resourcemap has no destination (controller.capnp:208-209 has its
# target fields commented out).
"resources": frozenset({"resourcemap"}),
"queues": frozenset(
{
"backfill",

View file

@ -63,19 +63,22 @@ reason -- never a silent drop.
there is nothing to exclude.
``Resource`` (``resources.yaml``)
``idx``, ``valtype``, ``ordtype``, ``flags``, ``lastReport`` are left
unset. The YAML ``type``/``order``/``consumable``/``release`` keys
would need to encode into ``valtype``/``ordtype``/``flags`` as
specific integers, but neither ``controller.capnp`` nor design.md
documents that encoding within this task's boundary -- inventing one
risks silently miswriting a value the controller then misinterprets,
strictly worse than shipping it unset. ``idx`` has no YAML source at
all (controller/runtime-assigned). ``lastReport`` is a runtime
timestamp field (epoch seconds of last report) -- excluded doubly by
``valtype``/``ordtype``/``flags`` ARE mapped from YAML
``type``/``order``/``consumable``/``release`` -- the encoding is
documented, just not in ``controller.capnp``/design.md: it lives in
``include/xce/types.h`` (``XCE_VAL_*``, ``xce_order_t``,
``XCE_RESF_CONSUMABLE``/``XCE_RESF_RELEASE``) and is actively read at
runtime (e.g. ``scheduler.c`` copies these three fields straight out
of the parsed resource). See :data:`_RESOURCE_VALTYPE`,
:data:`_RESOURCE_ORDTYPE`, and the flag constants below for the exact
values. ``idx`` and ``lastReport`` remain unset: ``idx`` has no YAML
source at all (controller/runtime-assigned), and ``lastReport`` is a
runtime timestamp field (epoch seconds of last report) -- excluded by
Requirement 9.2's "no timestamp" rule. ``resourcemap`` has no
destination: the capnp fields it would map to (``distributions``,
``hostBased``) are commented out of the ``Resource`` struct
(``controller.capnp:208-209``).
(``controller.capnp:208-209``) -- mapping it needs a schema change
first, out of this module's scope.
``QueueConf`` (``queues.yaml``)
``attrs`` (raw ``UInt32`` bitfield), ``status``, ``acceptIntervals``,
@ -170,6 +173,17 @@ _XCE_SCALAR_FIELDS: tuple[tuple[str, str], ...] = (
("netio_timeout", "netioTimeout"),
)
# resources.yaml ``type``/``order`` -> Resource.valtype/ordtype, mirroring
# include/xce/types.h's XCE_VAL_*/xce_order_t (both plain capnp Int32
# fields, not capnp enums -- see controller.capnp:203-205).
_RESOURCE_VALTYPE: dict[str, int] = {"string": 0, "numeric": 1, "boolean": 2}
_RESOURCE_ORDTYPE: dict[str, int] = {"increase": 0, "decrease": 1, "na": 2}
# resources.yaml ``consumable``/``release`` -> Resource.flags bits, mirroring
# include/xce/types.h's XCE_RESF_CONSUMABLE/XCE_RESF_RELEASE.
_RESOURCE_FLAG_CONSUMABLE = 0x80
_RESOURCE_FLAG_RELEASE = 0x20
def new_cluster_config_builder() -> "capnp._DynamicStructBuilder":
"""Convenience constructor: ``load_schema().ClusterConfig.new_message()``.
@ -294,6 +308,16 @@ def _build_resource(item: dict, target: "capnp._DynamicStructBuilder") -> None:
target.desc = item["description"]
if "interval" in item:
target.interval = item["interval"]
if "type" in item:
target.valtype = _RESOURCE_VALTYPE[item["type"]]
if "order" in item:
target.ordtype = _RESOURCE_ORDTYPE[item["order"]]
flags = 0
if item.get("consumable"):
flags |= _RESOURCE_FLAG_CONSUMABLE
if item.get("release"):
flags |= _RESOURCE_FLAG_RELEASE
target.flags = flags
def _build_queue(item: dict, target: "capnp._DynamicStructBuilder") -> None:

Binary file not shown.

View file

@ -117,7 +117,11 @@ def test_build_raises_duplicate_entry_error(tmp_path):
# ---------------------------------------------------------------------------
def test_build_warns_not_fails_on_schema_valid_unmapped_resource_fields(tmp_path, capsys):
def test_build_maps_resource_type_order_consumable_release(tmp_path, capsys):
"""type/order/consumable/release map to valtype/ordtype/flags (see
xcecommon.mapping's _RESOURCE_VALTYPE/_RESOURCE_ORDTYPE/flag constants)
-- no longer an "unmapped field" warning.
"""
_write_yaml(
tmp_path / "resources.yaml",
{
@ -127,7 +131,41 @@ def test_build_warns_not_fails_on_schema_valid_unmapped_resource_fields(tmp_path
"type": "numeric",
"order": "increase",
"consumable": True,
"release": False,
"release": True,
}
]
},
)
data = build.build_cluster_config(str(tmp_path)) # must not raise
mod = load_schema()
with mod.ClusterConfig.from_bytes(data) as reader:
resource = reader.resources[0]
assert resource.name == "gpus"
assert resource.valtype == 1 # XCE_VAL_NUMERIC
assert resource.ordtype == 0 # XCE_ORD_INCR
assert resource.flags == 0x80 | 0x20 # XCE_RESF_CONSUMABLE | XCE_RESF_RELEASE
stderr = capsys.readouterr().err
assert "'type'" not in stderr
assert "'order'" not in stderr
assert "'consumable'" not in stderr
assert "'release'" not in stderr
def test_build_warns_not_fails_on_schema_valid_unmapped_resourcemap_field(tmp_path, capsys):
"""resourcemap has no destination -- controller.capnp's distributions/
hostBased fields are commented out (needs a schema change first).
"""
_write_yaml(
tmp_path / "resources.yaml",
{
"resources": [
{
"name": "gpus",
"type": "numeric",
"resourcemap": [{"location": ["host1"], "value": 1}],
}
]
},
@ -141,10 +179,7 @@ def test_build_warns_not_fails_on_schema_valid_unmapped_resource_fields(tmp_path
stderr = capsys.readouterr().err
assert "resources.yaml" in stderr
assert "'type'" in stderr
assert "'order'" in stderr
assert "'consumable'" in stderr
assert "'release'" in stderr
assert "'resourcemap'" in stderr
assert "gpus" in stderr
@ -215,10 +250,13 @@ def test_build_real_repo_configs_succeeds_and_warns_on_known_gaps(capsys):
with mod.ClusterConfig.from_bytes(data) as reader:
assert reader.clusterName == "eda1"
# src/configs/hosts.yaml has 2 pre-existing undefined refs (resource
# "mg", host "h3" -- see task 2.1's Implementation Notes) and every
# resources.yaml entry has a schema-required "type" field with no
# ClusterConfig destination -- both are expected warnings, not errors.
# queues.yaml/hosts.yaml still have several schema-valid-but-unmapped
# fields (description, nice, fairshare, ... -- see build.py's
# _UNMAPPED_BUT_VALID_ITEM_KEYS) and limits.yaml/queues.yaml reference
# bare usernames (e.g. "rshen") that the undefined-ref check can never
# resolve since users.yaml has no individual-user roster, only
# usergroups (Requirement 4.4 scopes that check to groups only) --
# both are expected warnings, not errors.
stderr = capsys.readouterr().err
assert "warning:" in stderr

View file

@ -153,14 +153,17 @@ def test_map_list_section_resources_populates_mapped_fields_only(cluster_config)
assert r0.name == "slots"
assert r0.desc == "job slot count"
assert r0.interval == 0
assert r0.valtype == 1 # XCE_VAL_NUMERIC
assert r0.ordtype == 1 # XCE_ORD_DECR
assert r0.flags == 0x80 # XCE_RESF_CONSUMABLE ("release" absent -> not OR'd in)
assert r1.name == "mem"
assert r1.interval == 60
assert r1.valtype == 1 # XCE_VAL_NUMERIC ("type" present)
assert r1.ordtype == 0 # "order" absent on this item -> untouched default
assert r1.flags == 0 # "consumable"/"release" both absent -> no bits set
# Deliberately-unmapped fields (see module docstring) stay at default.
assert r0.idx == 0
assert r0.valtype == 0
assert r0.ordtype == 0
assert r0.flags == 0
assert r0.lastReport == 0