diff --git a/src/configs/hosts.yaml b/src/configs/hosts.yaml index d9d57167..2792f694 100644 --- a/src/configs/hosts.yaml +++ b/src/configs/hosts.yaml @@ -6,7 +6,7 @@ hosts: max_slot: -1 description: "default host" resource: - - name: mg + - name: mem - name: maxswap value: 10000 sched_stop: @@ -30,10 +30,20 @@ hosts: - name: gstring value: "This is a G string" + - hostname: host3 + type: "x86_64-linux-gnu" + model: "x86_64-linux-gnu" + max_slot: 128 + hostgroups: - groupname: hg1 members: - all - exclude_members: [h3] - + exclude_members: [host3] + + - groupname: hg2 + members: + - worker2 + - host3 + diff --git a/src/configs/queues.yaml b/src/configs/queues.yaml index 9bafd74c..3ffa3033 100644 --- a/src/configs/queues.yaml +++ b/src/configs/queues.yaml @@ -68,8 +68,8 @@ queues: - user3 hosts: members: - - hgroup1 - - hgroup2 + - hg1 + - hg2 exclude_members: - host3 @@ -87,8 +87,8 @@ queues: backfill: true users: members: - - ugroup1 + - ug1 hosts: members: - - hgroup1 + - hg1 diff --git a/src/configs/resources.yaml b/src/configs/resources.yaml index f12b9f82..2bebb32c 100644 --- a/src/configs/resources.yaml +++ b/src/configs/resources.yaml @@ -15,6 +15,29 @@ resources: order: decrease description: "available memory in MB (built-in dynamic load index)" + # built-in: worker load indices registered by + # worker/src/resource_sync.c init_builtin_load_indices() + - name: r1m + type: numeric + interval: 60 + consumable: false + order: decrease + description: "1-minute load average (built-in dynamic load index)" + + - name: r15s + type: numeric + interval: 60 + consumable: false + order: decrease + description: "15-second load average (built-in dynamic load index)" + + - name: tmp + type: numeric + interval: 60 + consumable: false + order: decrease + description: "available tmp disk space in MB (built-in dynamic load index)" + # built-in host attributes, registered per-worker by # controller/worker.c add_hostattrs_resvals() - name: type diff --git a/src/configs/users.yaml b/src/configs/users.yaml index b8841ee2..b0f3542f 100644 --- a/src/configs/users.yaml +++ b/src/configs/users.yaml @@ -1,8 +1,13 @@ --- -#define usergroup share +#define usergroup share usergroups: - groupname: ug1 members: - user1 - user2 + - groupname: ug2 + members: + - user3 + - user4 + diff --git a/src/xceadmin/src/xceadmin/build.py b/src/xceadmin/src/xceadmin/build.py index d26dd5b5..92ec1e8c 100644 --- a/src/xceadmin/src/xceadmin/build.py +++ b/src/xceadmin/src/xceadmin/build.py @@ -32,7 +32,11 @@ from xcecommon import mapping, validation #: allows. Keyed by SECTION_MAP's ``capnp_field`` (not ``yaml_file``, #: since ``hosts.yaml`` feeds two fields with two different item shapes). _UNMAPPED_BUT_VALID_ITEM_KEYS: dict[str, frozenset[str]] = { - "resources": frozenset({"type", "order", "consumable", "release", "resourcemap"}), + # type/order/consumable/release ARE mapped (mapping.py's + # _RESOURCE_VALTYPE/_RESOURCE_ORDTYPE/flag constants) -- only + # resourcemap has no destination (controller.capnp:208-209 has its + # target fields commented out). + "resources": frozenset({"resourcemap"}), "queues": frozenset( { "backfill", diff --git a/src/xceadmin/src/xcecommon/mapping.py b/src/xceadmin/src/xcecommon/mapping.py index 681f5a92..3e6f95d8 100644 --- a/src/xceadmin/src/xcecommon/mapping.py +++ b/src/xceadmin/src/xcecommon/mapping.py @@ -63,19 +63,22 @@ reason -- never a silent drop. there is nothing to exclude. ``Resource`` (``resources.yaml``) - ``idx``, ``valtype``, ``ordtype``, ``flags``, ``lastReport`` are left - unset. The YAML ``type``/``order``/``consumable``/``release`` keys - would need to encode into ``valtype``/``ordtype``/``flags`` as - specific integers, but neither ``controller.capnp`` nor design.md - documents that encoding within this task's boundary -- inventing one - risks silently miswriting a value the controller then misinterprets, - strictly worse than shipping it unset. ``idx`` has no YAML source at - all (controller/runtime-assigned). ``lastReport`` is a runtime - timestamp field (epoch seconds of last report) -- excluded doubly by + ``valtype``/``ordtype``/``flags`` ARE mapped from YAML + ``type``/``order``/``consumable``/``release`` -- the encoding is + documented, just not in ``controller.capnp``/design.md: it lives in + ``include/xce/types.h`` (``XCE_VAL_*``, ``xce_order_t``, + ``XCE_RESF_CONSUMABLE``/``XCE_RESF_RELEASE``) and is actively read at + runtime (e.g. ``scheduler.c`` copies these three fields straight out + of the parsed resource). See :data:`_RESOURCE_VALTYPE`, + :data:`_RESOURCE_ORDTYPE`, and the flag constants below for the exact + values. ``idx`` and ``lastReport`` remain unset: ``idx`` has no YAML + source at all (controller/runtime-assigned), and ``lastReport`` is a + runtime timestamp field (epoch seconds of last report) -- excluded by Requirement 9.2's "no timestamp" rule. ``resourcemap`` has no destination: the capnp fields it would map to (``distributions``, ``hostBased``) are commented out of the ``Resource`` struct - (``controller.capnp:208-209``). + (``controller.capnp:208-209``) -- mapping it needs a schema change + first, out of this module's scope. ``QueueConf`` (``queues.yaml``) ``attrs`` (raw ``UInt32`` bitfield), ``status``, ``acceptIntervals``, @@ -170,6 +173,17 @@ _XCE_SCALAR_FIELDS: tuple[tuple[str, str], ...] = ( ("netio_timeout", "netioTimeout"), ) +# resources.yaml ``type``/``order`` -> Resource.valtype/ordtype, mirroring +# include/xce/types.h's XCE_VAL_*/xce_order_t (both plain capnp Int32 +# fields, not capnp enums -- see controller.capnp:203-205). +_RESOURCE_VALTYPE: dict[str, int] = {"string": 0, "numeric": 1, "boolean": 2} +_RESOURCE_ORDTYPE: dict[str, int] = {"increase": 0, "decrease": 1, "na": 2} + +# resources.yaml ``consumable``/``release`` -> Resource.flags bits, mirroring +# include/xce/types.h's XCE_RESF_CONSUMABLE/XCE_RESF_RELEASE. +_RESOURCE_FLAG_CONSUMABLE = 0x80 +_RESOURCE_FLAG_RELEASE = 0x20 + def new_cluster_config_builder() -> "capnp._DynamicStructBuilder": """Convenience constructor: ``load_schema().ClusterConfig.new_message()``. @@ -294,6 +308,16 @@ def _build_resource(item: dict, target: "capnp._DynamicStructBuilder") -> None: target.desc = item["description"] if "interval" in item: target.interval = item["interval"] + if "type" in item: + target.valtype = _RESOURCE_VALTYPE[item["type"]] + if "order" in item: + target.ordtype = _RESOURCE_ORDTYPE[item["order"]] + flags = 0 + if item.get("consumable"): + flags |= _RESOURCE_FLAG_CONSUMABLE + if item.get("release"): + flags |= _RESOURCE_FLAG_RELEASE + target.flags = flags def _build_queue(item: dict, target: "capnp._DynamicStructBuilder") -> None: diff --git a/src/xceadmin/tests/fixtures/full/expected.bin b/src/xceadmin/tests/fixtures/full/expected.bin index 13aa3878..cfb8788b 100644 Binary files a/src/xceadmin/tests/fixtures/full/expected.bin and b/src/xceadmin/tests/fixtures/full/expected.bin differ diff --git a/src/xceadmin/tests/test_config_build.py b/src/xceadmin/tests/test_config_build.py index 5610c725..937b89c5 100644 --- a/src/xceadmin/tests/test_config_build.py +++ b/src/xceadmin/tests/test_config_build.py @@ -117,7 +117,11 @@ def test_build_raises_duplicate_entry_error(tmp_path): # --------------------------------------------------------------------------- -def test_build_warns_not_fails_on_schema_valid_unmapped_resource_fields(tmp_path, capsys): +def test_build_maps_resource_type_order_consumable_release(tmp_path, capsys): + """type/order/consumable/release map to valtype/ordtype/flags (see + xcecommon.mapping's _RESOURCE_VALTYPE/_RESOURCE_ORDTYPE/flag constants) + -- no longer an "unmapped field" warning. + """ _write_yaml( tmp_path / "resources.yaml", { @@ -127,7 +131,41 @@ def test_build_warns_not_fails_on_schema_valid_unmapped_resource_fields(tmp_path "type": "numeric", "order": "increase", "consumable": True, - "release": False, + "release": True, + } + ] + }, + ) + + data = build.build_cluster_config(str(tmp_path)) # must not raise + + mod = load_schema() + with mod.ClusterConfig.from_bytes(data) as reader: + resource = reader.resources[0] + assert resource.name == "gpus" + assert resource.valtype == 1 # XCE_VAL_NUMERIC + assert resource.ordtype == 0 # XCE_ORD_INCR + assert resource.flags == 0x80 | 0x20 # XCE_RESF_CONSUMABLE | XCE_RESF_RELEASE + + stderr = capsys.readouterr().err + assert "'type'" not in stderr + assert "'order'" not in stderr + assert "'consumable'" not in stderr + assert "'release'" not in stderr + + +def test_build_warns_not_fails_on_schema_valid_unmapped_resourcemap_field(tmp_path, capsys): + """resourcemap has no destination -- controller.capnp's distributions/ + hostBased fields are commented out (needs a schema change first). + """ + _write_yaml( + tmp_path / "resources.yaml", + { + "resources": [ + { + "name": "gpus", + "type": "numeric", + "resourcemap": [{"location": ["host1"], "value": 1}], } ] }, @@ -141,10 +179,7 @@ def test_build_warns_not_fails_on_schema_valid_unmapped_resource_fields(tmp_path stderr = capsys.readouterr().err assert "resources.yaml" in stderr - assert "'type'" in stderr - assert "'order'" in stderr - assert "'consumable'" in stderr - assert "'release'" in stderr + assert "'resourcemap'" in stderr assert "gpus" in stderr @@ -215,10 +250,13 @@ def test_build_real_repo_configs_succeeds_and_warns_on_known_gaps(capsys): with mod.ClusterConfig.from_bytes(data) as reader: assert reader.clusterName == "eda1" - # src/configs/hosts.yaml has 2 pre-existing undefined refs (resource - # "mg", host "h3" -- see task 2.1's Implementation Notes) and every - # resources.yaml entry has a schema-required "type" field with no - # ClusterConfig destination -- both are expected warnings, not errors. + # queues.yaml/hosts.yaml still have several schema-valid-but-unmapped + # fields (description, nice, fairshare, ... -- see build.py's + # _UNMAPPED_BUT_VALID_ITEM_KEYS) and limits.yaml/queues.yaml reference + # bare usernames (e.g. "rshen") that the undefined-ref check can never + # resolve since users.yaml has no individual-user roster, only + # usergroups (Requirement 4.4 scopes that check to groups only) -- + # both are expected warnings, not errors. stderr = capsys.readouterr().err assert "warning:" in stderr diff --git a/src/xceadmin/tests/test_mapping.py b/src/xceadmin/tests/test_mapping.py index 7e999334..90ad6487 100644 --- a/src/xceadmin/tests/test_mapping.py +++ b/src/xceadmin/tests/test_mapping.py @@ -153,14 +153,17 @@ def test_map_list_section_resources_populates_mapped_fields_only(cluster_config) assert r0.name == "slots" assert r0.desc == "job slot count" assert r0.interval == 0 + assert r0.valtype == 1 # XCE_VAL_NUMERIC + assert r0.ordtype == 1 # XCE_ORD_DECR + assert r0.flags == 0x80 # XCE_RESF_CONSUMABLE ("release" absent -> not OR'd in) assert r1.name == "mem" assert r1.interval == 60 + assert r1.valtype == 1 # XCE_VAL_NUMERIC ("type" present) + assert r1.ordtype == 0 # "order" absent on this item -> untouched default + assert r1.flags == 0 # "consumable"/"release" both absent -> no bits set # Deliberately-unmapped fields (see module docstring) stay at default. assert r0.idx == 0 - assert r0.valtype == 0 - assert r0.ordtype == 0 - assert r0.flags == 0 assert r0.lastReport == 0