feat(admin-config-tool): task 2.3 - xcecommon.mapping

Map validated YAML section dicts into ClusterConfig capnp field values via a declarative SECTION_MAP driving two generic mapper shapes (list/scalar). Preserves YAML source order, sources no timestamp/hostname/username, and is proven byte-deterministic against both synthetic and real src/configs/*.yaml fixtures.
2026-07-22 07:36:30 +00:00 · 2026-07-22 07:36:30 +00:00 · 666a838fb2
commit 666a838fb2
parent dbf5c6c54d
3 changed files with 1043 additions and 1 deletions
--- a/src/xceadmin/tests/test_mapping.py
+++ b/src/xceadmin/tests/test_mapping.py
@ -0,0 +1,585 @@
+"""Unit tests for ``xcecommon.mapping`` (design.md, ``xcecommon.mapping``;
+Requirements 1.1, 1.7, 1.8, 1.9, 1.10, 1.11, 1.12, 5.3, 9.1, 9.2, 9.3).
+
+Pure unit tests against in-memory dicts (no file I/O) per task 2.3/5.1 --
+except where a test deliberately loads the real ``src/configs/*.yaml``
+fixtures to prove the mapper handles this repo's actual example configs,
+not just synthetic minimal dicts (task 2.3's TDD approach explicitly asks
+for both).
+
+These tests build real capnp ``ClusterConfig`` messages via
+``xcecommon.schema.load_schema()`` -- the same loaded module ``mapping.py``
+targets -- so any drift between this module's hardcoded field names and the
+real ``controller.capnp`` schema surfaces as an ``AttributeError``/
+``KjException`` here, not silently.
+"""
+
+from __future__ import annotations
+
+import inspect
+from pathlib import Path
+
+import pytest
+import yaml
+
+from xcecommon import mapping
+from xcecommon.schema import load_schema
+
+REPO_CONFIGS_DIR = Path(__file__).resolve().parents[2] / "configs"
+
+
+@pytest.fixture
+def cluster_config():
+    """A fresh ``ClusterConfig`` builder -- never reused across tests, so
+    every test starts from an all-default message.
+    """
+    return mapping.new_cluster_config_builder()
+
+
+def _load_real_yaml(name: str) -> dict:
+    path = REPO_CONFIGS_DIR / name
+    with open(path, encoding="utf-8") as fh:
+        return yaml.safe_load(fh)
+
+
+# ---------------------------------------------------------------------------
+# SECTION_MAP shape
+# ---------------------------------------------------------------------------
+
+
+def test_section_map_has_eight_entries_one_per_yaml_capnp_field_pair():
+    assert len(mapping.SECTION_MAP) == 8
+
+
+def test_section_map_hosts_yaml_feeds_two_distinct_capnp_fields():
+    hosts_entries = [s for s in mapping.SECTION_MAP if s["yaml_file"] == "hosts.yaml"]
+    assert len(hosts_entries) == 2
+    capnp_fields = {s["capnp_field"] for s in hosts_entries}
+    assert capnp_fields == {"hgroups", "hostConfigs"}
+    yaml_keys = {s["yaml_key"] for s in hosts_entries}
+    assert yaml_keys == {"hostgroups", "hosts"}
+
+
+def test_section_map_covers_all_13_clusterconfig_fields_or_documents_exclusion():
+    # 5 scalar (xce.yaml) + 7 list capnp_fields covers 12 of the 13
+    # ClusterConfig fields; configGeneration (@12) is the one deliberately
+    # excluded field (Requirement 1.9), documented in the module docstring,
+    # not present in SECTION_MAP at all.
+    list_fields = {s["capnp_field"] for s in mapping.SECTION_MAP if s["kind"] == "list"}
+    assert list_fields == {
+        "resources",
+        "queues",
+        "ugroups",
+        "hgroups",
+        "hostConfigs",
+        "hproviders",
+        "limits",
+    }
+    scalar_entries = [s for s in mapping.SECTION_MAP if s["kind"] == "scalar"]
+    assert len(scalar_entries) == 1
+    assert {f[1] for f in mapping._XCE_SCALAR_FIELDS} == {
+        "clusterName",
+        "heartbeatInterval",
+        "inactiveHeartbeatCount",
+        "fetchJobInterval",
+        "netioTimeout",
+    }
+
+
+# ---------------------------------------------------------------------------
+# map_scalar_section (xce.yaml)
+# ---------------------------------------------------------------------------
+
+
+def test_map_scalar_section_sets_all_5_fields(cluster_config):
+    spec = mapping.SECTION_MAP[0]
+    values = {
+        "cluster_name": "eda1",
+        "heartbeat_interval": 30,
+        "inactive_heartbeat_count": 3,
+        "fetch_job_interval": 10,
+        "netio_timeout": 30,
+    }
+    mapping.map_scalar_section(spec, values, cluster_config)
+
+    assert cluster_config.clusterName == "eda1"
+    assert cluster_config.heartbeatInterval == 30
+    assert cluster_config.inactiveHeartbeatCount == 3
+    assert cluster_config.fetchJobInterval == 10
+    assert cluster_config.netioTimeout == 30
+
+
+def test_map_scalar_section_leaves_missing_optional_keys_at_default(cluster_config):
+    spec = mapping.SECTION_MAP[0]
+    values = {"cluster_name": "eda1", "heartbeat_interval": 30, "inactive_heartbeat_count": 3}
+    mapping.map_scalar_section(spec, values, cluster_config)
+
+    assert cluster_config.clusterName == "eda1"
+    assert cluster_config.fetchJobInterval == 0
+    assert cluster_config.netioTimeout == 0
+
+
+@pytest.mark.parametrize("empty", [None, {}])
+def test_map_scalar_section_noop_on_empty_values(cluster_config, empty):
+    spec = mapping.SECTION_MAP[0]
+    mapping.map_scalar_section(spec, empty, cluster_config)
+
+    assert cluster_config.clusterName == ""
+    assert cluster_config.heartbeatInterval == 0
+
+
+# ---------------------------------------------------------------------------
+# map_list_section: resources.yaml -> resources
+# ---------------------------------------------------------------------------
+
+
+def _spec_for(capnp_field: str) -> mapping.SectionSpec:
+    for spec in mapping.SECTION_MAP:
+        if spec["capnp_field"] == capnp_field:
+            return spec
+    raise AssertionError(f"no SECTION_MAP entry for {capnp_field!r}")
+
+
+def test_map_list_section_resources_populates_mapped_fields_only(cluster_config):
+    items = [
+        {"name": "slots", "type": "numeric", "order": "decrease", "consumable": True,
+         "description": "job slot count", "interval": 0},
+        {"name": "mem", "type": "numeric", "interval": 60, "description": "memory"},
+    ]
+    mapping.map_list_section(_spec_for("resources"), items, cluster_config)
+
+    assert len(cluster_config.resources) == 2
+    r0, r1 = cluster_config.resources
+    assert r0.name == "slots"
+    assert r0.desc == "job slot count"
+    assert r0.interval == 0
+    assert r1.name == "mem"
+    assert r1.interval == 60
+
+    # Deliberately-unmapped fields (see module docstring) stay at default.
+    assert r0.idx == 0
+    assert r0.valtype == 0
+    assert r0.ordtype == 0
+    assert r0.flags == 0
+    assert r0.lastReport == 0
+
+
+def test_map_list_section_noop_on_empty_or_missing_items(cluster_config):
+    mapping.map_list_section(_spec_for("resources"), [], cluster_config)
+    mapping.map_list_section(_spec_for("resources"), None, cluster_config)
+    assert len(cluster_config.resources) == 0
+
+
+# ---------------------------------------------------------------------------
+# map_list_section: users.yaml/hosts.yaml -> ugroups/hgroups (named groups)
+# ---------------------------------------------------------------------------
+
+
+def test_map_list_section_ugroups_maps_groupname_members_exclude(cluster_config):
+    items = [{"groupname": "ug1", "members": ["user1", "user2"], "exclude_members": ["user3"]}]
+    mapping.map_list_section(_spec_for("ugroups"), items, cluster_config)
+
+    g = cluster_config.ugroups[0]
+    assert g.name == "ug1"
+    assert list(g.members) == ["user1", "user2"]
+    assert list(g.exmembers) == ["user3"]
+
+
+def test_map_list_section_hgroups_omits_exclude_members_when_absent(cluster_config):
+    items = [{"groupname": "hg1", "members": ["all"]}]
+    mapping.map_list_section(_spec_for("hgroups"), items, cluster_config)
+
+    g = cluster_config.hgroups[0]
+    assert g.name == "hg1"
+    assert list(g.members) == ["all"]
+    assert list(g.exmembers) == []
+
+
+# ---------------------------------------------------------------------------
+# map_list_section: hosts.yaml -> hostConfigs (the "feeds two fields" file)
+# ---------------------------------------------------------------------------
+
+
+def test_hosts_yaml_feeds_hgroups_and_hostconfigs_independently(cluster_config):
+    """The one file that feeds two separate output fields (design.md /
+    task 5.1) -- build both from the same parsed hosts.yaml dict and
+    assert each populated correctly and independently.
+    """
+    hosts_data = {
+        "hosts": [{"hostname": "h1", "type": "linux", "max_slot": 4}],
+        "hostgroups": [{"groupname": "hg1", "members": ["all"]}],
+    }
+    mapping.map_list_section(_spec_for("hgroups"), hosts_data.get("hostgroups"), cluster_config)
+    mapping.map_list_section(_spec_for("hostConfigs"), hosts_data.get("hosts"), cluster_config)
+
+    assert cluster_config.hgroups[0].name == "hg1"
+    assert cluster_config.hostConfigs[0].name == "h1"
+    assert cluster_config.hostConfigs[0].mxj == 4
+    # Independence: touching hostConfigs must not disturb hgroups.
+    assert len(cluster_config.hgroups) == 1
+
+
+def test_map_list_section_host_config_full_field_mapping(cluster_config):
+    items = [
+        {
+            "hostname": "default",
+            "type": "x86_64-linux-gnu",
+            "model": "x86_64-linux-gnu",
+            "max_slot": -1,
+            "description": "default host",
+            "resource": [{"name": "mg"}, {"name": "maxswap", "value": 10000}],
+            "sched_stop": [
+                {"resname": "r1m", "sched": 3.5, "stop": 4.5},
+                {"resname": "r15s", "sched": 4.0, "stop": 10.0},
+            ],
+            "dispatch_window": [
+                {"start_time": "9:00", "end_time": "18:00"},
+                {"start_time": "21:00", "end_time": "7:00"},
+            ],
+        },
+    ]
+    mapping.map_list_section(_spec_for("hostConfigs"), items, cluster_config)
+
+    h = cluster_config.hostConfigs[0]
+    assert h.name == "default"
+    assert h.type == "x86_64-linux-gnu"
+    assert h.model == "x86_64-linux-gnu"
+    # max_slot == -1 (auto-detect sentinel) cannot be represented by
+    # UInt32 mxj -- deliberately left at default (see module docstring).
+    assert h.mxj == 0
+
+    assert [r.name for r in h.resources] == ["mg", "maxswap"]
+    assert h.resources[0].value == ""
+    assert h.resources[1].value == "10000"
+
+    assert [t.name for t in h.thresholds] == ["r1m", "r15s"]
+    assert h.thresholds[0].sched == 3.5
+    assert h.thresholds[0].stop == 4.5
+
+    segs = h.dispatchwindows.segs
+    assert len(segs) == 2
+    assert segs[0].open.hour == 9
+    assert segs[0].open.min == 0
+    assert segs[0].close.hour == 18
+    assert segs[1].open.hour == 21
+    assert segs[1].close.hour == 7
+
+
+def test_map_list_section_host_config_sets_mxj_when_max_slot_nonnegative(cluster_config):
+    items = [{"hostname": "worker2", "max_slot": 256}]
+    mapping.map_list_section(_spec_for("hostConfigs"), items, cluster_config)
+
+    assert cluster_config.hostConfigs[0].mxj == 256
+
+
+def test_time_window_day_prefixed_spec_parses_day_hour_minute(cluster_config):
+    items = [
+        {
+            "name": "night",
+            "priority": 30,
+            "run_window": [
+                {"start_time": "5:19:00", "end_time": "1:8:30"},
+            ],
+        }
+    ]
+    mapping.map_list_section(_spec_for("queues"), items, cluster_config)
+
+    seg = cluster_config.queues[0].runWindow.segs[0]
+    assert seg.open.day == 5
+    assert seg.open.hour == 19
+    assert seg.open.min == 0
+    assert seg.close.day == 1
+    assert seg.close.hour == 8
+    assert seg.close.min == 30
+
+
+# ---------------------------------------------------------------------------
+# map_list_section: queues.yaml -> queues
+# ---------------------------------------------------------------------------
+
+
+def test_map_list_section_queue_full_field_mapping(cluster_config):
+    items = [
+        {
+            "name": "night",
+            "description": "night queue",
+            "priority": 30,
+            "res_req": "mem>1000 rusage[mem=1000]",
+            "gpu_req": "num=1:mode=shared",
+            "sched_stop": [{"resname": "tmp", "sched": 2000, "stop": 1000}],
+            "dispatch_window": [{"start_time": "19:00", "end_time": "8:30"}],
+            "run_window": [{"start_time": "20:00", "end_time": "8:30"}],
+            "pre_exec": "/path/to/preexec.sh",
+            "post_exec": "/path/to/postexec.sh",
+            "users": {"members": ["ug1", "ug2"], "exclude_members": ["user3"]},
+            "hosts": {"members": ["hgroup1", "hgroup2"], "exclude_members": ["host3"]},
+        },
+    ]
+    mapping.map_list_section(_spec_for("queues"), items, cluster_config)
+
+    q = cluster_config.queues[0]
+    assert q.name == "night"
+    assert q.priority == 30
+    assert q.resreq == "mem>1000 rusage[mem=1000]"
+    assert q.preExec == "/path/to/preexec.sh"
+    assert q.postExec == "/path/to/postexec.sh"
+    assert list(q.users.members) == ["ug1", "ug2"]
+    assert list(q.users.exmembers) == ["user3"]
+    assert q.users.name == ""  # embedded Membership never carries a name
+    assert list(q.hosts.members) == ["hgroup1", "hgroup2"]
+    assert q.thresholds[0].name == "tmp"
+    assert q.thresholds[0].sched == 2000.0
+    assert q.dispatchWindow.segs[0].open.hour == 19
+    assert q.runWindow.segs[0].open.hour == 20
+
+    # Deliberately-unmapped fields: description, gpu_req, attrs, status,
+    # acceptIntervals, schedDelay have no destination or no data source.
+    assert q.attrs == 0
+    assert q.status == 0
+    assert q.acceptIntervals == 0
+    assert q.schedDelay == 0
+
+
+def test_map_list_section_queue_minimal_required_fields_only(cluster_config):
+    items = [{"name": "xce", "priority": 30}]
+    mapping.map_list_section(_spec_for("queues"), items, cluster_config)
+
+    q = cluster_config.queues[0]
+    assert q.name == "xce"
+    assert q.priority == 30
+    assert q.resreq == ""
+    assert len(q.thresholds) == 0
+    assert len(q.dispatchWindow.segs) == 0
+
+
+# ---------------------------------------------------------------------------
+# map_list_section: scaler.yaml -> hproviders
+# ---------------------------------------------------------------------------
+
+
+def test_map_list_section_scaler_config_full_field_mapping(cluster_config):
+    items = [
+        {
+            "name": "aws-asg",
+            "type": "script",
+            "conf_path": "/etc/xce/scalers/aws-asg.conf",
+            "script_path": "/opt/xce/scalers/aws_asg.sh",
+            "script_option": "--region us-east-1",
+        },
+        {"name": "k8s-pool", "type": "script", "script_path": "/opt/xce/scalers/k8s_pool.sh"},
+    ]
+    mapping.map_list_section(_spec_for("hproviders"), items, cluster_config)
+
+    p0, p1 = cluster_config.hproviders
+    assert p0.name == "aws-asg"
+    assert p0.type == "script"
+    assert p0.confPath == "/etc/xce/scalers/aws-asg.conf"
+    assert p0.scriptPath == "/opt/xce/scalers/aws_asg.sh"
+    assert p0.scriptOption == "--region us-east-1"
+
+    assert p1.name == "k8s-pool"
+    assert p1.confPath == ""
+    assert p1.scriptOption == ""
+
+
+# ---------------------------------------------------------------------------
+# map_list_section: limits.yaml -> limits
+# ---------------------------------------------------------------------------
+
+
+def test_map_list_section_limit_config_full_field_mapping(cluster_config):
+    items = [
+        {
+            "name": "ug2_slots_per_user",
+            "users": {"members": ["ug2"], "exclude_members": ["user3"]},
+            "resname": "slots",
+            "value": 4,
+            "per_user": True,
+        },
+        {
+            "name": "night_mem_pct",
+            "queues": {"members": ["night"]},
+            "resname": "mem",
+            "value": 50,
+            "percent": True,
+        },
+        {
+            "name": "ug1_normal_slots",
+            "users": {"members": ["ug1"]},
+            "queues": {"members": ["normal"]},
+            "resname": "slots",
+            "value": 8,
+        },
+    ]
+    mapping.map_list_section(_spec_for("limits"), items, cluster_config)
+
+    l0, l1, l2 = cluster_config.limits
+    assert l0.name == "ug2_slots_per_user"
+    assert list(l0.users.members) == ["ug2"]
+    assert list(l0.users.exmembers) == ["user3"]
+    assert l0.val.resname == "slots"
+    assert l0.val.value == 4.0
+    assert l0.val.percentage is False
+    assert l0.perUser is True
+    assert l0.perQueue is False
+
+    assert l1.val.resname == "mem"
+    assert l1.val.percentage is True
+
+    assert list(l2.users.members) == ["ug1"]
+    assert list(l2.queues.members) == ["normal"]
+
+
+# ---------------------------------------------------------------------------
+# Order preservation (Requirement 9.1)
+# ---------------------------------------------------------------------------
+
+
+def test_list_order_preserved_not_sorted(cluster_config):
+    items = [{"name": n} for n in ["zebra", "apple", "mango"]]
+    mapping.map_list_section(_spec_for("resources"), items, cluster_config)
+
+    assert [r.name for r in cluster_config.resources] == ["zebra", "apple", "mango"]
+
+
+def test_group_membership_order_preserved(cluster_config):
+    items = [{"groupname": "hg1", "members": ["h3", "h1", "h2"]}]
+    mapping.map_list_section(_spec_for("hgroups"), items, cluster_config)
+
+    assert list(cluster_config.hgroups[0].members) == ["h3", "h1", "h2"]
+
+
+# ---------------------------------------------------------------------------
+# Determinism (Requirements 9.1, 9.2, 9.3)
+# ---------------------------------------------------------------------------
+
+
+def _build_full_sections() -> dict[str, dict]:
+    return {
+        "xce": {
+            "cluster_name": "eda1",
+            "heartbeat_interval": 30,
+            "inactive_heartbeat_count": 3,
+            "fetch_job_interval": 10,
+            "netio_timeout": 30,
+        },
+        "resources": {"resources": [{"name": "slots", "type": "numeric"}, {"name": "mem", "type": "numeric"}]},
+        "queues": {"queues": [{"name": "xce", "priority": 30, "users": {"members": ["all"]}}]},
+        "users": {"usergroups": [{"groupname": "ug1", "members": ["user1", "user2"]}]},
+        "hosts": {
+            "hosts": [{"hostname": "h1", "max_slot": 4}],
+            "hostgroups": [{"groupname": "hg1", "members": ["all"]}],
+        },
+        "scaler": {"scalers": [{"name": "aws-asg", "type": "script", "script_path": "/x.sh"}]},
+        "limits": {"limits": [{"name": "l1", "resname": "slots", "value": 10}]},
+    }
+
+
+def test_map_all_sections_building_same_input_twice_produces_identical_bytes():
+    sections = _build_full_sections()
+
+    target1 = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections, target1)
+    bytes1 = target1.to_bytes()
+
+    target2 = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections, target2)
+    bytes2 = target2.to_bytes()
+
+    assert bytes1 == bytes2
+
+
+def test_map_all_sections_missing_file_matches_present_empty_section():
+    """A schema_name absent from `sections` (missing YAML file) must
+    produce byte-identical output to that section being present but
+    empty -- Requirement 1.5's rule, enforced structurally.
+    """
+    sections_missing = {
+        "xce": {"cluster_name": "c", "heartbeat_interval": 1, "inactive_heartbeat_count": 1}
+    }
+    sections_present_empty = dict(sections_missing, resources={"resources": []})
+
+    target_missing = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections_missing, target_missing)
+
+    target_empty = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections_present_empty, target_empty)
+
+    assert target_missing.to_bytes() == target_empty.to_bytes()
+
+
+def test_map_all_sections_populates_all_capnp_fields_and_skips_configgeneration():
+    sections = _build_full_sections()
+    target = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections, target)
+
+    assert target.clusterName == "eda1"
+    assert len(target.resources) == 2
+    assert len(target.queues) == 1
+    assert len(target.ugroups) == 1
+    assert len(target.hgroups) == 1
+    assert len(target.hostConfigs) == 1
+    assert len(target.hproviders) == 1
+    assert len(target.limits) == 1
+    # configGeneration (@12): controller-assigned, never touched (1.9).
+    assert target.configGeneration == 0
+
+
+def test_no_timestamp_hostname_or_username_source_anywhere_in_module():
+    """Static regression guard for Requirement 9.2: no wall-clock,
+    hostname, or username source is ever imported or called by this
+    module.
+    """
+    source = inspect.getsource(mapping)
+    forbidden_tokens = [
+        "datetime",
+        "time.time",
+        "gethostname",
+        "getlogin",
+        "getuser",
+        "uuid.uuid",
+        "os.environ",
+    ]
+    for token in forbidden_tokens:
+        assert token not in source, f"forbidden non-deterministic source {token!r} found in mapping.py"
+
+
+# ---------------------------------------------------------------------------
+# Real src/configs/*.yaml fixtures
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.skipif(not REPO_CONFIGS_DIR.is_dir(), reason="src/configs/ fixtures not present")
+def test_maps_real_repo_config_fixtures_without_error():
+    sections = {
+        "xce": _load_real_yaml("xce.yaml"),
+        "resources": _load_real_yaml("resources.yaml"),
+        "queues": _load_real_yaml("queues.yaml"),
+        "users": _load_real_yaml("users.yaml"),
+        "hosts": _load_real_yaml("hosts.yaml"),
+        "scaler": _load_real_yaml("scaler.yaml"),
+        "limits": _load_real_yaml("limits.yaml"),
+    }
+    target = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections, target)
+
+    assert target.clusterName == "eda1"
+    assert len(target.resources) == len(sections["resources"]["resources"])
+    assert len(target.queues) == len(sections["queues"]["queues"])
+    assert len(target.hostConfigs) == len(sections["hosts"]["hosts"])
+    assert len(target.hgroups) == len(sections["hosts"]["hostgroups"])
+    assert len(target.limits) == len(sections["limits"]["limits"])
+
+    # Round-trip through to_bytes()/from_bytes() using the real loaded
+    # schema module -- catches drift between mapping.py's hardcoded field
+    # names and the real controller.capnp schema.
+    mod = load_schema()
+    target_bytes = target.to_bytes()
+    with mod.ClusterConfig.from_bytes(target_bytes) as reader:
+        assert reader.clusterName == "eda1"
+        assert reader.queues[3].name == "short"
+
+    # Determinism against the real fixtures too.
+    target2 = mapping.new_cluster_config_builder()
+    mapping.map_all_sections(sections, target2)
+    assert target_bytes == target2.to_bytes()