feat(admin-config-tool): task 2.1 - xcecommon.validation
Validate operator-authored YAML against the 7 bundled JSON schemas, enforce reserved-name and duplicate-name rejection, and flag undefined group/resource cross-references as non-blocking warnings.
This commit is contained in:
parent
798f19b83a
commit
12b13a54a6
3 changed files with 862 additions and 1 deletions
528
src/xceadmin/tests/test_validation.py
Normal file
528
src/xceadmin/tests/test_validation.py
Normal file
|
|
@ -0,0 +1,528 @@
|
|||
"""Tests for xcecommon.validation (task 2.1).
|
||||
|
||||
Covers Requirements 4.1, 4.2, 4.3, 4.4, 4.5, 5.5: JSON Schema validation of
|
||||
operator-authored YAML against the 7 bundled schemas, plus business-rule
|
||||
checks (reserved names, duplicates, undefined cross-references) that JSON
|
||||
Schema alone cannot express.
|
||||
|
||||
Two fixture strategies are used deliberately:
|
||||
- `validate_yaml()` is exercised against the *real* bundled schemas
|
||||
(`default_schema_dir()`) and the *real* example configs under
|
||||
`src/configs/*.yaml` — proving the actual shipped schemas accept actual
|
||||
shipped example data, not just hand-rolled inline schema dicts.
|
||||
- `check_reserved_names()` / `check_duplicates()` / `check_undefined_refs()`
|
||||
are pure functions over already-validated section dicts, so they are
|
||||
exercised with small in-memory fixtures built directly in this file —
|
||||
no file I/O needed for business-rule logic that never touches a schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from xcecommon.errors import DuplicateEntryError, ReservedNameError, ValidationError
|
||||
from xcecommon.validation import (
|
||||
check_duplicates,
|
||||
check_reserved_names,
|
||||
check_undefined_refs,
|
||||
default_schema_dir,
|
||||
validate_yaml,
|
||||
)
|
||||
|
||||
# The real, operator-facing example configs this repo ships under
|
||||
# src/configs/ — used as realistic valid-case fixtures per design.md's
|
||||
# Testing Strategy ("full fixture... covers all 7 YAML files"). Failing
|
||||
# loud (not skipping) if this repo layout assumption is wrong is
|
||||
# intentional: a silently-skipped test would hide a real fixture gap.
|
||||
REPO_CONFIGS_DIR = Path(__file__).resolve().parent.parent.parent / "configs"
|
||||
|
||||
ALL_SCHEMA_NAMES = [
|
||||
"hosts",
|
||||
"queues",
|
||||
"resources",
|
||||
"users",
|
||||
"limits",
|
||||
"xce",
|
||||
"scaler",
|
||||
]
|
||||
|
||||
|
||||
def _real_config_path(schema_name: str) -> Path:
|
||||
path = REPO_CONFIGS_DIR / f"{schema_name}.yaml"
|
||||
assert path.is_file(), f"expected real example config at {path}"
|
||||
return path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# default_schema_dir()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_default_schema_dir_resolves_to_real_bundled_schemas() -> None:
|
||||
"""default_schema_dir() must resolve to the packaged schema_files/json/
|
||||
directory, containing all 7 bundled schema files (Requirement 5.5)."""
|
||||
schema_dir = Path(default_schema_dir())
|
||||
|
||||
assert schema_dir.is_dir()
|
||||
for schema_name in ALL_SCHEMA_NAMES:
|
||||
schema_file = schema_dir / f"{schema_name}.schema.json"
|
||||
assert schema_file.is_file(), f"missing bundled schema: {schema_file}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# validate_yaml() — real bundled schemas against real example YAML
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("schema_name", ALL_SCHEMA_NAMES)
|
||||
def test_validate_yaml_accepts_real_example_config(schema_name: str) -> None:
|
||||
"""Every one of the 7 real src/configs/*.yaml example files validates
|
||||
successfully against its corresponding bundled schema — proving the
|
||||
actual shipped schema accepts actual shipped example data, not just an
|
||||
inline hand-rolled schema dict."""
|
||||
data = validate_yaml(str(_real_config_path(schema_name)), schema_name)
|
||||
|
||||
assert isinstance(data, dict)
|
||||
|
||||
|
||||
def test_validate_yaml_no_schema_subdirectory_needed_in_configs_dir(tmp_path: Path) -> None:
|
||||
"""Observable from tasks.md: validating a config directory with no
|
||||
schema files present anywhere in it still succeeds — schemas are
|
||||
bundled with xceadmin (Requirement 5.5), never read from configs_dir."""
|
||||
configs_dir = tmp_path / "configs"
|
||||
configs_dir.mkdir()
|
||||
hosts_yaml = configs_dir / "hosts.yaml"
|
||||
hosts_yaml.write_text(_real_config_path("hosts").read_text())
|
||||
|
||||
# No "schema/" subdirectory anywhere under configs_dir.
|
||||
assert not any(configs_dir.rglob("*schema*"))
|
||||
|
||||
data = validate_yaml(str(hosts_yaml), "hosts")
|
||||
|
||||
assert "hosts" in data
|
||||
|
||||
|
||||
def test_validate_yaml_reports_file_pointer_and_constraint_on_schema_violation(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Requirement 4.1: a schema violation reports the file name, the
|
||||
JSON-pointer field path, and the violated constraint, then the caller
|
||||
can treat it as a hard failure."""
|
||||
bad_queues = tmp_path / "queues.yaml"
|
||||
bad_queues.write_text(
|
||||
"queues:\n"
|
||||
" - name: badqueue\n"
|
||||
" priority: 30\n"
|
||||
" preemption: not-a-real-value\n"
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
validate_yaml(str(bad_queues), "queues")
|
||||
|
||||
caught = exc_info.value
|
||||
assert caught.file == str(bad_queues)
|
||||
assert "preemption" in caught.json_pointer
|
||||
assert caught.constraint == "enum"
|
||||
|
||||
|
||||
def test_validate_yaml_reports_missing_required_field(tmp_path: Path) -> None:
|
||||
"""Requirement 4.1: a missing required field (queue with no
|
||||
"priority") is a schema violation, not silently accepted."""
|
||||
bad_queues = tmp_path / "queues.yaml"
|
||||
bad_queues.write_text("queues:\n - name: no-priority\n")
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
validate_yaml(str(bad_queues), "queues")
|
||||
|
||||
assert exc_info.value.constraint == "required"
|
||||
|
||||
|
||||
def test_validate_yaml_missing_yaml_file_is_not_this_functions_concern(tmp_path: Path) -> None:
|
||||
"""A missing individual YAML file is handled by the caller (build
|
||||
orchestration treats it as an empty section, Requirement 1.5) — but if
|
||||
validate_yaml is asked to open a nonexistent file directly, it must
|
||||
fail loud with a ValidationError, not an unhandled OSError leaking out
|
||||
of the validation boundary."""
|
||||
missing = tmp_path / "does-not-exist.yaml"
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
validate_yaml(str(missing), "queues")
|
||||
|
||||
|
||||
def test_validate_yaml_reports_malformed_yaml_syntax(tmp_path: Path) -> None:
|
||||
"""Invalid YAML syntax is a validation failure too, not an unhandled
|
||||
parser exception escaping the boundary."""
|
||||
broken = tmp_path / "queues.yaml"
|
||||
broken.write_text("queues: [this is not: valid yaml")
|
||||
|
||||
with pytest.raises(ValidationError):
|
||||
validate_yaml(str(broken), "queues")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_reserved_names() — Requirement 4.2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_check_reserved_names_rejects_reserved_queue_name() -> None:
|
||||
sections = {
|
||||
"queues": {"queues": [{"name": "lost_and_found", "priority": 10}]},
|
||||
}
|
||||
|
||||
with pytest.raises(ReservedNameError):
|
||||
check_reserved_names(sections)
|
||||
|
||||
|
||||
def test_check_reserved_names_rejects_reserved_host_name() -> None:
|
||||
sections = {
|
||||
"hosts": {"hosts": [{"hostname": "lost_and_found"}]},
|
||||
}
|
||||
|
||||
with pytest.raises(ReservedNameError):
|
||||
check_reserved_names(sections)
|
||||
|
||||
|
||||
def test_check_reserved_names_allows_ordinary_names() -> None:
|
||||
sections = {
|
||||
"hosts": {"hosts": [{"hostname": "worker1"}]},
|
||||
"queues": {"queues": [{"name": "normal", "priority": 10}]},
|
||||
}
|
||||
|
||||
check_reserved_names(sections) # must not raise
|
||||
|
||||
|
||||
def test_check_reserved_names_ignores_missing_sections() -> None:
|
||||
"""Sections not present in the dict (file was absent, Requirement 1.5)
|
||||
must not crash the check."""
|
||||
check_reserved_names({}) # must not raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_duplicates() — Requirement 4.3
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_check_duplicates_rejects_duplicate_queue_names() -> None:
|
||||
sections = {
|
||||
"queues": {
|
||||
"queues": [
|
||||
{"name": "batch", "priority": 10},
|
||||
{"name": "batch", "priority": 20},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(DuplicateEntryError):
|
||||
check_duplicates(sections)
|
||||
|
||||
|
||||
def test_check_duplicates_rejects_duplicate_host_names() -> None:
|
||||
sections = {
|
||||
"hosts": {
|
||||
"hosts": [{"hostname": "worker1"}, {"hostname": "worker1"}],
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(DuplicateEntryError):
|
||||
check_duplicates(sections)
|
||||
|
||||
|
||||
def test_check_duplicates_rejects_duplicate_resource_names() -> None:
|
||||
sections = {
|
||||
"resources": {
|
||||
"resources": [
|
||||
{"name": "gpu", "type": "numeric"},
|
||||
{"name": "gpu", "type": "numeric"},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(DuplicateEntryError):
|
||||
check_duplicates(sections)
|
||||
|
||||
|
||||
def test_check_duplicates_rejects_duplicate_host_group_names() -> None:
|
||||
sections = {
|
||||
"hosts": {
|
||||
"hosts": [],
|
||||
"hostgroups": [
|
||||
{"groupname": "hg1", "members": ["all"]},
|
||||
{"groupname": "hg1", "members": ["all"]},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(DuplicateEntryError):
|
||||
check_duplicates(sections)
|
||||
|
||||
|
||||
def test_check_duplicates_rejects_duplicate_user_group_names() -> None:
|
||||
sections = {
|
||||
"users": {
|
||||
"usergroups": [
|
||||
{"groupname": "ug1", "members": ["u1"]},
|
||||
{"groupname": "ug1", "members": ["u2"]},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(DuplicateEntryError):
|
||||
check_duplicates(sections)
|
||||
|
||||
|
||||
def test_check_duplicates_rejects_duplicate_limit_names() -> None:
|
||||
sections = {
|
||||
"limits": {
|
||||
"limits": [
|
||||
{"name": "lim1", "resname": "slots", "value": 1},
|
||||
{"name": "lim1", "resname": "mem", "value": 2},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
with pytest.raises(DuplicateEntryError):
|
||||
check_duplicates(sections)
|
||||
|
||||
|
||||
def test_check_duplicates_allows_limit_name_to_collide_with_queue_name() -> None:
|
||||
"""Requirement 4.3, scoped intentionally: a limit name is not required
|
||||
to differ from a queue, host, or group name — duplicates are checked
|
||||
within their own file/section only."""
|
||||
sections = {
|
||||
"queues": {"queues": [{"name": "shared_name", "priority": 10}]},
|
||||
"limits": {
|
||||
"limits": [{"name": "shared_name", "resname": "slots", "value": 1}]
|
||||
},
|
||||
}
|
||||
|
||||
check_duplicates(sections) # must not raise
|
||||
|
||||
|
||||
def test_check_duplicates_allows_host_group_and_user_group_same_name() -> None:
|
||||
"""Host groups and user groups are separate files/sections — a name
|
||||
used in both is not a duplicate."""
|
||||
sections = {
|
||||
"hosts": {"hostgroups": [{"groupname": "shared", "members": ["all"]}]},
|
||||
"users": {"usergroups": [{"groupname": "shared", "members": ["u1"]}]},
|
||||
}
|
||||
|
||||
check_duplicates(sections) # must not raise
|
||||
|
||||
|
||||
def test_check_duplicates_ignores_missing_sections() -> None:
|
||||
check_duplicates({}) # must not raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# check_undefined_refs() — Requirement 4.4, 4.5
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_check_undefined_refs_never_raises_on_missing_sections() -> None:
|
||||
"""This is the one intentionally non-fatal check (4.4) — it must never
|
||||
raise, even with an empty/partial sections dict."""
|
||||
warnings = check_undefined_refs({})
|
||||
|
||||
assert warnings == []
|
||||
|
||||
|
||||
def test_check_undefined_refs_warns_on_undefined_host_group_in_queue() -> None:
|
||||
"""Requirement 4.4 example: a queue references a host group that is not
|
||||
defined."""
|
||||
sections = {
|
||||
"hosts": {"hosts": [], "hostgroups": [{"groupname": "hg1", "members": ["all"]}]},
|
||||
"queues": {
|
||||
"queues": [
|
||||
{
|
||||
"name": "batch",
|
||||
"priority": 10,
|
||||
"hosts": {"members": ["hg_not_defined"]},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert any("hg_not_defined" in w for w in warnings)
|
||||
|
||||
|
||||
def test_check_undefined_refs_does_not_warn_on_defined_host_group_in_queue() -> None:
|
||||
sections = {
|
||||
"hosts": {"hosts": [], "hostgroups": [{"groupname": "hg1", "members": ["all"]}]},
|
||||
"queues": {
|
||||
"queues": [{"name": "batch", "priority": 10, "hosts": {"members": ["hg1"]}}]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert warnings == []
|
||||
|
||||
|
||||
def test_check_undefined_refs_does_not_warn_on_literal_all() -> None:
|
||||
sections = {
|
||||
"hosts": {"hosts": [], "hostgroups": []},
|
||||
"queues": {
|
||||
"queues": [{"name": "batch", "priority": 10, "hosts": {"members": ["all"]}}]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert warnings == []
|
||||
|
||||
|
||||
def test_check_undefined_refs_warns_on_undefined_user_group_in_queue() -> None:
|
||||
sections = {
|
||||
"users": {"usergroups": [{"groupname": "ug1", "members": ["u1"]}]},
|
||||
"queues": {
|
||||
"queues": [
|
||||
{
|
||||
"name": "batch",
|
||||
"priority": 10,
|
||||
"users": {"members": ["ug_not_defined"]},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert any("ug_not_defined" in w for w in warnings)
|
||||
|
||||
|
||||
def test_check_undefined_refs_warns_on_undefined_resname_in_limit() -> None:
|
||||
"""Requirement 4.4 example: a limit references a resname not defined in
|
||||
resources.yaml."""
|
||||
sections = {
|
||||
"resources": {"resources": [{"name": "slots", "type": "numeric"}]},
|
||||
"limits": {
|
||||
"limits": [{"name": "lim1", "resname": "not_a_real_resource", "value": 1}]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert any("not_a_real_resource" in w for w in warnings)
|
||||
|
||||
|
||||
def test_check_undefined_refs_warns_on_undefined_queue_in_limit() -> None:
|
||||
sections = {
|
||||
"queues": {"queues": [{"name": "batch", "priority": 10}]},
|
||||
"limits": {
|
||||
"limits": [
|
||||
{
|
||||
"name": "lim1",
|
||||
"resname": "slots",
|
||||
"value": 1,
|
||||
"queues": {"members": ["not_a_real_queue"]},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert any("not_a_real_queue" in w for w in warnings)
|
||||
|
||||
|
||||
def test_check_undefined_refs_built_in_resource_name_not_implicitly_exempt() -> None:
|
||||
"""Requirement 4.5: resources.yaml is the exhaustive resource catalog —
|
||||
built-in/dynamic resource names (slots, mem, and the 8
|
||||
add_hostattrs_resvals() host attributes) are NOT treated as implicit
|
||||
built-ins exempt from the catalog. A limit referencing "slots" when
|
||||
resources.yaml does not declare it must still warn."""
|
||||
sections = {
|
||||
"resources": {"resources": [{"name": "mem", "type": "numeric"}]}, # no "slots"
|
||||
"limits": {
|
||||
"limits": [{"name": "lim1", "resname": "slots", "value": 1}]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert any("slots" in w for w in warnings)
|
||||
|
||||
|
||||
def test_check_undefined_refs_all_ten_built_in_names_pass_when_declared() -> None:
|
||||
"""The converse of the above: once resources.yaml actually declares
|
||||
all the built-in/dynamic names (as the real example resources.yaml
|
||||
does), referencing them from a limit produces no warning."""
|
||||
built_in_names = [
|
||||
"slots",
|
||||
"mem",
|
||||
"type",
|
||||
"model",
|
||||
"cpuf",
|
||||
"maxcpus",
|
||||
"maxmem",
|
||||
"maxswap",
|
||||
"maxtmp",
|
||||
"ndisks",
|
||||
]
|
||||
sections = {
|
||||
"resources": {
|
||||
"resources": [{"name": name, "type": "numeric"} for name in built_in_names]
|
||||
},
|
||||
"limits": {
|
||||
"limits": [
|
||||
{"name": f"lim_{name}", "resname": name, "value": 1}
|
||||
for name in built_in_names
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert warnings == []
|
||||
|
||||
|
||||
def test_check_undefined_refs_does_not_block_on_multiple_violations() -> None:
|
||||
"""4.4: undefined references warn but never raise, regardless of how
|
||||
many violations are found."""
|
||||
sections = {
|
||||
"queues": {
|
||||
"queues": [
|
||||
{
|
||||
"name": "batch",
|
||||
"priority": 10,
|
||||
"hosts": {"members": ["ghost_group"]},
|
||||
"users": {"members": ["ghost_users"]},
|
||||
}
|
||||
]
|
||||
},
|
||||
"limits": {
|
||||
"limits": [{"name": "lim1", "resname": "ghost_resource", "value": 1}]
|
||||
},
|
||||
}
|
||||
|
||||
warnings = check_undefined_refs(sections) # must not raise
|
||||
|
||||
assert len(warnings) >= 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration: real example configs, parsed via validate_yaml, produce no
|
||||
# reserved-name / duplicate / undefined-ref surprises against each other.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_business_rule_checks_run_clean_against_all_real_example_sections() -> None:
|
||||
"""Load every real src/configs/*.yaml section through validate_yaml,
|
||||
then run all three business-rule checks against the combined sections
|
||||
dict — check_reserved_names/check_duplicates must not raise (the real
|
||||
fixtures use no reserved or duplicate names), and check_undefined_refs
|
||||
must return a list without raising."""
|
||||
sections = {
|
||||
schema_name: validate_yaml(str(_real_config_path(schema_name)), schema_name)
|
||||
for schema_name in ALL_SCHEMA_NAMES
|
||||
}
|
||||
|
||||
check_reserved_names(sections)
|
||||
check_duplicates(sections)
|
||||
warnings = check_undefined_refs(sections)
|
||||
|
||||
assert isinstance(warnings, list)
|
||||
Loading…
Add table
Add a link
Reference in a new issue