feat(scheduler-core): build xceadmin scheduler dump command (task 10.4)

Adds dump() to scheduler.py -- a pure local .replay file decoder, zero
network I/O. Key finding: pycapnp's from_bytes() does not validate a
message against the schema type you ask it to decode as (trial-decoding
SnapshotEntry bytes as ReplayEvent raises nothing, returns plausible
garbage), ruling out trial-decode-and-catch for message-type
discrimination. Instead reads each message's raw segment-table/
root-pointer framing to extract its (dataWordCount, pointerCount) shape,
computed live from the schema, and matches against the four replay
message types' distinct shapes before decoding. AC 20's stream-ordering
rules (msg 1 = ReplayHeader, msg 2 = SnapshotEntry) are enforced
structurally, not inferred.

Adversarial review reproduced a real (if forward-looking) gap: an actual
modified schema with a shape collision caused an uncaught raw capnp
exception to escape dump(). Closed with a pairwise-distinctness
assertion right after the four shapes are computed, raising a typed
ReplayFileError before any message is decoded if a future schema change
ever makes two shapes collide -- restoring dump()'s own docstring
guarantee unconditionally. New regression test simulates the exact
collision the review found.

201 passed, 3 skipped (up from 188 pre-task).
This commit is contained in:
Rongsong Shen 2026-08-06 05:29:15 +00:00
parent b64875f4c3
commit 8034fb6e1f
Signed by: rshen
SSH key fingerprint: SHA256:8Vf2pZ3h6rJa7EhGK+WaAQ9Mta/apZvteW6g9U/WKKk
4 changed files with 679 additions and 13 deletions

View file

@ -1,5 +1,7 @@
"""Mock-controller tests for ``xceadmin.scheduler``'s record-control client
functions (design.md, ``xceadmin.scheduler``; Requirements 5.10).
functions (design.md, ``xceadmin.scheduler``; Requirements 5.10) AND
golden-file tests for the local ``.replay`` file decoder, ``dump()`` (task
10.4, Requirement 5.20).
Mirrors ``test_config_push_mock.py``'s mock TLS server pattern exactly
(same ``mock_controller``/``wrap_reply``/``tls_certs`` fixtures from
@ -10,11 +12,16 @@ test environment), so these are tested against a real local TLS mock
server built from the actual bundled schema, exactly as
``test_config_push_mock.py`` does for ``push_config``/``probe_config``.
Only ``record_start``/``record_stop``/``record_status`` (task 10.3) are
covered here. ``dump()`` (task 10.4, not yet built) will get its own
golden-file tests appended to this file later (design.md's File Structure
Plan places both in one ``test_scheduler.py``) -- not duplicated or
anticipated here.
``dump()`` tests (bottom of this file) are a completely separate concern
-- no network, no mock controller. No pre-existing ``.replay`` fixture
file exists anywhere in this repo (confirmed by a repo-wide search before
writing these tests) and the C-side simulation driver that would consume
one doesn't exist yet either (Group 11+, out of scope) -- so the fixture
stream is built directly with pycapnp's write-side API
(``mod.X.new_message()`` + ``.to_bytes()``, concatenated with NO extra
framing), mirroring exactly what the real C writer
(``recording.c``/``recording_events.c``/``recording_session.c``'s
``write_capnp_message()``) produces on the wire.
"""
from __future__ import annotations
@ -27,7 +34,7 @@ from pathlib import Path
import pytest
from xcecommon import wire
from xcecommon.errors import NoPrimaryError, ReplyDecodeError, SchedAdminError
from xcecommon.errors import NoPrimaryError, ReplayFileError, ReplyDecodeError, SchedAdminError
from xcecommon.schema import load_schema
from xceadmin import scheduler
@ -408,3 +415,282 @@ def test_record_explicit_port_parsed():
host, port = scheduler._split_controller("somehost:1234")
assert host == "somehost"
assert port == 1234
# ---------------------------------------------------------------------------
# dump(): local .replay file decoder (task 10.4, Requirement 5.20).
#
# No mock controller involved -- these tests build a ``.replay`` byte
# stream directly with pycapnp and write it to a real file on disk, then
# call ``scheduler.dump()`` against that real path (module docstring).
# ---------------------------------------------------------------------------
def _build_full_replay_stream(mod) -> bytes:
"""Builds one complete, well-formed ``.replay`` stream covering every
message kind AC 20 documents, in the order the real C recorder
produces them: ``ReplayHeader`` -> initial ``SnapshotEntry`` ->
``ReplayEvent`` (jobsPush) -> a periodic checkpoint ``SnapshotEntry``
(task 9.3's feature, AC 11) -> ``ReplaySession`` (one dispatch + one
pend decision) -> ``ReplayEvent{endSession}``. Used by both the "full
Observable" test and the "periodic checkpoint" test below (the
checkpoint is always present in this stream) -- avoids building two
near-identical fixture streams.
Every message is serialized with ``.to_bytes()`` and concatenated with
NO extra framing, exactly matching ``write_capnp_message()``'s "single
stream of consecutive capnp messages" contract (recording.c).
"""
parts: list[bytes] = []
hdr = mod.ReplayHeader.new_message()
hdr.recorderVersion = "1.0"
hdr.recordingStartTs = 1000
hdr.recordingStartWall = "2026-08-05T00:00:00Z"
hdr.schedPolicy = "builtin_fcfs"
hdr.cluster.name = "testcluster"
hdr.cluster.recordingHost = "ctrl-1"
hdr.cluster.init("controllers", 1)
hdr.cluster.controllers[0] = "127.0.0.1:9991"
parts.append(hdr.to_bytes())
snap = mod.SnapshotEntry.new_message()
snap.ts = 1000
snap.data.config.clustername = "testcluster"
snap.data.jobs.ntotal = 1
jobs = snap.data.jobs.init("jobs", 1)
jobs[0].jobid = 501
jobs[0].state = 1
queues = snap.data.init("queues", 1)
queues[0].name = "normal"
queues[0].priority = 10
snap.data.workers.ntotal = 1
workers = snap.data.workers.init("workers", 1)
workers[0].name = "worker1"
snap.data.resources.ntotal = 1
resources = snap.data.resources.init("resources", 1)
resources[0].name = "cpu"
snap.data.resvals.init("vals", 0)
users = snap.data.init("users", 1)
users[0].name = "alice"
projects = snap.data.init("projects", 1)
projects[0].name = "proj1"
projects[0].priority = 5
snap.data.prngSeed = 42
snap.data.internalVersion = 1
parts.append(snap.to_bytes())
ev_push = mod.ReplayEvent.new_message()
ev_push.ts = 1001
jobs_push = ev_push.body.init("jobsPush")
jobs_push.ntotal = 1
jp_jobs = jobs_push.init("jobs", 1)
jp_jobs[0].jobid = 502
parts.append(ev_push.to_bytes())
checkpoint = mod.SnapshotEntry.new_message()
checkpoint.ts = 1005
checkpoint.data.prngSeed = 99
parts.append(checkpoint.to_bytes())
sess = mod.ReplaySession.new_message()
sess.beginRound = 1010
decisions = sess.init("decisions", 2)
decisions[0].ts = 1010
dispatch = decisions[0].body.init("dispatch")
dispatch.alloc.jobid = 502
host_allocs = dispatch.alloc.init("allocs", 1)
host_allocs[0].host = "worker1"
decisions[1].ts = 1011
pend = decisions[1].body.init("pend")
pend.jobid = 503
pend.reason.mainReason = 7
sess.summary.durationNs = 500
sess.summary.nDecisions = 2
sess.summary.nDispatched = 1
sess.summary.nPended = 1
parts.append(sess.to_bytes())
ev_end = mod.ReplayEvent.new_message()
ev_end.ts = 1020
ev_end.body.endSession = None
parts.append(ev_end.to_bytes())
return b"".join(parts)
@pytest.fixture
def full_replay_file(tmp_path: Path) -> Path:
mod = load_schema()
path = tmp_path / "full.replay"
path.write_bytes(_build_full_replay_stream(mod))
return path
def test_dump_lists_header_fields(full_replay_file):
text = scheduler.dump(str(full_replay_file))
assert "=== ReplayHeader ===" in text
assert "recorderVersion: 1.0" in text
assert "schedPolicy: builtin_fcfs" in text
assert "cluster.name: testcluster" in text
assert "127.0.0.1:9991" in text
def test_dump_lists_initial_snapshot_entry(full_replay_file):
text = scheduler.dump(str(full_replay_file))
assert "=== SnapshotEntry (initial) ===" in text
assert "config.clustername: testcluster" in text
assert "jobs: [501]" in text
assert "worker1" in text
assert "proj1" in text
def test_dump_lists_each_event(full_replay_file):
text = scheduler.dump(str(full_replay_file))
assert "=== ReplayEvent ===" in text
assert "type: jobsPush" in text
assert "jobids: [502]" in text
assert "type: endSession" in text
assert "-- end of recording --" in text
def test_dump_lists_session_and_each_decision(full_replay_file):
text = scheduler.dump(str(full_replay_file))
assert "=== ReplaySession ===" in text
assert "beginRound: 1010" in text
assert "decisions: 2" in text
assert "type=dispatch jobid=502 hosts=['worker1']" in text
assert "type=pend jobid=503 mainReason=7" in text
assert "nDispatched=1" in text
assert "nPended=1" in text
def test_dump_periodic_checkpoint_snapshot_still_decodes(full_replay_file):
"""Task 9.3's periodic-checkpoint feature: a mid-stream SnapshotEntry
(not just the initial one at message 2) must still decode correctly.
"""
text = scheduler.dump(str(full_replay_file))
assert text.count("=== SnapshotEntry") == 2
assert "=== SnapshotEntry (initial) ===" in text
assert "=== SnapshotEntry (checkpoint) ===" in text
# the checkpoint carries its own distinct ts/prngSeed, proving it was
# actually decoded (not just a relabeled copy of the initial entry)
assert "ts: 1005" in text
def test_dump_output_ends_with_trailing_newline(full_replay_file):
text = scheduler.dump(str(full_replay_file))
assert text.endswith("\n")
# ---------------------------------------------------------------------------
# dump(): malformed/truncated/empty input handled gracefully (typed error,
# never an uncontrolled crash).
# ---------------------------------------------------------------------------
def test_dump_missing_file_raises_replay_file_error(tmp_path):
missing = tmp_path / "does-not-exist.replay"
with pytest.raises(ReplayFileError) as exc_info:
scheduler.dump(str(missing))
assert exc_info.value.path == str(missing)
def test_dump_empty_file_raises_replay_file_error(tmp_path):
empty = tmp_path / "empty.replay"
empty.write_bytes(b"")
with pytest.raises(ReplayFileError) as exc_info:
scheduler.dump(str(empty))
assert "empty" in exc_info.value.reason
def test_dump_garbage_bytes_raises_replay_file_error_not_crash(tmp_path):
garbage = tmp_path / "garbage.replay"
garbage.write_bytes(b"this is not a capnp message at all, just plain text bytes" * 4)
with pytest.raises(ReplayFileError):
scheduler.dump(str(garbage))
def test_dump_truncated_mid_header_raises_replay_file_error(full_replay_file, tmp_path):
full_bytes = full_replay_file.read_bytes()
truncated = tmp_path / "truncated.replay"
# Cut off partway through the first message -- past its segment table
# (so _peek_message's length calc succeeds) but before all declared
# bytes are actually present.
truncated.write_bytes(full_bytes[:40])
with pytest.raises(ReplayFileError) as exc_info:
scheduler.dump(str(truncated))
assert exc_info.value.reason == "truncated message"
def test_dump_second_message_not_snapshot_entry_raises_replay_file_error(tmp_path):
"""AC 20: the second message MUST be a SnapshotEntry. A file whose
second message is something else (here: a ReplayEvent, wrong shape)
is a format violation dump() must reject, not silently misinterpret --
catches exactly the failure mode this task's own research proved
trial-decoding cannot reliably detect (module docstring in
scheduler.py).
"""
mod = load_schema()
hdr = mod.ReplayHeader.new_message()
hdr.recorderVersion = "1.0"
ev = mod.ReplayEvent.new_message()
ev.ts = 1
ev.body.endSession = None
bad = tmp_path / "bad-second-message.replay"
bad.write_bytes(hdr.to_bytes() + ev.to_bytes())
with pytest.raises(ReplayFileError) as exc_info:
scheduler.dump(str(bad))
assert exc_info.value.reason == "unrecognized message shape"
def test_dump_first_message_not_header_raises_replay_file_error(tmp_path):
"""A file that starts with a SnapshotEntry instead of a ReplayHeader
(wrong shape at message 1) is rejected rather than silently decoded
as a garbage ReplayHeader (empty text fields, no crash -- exactly the
silent-misinterpretation failure mode this task's research ruled out
trial-decoding for; see scheduler.py's dump() section docstring).
"""
mod = load_schema()
snap = mod.SnapshotEntry.new_message()
snap.ts = 1
bad = tmp_path / "bad-first-message.replay"
bad.write_bytes(snap.to_bytes())
with pytest.raises(ReplayFileError) as exc_info:
scheduler.dump(str(bad))
assert exc_info.value.reason == "unrecognized message shape"
def test_dump_raises_on_ambiguous_message_shapes(monkeypatch, full_replay_file):
"""Message-type discrimination relies entirely on ReplayHeader/
SnapshotEntry/ReplayEvent/ReplaySession having pairwise-distinct
(dataWords, pointers) shapes -- verified distinct for the real schema
elsewhere in this file. If a future schema edit ever made two of them
collide, trial-decoding would not raise (this task's own research
finding), so dump() must detect the collision itself and fail loud
with a clear, typed error -- not let an uncaught capnp exception
escape from deep inside from_bytes(). Simulate that collision by
monkeypatching _shape_of() rather than editing the real schema.
"""
real_shape_of = scheduler._shape_of
mod = load_schema()
def _colliding_shape_of(cls):
if cls is mod.SnapshotEntry:
return real_shape_of(mod.ReplaySession)
return real_shape_of(cls)
monkeypatch.setattr(scheduler, "_shape_of", _colliding_shape_of)
with pytest.raises(ReplayFileError) as exc_info:
scheduler.dump(str(full_replay_file))
assert exc_info.value.reason == "ambiguous message shapes"
assert "SnapshotEntry" in str(exc_info.value)
assert "ReplaySession" in str(exc_info.value)