feat(scheduler-sim-decision-trace): capture the trace-off REPL transcript baseline
Task 1.1: scripted REPL session over a contention replay (license, cluster and host limits, parallel and alternative jobs); transcript, recordings and --sim-verify result committed as fixtures and compared byte for byte. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
bf1e4b7245
commit
e2d5662d28
GPG key ID:
80533536B0EB651C
9 changed files with 1206 additions and 1 deletions
Binary file not shown.
|
|
@ -0,0 +1,41 @@
|
|||
help
|
||||
info
|
||||
info config
|
||||
step
|
||||
info session
|
||||
info summary
|
||||
info jobs
|
||||
info job 104
|
||||
info worker all
|
||||
info resource license
|
||||
info limit all
|
||||
set units raw
|
||||
info worker w01
|
||||
set units human
|
||||
info worker w01
|
||||
info resource all
|
||||
set units both
|
||||
info worker w02
|
||||
info queue all
|
||||
set verify off
|
||||
info config
|
||||
set verify on
|
||||
snapshot snap.replay
|
||||
set output set_output.replay
|
||||
break DECISION jobid=104 type=DISPATCH
|
||||
info break
|
||||
continue
|
||||
info session
|
||||
info job 104
|
||||
delete 1
|
||||
break BEGIN_ROUND
|
||||
continue
|
||||
info session
|
||||
info summary
|
||||
delete all
|
||||
continue
|
||||
info summary
|
||||
info jobs
|
||||
info config
|
||||
set output off
|
||||
quit
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,28 @@
|
|||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-101 nhosts=2 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=1 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-102 nhosts=1 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=2 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-103 nhosts=1 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=3 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-104 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=4 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-105 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=5 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-106 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=2000 decision_index=6 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=3000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-104 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=3000 decision_index=1 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-105 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=3000 decision_index=2 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-106 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=3000 decision_index=3 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=4000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-104 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=4000 decision_index=1 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-105 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=4000 decision_index=2 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-106 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=4000 decision_index=3 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=5000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-104 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=5000 decision_index=1 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-105 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=5000 decision_index=2 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-106 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=5000 decision_index=3 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=6000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-104 nhosts=1 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=6000 decision_index=1 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-105 nhosts=1 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=6000 decision_index=2 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-106 nhosts=2 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=6000 decision_index=3 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=7000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=8000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=9000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=9000 decision_index=1 expected_ts=0 -- expected: <none> -- actual: DISPATCH jobid=0-0-0-108 nhosts=1 -- diff: no recorded decision at this index
|
||||
sim_log_mismatch decision verification MISMATCH at begin_round=10000 decision_index=0 expected_ts=0 -- expected: <none> -- actual: PEND jobid=0-0-0-107 main_reason=0 -- diff: no recorded decision at this index
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
[sim] replay file contains 9 sessions
|
||||
[sim] loaded: 0 jobs, snapshot @ ts=1000 (1970-01-01T00:00:00Z)
|
||||
[sim] (jobs above reflect this snapshot only -- step/continue to replay later events)
|
||||
[sim] ready. type 'help' for commands.
|
||||
[sim] available commands:
|
||||
|
||||
Execution Control:
|
||||
step [N] Advance N sessions (default 1), pause after.
|
||||
continue Run until next breakpoint hit or EOF.
|
||||
quit Exit the scheduler immediately.
|
||||
help Print this help listing.
|
||||
|
||||
Breakpoints:
|
||||
break <EVENT_TYPE> [key=value ...] Set a breakpoint. Valid event types: CONFIG_PUSH, JOB_PUSH, WORKERS_PUSH, RESOURCES_PUSH, RESVALS_FULL, RESVALS_DELTA, JOB_STATE_CHANGE, JOB_UPDATE, QUEUE_UPDATE, JOB_RUSAGE, BEGIN_ROUND, DECISION.
|
||||
info break List breakpoints.
|
||||
delete <id>|all Delete breakpoint(s).
|
||||
|
||||
State Inspection:
|
||||
info session Simulation position.
|
||||
info job <jobid> Dump one job.
|
||||
info jobs List all job IDs.
|
||||
info queue <name>|all Dump queue(s).
|
||||
info worker <name>|all Dump worker(s).
|
||||
info resource <name>|all Dump resource(s).
|
||||
info summary Quick overview.
|
||||
info config Show sim parameters.
|
||||
info limit <name>|all Show limit setting(s) and usage.
|
||||
|
||||
Snapshot & Parameters:
|
||||
snapshot [FILE] Write a standalone snapshot.
|
||||
set output <FILE>|off Start/stop writing decisions.
|
||||
set verify on|off Toggle decision verification.
|
||||
set units raw|human|both Show resource values raw, human-scaled, or both (default: both).
|
||||
[sim] info: usage: info <break|session|config|job|jobs|queue|worker|resource|summary|limit> [arg]
|
||||
[sim] config: output=(active, path unknown) verify=on events_processed=0 current_ts=1000 (1970-01-01T00:00:00Z) sessions=0
|
||||
[sim] t=1001 (1970-01-01T00:00:00Z) session=1
|
||||
[sim] event: WORKERS_PUSH nworkers=2 names=[w01,w02]
|
||||
[sim] event: QUEUE_UPDATE queue=normal
|
||||
[sim] event: RESVALS_FULL n=1
|
||||
[sim] event: JOB_PUSH njobs=7 jobids=[0-0-0-101,0-0-0-102,0-0-0-103,0-0-0-104,0-0-0-105,0-0-0-106,0-0-0-107]
|
||||
[sim] DISPATCH jobid=0-0-0-101
|
||||
[sim] DISPATCH jobid=0-0-0-102
|
||||
[sim] DISPATCH jobid=0-0-0-103
|
||||
[sim] PEND jobid=0-0-0-104
|
||||
[sim] PEND jobid=0-0-0-105
|
||||
[sim] PEND jobid=0-0-0-106
|
||||
[sim] PEND jobid=0-0-0-107
|
||||
[sim] session: ts=2000 (1970-01-01T00:00:00Z) event_index=4 sessions=1
|
||||
[sim] last session: round=2000 dispatch=3 pend=4 preempt=0
|
||||
[sim] summary: 7 jobs (4 pending, 3 running), 2 workers (2 online), 1 queues, 5 resources
|
||||
[sim] next non-dummy session: 5 -- run 'step 4' to reach it
|
||||
[sim] jobs:
|
||||
0-0-0-101 RUN normal
|
||||
0-0-0-102 RUN normal
|
||||
0-0-0-103 RUN normal
|
||||
0-0-0-104 PEND normal
|
||||
0-0-0-105 PEND normal
|
||||
0-0-0-106 PEND normal
|
||||
0-0-0-107 PEND normal
|
||||
[sim] info jobs: 7 job(s)
|
||||
[sim] job 104 (0-0-0-104): state=PEND queue=normal priority=0 queue_time=1104 resreq=Primary:
|
||||
Tasks: min = 1, max = 1
|
||||
Rusages:
|
||||
res license value 1.000000 per 3 duration -1.000000 decay 0
|
||||
allocs=0
|
||||
[sim] worker w01: status=online
|
||||
[sim] -- Consumable/Capacity --
|
||||
[sim] resource slots: value=4.00 used=3.00
|
||||
[sim] resource mem: value=16384.000000 (16.00 KB) used=2048.000000 (2.00 KB)
|
||||
[sim] resource limit=host_slots: value=3.00 used=3.00
|
||||
[sim] assigned job: jobid=0-0-0-101*2
|
||||
[sim] assigned job: jobid=0-0-0-102
|
||||
[sim] worker w02: status=online
|
||||
[sim] -- Consumable/Capacity --
|
||||
[sim] resource slots: value=4.00 used=3.00
|
||||
[sim] resource mem: value=16384.000000 (16.00 KB) used=2048.000000 (2.00 KB)
|
||||
[sim] resource limit=host_slots: value=3.00 used=3.00
|
||||
[sim] assigned job: jobid=0-0-0-101*2
|
||||
[sim] assigned job: jobid=0-0-0-103
|
||||
[sim] resource license: desc= valtype=numeric ordtype=incr flags=GLOBAL,SHARED,CONSUMABLE interval=0
|
||||
[sim] value=2.00 used=2.00
|
||||
[sim] limit: cluster_slots
|
||||
[sim] USER = all
|
||||
[sim] QUEUE = all
|
||||
[sim] PROJECT = all
|
||||
[sim] HOST = all
|
||||
[sim]
|
||||
[sim] slots = 6.000000
|
||||
[sim] usage limit=cluster_slots: accounted=6.00 capacity=6.00
|
||||
[sim] limit: host_slots
|
||||
[sim] USER = all
|
||||
[sim] QUEUE = all
|
||||
[sim] PROJECT = all
|
||||
[sim] HOST = all
|
||||
[sim]
|
||||
[sim] slots = 75.000000%
|
||||
[sim] note: this limit's value is a percentage of a resource that is not shared across the cluster, so usage for it is tracked separately per host even though the configuration above does not show an explicit host scope
|
||||
[sim] usage limit=host_slots host=w01: accounted=3.00 capacity=3.00
|
||||
[sim] usage limit=host_slots host=w02: accounted=3.00 capacity=3.00
|
||||
[sim] units set to raw
|
||||
[sim] worker w01: status=online
|
||||
[sim] -- Consumable/Capacity --
|
||||
[sim] resource slots: value=4.00 used=3.00
|
||||
[sim] resource mem: value=16384.000000 used=2048.000000
|
||||
[sim] resource limit=host_slots: value=3.00 used=3.00
|
||||
[sim] assigned job: jobid=0-0-0-101*2
|
||||
[sim] assigned job: jobid=0-0-0-102
|
||||
[sim] units set to human
|
||||
[sim] worker w01: status=online
|
||||
[sim] -- Consumable/Capacity --
|
||||
[sim] resource slots: value=4.00 used=3.00
|
||||
[sim] resource mem: value=16.00 KB used=2.00 KB
|
||||
[sim] resource limit=host_slots: value=3.00 used=3.00
|
||||
[sim] assigned job: jobid=0-0-0-101*2
|
||||
[sim] assigned job: jobid=0-0-0-102
|
||||
[sim] -- Consumable/Capacity --
|
||||
[sim] resource license: desc= valtype=numeric ordtype=incr flags=GLOBAL,SHARED,CONSUMABLE interval=0
|
||||
[sim] value=2.00 used=2.00
|
||||
[sim] resource limit=cluster_slots: desc=(none) valtype=numeric ordtype=decr flags=GLOBAL,SHARED,RELEASE,CONSUMABLE,LIMIT interval=0
|
||||
[sim] value=6.00 used=6.00
|
||||
[sim] units set to both
|
||||
[sim] worker w02: status=online
|
||||
[sim] -- Consumable/Capacity --
|
||||
[sim] resource slots: value=4.00 used=3.00
|
||||
[sim] resource mem: value=16384.000000 (16.00 KB) used=2048.000000 (2.00 KB)
|
||||
[sim] resource limit=host_slots: value=3.00 used=3.00
|
||||
[sim] assigned job: jobid=0-0-0-101*2
|
||||
[sim] assigned job: jobid=0-0-0-103
|
||||
[sim] queue normal: priority=1 status=open RUNLIMIT=n/a MAX_RUNNING=n/a job_pick_plugin=(inherit)
|
||||
[sim] queued job: jobid=0-0-0-104 priority=0
|
||||
[sim] queued job: jobid=0-0-0-105 priority=0
|
||||
[sim] queued job: jobid=0-0-0-106 priority=0
|
||||
[sim] queued job: jobid=0-0-0-107 priority=0
|
||||
[sim] verify set to off
|
||||
[sim] config: output=(active, path unknown) verify=off events_processed=4 current_ts=2000 (1970-01-01T00:00:00Z) sessions=1
|
||||
[sim] verify set to on
|
||||
[sim] snapshot written: snap.replay (ts=2000, 1970-01-01T00:00:00Z)
|
||||
[sim] output started: set_output.replay
|
||||
[sim] breakpoint 1 set: DECISION jobid=104 type=DISPATCH
|
||||
[sim] breakpoints:
|
||||
[sim] 1: DECISION jobid=104 type=DISPATCH
|
||||
[sim] round 2-4 (3 round(s), t=1970-01-01T00:00:00Z..1970-01-01T00:00:00Z): 0 dispatch, 4 pend, 0 preempt -- jobids=0-0-0-104,0-0-0-105,0-0-0-106,0-0-0-107
|
||||
[sim] hit breakpoint 1 @ decision: DISPATCH jobid=0-0-0-104
|
||||
[sim] session: ts=6000 (1970-01-01T00:00:00Z) event_index=6 sessions=5
|
||||
[sim] last session: round=6000 dispatch=1 pend=0 preempt=0
|
||||
[sim] job 104 (0-0-0-104): state=RUN queue=normal priority=0 queue_time=1104 resreq=Primary:
|
||||
Tasks: min = 1, max = 1
|
||||
Rusages:
|
||||
res license value 1.000000 per 3 duration -1.000000 decay 0
|
||||
allocs=1
|
||||
[sim] breakpoint 1 deleted
|
||||
[sim] breakpoint 2 set: BEGIN_ROUND
|
||||
[sim] round 5-5 (1 round(s), t=1970-01-01T00:00:00Z..1970-01-01T00:00:00Z): 3 dispatch, 1 pend, 0 preempt -- jobids=0-0-0-107
|
||||
[sim] hit breakpoint 2 @ BEGIN_ROUND round=7000
|
||||
[sim] session: ts=6000 (1970-01-01T00:00:00Z) event_index=6 sessions=6
|
||||
[sim] last session: round=7000 dispatch=0 pend=0 preempt=0
|
||||
[sim] summary: 7 jobs (1 pending, 4 running), 2 workers (2 online), 1 queues, 5 resources
|
||||
[sim] next non-dummy session: 8 -- run 'step 2' to reach it
|
||||
[sim] deleted 1 breakpoint(s)
|
||||
[sim] round 6-7 (2 round(s), t=1970-01-01T00:00:00Z..1970-01-01T00:00:00Z): 0 dispatch, 1 pend, 0 preempt -- jobids=0-0-0-107
|
||||
[sim] t=8500 (1970-01-01T00:00:00Z) session=8
|
||||
[sim] event: JOB_PUSH njobs=1 jobids=[0-0-0-108]
|
||||
[sim] round 8-8 (1 round(s), t=1970-01-01T00:00:00Z..1970-01-01T00:00:00Z): 1 dispatch, 1 pend, 0 preempt -- jobids=0-0-0-107
|
||||
[sim] round 9-9 (1 round(s), t=1970-01-01T00:00:00Z..1970-01-01T00:00:00Z): 0 dispatch, 1 pend, 0 preempt -- jobids=0-0-0-107
|
||||
[sim] replay complete. 9 session(s) processed.
|
||||
[sim] output written to: set_output.replay
|
||||
[sim] simulation finished -- state remains available for inspection (info job/queue/worker/resource/summary). type 'quit' to exit.
|
||||
[sim] summary: 8 jobs (1 pending, 4 running), 2 workers (2 online), 1 queues, 5 resources
|
||||
[sim] next non-dummy session: none remaining
|
||||
[sim] jobs:
|
||||
0-0-0-101 DONE normal
|
||||
0-0-0-102 EXIT normal
|
||||
0-0-0-103 DONE normal
|
||||
0-0-0-104 RUN normal
|
||||
0-0-0-105 RUN normal
|
||||
0-0-0-106 RUN normal
|
||||
0-0-0-107 PEND normal
|
||||
0-0-0-108 RUN normal
|
||||
[sim] info jobs: 8 job(s)
|
||||
[sim] config: output=set_output.replay verify=on events_processed=8 current_ts=10000 (1970-01-01T00:00:00Z) sessions=9
|
||||
[sim] output stopped: set_output.replay (4 event(s) written)
|
||||
|
|
@ -2474,3 +2474,38 @@ test('scheduler sim discrete clock test',
|
|||
sched_sim_discrete_clock_test, suite: 'scheduler',
|
||||
env: ['ASAN_OPTIONS=detect_leaks=0'],
|
||||
depends: scheduler)
|
||||
|
||||
# scheduler-sim-decision-trace spec, task 1.1 (Req 10.4, 10.5): the
|
||||
# trace-off REPL transcript baseline. Runs the built xce_scheduler
|
||||
# (XCE_SCHEDULER_BIN; `depends: scheduler` builds it first) in interactive
|
||||
# simulation mode on the committed resource-contention replay under
|
||||
# fixtures/repl_trace_baseline/ with a scripted session, and compares the
|
||||
# console transcript, the --sim-output and `set output` recordings and the
|
||||
# --sim-verify result with the committed baseline (wall-clock metadata
|
||||
# excluded). The replay writer links the real scheduler_srcs, same
|
||||
# technique as sched_sim_e2e_limit_visibility_test. Set
|
||||
# XCE_REPL_BASELINE_REGEN=1 to regenerate the fixtures (see the file's
|
||||
# header comment).
|
||||
sched_repl_trace_baseline_test_srcs = ['test_repl_trace_baseline.c']
|
||||
foreach f : scheduler_srcs
|
||||
if f != 'src/main.c'
|
||||
sched_repl_trace_baseline_test_srcs += ['../' + f]
|
||||
endif
|
||||
endforeach
|
||||
|
||||
sched_repl_trace_baseline_test = executable(
|
||||
'sched_repl_trace_baseline_test',
|
||||
sched_repl_trace_baseline_test_srcs,
|
||||
include_directories:[incdir],
|
||||
dependencies: [scheduler_test_deps],
|
||||
c_args: ['-DXCE_SCHEDULER_BIN="' + scheduler.full_path() + '"',
|
||||
'-DXCE_BASELINE_DIR="'
|
||||
+ (meson.current_source_dir() / 'fixtures'
|
||||
/ 'repl_trace_baseline')
|
||||
+ '"'],
|
||||
install: false)
|
||||
|
||||
test('scheduler repl trace baseline test',
|
||||
sched_repl_trace_baseline_test, suite: 'scheduler',
|
||||
env: ['ASAN_OPTIONS=detect_leaks=0'],
|
||||
depends: scheduler)
|
||||
|
|
|
|||
917
src/scheduler/testsuite/test_repl_trace_baseline.c
Normal file
917
src/scheduler/testsuite/test_repl_trace_baseline.c
Normal file
|
|
@ -0,0 +1,917 @@
|
|||
/*
|
||||
* test_repl_trace_baseline.c - scheduler-sim-decision-trace spec, task 1.1
|
||||
* (Requirements 10.4, 10.5; design.md "Testing Strategy" -> "Equivalence
|
||||
* Tests (Requirement 10)" item 2): the trace-off REPL transcript baseline.
|
||||
*
|
||||
* A scripted REPL session runs the real, built xce_scheduler binary
|
||||
* (XCE_SCHEDULER_BIN, from testsuite/meson.build) in interactive simulation
|
||||
* mode over a committed resource-contention replay. Its console transcript,
|
||||
* its --sim-output recording, the recording a `set output` command opens
|
||||
* midway, and its --sim-verify result are compared with committed fixtures
|
||||
* captured before any decision-trace code existed. Later tasks of the spec
|
||||
* use this test to prove that, with tracing off, the REPL output is
|
||||
* unchanged (task 5.2 updates exactly the help and info-usage lines).
|
||||
*
|
||||
* Fixtures (testsuite/fixtures/repl_trace_baseline/, XCE_BASELINE_DIR):
|
||||
* contention.replay input replay (written by write_contention_replay())
|
||||
* script.txt the REPL command script fed on stdin
|
||||
* transcript.txt expected stdout, byte for byte
|
||||
* sim_output.replay expected --sim-output recording
|
||||
* set_output.replay expected recording opened by `set output`
|
||||
*
|
||||
* sim_output.replay holds only the header, the snapshot and the input
|
||||
* events, with no session decisions. This is existing behaviour: the
|
||||
* script's `set output` runs in xce_repl_post_session_pause(), before
|
||||
* sched_record_session_finalize() writes session 1 (sim_loop.c), so session
|
||||
* 1's decisions land in set_output.replay, as do all later sessions.
|
||||
* Decision coverage therefore comes from set_output.replay and
|
||||
* sim_verify.txt. The file written by the script's `snapshot snap.replay`
|
||||
* is not compared; only its transcript line is.
|
||||
* sim_verify.txt expected --sim-verify result: the mismatch lines of
|
||||
* the log, each cut to start at its message
|
||||
* ("sim_log_mismatch decision verification MISMATCH")
|
||||
* so the logger's time/thread-id prefix is dropped
|
||||
*
|
||||
* The only metadata the comparison ignores: ReplayHeader.
|
||||
* recordingStartWall in both recordings, and the log-line prefix (time and
|
||||
* thread id) of the verify lines. Everything else is compared byte for
|
||||
* byte.
|
||||
*
|
||||
* The contention replay: two workers (4 slots, 16384 mem each), a 2-unit
|
||||
* cluster license, a cluster-wide limit of 6 slots and a host-scoped limit
|
||||
* of 75% of each host's slots. Jobs compete for the license, both limits
|
||||
* and the hosts: license jobs, a parallel job (2..4 tasks), an alternative-
|
||||
* requirement job whose first alternative cannot fit, and slot-heavy jobs.
|
||||
* Later rounds finish jobs (JOB_STATE_CHANGE) and push a late job, with
|
||||
* idle rounds in between so `continue` collapses ranges. Every session's
|
||||
* recorded decision list is empty, so --sim-verify reports each live
|
||||
* decision as unexpected and no recorded-decision reconciliation runs.
|
||||
*
|
||||
* The child runs in a fresh temporary directory holding copies of the
|
||||
* replay and script, with relative --sim-output/--logfile/`set output`
|
||||
* paths, so no temporary path reaches the transcript.
|
||||
*
|
||||
* Regenerating (only when a spec task deliberately changes the expected
|
||||
* output, e.g. task 5.2's help lines): run this binary with
|
||||
* XCE_REPL_BASELINE_REGEN=1. It rewrites contention.replay and every
|
||||
* expected-output fixture from the current tree; script.txt is hand
|
||||
* written and never regenerated. Review the fixture diff before
|
||||
* committing.
|
||||
*
|
||||
* Link style: the real scheduler_srcs list (minus main.c), same as
|
||||
* test_sim_e2e_limit_visibility.c, which this file's replay writer
|
||||
* follows (resources, license value and limit config baked into the
|
||||
* snapshot; workers, queue and jobs as wire events).
|
||||
*/
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <limits.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "controller.capnp.h"
|
||||
#include "recording.h"
|
||||
#include "resobj.h"
|
||||
#include "scheduler.h"
|
||||
#include "sim.h"
|
||||
#include "xcelimit.h"
|
||||
#include "xstring.h"
|
||||
|
||||
#if !defined(XCE_SCHEDULER_BIN) || !defined(XCE_BASELINE_DIR)
|
||||
#error "XCE_SCHEDULER_BIN and XCE_BASELINE_DIR must be defined by meson"
|
||||
#endif
|
||||
|
||||
extern int write_capnp_message(FILE* f, struct capn* c);
|
||||
|
||||
#define REPLAY_NAME "contention.replay"
|
||||
#define SCRIPT_NAME "script.txt"
|
||||
#define TRANSCRIPT_NAME "transcript.txt"
|
||||
#define SIM_OUT_NAME "sim_output.replay"
|
||||
#define SET_OUT_NAME "set_output.replay"
|
||||
#define VERIFY_NAME "sim_verify.txt"
|
||||
#define LOG_NAME "sched.log"
|
||||
#define STDERR_NAME "stderr.txt"
|
||||
#define SNAP_NAME "snap.replay"
|
||||
|
||||
#define QUEUE_NAME "normal"
|
||||
#define WORKER_A "w01"
|
||||
#define WORKER_B "w02"
|
||||
#define LICENSE_RES "license"
|
||||
#define SNAPSHOT_TS 1000ULL
|
||||
#define GLIMIT_NAME "cluster_slots"
|
||||
#define HLIMIT_NAME "host_slots"
|
||||
|
||||
#define MISMATCH_TAG "decision verification MISMATCH"
|
||||
#define VERIFY_MARK "sim_log_mismatch " MISMATCH_TAG
|
||||
|
||||
void setUp(void) { return; }
|
||||
void tearDown(void) { return; }
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* File helpers */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
/* Reads a whole file. Returns NULL (and *len = 0) when it cannot be read,
|
||||
* so a missing fixture fails the caller's assertion, not this helper. */
|
||||
static char* slurp(const char* path, long* len)
|
||||
{
|
||||
FILE* f = fopen(path, "rb");
|
||||
char* buf;
|
||||
long n;
|
||||
|
||||
*len = 0;
|
||||
if (f == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
if ((fseek(f, 0, SEEK_END) != 0) || ((n = ftell(f)) < 0)) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
rewind(f);
|
||||
buf = (char*)malloc((size_t)n + 1);
|
||||
if (buf == NULL) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
if ((n > 0) && (fread(buf, 1, (size_t)n, f) != (size_t)n)) {
|
||||
free(buf);
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
buf[n] = '\0';
|
||||
fclose(f);
|
||||
*len = n;
|
||||
return buf;
|
||||
}
|
||||
|
||||
static void spit(const char* path, const char* buf, long len)
|
||||
{
|
||||
FILE* f = fopen(path, "wb");
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(f, path);
|
||||
if (len > 0) {
|
||||
TEST_ASSERT_EQUAL_UINT((size_t)len, fwrite(buf, 1, (size_t)len, f));
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(0, fclose(f));
|
||||
}
|
||||
|
||||
static void copy_file(const char* from, const char* to)
|
||||
{
|
||||
long len;
|
||||
char* buf = slurp(from, &len);
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(buf, from);
|
||||
spit(to, buf, len);
|
||||
free(buf);
|
||||
}
|
||||
|
||||
static void join_path(char* out, size_t outsz, const char* dir,
|
||||
const char* name)
|
||||
{
|
||||
int n = snprintf(out, outsz, "%s/%s", dir, name);
|
||||
|
||||
TEST_ASSERT_TRUE((n > 0) && ((size_t)n < outsz));
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* Contention replay writer */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
static xce_resource_t* bake_resource(var resmgr, const char* name, int flags)
|
||||
{
|
||||
xce_resource_t res;
|
||||
xce_resource_t* r;
|
||||
|
||||
memset(&res, 0, sizeof(res));
|
||||
res.name = strdup(name);
|
||||
res.valtype = XCE_VAL_NUMERIC;
|
||||
res.flags = flags;
|
||||
|
||||
r = makeResource(resmgr, &res);
|
||||
TEST_ASSERT_NOT_NULL(r);
|
||||
return r;
|
||||
}
|
||||
|
||||
static xce_name_val_t* make_nv(const char* name, float v)
|
||||
{
|
||||
xce_name_val_t* nv = (xce_name_val_t*)calloc(1, sizeof(*nv));
|
||||
xce_val_t* val = (xce_val_t*)calloc(1, sizeof(*val));
|
||||
|
||||
TEST_ASSERT_NOT_NULL(nv);
|
||||
TEST_ASSERT_NOT_NULL(val);
|
||||
val->which = Value_u_fvalue;
|
||||
val->u.fvalue = v;
|
||||
nv->name = strdup(name);
|
||||
nv->val = val;
|
||||
return nv;
|
||||
}
|
||||
|
||||
static xce_worker_t* make_wire_worker(const char* name, float slots,
|
||||
float mem)
|
||||
{
|
||||
xce_worker_t* w = (xce_worker_t*)calloc(1, sizeof(*w));
|
||||
xce_name_val_t** nv = (xce_name_val_t**)calloc(2, sizeof(*nv));
|
||||
|
||||
TEST_ASSERT_NOT_NULL(w);
|
||||
TEST_ASSERT_NOT_NULL(nv);
|
||||
nv[0] = make_nv("slots", slots);
|
||||
nv[1] = make_nv("mem", mem);
|
||||
w->name = strdup(name);
|
||||
w->status = 0;
|
||||
w->nresvals_to_sched = 2;
|
||||
w->resvals_to_sched = nv;
|
||||
return w;
|
||||
}
|
||||
|
||||
static xce_job_t* make_push_job(uint64_t jobid, uint64_t sub_time,
|
||||
const char* resreq, const char* user,
|
||||
uint32_t mintasks, uint32_t maxtasks)
|
||||
{
|
||||
xce_job_t* j = (xce_job_t*)calloc(1, sizeof(*j));
|
||||
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
j->jobid = jobid;
|
||||
j->state = JOB_STAT_PEND;
|
||||
j->sub_time = sub_time;
|
||||
j->queue_time = sub_time;
|
||||
|
||||
j->submit = (job_submit_data_t*)calloc(1, sizeof(*(j->submit)));
|
||||
TEST_ASSERT_NOT_NULL(j->submit);
|
||||
j->submit->queue = (char*)QUEUE_NAME;
|
||||
j->submit->user = (char*)user;
|
||||
j->submit->project_name = (char*)"proj1";
|
||||
j->submit->resreq = (char*)resreq;
|
||||
j->submit->mintasks = mintasks;
|
||||
j->submit->maxtasks = maxtasks;
|
||||
return j;
|
||||
}
|
||||
|
||||
static void write_event(FILE* f, replay_event_t* src)
|
||||
{
|
||||
struct capn c;
|
||||
struct capn_segment* cs;
|
||||
ReplayEvent_ptr p;
|
||||
|
||||
capn_init_malloc(&c);
|
||||
cs = capn_root(&c).seg;
|
||||
encode_ReplayEvent_ptr(cs, &p, src);
|
||||
capn_setp(capn_root(&c), 0, p.p);
|
||||
TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c));
|
||||
capn_free(&c);
|
||||
}
|
||||
|
||||
static void append_workers(FILE* f, uint64_t ts, xce_worker_t** w, int n)
|
||||
{
|
||||
xce_workers_data_t wrap;
|
||||
replay_event_t src;
|
||||
|
||||
memset(&wrap, 0, sizeof(wrap));
|
||||
wrap.n_total = n;
|
||||
wrap.count = n;
|
||||
wrap.workers = w;
|
||||
|
||||
memset(&src, 0, sizeof(src));
|
||||
src.ts = ts;
|
||||
src.body_which = ReplayEvent_body_workersPush;
|
||||
src.body.workers_push = &wrap;
|
||||
write_event(f, &src);
|
||||
}
|
||||
|
||||
static void append_queue(FILE* f, uint64_t ts, const char* name)
|
||||
{
|
||||
xce_queue_t q;
|
||||
replay_event_t src;
|
||||
|
||||
memset(&q, 0, sizeof(q));
|
||||
q.queue = (char*)name;
|
||||
q.status = QUEUE_STAT_OPEN;
|
||||
q.priority = 1;
|
||||
|
||||
memset(&src, 0, sizeof(src));
|
||||
src.ts = ts;
|
||||
src.body_which = ReplayEvent_body_queueUpdate;
|
||||
src.body.queue_update = &q;
|
||||
write_event(f, &src);
|
||||
}
|
||||
|
||||
/* The license's cluster-wide value, through the same RESVALS_FULL ingest
|
||||
* the live driver records. bug-1923: shared resvals baked into the
|
||||
* snapshot are dropped on load, so the snapshot cannot carry it. */
|
||||
static void append_license_value(FILE* f, uint64_t ts, float total)
|
||||
{
|
||||
xce_val_t val;
|
||||
xce_resval_t rv;
|
||||
xce_resval_t* rvs[1];
|
||||
xce_resval_list_t list;
|
||||
replay_event_t src;
|
||||
|
||||
memset(&val, 0, sizeof(val));
|
||||
val.which = Value_u_fvalue;
|
||||
val.u.fvalue = total;
|
||||
|
||||
memset(&rv, 0, sizeof(rv));
|
||||
rv.name = (char*)LICENSE_RES;
|
||||
rv.val = &val;
|
||||
rvs[0] = &rv;
|
||||
|
||||
memset(&list, 0, sizeof(list));
|
||||
list.count = 1;
|
||||
list.vals = rvs;
|
||||
|
||||
memset(&src, 0, sizeof(src));
|
||||
src.ts = ts;
|
||||
src.body_which = ReplayEvent_body_resvalsFull;
|
||||
src.body.resvals_full = &list;
|
||||
write_event(f, &src);
|
||||
}
|
||||
|
||||
static void append_jobs(FILE* f, uint64_t ts, xce_job_t** jobs, int n)
|
||||
{
|
||||
xce_jobs_data_t wrap;
|
||||
replay_event_t src;
|
||||
|
||||
memset(&wrap, 0, sizeof(wrap));
|
||||
wrap.n_total = n;
|
||||
wrap.count = n;
|
||||
wrap.jobs = jobs;
|
||||
|
||||
memset(&src, 0, sizeof(src));
|
||||
src.ts = ts;
|
||||
src.body_which = ReplayEvent_body_jobsPush;
|
||||
src.body.jobs_push = &wrap;
|
||||
write_event(f, &src);
|
||||
}
|
||||
|
||||
static void append_state_change(FILE* f, uint64_t ts, uint64_t jobid,
|
||||
uint32_t old_state, uint32_t new_state)
|
||||
{
|
||||
job_state_change_event_t change;
|
||||
replay_event_t src;
|
||||
|
||||
memset(&change, 0, sizeof(change));
|
||||
change.jobid = jobid;
|
||||
change.old_state = old_state;
|
||||
change.new_state = new_state;
|
||||
|
||||
memset(&src, 0, sizeof(src));
|
||||
src.ts = ts;
|
||||
src.body_which = ReplayEvent_body_jobStateChange;
|
||||
src.body.job_state_change = &change;
|
||||
write_event(f, &src);
|
||||
}
|
||||
|
||||
/* An empty decisions list: see this file's header comment. */
|
||||
static void append_session(FILE* f, uint64_t begin_round)
|
||||
{
|
||||
struct capn c;
|
||||
struct capn_segment* cs;
|
||||
ReplaySession_ptr sp;
|
||||
|
||||
capn_init_malloc(&c);
|
||||
cs = capn_root(&c).seg;
|
||||
sp = new_ReplaySession(cs);
|
||||
ReplaySession_set_beginRound(sp, begin_round);
|
||||
capn_setp(capn_root(&c), 0, sp.p);
|
||||
TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c));
|
||||
capn_free(&c);
|
||||
}
|
||||
|
||||
static void append_end_of_stream(FILE* f)
|
||||
{
|
||||
replay_event_t src;
|
||||
|
||||
memset(&src, 0, sizeof(src));
|
||||
src.body_which = ReplayEvent_body_endSession;
|
||||
write_event(f, &src);
|
||||
}
|
||||
|
||||
/* Header + snapshot via the real sched_record_start(), with the resources
|
||||
* and both limits baked into the snapshot (the limit plugin is seeded from
|
||||
* the snapshot before any replay event runs; see
|
||||
* test_sim_e2e_limit_visibility.c's header comment). Records into a
|
||||
* scratch directory; the caller moves the file to its final place. */
|
||||
static FILE* write_header_and_snapshot(char* out_path, size_t out_len,
|
||||
const char* record_dir)
|
||||
{
|
||||
xce_scheduler_t sched;
|
||||
xce_sched_plugin_t* fcfs;
|
||||
xce_limit_val_t gval = { .resname = (char*)"slots",
|
||||
.percent = 0,
|
||||
.val = 6.0 };
|
||||
xce_limit_config_t gcfg = { .name = (char*)GLIMIT_NAME, .val = &gval };
|
||||
xce_limit_val_t hval = { .resname = (char*)"slots",
|
||||
.percent = 1,
|
||||
.val = 0.75 };
|
||||
xce_limit_config_t hcfg = { .name = (char*)HLIMIT_NAME, .val = &hval };
|
||||
xce_limit_config_t* limits[2] = { &gcfg, &hcfg };
|
||||
xce_conf4sched_t config;
|
||||
|
||||
memset(&sched, 0, sizeof(sched));
|
||||
mylock_init(&(sched.record_lock));
|
||||
TEST_ASSERT_EQUAL_INT(0, xce_sched_core_init(&(sched.core)));
|
||||
|
||||
fcfs = builtin_fcfs_new_plugin();
|
||||
TEST_ASSERT_NOT_NULL(fcfs);
|
||||
TEST_ASSERT_TRUE(plugin_register_core(&(sched.core), fcfs) >= 0);
|
||||
|
||||
(void)bake_resource(sched.core.resmgr, "slots", XCE_RESF_CONSUMABLE);
|
||||
(void)bake_resource(sched.core.resmgr, "mem", XCE_RESF_CONSUMABLE);
|
||||
(void)bake_resource(sched.core.resmgr, LICENSE_RES,
|
||||
XCE_RESF_GLOBAL | XCE_RESF_SHARED
|
||||
| XCE_RESF_CONSUMABLE);
|
||||
|
||||
memset(&config, 0, sizeof(config));
|
||||
config.nlimits = 2;
|
||||
config.limits = limits;
|
||||
sched.core.config = &config;
|
||||
|
||||
/* A frozen logical clock (speed 0, the simulator's own discrete-clock
|
||||
* setting) pins the header's recordingStartTs and the snapshot ts to
|
||||
* SNAPSHOT_TS instead of a CLOCK_MONOTONIC reading. */
|
||||
atomic_store(&(sched.core.virtual_clock_base), SNAPSHOT_TS);
|
||||
atomic_store(&(sched.core.speed_num), 0ULL);
|
||||
atomic_store(&(sched.core.speed_den), 1ULL);
|
||||
|
||||
sched.core.job_pick_plugin = (char*)"builtin_fcfs";
|
||||
sched.record_dir = (char*)record_dir;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, sched_record_start(&sched, REPLAY_NAME));
|
||||
TEST_ASSERT_TRUE(strlen(sched.record_file_path) < out_len);
|
||||
strcpy(out_path, sched.record_file_path);
|
||||
return sched.record_file;
|
||||
}
|
||||
|
||||
/*
|
||||
* write_contention_replay - the input fixture (regeneration only). The
|
||||
* license's value (2) arrives as a RESVALS_FULL event after the workers.
|
||||
* Rounds (beginRound = ts) and what the baseline shows for them:
|
||||
* 2000 jobs 101..107 pending, in FCFS order: 101 is a 2..4-task parallel
|
||||
* job (dispatches 4 tasks over both hosts), 102/103/104 want one
|
||||
* license each (104 pends: license exhausted), 105 wants an
|
||||
* alternative whose first branch (4 tasks x 12288 mem) cannot fit,
|
||||
* 106/107 want 3 slots each; the cluster-wide limit (6) and the
|
||||
* host limit (3 per host) are then full, so 105..107 pend
|
||||
* 3000, 4000, 5000 identical repeats (collapsed by `continue`)
|
||||
* 6000 after 101 finishes and 102 exits: 104, 105 and 106 dispatch
|
||||
* 7000, 8000 identical repeats
|
||||
* 9000 after a late license job 108 is pushed and 103 finishes
|
||||
* 10000 final round
|
||||
*/
|
||||
static void write_contention_replay(const char* dest)
|
||||
{
|
||||
char record_dir[PATH_MAX];
|
||||
char path[PATH_MAX];
|
||||
FILE* f;
|
||||
xce_worker_t* workers[2];
|
||||
xce_job_t* jobs[7];
|
||||
xce_job_t* late[1];
|
||||
|
||||
snprintf(record_dir, sizeof(record_dir), "/tmp/sched_trbase_gen_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(record_dir));
|
||||
|
||||
f = write_header_and_snapshot(path, sizeof(path), record_dir);
|
||||
|
||||
workers[0] = make_wire_worker(WORKER_A, 4.0f, 16384.0f);
|
||||
workers[1] = make_wire_worker(WORKER_B, 4.0f, 16384.0f);
|
||||
append_workers(f, 1001ULL, workers, 2);
|
||||
append_queue(f, 1002ULL, QUEUE_NAME);
|
||||
append_license_value(f, 1002ULL, 2.0f);
|
||||
|
||||
jobs[0] = make_push_job(101, 1101, "rusage[mem=1024]", "alice", 2, 4);
|
||||
jobs[1] = make_push_job(102, 1102, "rusage[license=1]", "bob", 1, 1);
|
||||
jobs[2] = make_push_job(103, 1103, "rusage[license=1]", "alice", 1, 1);
|
||||
jobs[3] = make_push_job(104, 1104, "rusage[license=1]", "bob", 1, 1);
|
||||
jobs[4] = make_push_job(105, 1105,
|
||||
"{4*{rusage[mem=12288]}} || "
|
||||
"{1*{rusage[mem=2048]}}",
|
||||
"alice", 1, 1);
|
||||
jobs[5] = make_push_job(106, 1106, "rusage[mem=512]", "bob", 3, 3);
|
||||
jobs[6] = make_push_job(107, 1107, "rusage[mem=512]", "alice", 3, 3);
|
||||
append_jobs(f, 1003ULL, jobs, 7);
|
||||
|
||||
append_session(f, 2000ULL);
|
||||
append_session(f, 3000ULL);
|
||||
append_session(f, 4000ULL);
|
||||
append_session(f, 5000ULL);
|
||||
|
||||
append_state_change(f, 5500ULL, 101, JOB_STAT_RUN, JOB_STAT_DONE);
|
||||
append_state_change(f, 5600ULL, 102, JOB_STAT_RUN, JOB_STAT_EXIT);
|
||||
append_session(f, 6000ULL);
|
||||
append_session(f, 7000ULL);
|
||||
append_session(f, 8000ULL);
|
||||
|
||||
late[0] = make_push_job(108, 8500, "rusage[license=1]", "bob", 1, 1);
|
||||
append_jobs(f, 8500ULL, late, 1);
|
||||
append_state_change(f, 8600ULL, 103, JOB_STAT_RUN, JOB_STAT_DONE);
|
||||
append_session(f, 9000ULL);
|
||||
append_session(f, 10000ULL);
|
||||
|
||||
append_end_of_stream(f);
|
||||
TEST_ASSERT_EQUAL_INT(0, fclose(f));
|
||||
|
||||
copy_file(path, dest);
|
||||
unlink(path);
|
||||
rmdir(record_dir);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* Running the scripted REPL session */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
static char g_workdir[PATH_MAX];
|
||||
static int g_ran;
|
||||
|
||||
/* Runs the built scheduler in a fresh work directory: stdin = script,
|
||||
* stdout = transcript, stderr kept apart (sanitizer and startup noise is
|
||||
* not part of the transcript). Returns the child's exit status. */
|
||||
static int run_session(const char* replay_src)
|
||||
{
|
||||
char path[PATH_MAX];
|
||||
pid_t pid;
|
||||
int status;
|
||||
|
||||
snprintf(g_workdir, sizeof(g_workdir), "/tmp/sched_trbase_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(g_workdir));
|
||||
|
||||
join_path(path, sizeof(path), g_workdir, REPLAY_NAME);
|
||||
copy_file(replay_src, path);
|
||||
{
|
||||
char src[PATH_MAX];
|
||||
|
||||
join_path(src, sizeof(src), XCE_BASELINE_DIR, SCRIPT_NAME);
|
||||
join_path(path, sizeof(path), g_workdir, SCRIPT_NAME);
|
||||
copy_file(src, path);
|
||||
}
|
||||
|
||||
fflush(stdout);
|
||||
fflush(stderr);
|
||||
pid = fork();
|
||||
TEST_ASSERT_TRUE_MESSAGE(pid >= 0, "fork must succeed");
|
||||
if (pid == 0) {
|
||||
if ((chdir(g_workdir) != 0)
|
||||
|| (freopen(SCRIPT_NAME, "r", stdin) == NULL)
|
||||
|| (freopen(TRANSCRIPT_NAME, "w", stdout) == NULL)
|
||||
|| (freopen(STDERR_NAME, "w", stderr) == NULL)) {
|
||||
_exit(120);
|
||||
}
|
||||
setenv("TERM", "dumb", 1);
|
||||
execl(XCE_SCHEDULER_BIN, XCE_SCHEDULER_BIN, "--sim", REPLAY_NAME,
|
||||
"--interactive", "--sim-verify", "--sim-output", SIM_OUT_NAME,
|
||||
"--logfile", LOG_NAME, (char*)NULL);
|
||||
_exit(121);
|
||||
}
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(pid, waitpid(pid, &status, 0));
|
||||
TEST_ASSERT_TRUE_MESSAGE(WIFEXITED(status),
|
||||
"the scheduler must exit normally");
|
||||
return WEXITSTATUS(status);
|
||||
}
|
||||
|
||||
/* The --sim-verify result: every mismatch line of the log, cut to start at
|
||||
* the message itself, which begins with the reporting function's name.
|
||||
* This drops the logger's prefix: the time, the thread id, the level and
|
||||
* the padded function-name column. */
|
||||
static char* extract_verify_result(const char* log_path, long* out_len)
|
||||
{
|
||||
FILE* f = fopen(log_path, "r");
|
||||
char* out = NULL;
|
||||
size_t cap = 0;
|
||||
size_t len = 0;
|
||||
char line[8192];
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(f, "the scheduler log must exist");
|
||||
out = (char*)calloc(1, 1);
|
||||
TEST_ASSERT_NOT_NULL(out);
|
||||
while (fgets(line, sizeof(line), f) != NULL) {
|
||||
char* start;
|
||||
size_t n;
|
||||
|
||||
if (strstr(line, MISMATCH_TAG) == NULL) {
|
||||
continue;
|
||||
}
|
||||
start = strstr(line, VERIFY_MARK);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(start, "mismatch line must name "
|
||||
VERIFY_MARK);
|
||||
n = strlen(start);
|
||||
if (len + n + 1 > cap) {
|
||||
cap = (len + n + 1) * 2;
|
||||
out = (char*)realloc(out, cap);
|
||||
TEST_ASSERT_NOT_NULL(out);
|
||||
}
|
||||
memcpy(out + len, start, n + 1);
|
||||
len += n;
|
||||
}
|
||||
fclose(f);
|
||||
*out_len = (long)len;
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Runs the session once per process; in regeneration mode first rewrites
|
||||
* the input replay, then copies every produced output into the fixtures. */
|
||||
static void ensure_session_ran(void)
|
||||
{
|
||||
char replay[PATH_MAX];
|
||||
int regen = (getenv("XCE_REPL_BASELINE_REGEN") != NULL);
|
||||
int rc;
|
||||
|
||||
if (g_ran) {
|
||||
return;
|
||||
}
|
||||
g_ran = 1;
|
||||
|
||||
join_path(replay, sizeof(replay), XCE_BASELINE_DIR, REPLAY_NAME);
|
||||
if (regen) {
|
||||
write_contention_replay(replay);
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, access(replay, R_OK),
|
||||
"committed contention.replay fixture "
|
||||
"must exist");
|
||||
|
||||
rc = run_session(replay);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "the scripted session must end "
|
||||
"with quit's exit(0)");
|
||||
|
||||
if (regen) {
|
||||
static const char* const outs[] = { TRANSCRIPT_NAME, SIM_OUT_NAME,
|
||||
SET_OUT_NAME };
|
||||
char from[PATH_MAX];
|
||||
char to[PATH_MAX];
|
||||
char* verify;
|
||||
long vlen;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 3; i++) {
|
||||
join_path(from, sizeof(from), g_workdir, outs[i]);
|
||||
join_path(to, sizeof(to), XCE_BASELINE_DIR, outs[i]);
|
||||
copy_file(from, to);
|
||||
}
|
||||
join_path(from, sizeof(from), g_workdir, LOG_NAME);
|
||||
verify = extract_verify_result(from, &vlen);
|
||||
join_path(to, sizeof(to), XCE_BASELINE_DIR, VERIFY_NAME);
|
||||
spit(to, verify, vlen);
|
||||
free(verify);
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* Comparisons */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
/* Reports the first differing line of two texts, for a readable failure. */
|
||||
static void assert_text_equal(const char* what, const char* exp, long elen,
|
||||
const char* got, long glen)
|
||||
{
|
||||
long i = 0;
|
||||
long line = 1;
|
||||
long bol = 0;
|
||||
char msg[1024];
|
||||
|
||||
while ((i < elen) && (i < glen) && (exp[i] == got[i])) {
|
||||
if (exp[i] == '\n') {
|
||||
line++;
|
||||
bol = i + 1;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
if ((i == elen) && (i == glen)) {
|
||||
return;
|
||||
}
|
||||
{
|
||||
const char* ee = memchr(exp + bol, '\n', (size_t)(elen - bol));
|
||||
const char* ge = memchr(got + bol, '\n', (size_t)(glen - bol));
|
||||
int el = ee ? (int)(ee - (exp + bol)) : (int)(elen - bol);
|
||||
int gl = ge ? (int)(ge - (got + bol)) : (int)(glen - bol);
|
||||
|
||||
snprintf(msg, sizeof(msg),
|
||||
"%s differs from the baseline at line %ld:\n"
|
||||
" expected: %.*s\n actual: %.*s",
|
||||
what, line, (el > 400) ? 400 : el, exp + bol,
|
||||
(gl > 400) ? 400 : gl, got + bol);
|
||||
}
|
||||
TEST_FAIL_MESSAGE(msg);
|
||||
}
|
||||
|
||||
/* A recording's header fields other than recordingStartWall, plus the
|
||||
* offset just past the header message. */
|
||||
static void read_header(const char* path, replay_header_t* hdr,
|
||||
long* hdr_end)
|
||||
{
|
||||
FILE* f = fopen(path, "rb");
|
||||
struct capn c;
|
||||
ReplayHeader_ptr p;
|
||||
struct ReplayHeader wire;
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(f, path);
|
||||
TEST_ASSERT_EQUAL_INT(0, capn_init_fp(&c, f, 0));
|
||||
p.p = capn_getp(capn_root(&c), 0, 1);
|
||||
read_ReplayHeader(&wire, p);
|
||||
memset(hdr, 0, sizeof(*hdr));
|
||||
decode_ReplayHeader(hdr, &wire);
|
||||
capn_free(&c);
|
||||
*hdr_end = ftell(f);
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
static void assert_recording_equal(const char* name)
|
||||
{
|
||||
char exp_path[PATH_MAX];
|
||||
char got_path[PATH_MAX];
|
||||
replay_header_t eh;
|
||||
replay_header_t gh;
|
||||
long e_end;
|
||||
long g_end;
|
||||
long elen;
|
||||
long glen;
|
||||
char* eb;
|
||||
char* gb;
|
||||
|
||||
join_path(exp_path, sizeof(exp_path), XCE_BASELINE_DIR, name);
|
||||
join_path(got_path, sizeof(got_path), g_workdir, name);
|
||||
|
||||
eb = slurp(exp_path, &elen);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(eb, "baseline recording fixture missing");
|
||||
TEST_ASSERT_TRUE_MESSAGE(elen > 0, "baseline recording fixture empty");
|
||||
gb = slurp(got_path, &glen);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(gb, "the session must write the recording");
|
||||
|
||||
read_header(exp_path, &eh, &e_end);
|
||||
read_header(got_path, &gh, &g_end);
|
||||
|
||||
TEST_ASSERT_EQUAL_STRING(eh.recorder_version, gh.recorder_version);
|
||||
TEST_ASSERT_EQUAL_UINT64(eh.recording_start_ts, gh.recording_start_ts);
|
||||
TEST_ASSERT_EQUAL_STRING(eh.sched_policy, gh.sched_policy);
|
||||
TEST_ASSERT_EQUAL_PTR(NULL, eh.cluster);
|
||||
TEST_ASSERT_EQUAL_PTR(NULL, gh.cluster);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT64_MESSAGE(elen - e_end, glen - g_end,
|
||||
"recording length after the header "
|
||||
"must match the baseline");
|
||||
TEST_ASSERT_EQUAL_MEMORY_MESSAGE(eb + e_end, gb + g_end,
|
||||
(size_t)(elen - e_end),
|
||||
"recording bytes after the header "
|
||||
"must match the baseline");
|
||||
|
||||
free_ReplayHeader(&eh);
|
||||
free_ReplayHeader(&gh);
|
||||
free(eb);
|
||||
free(gb);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* Tests */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
/* The console transcript is byte-identical to the baseline (10.4, 10.5). */
|
||||
static void test_transcript_matches_baseline(void)
|
||||
{
|
||||
char path[PATH_MAX];
|
||||
char* exp;
|
||||
char* got;
|
||||
long elen;
|
||||
long glen;
|
||||
|
||||
ensure_session_ran();
|
||||
|
||||
join_path(path, sizeof(path), XCE_BASELINE_DIR, TRANSCRIPT_NAME);
|
||||
exp = slurp(path, &elen);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(exp, "baseline transcript fixture missing");
|
||||
TEST_ASSERT_TRUE_MESSAGE(elen > 0, "baseline transcript fixture empty");
|
||||
|
||||
join_path(path, sizeof(path), g_workdir, TRANSCRIPT_NAME);
|
||||
got = slurp(path, &glen);
|
||||
TEST_ASSERT_NOT_NULL(got);
|
||||
|
||||
assert_text_equal("transcript", exp, elen, got, glen);
|
||||
free(exp);
|
||||
free(got);
|
||||
}
|
||||
|
||||
/* The baseline really covers what task 1.1 asks the session to exercise
|
||||
* and the contention it asks the fixture to have, so a regenerated fixture
|
||||
* cannot silently degrade to a trivial one. */
|
||||
static void test_baseline_covers_required_commands_and_contention(void)
|
||||
{
|
||||
static const char* const needles[] = {
|
||||
"[sim] available commands:", /* help */
|
||||
"[sim] info: usage: info <", /* info usage */
|
||||
"[sim] summary: ", /* info subcommands */
|
||||
"[sim] jobs:",
|
||||
"[sim] job 104 (0-0-0-104): state=PEND",
|
||||
"[sim] worker w01: status=online",
|
||||
"[sim] resource license:",
|
||||
"[sim] limit: cluster_slots",
|
||||
"[sim] limit: host_slots",
|
||||
"[sim] units set to raw", /* set units */
|
||||
"[sim] units set to human",
|
||||
"[sim] units set to both",
|
||||
"resource mem: value=16.00 KB used=2.00 KB", /* human */
|
||||
"resource mem: value=16384.000000 (16.00 KB)", /* both */
|
||||
"[sim] queue normal: priority=1", /* info queue all */
|
||||
"[sim] verify set to off",
|
||||
"[sim] verify set to on",
|
||||
"[sim] snapshot written: snap.replay",
|
||||
"[sim] output started: set_output.replay", /* set output */
|
||||
"[sim] output stopped: set_output.replay",
|
||||
"last session: round=2000", /* step */
|
||||
"[sim] round 2-4 (3 round(s)", /* collapsed range */
|
||||
"[sim] hit breakpoint 1 @ decision: DISPATCH jobid=0-0-0-104",
|
||||
"[sim] hit breakpoint 2 @ BEGIN_ROUND round=7000",
|
||||
"[sim] replay complete.",
|
||||
/* contention: parallel job over both hosts, license exhausted,
|
||||
* cluster-wide and host-scoped limits full */
|
||||
"assigned job: jobid=0-0-0-101*2",
|
||||
"[sim] PEND jobid=0-0-0-104",
|
||||
" value=2.00 used=2.00",
|
||||
"usage limit=cluster_slots: accounted=6.00 capacity=6.00",
|
||||
"usage limit=host_slots host=w01: accounted=3.00 capacity=3.00",
|
||||
};
|
||||
char path[PATH_MAX];
|
||||
char* t;
|
||||
long len;
|
||||
size_t i;
|
||||
|
||||
ensure_session_ran();
|
||||
join_path(path, sizeof(path), XCE_BASELINE_DIR, TRANSCRIPT_NAME);
|
||||
t = slurp(path, &len);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(t, "baseline transcript fixture missing");
|
||||
for (i = 0; i < sizeof(needles) / sizeof(needles[0]); i++) {
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(strstr(t, needles[i]), needles[i]);
|
||||
}
|
||||
free(t);
|
||||
}
|
||||
|
||||
/* The --sim-output and `set output` recordings match apart from
|
||||
* ReplayHeader.recordingStartWall (10.4, 10.5). */
|
||||
static void test_recordings_match_baseline(void)
|
||||
{
|
||||
ensure_session_ran();
|
||||
assert_recording_equal(SIM_OUT_NAME);
|
||||
assert_recording_equal(SET_OUT_NAME);
|
||||
}
|
||||
|
||||
/* The --sim-verify result matches the baseline (10.4, 10.5). */
|
||||
static void test_sim_verify_result_matches_baseline(void)
|
||||
{
|
||||
char path[PATH_MAX];
|
||||
char* exp;
|
||||
char* got;
|
||||
long elen;
|
||||
long glen;
|
||||
|
||||
ensure_session_ran();
|
||||
|
||||
join_path(path, sizeof(path), XCE_BASELINE_DIR, VERIFY_NAME);
|
||||
exp = slurp(path, &elen);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(exp, "baseline sim_verify fixture missing");
|
||||
TEST_ASSERT_TRUE_MESSAGE(elen > 0, "baseline sim_verify fixture empty");
|
||||
|
||||
join_path(path, sizeof(path), g_workdir, LOG_NAME);
|
||||
got = extract_verify_result(path, &glen);
|
||||
|
||||
assert_text_equal("--sim-verify result", exp, elen, got, glen);
|
||||
free(exp);
|
||||
free(got);
|
||||
}
|
||||
|
||||
/* Removes the work directory and the files the session leaves in it. */
|
||||
static void remove_workdir(void)
|
||||
{
|
||||
static const char* const names[] = { REPLAY_NAME, SCRIPT_NAME,
|
||||
TRANSCRIPT_NAME, SIM_OUT_NAME,
|
||||
SET_OUT_NAME, LOG_NAME,
|
||||
STDERR_NAME, SNAP_NAME };
|
||||
char path[PATH_MAX];
|
||||
size_t i;
|
||||
|
||||
if (g_workdir[0] == '\0') {
|
||||
return;
|
||||
}
|
||||
for (i = 0; i < sizeof(names) / sizeof(names[0]); i++) {
|
||||
snprintf(path, sizeof(path), "%s/%s", g_workdir, names[i]);
|
||||
unlink(path);
|
||||
}
|
||||
rmdir(g_workdir);
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
UNITY_BEGIN();
|
||||
RUN_TEST(test_transcript_matches_baseline);
|
||||
RUN_TEST(test_baseline_covers_required_commands_and_contention);
|
||||
RUN_TEST(test_recordings_match_baseline);
|
||||
RUN_TEST(test_sim_verify_result_matches_baseline);
|
||||
rc = UNITY_END();
|
||||
remove_workdir();
|
||||
return rc;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue