diff --git a/.kiro/specs/scheduler-sim-decision-trace/tasks.md b/.kiro/specs/scheduler-sim-decision-trace/tasks.md index c2e96230..e136afa0 100644 --- a/.kiro/specs/scheduler-sim-decision-trace/tasks.md +++ b/.kiro/specs/scheduler-sim-decision-trace/tasks.md @@ -302,7 +302,7 @@ - _Requirements: 1.6, 2.5, 7.1, 7.2, 7.3, 7.5, 10.1, 11.3_ - [ ] 6. Validation -- [ ] 6.1 End-to-end trace scenarios over the contention fixture +- [x] 6.1 End-to-end trace scenarios over the contention fixture - Drive the real simulator REPL with tracing on for selected jobs and for all jobs, at both levels. - Assert that every section of Requirements 2 to 6 appears with the expected content: - alternative and compound labels; @@ -373,3 +373,4 @@ - 5.1: all 26 tests that include repl.c/sim_loop.c now `#include` the five REPL trace sources (repl_trace_render.c, repl_trace_render_outcome.c, repl_trace_store.c, repl_trace.c, repl_trace_emit.c) and the four core trace sources, exactly once. sched_sim_isolation_lib carries all nine trace sources. test_sim_isolation.c's `test_sim_trace_repl_sources_are_isolated` scans the REPL trace files plus repl_trace_render.h; stdio is allowed there, controller symbols are not. A new REPL trace file must be added to the 26 tests, the lib and the scan. test_sim_startup.c includes only sim.c, so if sim.c starts calling trace code it needs the sources too. - 5.2: repl.h now has the interactive-only gates `xce_repl_trace_session_begin/end(sim)` and `xce_repl_check_trace/xce_repl_check_trace_only(sim, d)`; the two check gates also no-op when d->trace is NULL. 5.3 calls these from sim_loop.c. `trace` is routed with the raw line, `info trace` is in usage and help, `set output` checks `repl_trace_file_conflicts` first, and quit/EOF call `repl_trace_shutdown`. The continue-mode rollup prints a traced round on its own via `rs->trace.round_traced`. The baseline transcript changed only in the help and info-usage lines; regenerating also rewrites the .replay fixtures' wall-clock fields, so restore those. Pre-existing: repl_count_jobs (repl.c) never destroys its iterator, leaking 16 B per continue round; the rollup tests run with detect_leaks=0. - 5.3: sim_loop.c calls the repl.h gates. Session begin runs after a successful begin-session and before reconcile/signal_go; session end runs after the drain and verify tail, before plugin_end_session. The drain loop handles TRACE first (skipping verify, record, echo and breakpoint), then echo gate, `check_trace`, `check_decision`, `rec_free`, free. The reconstructed block uses the new gate `xce_repl_trace_reconstructed` (`repl_trace_on_reconstructed` in repl_trace_emit.c). Settles the 1.7 OPEN item (both free sites). A reviewer's trace-on vs trace-off run of the contention fixture gave byte-identical --sim-output, identical verify and identical decisions. +- 6.1: the e2e test (test_repl_trace_e2e.c, its _fixture.c and .h) drives the built xce_scheduler: step/full/all with the file copy, continue/breakpoint/summary/info trace, and selected jobs over a run-time extra fixture with compounds, an array and a selection mismatch. `XCE_TRACE_E2E_DUMP=1` prints the transcript. Findings: a compound dispatch shows one entry per member on a shared host, which is faithful (merge_job_alloc keys on instid); `break DECISION jobid=` matches only raw decimal ids, a pre-existing repl_break.c bug logged separately. diff --git a/src/scheduler/testsuite/meson.build b/src/scheduler/testsuite/meson.build index 5c73840e..41f44ee9 100644 --- a/src/scheduler/testsuite/meson.build +++ b/src/scheduler/testsuite/meson.build @@ -2547,6 +2547,40 @@ test('scheduler repl trace baseline test', env: ['ASAN_OPTIONS=detect_leaks=0'], depends: scheduler) +# scheduler-sim-decision-trace task 6.1 (Requirements 1-8; design.md +# "Testing Strategy" -> "Integration Tests" item 2): end-to-end trace +# scenarios. Runs the built xce_scheduler (`depends: scheduler`) in +# interactive simulation mode with scripted REPL sessions over the task 1.1 +# contention fixture (read only) and an extra replay the test generates +# (compound requirements, a job array, a host-selection mismatch), and checks +# the inline traces, their placement in step/continue mode and at +# breakpoints, `info trace`, and the byte-identical trace file copy. The +# replay writer links the real scheduler_srcs, as the baseline test does. +sched_repl_trace_e2e_test_srcs = ['test_repl_trace_e2e.c', + 'test_repl_trace_e2e_fixture.c'] +foreach f : scheduler_srcs + if f != 'src/main.c' + sched_repl_trace_e2e_test_srcs += ['../' + f] + endif +endforeach + +sched_repl_trace_e2e_test = executable( + 'sched_repl_trace_e2e_test', + sched_repl_trace_e2e_test_srcs, + include_directories:[incdir], + dependencies: [scheduler_test_deps], + c_args: ['-DXCE_SCHEDULER_BIN="' + scheduler.full_path() + '"', + '-DXCE_BASELINE_DIR="' + + (meson.current_source_dir() / 'fixtures' + / 'repl_trace_baseline') + + '"'], + install: false) + +test('scheduler repl trace e2e test', + sched_repl_trace_e2e_test, suite: 'scheduler', + env: ['ASAN_OPTIONS=detect_leaks=0'], + depends: scheduler) + # scheduler-sim-decision-trace task 1.3: decision trace record model # (src/sched_trace.c) unit tests -- selection matching, record header and # per-record arena copying and size, record finishing. Links only the model diff --git a/src/scheduler/testsuite/test_repl_trace_e2e.c b/src/scheduler/testsuite/test_repl_trace_e2e.c new file mode 100644 index 00000000..6b16f1b9 --- /dev/null +++ b/src/scheduler/testsuite/test_repl_trace_e2e.c @@ -0,0 +1,674 @@ +/* + * test_repl_trace_e2e.c - scheduler-sim-decision-trace spec, task 6.1 + * (design.md "Testing Strategy" -> "Integration Tests" item 2, "System + * Flows", "TraceRenderer"): end-to-end decision-trace scenarios. + * + * Each scenario runs the real, built xce_scheduler binary + * (XCE_SCHEDULER_BIN) in interactive simulation mode with a scripted REPL + * session on stdin, the way test_repl_trace_baseline.c does, and checks the + * console transcript (and the trace file copy) for the expected trace + * content and its placement. Checks look for key lines and their order, + * never one golden blob, except the file-copy check, which is byte for + * byte. Nothing depends on the wall clock: the replays' times are fixed and + * the traces print only replay times. + * + * Replays (test_repl_trace_e2e_fixture.c): + * - contention.replay, the task 1.1 baseline fixture + * (fixtures/repl_trace_baseline/, read only): two 4-slot workers, a + * 2-unit cluster license, a cluster-wide limit of 6 slots, a host-scoped + * limit of 75% of each host's slots, a parallel job (101), license jobs + * (102..104), an alternative-requirement job (105) and slot jobs + * (106, 107). In its session 5, 105's first alternative fills two + * positions of a 4-position plan and then releases them (rollback). + * - extra.replay, generated per run, for what the baseline lacks: a + * satisfied compound requirement (201), a compound requirement that + * fails at its first member (202), a job array with two elements + * (0-0-0-32768[1] and [2]) and a host-selection mismatch (203 selects + * mem>20000, which only w02 has). + * + * Breakpoint filters compare the raw decimal jobid (repl_break.c + * repl_u64_str_eq), so `break DECISION jobid=105` names 0-0-0-105. + * + * Scenarios: + * 1. step mode, all jobs, level full, file copy (baseline replay, five + * sessions): every section of Requirements 2-6, each trace right after + * its echo line, and the file equal to the console traces. + * 2. continue mode with a breakpoint (baseline replay): traces up to the + * stopping decision only, `info trace` at the stop, a selection change + * at the breakpoint applying from the next session, summary level, + * traced rounds kept out of collapsed ranges, `info trace` after the + * job finished and after `set units raw`. + * 3. selected jobs incl. a job array (extra replay): compound labels, + * selection mismatch, host-limit rejection, shortfall next to a limit, + * `info trace` for an array, an element and an unknown job, and the + * file equal to the inline console traces. + */ +#include +#include +#include +#include + +#include + +#include "test_repl_trace_e2e.h" + +void setUp(void) { return; } +void tearDown(void) { return; } + +/* ---------------------------------------------------------------------- */ +/* Transcript helpers */ +/* ---------------------------------------------------------------------- */ + +/* Asserts the needles occur in `text` in the given order, each after the + * end of the previous one; returns the end of the last match. */ +static const char* assert_in_order(const char* text, const char* const* nd, + size_t n) +{ + const char* at = text; + size_t i; + + for (i = 0; i < n; i++) { + const char* hit = strstr(at, nd[i]); + + if (hit == NULL) { + char msg[1024]; + + snprintf(msg, sizeof(msg), "missing (or out of order): %s", + nd[i]); + TEST_FAIL_MESSAGE(msg); + } + /* A needle's closing newline may open the next needle's line. */ + at = hit + strlen(nd[i]); + if ((at > hit) && (at[-1] == '\n')) { + at--; + } + } + return at; +} + +#define IN_ORDER(text, ...) \ + do { \ + static const char* const nd_[] = { __VA_ARGS__ }; \ + (void)assert_in_order((text), nd_, sizeof(nd_) / sizeof(nd_[0])); \ + } while (0) + +/* Pointer to the line containing `needle` (start of line), or fails. */ +static const char* line_of(const char* text, const char* needle) +{ + const char* hit = strstr(text, needle); + + TEST_ASSERT_NOT_NULL_MESSAGE(hit, needle); + while ((hit > text) && (hit[-1] != '\n')) { + hit--; + } + return hit; +} + +/* True when the line at `p` continues a trace block: trace lines after the + * header are indented, and a truncation marker starts with "...". */ +static bool is_trace_body(const char* p) +{ + return (p[0] == ' ') || (strncmp(p, "...", 3) == 0); +} + +static const char* next_line(const char* p) +{ + const char* nl = strchr(p, '\n'); + + return (nl != NULL) ? nl + 1 : p + strlen(p); +} + +/* Copy of the whole trace block whose header line contains `header`. */ +static char* block_of(const char* text, const char* header) +{ + const char* start = line_of(text, header); + const char* end = next_line(start); + char* out; + + while ((*end != '\0') && is_trace_body(end)) { + end = next_line(end); + } + out = strndup(start, (size_t)(end - start)); + TEST_ASSERT_NOT_NULL(out); + return out; +} + +/* True when the line before `line` is the echo line of a decision for the + * job the trace header at `line` names: "[sim] jobid=\n" before + * "[trace] job ...". */ +static bool follows_its_echo(const char* text, const char* line) +{ + const char* prev; + const char* id = line + strlen("[trace] job "); + const char* id_end = strstr(id, " "); + const char* eq; + + if ((line == text) || (id_end == NULL)) { + return false; + } + prev = line - 1; /* the '\n' ending the previous line */ + while ((prev > text) && (prev[-1] != '\n')) { + prev--; + } + if (strncmp(prev, "[sim] ", 6) != 0) { + return false; + } + eq = strstr(prev, " jobid="); + if ((eq == NULL) || (eq > line)) { + return false; + } + eq += strlen(" jobid="); + return (strncmp(eq, id, (size_t)(id_end - id)) == 0) + && (eq[id_end - id] == '\n'); +} + +/* Concatenation of the trace blocks of `text` (up to `stop`, or the end + * when NULL). With `inline_only`, only blocks printed right after their + * decision's echo line, i.e. not `info trace` output. */ +static char* collect_traces(const char* text, const char* stop, + bool inline_only, long* len) +{ + size_t cap = strlen(text) + 1; + char* out = (char*)malloc(cap); + size_t n = 0; + const char* p = text; + + TEST_ASSERT_NOT_NULL(out); + while ((*p != '\0') && ((stop == NULL) || (p < stop))) { + if ((strncmp(p, "[trace] job ", 12) == 0) + && (!inline_only || follows_its_echo(text, p))) { + const char* end = next_line(p); + + while ((*end != '\0') && is_trace_body(end)) { + end = next_line(end); + } + memcpy(out + n, p, (size_t)(end - p)); + n += (size_t)(end - p); + p = end; + continue; + } + p = next_line(p); + } + out[n] = '\0'; + *len = (long)n; + return out; +} + +/* Every inline trace header before `stop` (NULL = end) follows its echo + * line (7.1, 7.3). `info trace` output is excluded by the caller choosing + * `stop` before the first `info trace`. */ +static void assert_every_trace_follows_echo(const char* text, + const char* stop) +{ + const char* p = text; + int n = 0; + + while ((p = strstr(p, "[trace] job ")) != NULL) { + if ((stop != NULL) && (p >= stop)) { + break; + } + if ((p == text) || (p[-1] == '\n')) { + char msg[256]; + + snprintf(msg, sizeof(msg), "trace not after its echo line: %.80s", + p); + TEST_ASSERT_TRUE_MESSAGE(follows_its_echo(text, p), msg); + n++; + } + p++; + } + TEST_ASSERT_TRUE_MESSAGE(n > 0, "expected at least one inline trace"); +} + +/* The trace file of run `r` equals `expect` byte for byte (8.1). */ +static void assert_file_equals(const run_t* r, const char* expect, long len) +{ + char path[PATH_MAX + 32]; + long flen; + char* file; + + snprintf(path, sizeof(path), "%s/%s", r->dir, TRACE_FILE); + file = slurp(path, &flen); + TEST_ASSERT_NOT_NULL_MESSAGE(file, "the trace file must exist"); + TEST_ASSERT_TRUE_MESSAGE(len > 0, "console traces must not be empty"); + TEST_ASSERT_EQUAL_INT64_MESSAGE(len, flen, + "trace file length must equal the " + "console traces"); + TEST_ASSERT_EQUAL_MEMORY_MESSAGE(expect, file, (size_t)len, + "trace file must equal the console " + "traces byte for byte"); + free(file); +} + +/* ---------------------------------------------------------------------- */ +/* 1. Step mode, all jobs, full level, file copy */ +/* ---------------------------------------------------------------------- */ + +static void test_step_mode_full_all_jobs_and_file_copy(void) +{ + run_t r; + char* b; + char* traces; + long tlen; + + run_repl(&r, BASE_REPLAY, + "trace on all level full file " TRACE_FILE "\n" + "step\nstep\nstep\nstep\nstep\nquit\n"); + + /* 1.1, 2.1, 7.1: every job of session 1 in pick order, each trace + * right after its echo line. */ + assert_every_trace_follows_echo(r.out, NULL); + IN_ORDER(r.out, + "[sim] DISPATCH jobid=0-0-0-101\n" + "[trace] job 0-0-0-101 session 1 position 1 level full\n", + "[sim] DISPATCH jobid=0-0-0-102\n" + "[trace] job 0-0-0-102 session 1 position 2 level full\n", + "[trace] job 0-0-0-103 session 1 position 3 level full\n", + "[sim] PEND jobid=0-0-0-104\n" + "[trace] job 0-0-0-104 session 1 position 4 level full\n", + "[trace] job 0-0-0-105 session 1 position 5 level full\n", + "[trace] job 0-0-0-106 session 1 position 6 level full\n", + "[trace] job 0-0-0-107 session 1 position 7 level full\n", + "[trace] job 0-0-0-104 session 2 position 1 level full\n", + "[trace] job 0-0-0-105 session 5 position 2 level full\n", + "[trace] job 0-0-0-106 session 5 position 3 level full\n"); + + /* 101: selection, host-scoped limit rejections, plans, positions, + * already-claimed, commit, dispatch with limit charges (7.6 order). */ + b = block_of(r.out, "[trace] job 0-0-0-101 session 1"); + IN_ORDER(b, + "\n selection: policy builtin_fcfs queue normal (priority 1)" + " job priority 0 queued 1970-01-01T00:18:21Z\n", + "\n cluster-wide limits: cluster_slots; host-scoped limits are " + "reported with each host's verdict\n", + "\n requirement single tasks 2..4 requests mem=1024.000000 " + "(1.00 KB)\n", + "\n cluster-wide limit cluster_slots on slots: value 6 usage 0" + " reserved 0 requested 4 passed\n", + "\n cluster-wide check passed\n", + "\n plan 1 hosts 1 tasks per position [4]\n", + "\n position 1 (4 tasks)\n", + "\n host w01 rejected (per-host check): limit exceeded\n" + " limit host_slots (h=w01) on slots: value 3 usage 0 " + "reserved 0 requested 4\n", + "\n host w02 rejected (per-host check): limit exceeded\n", + "\n position 1 not filled: no host accepted\n", + "\n plan 1 failed: position 1 could not be filled\n", + "\n plan 2 hosts 2 tasks per position [2, 2]\n", + "\n host w01 accepted\n", + "\n position 1 filled by w01\n", + "\n host w01 skipped: already claimed by an earlier " + "position\n", + "\n position 2 filled by w02\n", + "\n plan 2 succeeded\n", + "\n plan 2 committed; no further plans tried\n", + "\n plans: 2 generated, 2 tried, 0 discarded, 0 not tried\n", + "\n single requirement satisfied\n", + "\n outcome: DISPATCH\n", + "\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit charges: " + "host_slots (h=w01) slots 2\n", + "\n host w02: mem=2048.000000 (2.00 KB) slots=2 limit charges: " + "host_slots (h=w02) slots 2\n", + "\n shared: none limit charges: cluster_slots slots 4\n"); + /* 5.5: a limit hit is never also a resource shortfall. */ + TEST_ASSERT_NULL(strstr(b, "resource shortfall")); + free(b); + + /* 102: a shared resource row, shared allocation and its limit charge + * kept apart (4.1, 6.2, 6.3). */ + b = block_of(r.out, "[trace] job 0-0-0-102 session 1"); + IN_ORDER(b, + "\n cluster-wide: license total 2 in-use 0 reserved 0 " + "requested 1 passed\n", + "\n cluster-wide limit cluster_slots on slots: value 6 usage 4" + " reserved 0 requested 1 passed\n", + "\n host w01: slots=1 limit charges: host_slots (h=w01) " + "slots 1\n", + "\n shared: license=1 limit charges: cluster_slots slots 1\n"); + free(b); + + /* 104: cluster-wide shortfall and cluster-wide limit rejection, no + * plans, pend reasons (4.1, 4.2, 4.8, 6.4). */ + b = block_of(r.out, "[trace] job 0-0-0-104 session 1"); + IN_ORDER(b, + "\n cluster-wide: license total 2 in-use 2 reserved 0 " + "requested 1 failed\n", + "\n cluster-wide limit cluster_slots on slots: value 6 usage 6" + " reserved 0 requested 1 failed\n", + "\n cluster-wide check failed; no placement plans tried\n", + "\n single requirement not satisfied\n", + "\n outcome: PEND\n", + "\n resource license: NO_RESOURCE\n", + "\n resource limit=cluster_slots: NO_RESOURCE\n"); + TEST_ASSERT_NULL(strstr(b, "plan 1")); + free(b); + /* 4.2: the cluster-wide limit is never shown as a resource row. */ + TEST_ASSERT_NULL(strstr(r.out, "cluster-wide: limit=")); + + /* 105, session 1: alternative labels and every alternative's failure + * reasons (3.1, 3.2, 3.5). */ + b = block_of(r.out, "[trace] job 0-0-0-105 session 1"); + IN_ORDER(b, + "\n requirement alternative 1/2 (compound, 1 member)\n", + "\n requirement alternative 1/2 > member 1/1 tasks 4..4 " + "requests mem=12288.000000 (12.00 KB)\n", + "\n member 1/1 not satisfied\n", + "\n failure reasons: NO_RESOURCE x1 (resources: " + "limit=cluster_slots)\n", + "\n alternative 1/2 not satisfied\n", + "\n requirement alternative 2/2 (compound, 1 member)\n", + "\n failure reasons: NO_RESOURCE x1 (resources: " + "limit=cluster_slots)\n", + "\n alternative 2/2 not satisfied\n", + "\n no alternative chosen: none of the 2 alternatives was " + "satisfied\n", + "\n outcome: PEND\n"); + free(b); + + /* 105, session 5: per-host shortfall rows next to limit rows, the + * rollback, no plan placed, then the chosen alternative (3.4, 4.6, + * 5.4, 5.5, 5.8). */ + b = block_of(r.out, "[trace] job 0-0-0-105 session 5"); + IN_ORDER(b, + "\n host w01 rejected (per-host check): resource " + "shortfall, limit exceeded\n" + " mem total 16384.000000 (16.00 KB) in-use 0.000000 " + "(0.00 B) reserved 0.000000 (0.00 B) requested 49152.000000 " + "(48.00 KB)\n" + " slots total 4 in-use 1 reserved 0 requested 4\n" + " limit host_slots (h=w01) on slots: value 3 usage 1 " + "reserved 0 requested 4\n", + "\n plan 4 hosts 4 tasks per position [1, 1, 1, 1]\n", + "\n position 1 filled by w01\n", + "\n position 2 filled by w02\n", + "\n position 3 not filled: no host accepted\n" + " released hosts: w01, w02\n" + " plan 4 failed: position 3 could not be filled\n", + "\n plans: 4 generated, 4 tried, 0 discarded, 0 not tried; " + "no plan could be placed\n", + "\n alternative 1/2 not satisfied\n", + "\n plan 1 committed; no further plans tried\n", + "\n alternative 2/2 satisfied\n" + " alternative 2/2 chosen; no alternatives after it\n" + " outcome: DISPATCH\n" + " host w01: mem=2048.000000 (2.00 KB) slots=1 limit charges: " + "host_slots (h=w01) slots 1\n"); + TEST_ASSERT_NULL(strstr(b, "limit=host_slots total")); + free(b); + + /* 8.1: the file holds exactly the console's traces. */ + traces = collect_traces(r.out, NULL, false, &tlen); + assert_file_equals(&r, traces, tlen); + free(traces); + run_done(&r); +} + +/* ---------------------------------------------------------------------- */ +/* 2. Continue mode, breakpoint, summary level, info trace */ +/* ---------------------------------------------------------------------- */ + +static void test_continue_breakpoint_summary_and_info_trace(void) +{ + run_t r; + const char* hit; + const char* after; + char* b; + + run_repl(&r, BASE_REPLAY, + "trace on all level full\n" + "break DECISION jobid=105 type=PEND\n" + "continue\n" + "info trace 0-0-0-105\n" + "info trace 0-0-0-106\n" + "trace on job 0-0-0-105 0-0-0-106 level summary\n" + "delete all\n" + "continue\n" + "info trace 0-0-0-101\n" + "set units raw\n" + "info trace 0-0-0-101\n" + "info trace 0-0-0-105\n" + "quit\n"); + + /* 7.5: at the stop, the round's traces up to and including the + * stopping decision are printed, and nothing later. */ + hit = line_of(r.out, "[sim] hit breakpoint 1 @ decision: PEND " + "jobid=0-0-0-105"); + assert_every_trace_follows_echo(r.out, hit); + IN_ORDER(r.out, "[trace] job 0-0-0-101 session 1 position 1", + "[trace] job 0-0-0-104 session 1 position 4", + "[sim] PEND jobid=0-0-0-105\n" + "[trace] job 0-0-0-105 session 1 position 5 level full\n", + "[sim] hit breakpoint 1 @ decision: PEND jobid=0-0-0-105\n"); + TEST_ASSERT_TRUE(strstr(r.out, "[trace] job 0-0-0-106") > hit); + + /* 7.5, 7.8: `info trace` at the stop shows the stopping decision's + * trace; the job decided later in this round has none yet. */ + IN_ORDER(hit, "\n[trace] job 0-0-0-105 session 1 position 5 level " + "full\n", + "\n[sim] info trace: job 0-0-0-106: no trace available (state " + "PEND)\n", + "\n[sim] trace: enabled; applies from the next session\n"); + + /* 1.6, 7.3: the rest of session 1 keeps the settings it started with + * (all jobs, full), echo line then trace in continue mode, then the + * round's line; 106 and 107 are traced at level full. */ + after = strstr(hit, "[sim] deleted "); + TEST_ASSERT_NOT_NULL(after); + IN_ORDER(after, + "\n[sim] PEND jobid=0-0-0-106\n" + "[trace] job 0-0-0-106 session 1 position 6 level full\n", + "\n[sim] PEND jobid=0-0-0-107\n" + "[trace] job 0-0-0-107 session 1 position 7 level full\n", + "\n[sim] round 1-1 (", + /* 1.2, 1.4, 7.3, 7.4: from session 2 only 105 and 106, at + * summary level, each traced round reported on its own. */ + "\n[sim] PEND jobid=0-0-0-105\n" + "[trace] job 0-0-0-105 session 2 position 2 level summary\n", + "\n[sim] PEND jobid=0-0-0-106\n" + "[trace] job 0-0-0-106 session 2 position 3 level summary\n", + "\n[sim] round 2-2 (", + "[trace] job 0-0-0-105 session 3", "\n[sim] round 3-3 (", + "[trace] job 0-0-0-105 session 4", "\n[sim] round 4-4 (", + "\n[sim] DISPATCH jobid=0-0-0-105\n" + "[trace] job 0-0-0-105 session 5 position 2 level summary\n", + "\n[sim] DISPATCH jobid=0-0-0-106\n" + "[trace] job 0-0-0-106 session 5 position 3 level summary\n", + "\n[sim] round 5-5 (", + /* untraced rounds still collapse */ + "\n[sim] round 6-7 (2 round(s)", + "\n[sim] replay complete."); + TEST_ASSERT_NULL(strstr(r.out, "[sim] round 2-4")); + TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-104 session 2")); + TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-107 session 2")); + + /* 5.8, 5.9: summary level counts verdicts per position, names each + * limit hit, keeps the rollback, and lists no host lines. */ + b = block_of(r.out, "[trace] job 0-0-0-105 session 5"); + IN_ORDER(b, + "\n plan 1 hosts 1 tasks per position [4]\n" + " position 1 (4 tasks)\n" + " accepted none; rejected: selection mismatch 0, resource " + "shortfall 2, limit exceeded 2 (host_slots (h=w01) 1, " + "host_slots (h=w02) 1), already claimed 0, not evaluated " + "(internal error) 0\n", + "\n position 3 (1 task)\n" + " accepted none; rejected: selection mismatch 0, resource " + "shortfall 0, limit exceeded 0, already claimed 2, not evaluated " + "(internal error) 0\n" + " released hosts: w01, w02\n", + "\n alternative 2/2 chosen; no alternatives after it\n"); + TEST_ASSERT_NULL(strstr(b, " host w01 ")); + free(b); + b = block_of(r.out, "[trace] job 0-0-0-106 session 5"); + IN_ORDER(b, + "\n accepted w02; rejected: selection mismatch 0, resource " + "shortfall 0, limit exceeded 1 (host_slots (h=w01) 1), already " + "claimed 0, not evaluated (internal error) 0\n", + "\n host w02: mem=1024.000000 (1.00 KB) slots=2 limit " + "charges: host_slots (h=w02) slots 2\n"); + free(b); + + /* 7.8: after the replay (101 finished at 5500), `info trace` re-prints + * its stored full trace, then again with the raw units mode. 105's + * latest trace is the session 5 summary one. */ + after = strstr(r.out, "[sim] simulation finished"); + TEST_ASSERT_NOT_NULL(after); + IN_ORDER(after, + "\n[trace] job 0-0-0-101 session 1 position 1 level full\n", + "\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit " + "charges: host_slots (h=w01) slots 2\n", + "\n[sim] units set to raw\n" + "[trace] job 0-0-0-101 session 1 position 1 level full\n", + "\n requirement single tasks 2..4 requests mem=1024.000000\n", + "\n host w01: mem=2048.000000 slots=2 limit charges: " + "host_slots (h=w01) slots 2\n", + "\n[trace] job 0-0-0-105 session 5 position 2 level " + "summary\n", + "\n requirement alternative 1/2 > member 1/1 tasks 4..4 " + "requests mem=12288.000000\n"); + run_done(&r); +} + +/* ---------------------------------------------------------------------- */ +/* 3. Selected jobs, compound, array, info trace on an array */ +/* ---------------------------------------------------------------------- */ + +static void test_compound_array_selection_and_info_trace(void) +{ + run_t r; + const char* mark; + const char* s2; + char* b; + char* traces; + long tlen; + + run_repl(&r, EXTRA_REPLAY, + "trace on job 0-0-0-201 0-0-0-202 0-0-0-32768 0-0-0-203 " + "level full file " TRACE_FILE "\n" + "step\n" + "trace status\n" + "info trace 0-0-0-32768\n" + "info trace 0-0-0-32768[2]\n" + "info trace 0-0-0-999\n" + "step\n" + "info trace 0-0-0-32768\n" + "quit\n"); + + mark = line_of(r.out, "[sim] trace status"); + assert_every_trace_follows_echo(r.out, mark); + /* 1.2: the bare array id selects both elements. */ + IN_ORDER(r.out, + "[sim] DISPATCH jobid=0-0-0-201\n" + "[trace] job 0-0-0-201 session 1 position 1 level full\n", + "[sim] PEND jobid=0-0-0-202\n" + "[trace] job 0-0-0-202 session 1 position 2 level full\n", + "[sim] DISPATCH jobid=0-0-0-32768[1]\n" + "[trace] job 0-0-0-32768[1] session 1 position 3 level " + "full\n", + "[sim] DISPATCH jobid=0-0-0-32768[2]\n" + "[trace] job 0-0-0-32768[2] session 1 position 4 level " + "full\n", + "[sim] DISPATCH jobid=0-0-0-203\n" + "[trace] job 0-0-0-203 session 1 position 5 level full\n"); + + /* 201: compound labels, the reservation of member 1 seen by member 2's + * cluster-wide check, shared and host allocations (3.1, 3.2, 4.1, 6.1, + * 6.2). */ + b = block_of(r.out, "[trace] job 0-0-0-201 session 1"); + IN_ORDER(b, "\n requirement compound (2 members)\n", + "\n requirement compound > member 1/2 tasks 1..1 requests " + "license=1\n", + "\n cluster-wide: license total 2 in-use 0 reserved 0 " + "requested 1 passed\n", + "\n member 1/2 satisfied\n", + "\n requirement compound > member 2/2 tasks 2..2 requests " + "mem=1024.000000 (1.00 KB)\n", + "\n cluster-wide limit cluster_slots on slots: value 6 usage " + "0 reserved 1 requested 2 passed\n", + "\n member 2/2 satisfied\n", + "\n compound requirement satisfied\n", + "\n outcome: DISPATCH\n", + "\n shared: license=1 limit charges: cluster_slots slots 3\n"); + free(b); + + /* 202, session 1: the failing member and the members not tried + * (3.3). */ + b = block_of(r.out, "[trace] job 0-0-0-202 session 1"); + IN_ORDER(b, + "\n cluster-wide check failed; no placement plans tried\n", + "\n member 1/2 not satisfied; member 2/2 not tried\n", + "\n compound requirement not satisfied\n", + "\n outcome: PEND\n"); + free(b); + + /* [2]: host-scoped limit rejection; 203: selection mismatch from the + * pre-filter (5.2, 5.3, 5.5). */ + b = block_of(r.out, "[trace] job 0-0-0-32768[2] session 1"); + IN_ORDER(b, + "\n host w01 rejected (per-host check): limit exceeded\n" + " limit host_slots (h=w01) on slots: value 3 usage 3 " + "reserved 0 requested 1\n" + " host w02 accepted\n"); + free(b); + b = block_of(r.out, "[trace] job 0-0-0-203 session 1"); + IN_ORDER(b, + "\n host w01 rejected (pre-filter): selection mismatch; " + "per-host check not performed\n" + " host w02 accepted\n" + " position 1 filled by w02\n"); + free(b); + + /* 7.9, 7.11: `info trace` for the array prints each element's trace, + * for one element only that one, and an unknown job is reported. */ + IN_ORDER(mark, + "\n[trace] job 0-0-0-32768[1] session 1 position 3 level " + "full\n", + "\n[trace] job 0-0-0-32768[2] session 1 position 4 level " + "full\n", + "\n[trace] job 0-0-0-32768[2] session 1 position 4 level " + "full\n", + "\n[sim] info trace: job 0-0-0-999: unknown job\n"); + + /* Session 2: 202 again; a shortfall row beside a limit row on w02 and + * plan 2 committed for member 1 (5.4, 5.5, 4.5). */ + s2 = strstr(r.out, "[trace] job 0-0-0-202 session 2"); + TEST_ASSERT_NOT_NULL(s2); + TEST_ASSERT_TRUE(follows_its_echo(r.out, s2)); + b = block_of(s2, "[trace] job 0-0-0-202 session 2"); + IN_ORDER(b, + "\n host w02 rejected (per-host check): resource " + "shortfall, limit exceeded\n" + " slots total 4 in-use 1 reserved 0 requested 4\n" + " limit host_slots (h=w02) on slots: value 3 usage 1 " + "reserved 0 requested 4\n", + "\n plan 2 committed; no further plans tried\n", + "\n member 1/2 satisfied\n", + "\n cluster-wide limit cluster_slots on slots: value 6 usage " + "1 reserved 4 requested 1 passed\n", + "\n compound requirement satisfied\n"); + TEST_ASSERT_NULL(strstr(b, "limit=host_slots total")); + free(b); + + /* 7.8, 7.9: after both elements finished, their traces remain. */ + IN_ORDER(s2, + "\n[trace] job 0-0-0-32768[1] session 1 position 3 level " + "full\n", + "\n[trace] job 0-0-0-32768[2] session 1 position 4 level " + "full\n"); + + /* 8.1: the file holds the inline traces (not `info trace` output), + * byte for byte. */ + traces = collect_traces(r.out, NULL, true, &tlen); + assert_file_equals(&r, traces, tlen); + free(traces); + run_done(&r); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_step_mode_full_all_jobs_and_file_copy); + RUN_TEST(test_continue_breakpoint_summary_and_info_trace); + RUN_TEST(test_compound_array_selection_and_info_trace); + return UNITY_END(); +} diff --git a/src/scheduler/testsuite/test_repl_trace_e2e.h b/src/scheduler/testsuite/test_repl_trace_e2e.h new file mode 100644 index 00000000..1f6d6547 --- /dev/null +++ b/src/scheduler/testsuite/test_repl_trace_e2e.h @@ -0,0 +1,33 @@ +/* + * test_repl_trace_e2e.h - shared between test_repl_trace_e2e.c (the + * scenarios) and test_repl_trace_e2e_fixture.c (the replays and the REPL + * runner); scheduler-sim-decision-trace task 6.1. + */ +#ifndef TEST_REPL_TRACE_E2E_H +#define TEST_REPL_TRACE_E2E_H + +#include + +#define BASE_REPLAY "contention.replay" /* the task 1.1 fixture, read only */ +#define EXTRA_REPLAY "extra.replay" /* generated per run */ +#define TRACE_FILE "trace.txt" /* the scripts' `file` argument */ + +typedef struct { + char dir[PATH_MAX]; /* the run's work directory */ + char* out; /* console transcript (stdout) */ + long out_len; +} run_t; + +/* Reads a whole file (NULL and *len = 0 when it cannot be read). */ +char* slurp(const char* path, long* len); + +/* Runs the built scheduler on `replay` (BASE_REPLAY or EXTRA_REPLAY) in a + * fresh work directory with `script` on stdin; fills r->out with the + * console transcript. Asserts a normal exit with status 0. */ +void run_repl(run_t* r, const char* replay, const char* script); + +/* Frees the transcript and removes the work directory (kept when + * XCE_TRACE_E2E_DUMP is set, which also echoes each transcript). */ +void run_done(run_t* r); + +#endif diff --git a/src/scheduler/testsuite/test_repl_trace_e2e_fixture.c b/src/scheduler/testsuite/test_repl_trace_e2e_fixture.c new file mode 100644 index 00000000..fd3c846d --- /dev/null +++ b/src/scheduler/testsuite/test_repl_trace_e2e_fixture.c @@ -0,0 +1,522 @@ +/* + * test_repl_trace_e2e_fixture.c - the replays and the REPL runner of the + * end-to-end decision-trace test (test_repl_trace_e2e.c, which documents + * the scenarios; scheduler-sim-decision-trace task 6.1). + * + * write_extra_replay() generates the extra replay with the technique of + * test_repl_trace_baseline.c's write_contention_replay(): the header and + * snapshot through the real sched_record_start() (resources and both + * limits baked in), then workers, queue, the license value and jobs as + * wire events. run_repl() runs the built xce_scheduler on a replay with a + * script on stdin in a fresh work directory and returns the console + * transcript. Kept apart from the scenarios for the 800-line file cap. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "config.h" +#include "controller.capnp.h" +#include "recording.h" +#include "resobj.h" +#include "scheduler.h" +#include "sim.h" +#include "utils.h" +#include "xcelimit.h" +#include "xstring.h" + +#include "test_repl_trace_e2e.h" + +#if !defined(XCE_SCHEDULER_BIN) || !defined(XCE_BASELINE_DIR) +#error "XCE_SCHEDULER_BIN and XCE_BASELINE_DIR must be defined by meson" +#endif + +extern int write_capnp_message(FILE* f, struct capn* c); + +#define SCRIPT_NAME "script.txt" +#define OUT_NAME "console.txt" +#define ERR_NAME "stderr.txt" +#define LOG_NAME "sched.log" + +#define QUEUE_NAME "normal" +#define LICENSE_RES "license" +#define SNAPSHOT_TS 1000ULL +#define GLIMIT_NAME "cluster_slots" +#define HLIMIT_NAME "host_slots" +#define ARRAY_SEQ ((uint64_t)32768) + +/* ---------------------------------------------------------------------- */ +/* File helpers */ +/* ---------------------------------------------------------------------- */ + +char* slurp(const char* path, long* len) +{ + FILE* f = fopen(path, "rb"); + char* buf; + long n; + + *len = 0; + if (f == NULL) { + return NULL; + } + if ((fseek(f, 0, SEEK_END) != 0) || ((n = ftell(f)) < 0)) { + fclose(f); + return NULL; + } + rewind(f); + buf = (char*)malloc((size_t)n + 1); + if (buf == NULL) { + fclose(f); + return NULL; + } + if ((n > 0) && (fread(buf, 1, (size_t)n, f) != (size_t)n)) { + free(buf); + fclose(f); + return NULL; + } + buf[n] = '\0'; + fclose(f); + *len = n; + return buf; +} + +static void spit(const char* path, const char* buf, long len) +{ + FILE* f = fopen(path, "wb"); + + TEST_ASSERT_NOT_NULL_MESSAGE(f, path); + if (len > 0) { + TEST_ASSERT_EQUAL_UINT((size_t)len, fwrite(buf, 1, (size_t)len, f)); + } + TEST_ASSERT_EQUAL_INT(0, fclose(f)); +} + +static void copy_file(const char* from, const char* to) +{ + long len; + char* buf = slurp(from, &len); + + TEST_ASSERT_NOT_NULL_MESSAGE(buf, from); + spit(to, buf, len); + free(buf); +} + +static void join_path(char* out, size_t outsz, const char* dir, + const char* name) +{ + int n = snprintf(out, outsz, "%s/%s", dir, name); + + TEST_ASSERT_TRUE((n > 0) && ((size_t)n < outsz)); +} + +/* ---------------------------------------------------------------------- */ +/* Extra replay writer (test_repl_trace_baseline.c's technique) */ +/* ---------------------------------------------------------------------- */ + +static void bake_resource(var resmgr, const char* name, int flags) +{ + xce_resource_t res; + + memset(&res, 0, sizeof(res)); + res.name = strdup(name); + res.valtype = XCE_VAL_NUMERIC; + res.flags = flags; + TEST_ASSERT_NOT_NULL(makeResource(resmgr, &res)); +} + +static xce_name_val_t* make_nv(const char* name, float v) +{ + xce_name_val_t* nv = (xce_name_val_t*)calloc(1, sizeof(*nv)); + xce_val_t* val = (xce_val_t*)calloc(1, sizeof(*val)); + + TEST_ASSERT_NOT_NULL(nv); + TEST_ASSERT_NOT_NULL(val); + val->which = Value_u_fvalue; + val->u.fvalue = v; + nv->name = strdup(name); + nv->val = val; + return nv; +} + +static xce_worker_t* make_wire_worker(const char* name, float slots, + float mem) +{ + xce_worker_t* w = (xce_worker_t*)calloc(1, sizeof(*w)); + xce_name_val_t** nv = (xce_name_val_t**)calloc(2, sizeof(*nv)); + + TEST_ASSERT_NOT_NULL(w); + TEST_ASSERT_NOT_NULL(nv); + nv[0] = make_nv("slots", slots); + nv[1] = make_nv("mem", mem); + w->name = strdup(name); + w->nresvals_to_sched = 2; + w->resvals_to_sched = nv; + return w; +} + +static xce_job_t* make_push_job(uint64_t jobid, uint64_t sub_time, + const char* resreq, const char* user, + uint32_t mintasks, uint32_t maxtasks) +{ + xce_job_t* j = (xce_job_t*)calloc(1, sizeof(*j)); + + TEST_ASSERT_NOT_NULL(j); + j->jobid = jobid; + j->state = JOB_STAT_PEND; + j->sub_time = sub_time; + j->queue_time = sub_time; + + j->submit = (job_submit_data_t*)calloc(1, sizeof(*(j->submit))); + TEST_ASSERT_NOT_NULL(j->submit); + j->submit->queue = (char*)QUEUE_NAME; + j->submit->user = (char*)user; + j->submit->project_name = (char*)"proj1"; + j->submit->resreq = (char*)resreq; + j->submit->mintasks = mintasks; + j->submit->maxtasks = maxtasks; + return j; +} + +static void write_event(FILE* f, replay_event_t* src) +{ + struct capn c; + struct capn_segment* cs; + ReplayEvent_ptr p; + + capn_init_malloc(&c); + cs = capn_root(&c).seg; + encode_ReplayEvent_ptr(cs, &p, src); + capn_setp(capn_root(&c), 0, p.p); + TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); + capn_free(&c); +} + +static void append_workers(FILE* f, uint64_t ts, xce_worker_t** w, int n) +{ + xce_workers_data_t wrap; + replay_event_t src; + + memset(&wrap, 0, sizeof(wrap)); + wrap.n_total = n; + wrap.count = n; + wrap.workers = w; + + memset(&src, 0, sizeof(src)); + src.ts = ts; + src.body_which = ReplayEvent_body_workersPush; + src.body.workers_push = &wrap; + write_event(f, &src); +} + +static void append_queue(FILE* f, uint64_t ts) +{ + xce_queue_t q; + replay_event_t src; + + memset(&q, 0, sizeof(q)); + q.queue = (char*)QUEUE_NAME; + q.status = QUEUE_STAT_OPEN; + q.priority = 1; + + memset(&src, 0, sizeof(src)); + src.ts = ts; + src.body_which = ReplayEvent_body_queueUpdate; + src.body.queue_update = &q; + write_event(f, &src); +} + +static void append_license_value(FILE* f, uint64_t ts, float total) +{ + xce_val_t val; + xce_resval_t rv; + xce_resval_t* rvs[1]; + xce_resval_list_t list; + replay_event_t src; + + memset(&val, 0, sizeof(val)); + val.which = Value_u_fvalue; + val.u.fvalue = total; + + memset(&rv, 0, sizeof(rv)); + rv.name = (char*)LICENSE_RES; + rv.val = &val; + rvs[0] = &rv; + + memset(&list, 0, sizeof(list)); + list.count = 1; + list.vals = rvs; + + memset(&src, 0, sizeof(src)); + src.ts = ts; + src.body_which = ReplayEvent_body_resvalsFull; + src.body.resvals_full = &list; + write_event(f, &src); +} + +static void append_jobs(FILE* f, uint64_t ts, xce_job_t** jobs, int n) +{ + xce_jobs_data_t wrap; + replay_event_t src; + + memset(&wrap, 0, sizeof(wrap)); + wrap.n_total = n; + wrap.count = n; + wrap.jobs = jobs; + + memset(&src, 0, sizeof(src)); + src.ts = ts; + src.body_which = ReplayEvent_body_jobsPush; + src.body.jobs_push = &wrap; + write_event(f, &src); +} + +static void append_state_change(FILE* f, uint64_t ts, uint64_t jobid, + uint32_t old_state, uint32_t new_state) +{ + job_state_change_event_t change; + replay_event_t src; + + memset(&change, 0, sizeof(change)); + change.jobid = jobid; + change.old_state = old_state; + change.new_state = new_state; + + memset(&src, 0, sizeof(src)); + src.ts = ts; + src.body_which = ReplayEvent_body_jobStateChange; + src.body.job_state_change = &change; + write_event(f, &src); +} + +static void append_session(FILE* f, uint64_t begin_round) +{ + struct capn c; + struct capn_segment* cs; + ReplaySession_ptr sp; + + capn_init_malloc(&c); + cs = capn_root(&c).seg; + sp = new_ReplaySession(cs); + ReplaySession_set_beginRound(sp, begin_round); + capn_setp(capn_root(&c), 0, sp.p); + TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); + capn_free(&c); +} + +static void append_end_of_stream(FILE* f) +{ + replay_event_t src; + + memset(&src, 0, sizeof(src)); + src.body_which = ReplayEvent_body_endSession; + write_event(f, &src); +} + +/* Header + snapshot via the real sched_record_start(), resources and both + * limits baked into the snapshot (the baseline fixture's configuration). */ +static FILE* write_header_and_snapshot(char* out_path, size_t out_len, + const char* record_dir) +{ + xce_scheduler_t sched; + xce_sched_plugin_t* fcfs; + xce_limit_val_t gval = { .resname = (char*)"slots", + .percent = 0, + .val = 6.0 }; + xce_limit_config_t gcfg = { .name = (char*)GLIMIT_NAME, .val = &gval }; + xce_limit_val_t hval = { .resname = (char*)"slots", + .percent = 1, + .val = 0.75 }; + xce_limit_config_t hcfg = { .name = (char*)HLIMIT_NAME, .val = &hval }; + xce_limit_config_t* limits[2] = { &gcfg, &hcfg }; + xce_conf4sched_t config; + + memset(&sched, 0, sizeof(sched)); + mylock_init(&(sched.record_lock)); + TEST_ASSERT_EQUAL_INT(0, xce_sched_core_init(&(sched.core))); + + fcfs = builtin_fcfs_new_plugin(); + TEST_ASSERT_NOT_NULL(fcfs); + TEST_ASSERT_TRUE(plugin_register_core(&(sched.core), fcfs) >= 0); + + bake_resource(sched.core.resmgr, "slots", XCE_RESF_CONSUMABLE); + bake_resource(sched.core.resmgr, "mem", XCE_RESF_CONSUMABLE); + bake_resource(sched.core.resmgr, LICENSE_RES, + XCE_RESF_GLOBAL | XCE_RESF_SHARED | XCE_RESF_CONSUMABLE); + + memset(&config, 0, sizeof(config)); + config.nlimits = 2; + config.limits = limits; + sched.core.config = &config; + + atomic_store(&(sched.core.virtual_clock_base), SNAPSHOT_TS); + atomic_store(&(sched.core.speed_num), 0ULL); + atomic_store(&(sched.core.speed_den), 1ULL); + + sched.core.job_pick_plugin = (char*)"builtin_fcfs"; + sched.record_dir = (char*)record_dir; + + TEST_ASSERT_EQUAL_INT(0, sched_record_start(&sched, EXTRA_REPLAY)); + TEST_ASSERT_TRUE(strlen(sched.record_file_path) < out_len); + strcpy(out_path, sched.record_file_path); + return sched.record_file; +} + +static uint64_t array_elem(uint32_t k) +{ + return MAKE_JOBID(0, 0, 0, TAG_JOB_ARRAY, ARRAY_SEQ | (uint64_t)k); +} + +/* + * write_extra_replay - w01 (4 slots, 16384 mem) and w02 (4 slots, 32768 + * mem), the baseline's license and limits. Rounds: + * 2000 201 compound {license} + {2 x mem} (satisfied, 3 slots); + * 202 compound whose first member (4 tasks) breaks the cluster-wide + * limit (6 - 3 < 4), so its second member is not tried; + * array elements [1] and [2] (1 slot each, dispatched); + * 203 selects mem>20000 (w01 mismatches), pended by the + * cluster-wide limit (6 of 6 in use) + * 3000 after 201 and both elements finish: 202 and 203 are tried again + */ +static void write_extra_replay(const char* dest) +{ + char record_dir[PATH_MAX]; + char path[PATH_MAX]; + FILE* f; + xce_worker_t* workers[2]; + xce_job_t* jobs[5]; + + snprintf(record_dir, sizeof(record_dir), "/tmp/sched_tre2e_gen_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(record_dir)); + + f = write_header_and_snapshot(path, sizeof(path), record_dir); + + workers[0] = make_wire_worker("w01", 4.0f, 16384.0f); + workers[1] = make_wire_worker("w02", 4.0f, 32768.0f); + append_workers(f, 1001ULL, workers, 2); + append_queue(f, 1002ULL); + append_license_value(f, 1002ULL, 2.0f); + + jobs[0] = make_push_job(201, 1101, + "1*{rusage[license=1]} + " + "2*{rusage[mem=1024]}", + "alice", 1, 1); + jobs[1] = make_push_job(202, 1102, + "4*{rusage[mem=512]} + " + "1*{rusage[mem=512]}", + "bob", 1, 1); + jobs[2] = make_push_job(array_elem(1), 1103, "rusage[mem=256]", "bob", + 1, 1); + jobs[3] = make_push_job(array_elem(2), 1103, "rusage[mem=256]", "bob", + 1, 1); + jobs[4] = make_push_job(203, 1104, "select[mem>20000] rusage[mem=512]", + "alice", 1, 1); + append_jobs(f, 1003ULL, jobs, 5); + + append_session(f, 2000ULL); + append_state_change(f, 2500ULL, 201, JOB_STAT_RUN, JOB_STAT_DONE); + append_state_change(f, 2600ULL, array_elem(1), JOB_STAT_RUN, + JOB_STAT_DONE); + append_state_change(f, 2700ULL, array_elem(2), JOB_STAT_RUN, + JOB_STAT_DONE); + append_session(f, 3000ULL); + + append_end_of_stream(f); + TEST_ASSERT_EQUAL_INT(0, fclose(f)); + + copy_file(path, dest); + unlink(path); + rmdir(record_dir); +} + +/* ---------------------------------------------------------------------- */ +/* Running a scripted REPL session */ +/* ---------------------------------------------------------------------- */ + + +/* Runs the built scheduler on `replay` (BASE_REPLAY or EXTRA_REPLAY) in a + * fresh work directory with `script` on stdin and returns the console + * transcript. Paths are relative to the work directory. */ +void run_repl(run_t* r, const char* replay, const char* script) +{ + char path[PATH_MAX]; + pid_t pid; + int status; + + memset(r, 0, sizeof(*r)); + snprintf(r->dir, sizeof(r->dir), "/tmp/sched_tre2e_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(r->dir)); + + join_path(path, sizeof(path), r->dir, replay); + if (strcmp(replay, BASE_REPLAY) == 0) { + char src[PATH_MAX]; + + join_path(src, sizeof(src), XCE_BASELINE_DIR, BASE_REPLAY); + copy_file(src, path); + } else { + write_extra_replay(path); + } + join_path(path, sizeof(path), r->dir, SCRIPT_NAME); + spit(path, script, (long)strlen(script)); + + fflush(stdout); + fflush(stderr); + pid = fork(); + TEST_ASSERT_TRUE_MESSAGE(pid >= 0, "fork must succeed"); + if (pid == 0) { + if ((chdir(r->dir) != 0) + || (freopen(SCRIPT_NAME, "r", stdin) == NULL) + || (freopen(OUT_NAME, "w", stdout) == NULL) + || (freopen(ERR_NAME, "w", stderr) == NULL)) { + _exit(120); + } + setenv("TERM", "dumb", 1); + execl(XCE_SCHEDULER_BIN, XCE_SCHEDULER_BIN, "--sim", replay, + "--interactive", "--logfile", LOG_NAME, (char*)NULL); + _exit(121); + } + TEST_ASSERT_EQUAL_INT(pid, waitpid(pid, &status, 0)); + TEST_ASSERT_TRUE_MESSAGE(WIFEXITED(status), + "the scheduler must exit normally"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, WEXITSTATUS(status), + "the session must end with quit's exit(0)"); + + join_path(path, sizeof(path), r->dir, OUT_NAME); + r->out = slurp(path, &(r->out_len)); + TEST_ASSERT_NOT_NULL(r->out); + if (getenv("XCE_TRACE_E2E_DUMP") != NULL) { + fprintf(stderr, "==== %s (%s)\n%s", replay, r->dir, r->out); + } +} + +void run_done(run_t* r) +{ + static const char* const names[] = { BASE_REPLAY, EXTRA_REPLAY, + SCRIPT_NAME, OUT_NAME, ERR_NAME, + LOG_NAME, TRACE_FILE }; + char path[PATH_MAX]; + size_t i; + + free(r->out); + r->out = NULL; + if (getenv("XCE_TRACE_E2E_DUMP") != NULL) { + return; + } + for (i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + int n = snprintf(path, sizeof(path), "%s/%s", r->dir, names[i]); + + if ((n > 0) && ((size_t)n < sizeof(path))) { + unlink(path); + } + } + rmdir(r->dir); +}