test(scheduler-sim-decision-trace): end-to-end trace scenarios over the contention fixture
Task 6.1: drives the real simulator REPL with tracing on for all and selected jobs at both levels, asserting every Requirement 2-6 section, ordering in step and continue mode and at breakpoints, info trace at a breakpoint, after finish, for arrays and after a units change, and a byte-identical trace file copy. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
4d8d819201
commit
625663af59
GPG key ID:
80533536B0EB651C
5 changed files with 1265 additions and 1 deletions
674
src/scheduler/testsuite/test_repl_trace_e2e.c
Normal file
674
src/scheduler/testsuite/test_repl_trace_e2e.c
Normal file
|
|
@ -0,0 +1,674 @@
|
|||
/*
|
||||
* test_repl_trace_e2e.c - scheduler-sim-decision-trace spec, task 6.1
|
||||
* (design.md "Testing Strategy" -> "Integration Tests" item 2, "System
|
||||
* Flows", "TraceRenderer"): end-to-end decision-trace scenarios.
|
||||
*
|
||||
* Each scenario runs the real, built xce_scheduler binary
|
||||
* (XCE_SCHEDULER_BIN) in interactive simulation mode with a scripted REPL
|
||||
* session on stdin, the way test_repl_trace_baseline.c does, and checks the
|
||||
* console transcript (and the trace file copy) for the expected trace
|
||||
* content and its placement. Checks look for key lines and their order,
|
||||
* never one golden blob, except the file-copy check, which is byte for
|
||||
* byte. Nothing depends on the wall clock: the replays' times are fixed and
|
||||
* the traces print only replay times.
|
||||
*
|
||||
* Replays (test_repl_trace_e2e_fixture.c):
|
||||
* - contention.replay, the task 1.1 baseline fixture
|
||||
* (fixtures/repl_trace_baseline/, read only): two 4-slot workers, a
|
||||
* 2-unit cluster license, a cluster-wide limit of 6 slots, a host-scoped
|
||||
* limit of 75% of each host's slots, a parallel job (101), license jobs
|
||||
* (102..104), an alternative-requirement job (105) and slot jobs
|
||||
* (106, 107). In its session 5, 105's first alternative fills two
|
||||
* positions of a 4-position plan and then releases them (rollback).
|
||||
* - extra.replay, generated per run, for what the baseline lacks: a
|
||||
* satisfied compound requirement (201), a compound requirement that
|
||||
* fails at its first member (202), a job array with two elements
|
||||
* (0-0-0-32768[1] and [2]) and a host-selection mismatch (203 selects
|
||||
* mem>20000, which only w02 has).
|
||||
*
|
||||
* Breakpoint filters compare the raw decimal jobid (repl_break.c
|
||||
* repl_u64_str_eq), so `break DECISION jobid=105` names 0-0-0-105.
|
||||
*
|
||||
* Scenarios:
|
||||
* 1. step mode, all jobs, level full, file copy (baseline replay, five
|
||||
* sessions): every section of Requirements 2-6, each trace right after
|
||||
* its echo line, and the file equal to the console traces.
|
||||
* 2. continue mode with a breakpoint (baseline replay): traces up to the
|
||||
* stopping decision only, `info trace` at the stop, a selection change
|
||||
* at the breakpoint applying from the next session, summary level,
|
||||
* traced rounds kept out of collapsed ranges, `info trace` after the
|
||||
* job finished and after `set units raw`.
|
||||
* 3. selected jobs incl. a job array (extra replay): compound labels,
|
||||
* selection mismatch, host-limit rejection, shortfall next to a limit,
|
||||
* `info trace` for an array, an element and an unknown job, and the
|
||||
* file equal to the inline console traces.
|
||||
*/
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "test_repl_trace_e2e.h"
|
||||
|
||||
void setUp(void) { return; }
|
||||
void tearDown(void) { return; }
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* Transcript helpers */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
/* Asserts the needles occur in `text` in the given order, each after the
|
||||
* end of the previous one; returns the end of the last match. */
|
||||
static const char* assert_in_order(const char* text, const char* const* nd,
|
||||
size_t n)
|
||||
{
|
||||
const char* at = text;
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < n; i++) {
|
||||
const char* hit = strstr(at, nd[i]);
|
||||
|
||||
if (hit == NULL) {
|
||||
char msg[1024];
|
||||
|
||||
snprintf(msg, sizeof(msg), "missing (or out of order): %s",
|
||||
nd[i]);
|
||||
TEST_FAIL_MESSAGE(msg);
|
||||
}
|
||||
/* A needle's closing newline may open the next needle's line. */
|
||||
at = hit + strlen(nd[i]);
|
||||
if ((at > hit) && (at[-1] == '\n')) {
|
||||
at--;
|
||||
}
|
||||
}
|
||||
return at;
|
||||
}
|
||||
|
||||
#define IN_ORDER(text, ...) \
|
||||
do { \
|
||||
static const char* const nd_[] = { __VA_ARGS__ }; \
|
||||
(void)assert_in_order((text), nd_, sizeof(nd_) / sizeof(nd_[0])); \
|
||||
} while (0)
|
||||
|
||||
/* Pointer to the line containing `needle` (start of line), or fails. */
|
||||
static const char* line_of(const char* text, const char* needle)
|
||||
{
|
||||
const char* hit = strstr(text, needle);
|
||||
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(hit, needle);
|
||||
while ((hit > text) && (hit[-1] != '\n')) {
|
||||
hit--;
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
/* True when the line at `p` continues a trace block: trace lines after the
|
||||
* header are indented, and a truncation marker starts with "...". */
|
||||
static bool is_trace_body(const char* p)
|
||||
{
|
||||
return (p[0] == ' ') || (strncmp(p, "...", 3) == 0);
|
||||
}
|
||||
|
||||
static const char* next_line(const char* p)
|
||||
{
|
||||
const char* nl = strchr(p, '\n');
|
||||
|
||||
return (nl != NULL) ? nl + 1 : p + strlen(p);
|
||||
}
|
||||
|
||||
/* Copy of the whole trace block whose header line contains `header`. */
|
||||
static char* block_of(const char* text, const char* header)
|
||||
{
|
||||
const char* start = line_of(text, header);
|
||||
const char* end = next_line(start);
|
||||
char* out;
|
||||
|
||||
while ((*end != '\0') && is_trace_body(end)) {
|
||||
end = next_line(end);
|
||||
}
|
||||
out = strndup(start, (size_t)(end - start));
|
||||
TEST_ASSERT_NOT_NULL(out);
|
||||
return out;
|
||||
}
|
||||
|
||||
/* True when the line before `line` is the echo line of a decision for the
|
||||
* job the trace header at `line` names: "[sim] <OP> jobid=<id>\n" before
|
||||
* "[trace] job <id> ...". */
|
||||
static bool follows_its_echo(const char* text, const char* line)
|
||||
{
|
||||
const char* prev;
|
||||
const char* id = line + strlen("[trace] job ");
|
||||
const char* id_end = strstr(id, " ");
|
||||
const char* eq;
|
||||
|
||||
if ((line == text) || (id_end == NULL)) {
|
||||
return false;
|
||||
}
|
||||
prev = line - 1; /* the '\n' ending the previous line */
|
||||
while ((prev > text) && (prev[-1] != '\n')) {
|
||||
prev--;
|
||||
}
|
||||
if (strncmp(prev, "[sim] ", 6) != 0) {
|
||||
return false;
|
||||
}
|
||||
eq = strstr(prev, " jobid=");
|
||||
if ((eq == NULL) || (eq > line)) {
|
||||
return false;
|
||||
}
|
||||
eq += strlen(" jobid=");
|
||||
return (strncmp(eq, id, (size_t)(id_end - id)) == 0)
|
||||
&& (eq[id_end - id] == '\n');
|
||||
}
|
||||
|
||||
/* Concatenation of the trace blocks of `text` (up to `stop`, or the end
|
||||
* when NULL). With `inline_only`, only blocks printed right after their
|
||||
* decision's echo line, i.e. not `info trace` output. */
|
||||
static char* collect_traces(const char* text, const char* stop,
|
||||
bool inline_only, long* len)
|
||||
{
|
||||
size_t cap = strlen(text) + 1;
|
||||
char* out = (char*)malloc(cap);
|
||||
size_t n = 0;
|
||||
const char* p = text;
|
||||
|
||||
TEST_ASSERT_NOT_NULL(out);
|
||||
while ((*p != '\0') && ((stop == NULL) || (p < stop))) {
|
||||
if ((strncmp(p, "[trace] job ", 12) == 0)
|
||||
&& (!inline_only || follows_its_echo(text, p))) {
|
||||
const char* end = next_line(p);
|
||||
|
||||
while ((*end != '\0') && is_trace_body(end)) {
|
||||
end = next_line(end);
|
||||
}
|
||||
memcpy(out + n, p, (size_t)(end - p));
|
||||
n += (size_t)(end - p);
|
||||
p = end;
|
||||
continue;
|
||||
}
|
||||
p = next_line(p);
|
||||
}
|
||||
out[n] = '\0';
|
||||
*len = (long)n;
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Every inline trace header before `stop` (NULL = end) follows its echo
|
||||
* line (7.1, 7.3). `info trace` output is excluded by the caller choosing
|
||||
* `stop` before the first `info trace`. */
|
||||
static void assert_every_trace_follows_echo(const char* text,
|
||||
const char* stop)
|
||||
{
|
||||
const char* p = text;
|
||||
int n = 0;
|
||||
|
||||
while ((p = strstr(p, "[trace] job ")) != NULL) {
|
||||
if ((stop != NULL) && (p >= stop)) {
|
||||
break;
|
||||
}
|
||||
if ((p == text) || (p[-1] == '\n')) {
|
||||
char msg[256];
|
||||
|
||||
snprintf(msg, sizeof(msg), "trace not after its echo line: %.80s",
|
||||
p);
|
||||
TEST_ASSERT_TRUE_MESSAGE(follows_its_echo(text, p), msg);
|
||||
n++;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
TEST_ASSERT_TRUE_MESSAGE(n > 0, "expected at least one inline trace");
|
||||
}
|
||||
|
||||
/* The trace file of run `r` equals `expect` byte for byte (8.1). */
|
||||
static void assert_file_equals(const run_t* r, const char* expect, long len)
|
||||
{
|
||||
char path[PATH_MAX + 32];
|
||||
long flen;
|
||||
char* file;
|
||||
|
||||
snprintf(path, sizeof(path), "%s/%s", r->dir, TRACE_FILE);
|
||||
file = slurp(path, &flen);
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(file, "the trace file must exist");
|
||||
TEST_ASSERT_TRUE_MESSAGE(len > 0, "console traces must not be empty");
|
||||
TEST_ASSERT_EQUAL_INT64_MESSAGE(len, flen,
|
||||
"trace file length must equal the "
|
||||
"console traces");
|
||||
TEST_ASSERT_EQUAL_MEMORY_MESSAGE(expect, file, (size_t)len,
|
||||
"trace file must equal the console "
|
||||
"traces byte for byte");
|
||||
free(file);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* 1. Step mode, all jobs, full level, file copy */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
static void test_step_mode_full_all_jobs_and_file_copy(void)
|
||||
{
|
||||
run_t r;
|
||||
char* b;
|
||||
char* traces;
|
||||
long tlen;
|
||||
|
||||
run_repl(&r, BASE_REPLAY,
|
||||
"trace on all level full file " TRACE_FILE "\n"
|
||||
"step\nstep\nstep\nstep\nstep\nquit\n");
|
||||
|
||||
/* 1.1, 2.1, 7.1: every job of session 1 in pick order, each trace
|
||||
* right after its echo line. */
|
||||
assert_every_trace_follows_echo(r.out, NULL);
|
||||
IN_ORDER(r.out,
|
||||
"[sim] DISPATCH jobid=0-0-0-101\n"
|
||||
"[trace] job 0-0-0-101 session 1 position 1 level full\n",
|
||||
"[sim] DISPATCH jobid=0-0-0-102\n"
|
||||
"[trace] job 0-0-0-102 session 1 position 2 level full\n",
|
||||
"[trace] job 0-0-0-103 session 1 position 3 level full\n",
|
||||
"[sim] PEND jobid=0-0-0-104\n"
|
||||
"[trace] job 0-0-0-104 session 1 position 4 level full\n",
|
||||
"[trace] job 0-0-0-105 session 1 position 5 level full\n",
|
||||
"[trace] job 0-0-0-106 session 1 position 6 level full\n",
|
||||
"[trace] job 0-0-0-107 session 1 position 7 level full\n",
|
||||
"[trace] job 0-0-0-104 session 2 position 1 level full\n",
|
||||
"[trace] job 0-0-0-105 session 5 position 2 level full\n",
|
||||
"[trace] job 0-0-0-106 session 5 position 3 level full\n");
|
||||
|
||||
/* 101: selection, host-scoped limit rejections, plans, positions,
|
||||
* already-claimed, commit, dispatch with limit charges (7.6 order). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-101 session 1");
|
||||
IN_ORDER(b,
|
||||
"\n selection: policy builtin_fcfs queue normal (priority 1)"
|
||||
" job priority 0 queued 1970-01-01T00:18:21Z\n",
|
||||
"\n cluster-wide limits: cluster_slots; host-scoped limits are "
|
||||
"reported with each host's verdict\n",
|
||||
"\n requirement single tasks 2..4 requests mem=1024.000000 "
|
||||
"(1.00 KB)\n",
|
||||
"\n cluster-wide limit cluster_slots on slots: value 6 usage 0"
|
||||
" reserved 0 requested 4 passed\n",
|
||||
"\n cluster-wide check passed\n",
|
||||
"\n plan 1 hosts 1 tasks per position [4]\n",
|
||||
"\n position 1 (4 tasks)\n",
|
||||
"\n host w01 rejected (per-host check): limit exceeded\n"
|
||||
" limit host_slots (h=w01) on slots: value 3 usage 0 "
|
||||
"reserved 0 requested 4\n",
|
||||
"\n host w02 rejected (per-host check): limit exceeded\n",
|
||||
"\n position 1 not filled: no host accepted\n",
|
||||
"\n plan 1 failed: position 1 could not be filled\n",
|
||||
"\n plan 2 hosts 2 tasks per position [2, 2]\n",
|
||||
"\n host w01 accepted\n",
|
||||
"\n position 1 filled by w01\n",
|
||||
"\n host w01 skipped: already claimed by an earlier "
|
||||
"position\n",
|
||||
"\n position 2 filled by w02\n",
|
||||
"\n plan 2 succeeded\n",
|
||||
"\n plan 2 committed; no further plans tried\n",
|
||||
"\n plans: 2 generated, 2 tried, 0 discarded, 0 not tried\n",
|
||||
"\n single requirement satisfied\n",
|
||||
"\n outcome: DISPATCH\n",
|
||||
"\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit charges: "
|
||||
"host_slots (h=w01) slots 2\n",
|
||||
"\n host w02: mem=2048.000000 (2.00 KB) slots=2 limit charges: "
|
||||
"host_slots (h=w02) slots 2\n",
|
||||
"\n shared: none limit charges: cluster_slots slots 4\n");
|
||||
/* 5.5: a limit hit is never also a resource shortfall. */
|
||||
TEST_ASSERT_NULL(strstr(b, "resource shortfall"));
|
||||
free(b);
|
||||
|
||||
/* 102: a shared resource row, shared allocation and its limit charge
|
||||
* kept apart (4.1, 6.2, 6.3). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-102 session 1");
|
||||
IN_ORDER(b,
|
||||
"\n cluster-wide: license total 2 in-use 0 reserved 0 "
|
||||
"requested 1 passed\n",
|
||||
"\n cluster-wide limit cluster_slots on slots: value 6 usage 4"
|
||||
" reserved 0 requested 1 passed\n",
|
||||
"\n host w01: slots=1 limit charges: host_slots (h=w01) "
|
||||
"slots 1\n",
|
||||
"\n shared: license=1 limit charges: cluster_slots slots 1\n");
|
||||
free(b);
|
||||
|
||||
/* 104: cluster-wide shortfall and cluster-wide limit rejection, no
|
||||
* plans, pend reasons (4.1, 4.2, 4.8, 6.4). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-104 session 1");
|
||||
IN_ORDER(b,
|
||||
"\n cluster-wide: license total 2 in-use 2 reserved 0 "
|
||||
"requested 1 failed\n",
|
||||
"\n cluster-wide limit cluster_slots on slots: value 6 usage 6"
|
||||
" reserved 0 requested 1 failed\n",
|
||||
"\n cluster-wide check failed; no placement plans tried\n",
|
||||
"\n single requirement not satisfied\n",
|
||||
"\n outcome: PEND\n",
|
||||
"\n resource license: NO_RESOURCE\n",
|
||||
"\n resource limit=cluster_slots: NO_RESOURCE\n");
|
||||
TEST_ASSERT_NULL(strstr(b, "plan 1"));
|
||||
free(b);
|
||||
/* 4.2: the cluster-wide limit is never shown as a resource row. */
|
||||
TEST_ASSERT_NULL(strstr(r.out, "cluster-wide: limit="));
|
||||
|
||||
/* 105, session 1: alternative labels and every alternative's failure
|
||||
* reasons (3.1, 3.2, 3.5). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-105 session 1");
|
||||
IN_ORDER(b,
|
||||
"\n requirement alternative 1/2 (compound, 1 member)\n",
|
||||
"\n requirement alternative 1/2 > member 1/1 tasks 4..4 "
|
||||
"requests mem=12288.000000 (12.00 KB)\n",
|
||||
"\n member 1/1 not satisfied\n",
|
||||
"\n failure reasons: NO_RESOURCE x1 (resources: "
|
||||
"limit=cluster_slots)\n",
|
||||
"\n alternative 1/2 not satisfied\n",
|
||||
"\n requirement alternative 2/2 (compound, 1 member)\n",
|
||||
"\n failure reasons: NO_RESOURCE x1 (resources: "
|
||||
"limit=cluster_slots)\n",
|
||||
"\n alternative 2/2 not satisfied\n",
|
||||
"\n no alternative chosen: none of the 2 alternatives was "
|
||||
"satisfied\n",
|
||||
"\n outcome: PEND\n");
|
||||
free(b);
|
||||
|
||||
/* 105, session 5: per-host shortfall rows next to limit rows, the
|
||||
* rollback, no plan placed, then the chosen alternative (3.4, 4.6,
|
||||
* 5.4, 5.5, 5.8). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-105 session 5");
|
||||
IN_ORDER(b,
|
||||
"\n host w01 rejected (per-host check): resource "
|
||||
"shortfall, limit exceeded\n"
|
||||
" mem total 16384.000000 (16.00 KB) in-use 0.000000 "
|
||||
"(0.00 B) reserved 0.000000 (0.00 B) requested 49152.000000 "
|
||||
"(48.00 KB)\n"
|
||||
" slots total 4 in-use 1 reserved 0 requested 4\n"
|
||||
" limit host_slots (h=w01) on slots: value 3 usage 1 "
|
||||
"reserved 0 requested 4\n",
|
||||
"\n plan 4 hosts 4 tasks per position [1, 1, 1, 1]\n",
|
||||
"\n position 1 filled by w01\n",
|
||||
"\n position 2 filled by w02\n",
|
||||
"\n position 3 not filled: no host accepted\n"
|
||||
" released hosts: w01, w02\n"
|
||||
" plan 4 failed: position 3 could not be filled\n",
|
||||
"\n plans: 4 generated, 4 tried, 0 discarded, 0 not tried; "
|
||||
"no plan could be placed\n",
|
||||
"\n alternative 1/2 not satisfied\n",
|
||||
"\n plan 1 committed; no further plans tried\n",
|
||||
"\n alternative 2/2 satisfied\n"
|
||||
" alternative 2/2 chosen; no alternatives after it\n"
|
||||
" outcome: DISPATCH\n"
|
||||
" host w01: mem=2048.000000 (2.00 KB) slots=1 limit charges: "
|
||||
"host_slots (h=w01) slots 1\n");
|
||||
TEST_ASSERT_NULL(strstr(b, "limit=host_slots total"));
|
||||
free(b);
|
||||
|
||||
/* 8.1: the file holds exactly the console's traces. */
|
||||
traces = collect_traces(r.out, NULL, false, &tlen);
|
||||
assert_file_equals(&r, traces, tlen);
|
||||
free(traces);
|
||||
run_done(&r);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* 2. Continue mode, breakpoint, summary level, info trace */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
static void test_continue_breakpoint_summary_and_info_trace(void)
|
||||
{
|
||||
run_t r;
|
||||
const char* hit;
|
||||
const char* after;
|
||||
char* b;
|
||||
|
||||
run_repl(&r, BASE_REPLAY,
|
||||
"trace on all level full\n"
|
||||
"break DECISION jobid=105 type=PEND\n"
|
||||
"continue\n"
|
||||
"info trace 0-0-0-105\n"
|
||||
"info trace 0-0-0-106\n"
|
||||
"trace on job 0-0-0-105 0-0-0-106 level summary\n"
|
||||
"delete all\n"
|
||||
"continue\n"
|
||||
"info trace 0-0-0-101\n"
|
||||
"set units raw\n"
|
||||
"info trace 0-0-0-101\n"
|
||||
"info trace 0-0-0-105\n"
|
||||
"quit\n");
|
||||
|
||||
/* 7.5: at the stop, the round's traces up to and including the
|
||||
* stopping decision are printed, and nothing later. */
|
||||
hit = line_of(r.out, "[sim] hit breakpoint 1 @ decision: PEND "
|
||||
"jobid=0-0-0-105");
|
||||
assert_every_trace_follows_echo(r.out, hit);
|
||||
IN_ORDER(r.out, "[trace] job 0-0-0-101 session 1 position 1",
|
||||
"[trace] job 0-0-0-104 session 1 position 4",
|
||||
"[sim] PEND jobid=0-0-0-105\n"
|
||||
"[trace] job 0-0-0-105 session 1 position 5 level full\n",
|
||||
"[sim] hit breakpoint 1 @ decision: PEND jobid=0-0-0-105\n");
|
||||
TEST_ASSERT_TRUE(strstr(r.out, "[trace] job 0-0-0-106") > hit);
|
||||
|
||||
/* 7.5, 7.8: `info trace` at the stop shows the stopping decision's
|
||||
* trace; the job decided later in this round has none yet. */
|
||||
IN_ORDER(hit, "\n[trace] job 0-0-0-105 session 1 position 5 level "
|
||||
"full\n",
|
||||
"\n[sim] info trace: job 0-0-0-106: no trace available (state "
|
||||
"PEND)\n",
|
||||
"\n[sim] trace: enabled; applies from the next session\n");
|
||||
|
||||
/* 1.6, 7.3: the rest of session 1 keeps the settings it started with
|
||||
* (all jobs, full), echo line then trace in continue mode, then the
|
||||
* round's line; 106 and 107 are traced at level full. */
|
||||
after = strstr(hit, "[sim] deleted ");
|
||||
TEST_ASSERT_NOT_NULL(after);
|
||||
IN_ORDER(after,
|
||||
"\n[sim] PEND jobid=0-0-0-106\n"
|
||||
"[trace] job 0-0-0-106 session 1 position 6 level full\n",
|
||||
"\n[sim] PEND jobid=0-0-0-107\n"
|
||||
"[trace] job 0-0-0-107 session 1 position 7 level full\n",
|
||||
"\n[sim] round 1-1 (",
|
||||
/* 1.2, 1.4, 7.3, 7.4: from session 2 only 105 and 106, at
|
||||
* summary level, each traced round reported on its own. */
|
||||
"\n[sim] PEND jobid=0-0-0-105\n"
|
||||
"[trace] job 0-0-0-105 session 2 position 2 level summary\n",
|
||||
"\n[sim] PEND jobid=0-0-0-106\n"
|
||||
"[trace] job 0-0-0-106 session 2 position 3 level summary\n",
|
||||
"\n[sim] round 2-2 (",
|
||||
"[trace] job 0-0-0-105 session 3", "\n[sim] round 3-3 (",
|
||||
"[trace] job 0-0-0-105 session 4", "\n[sim] round 4-4 (",
|
||||
"\n[sim] DISPATCH jobid=0-0-0-105\n"
|
||||
"[trace] job 0-0-0-105 session 5 position 2 level summary\n",
|
||||
"\n[sim] DISPATCH jobid=0-0-0-106\n"
|
||||
"[trace] job 0-0-0-106 session 5 position 3 level summary\n",
|
||||
"\n[sim] round 5-5 (",
|
||||
/* untraced rounds still collapse */
|
||||
"\n[sim] round 6-7 (2 round(s)",
|
||||
"\n[sim] replay complete.");
|
||||
TEST_ASSERT_NULL(strstr(r.out, "[sim] round 2-4"));
|
||||
TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-104 session 2"));
|
||||
TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-107 session 2"));
|
||||
|
||||
/* 5.8, 5.9: summary level counts verdicts per position, names each
|
||||
* limit hit, keeps the rollback, and lists no host lines. */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-105 session 5");
|
||||
IN_ORDER(b,
|
||||
"\n plan 1 hosts 1 tasks per position [4]\n"
|
||||
" position 1 (4 tasks)\n"
|
||||
" accepted none; rejected: selection mismatch 0, resource "
|
||||
"shortfall 2, limit exceeded 2 (host_slots (h=w01) 1, "
|
||||
"host_slots (h=w02) 1), already claimed 0, not evaluated "
|
||||
"(internal error) 0\n",
|
||||
"\n position 3 (1 task)\n"
|
||||
" accepted none; rejected: selection mismatch 0, resource "
|
||||
"shortfall 0, limit exceeded 0, already claimed 2, not evaluated "
|
||||
"(internal error) 0\n"
|
||||
" released hosts: w01, w02\n",
|
||||
"\n alternative 2/2 chosen; no alternatives after it\n");
|
||||
TEST_ASSERT_NULL(strstr(b, " host w01 "));
|
||||
free(b);
|
||||
b = block_of(r.out, "[trace] job 0-0-0-106 session 5");
|
||||
IN_ORDER(b,
|
||||
"\n accepted w02; rejected: selection mismatch 0, resource "
|
||||
"shortfall 0, limit exceeded 1 (host_slots (h=w01) 1), already "
|
||||
"claimed 0, not evaluated (internal error) 0\n",
|
||||
"\n host w02: mem=1024.000000 (1.00 KB) slots=2 limit "
|
||||
"charges: host_slots (h=w02) slots 2\n");
|
||||
free(b);
|
||||
|
||||
/* 7.8: after the replay (101 finished at 5500), `info trace` re-prints
|
||||
* its stored full trace, then again with the raw units mode. 105's
|
||||
* latest trace is the session 5 summary one. */
|
||||
after = strstr(r.out, "[sim] simulation finished");
|
||||
TEST_ASSERT_NOT_NULL(after);
|
||||
IN_ORDER(after,
|
||||
"\n[trace] job 0-0-0-101 session 1 position 1 level full\n",
|
||||
"\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit "
|
||||
"charges: host_slots (h=w01) slots 2\n",
|
||||
"\n[sim] units set to raw\n"
|
||||
"[trace] job 0-0-0-101 session 1 position 1 level full\n",
|
||||
"\n requirement single tasks 2..4 requests mem=1024.000000\n",
|
||||
"\n host w01: mem=2048.000000 slots=2 limit charges: "
|
||||
"host_slots (h=w01) slots 2\n",
|
||||
"\n[trace] job 0-0-0-105 session 5 position 2 level "
|
||||
"summary\n",
|
||||
"\n requirement alternative 1/2 > member 1/1 tasks 4..4 "
|
||||
"requests mem=12288.000000\n");
|
||||
run_done(&r);
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------------- */
|
||||
/* 3. Selected jobs, compound, array, info trace on an array */
|
||||
/* ---------------------------------------------------------------------- */
|
||||
|
||||
static void test_compound_array_selection_and_info_trace(void)
|
||||
{
|
||||
run_t r;
|
||||
const char* mark;
|
||||
const char* s2;
|
||||
char* b;
|
||||
char* traces;
|
||||
long tlen;
|
||||
|
||||
run_repl(&r, EXTRA_REPLAY,
|
||||
"trace on job 0-0-0-201 0-0-0-202 0-0-0-32768 0-0-0-203 "
|
||||
"level full file " TRACE_FILE "\n"
|
||||
"step\n"
|
||||
"trace status\n"
|
||||
"info trace 0-0-0-32768\n"
|
||||
"info trace 0-0-0-32768[2]\n"
|
||||
"info trace 0-0-0-999\n"
|
||||
"step\n"
|
||||
"info trace 0-0-0-32768\n"
|
||||
"quit\n");
|
||||
|
||||
mark = line_of(r.out, "[sim] trace status");
|
||||
assert_every_trace_follows_echo(r.out, mark);
|
||||
/* 1.2: the bare array id selects both elements. */
|
||||
IN_ORDER(r.out,
|
||||
"[sim] DISPATCH jobid=0-0-0-201\n"
|
||||
"[trace] job 0-0-0-201 session 1 position 1 level full\n",
|
||||
"[sim] PEND jobid=0-0-0-202\n"
|
||||
"[trace] job 0-0-0-202 session 1 position 2 level full\n",
|
||||
"[sim] DISPATCH jobid=0-0-0-32768[1]\n"
|
||||
"[trace] job 0-0-0-32768[1] session 1 position 3 level "
|
||||
"full\n",
|
||||
"[sim] DISPATCH jobid=0-0-0-32768[2]\n"
|
||||
"[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
||||
"full\n",
|
||||
"[sim] DISPATCH jobid=0-0-0-203\n"
|
||||
"[trace] job 0-0-0-203 session 1 position 5 level full\n");
|
||||
|
||||
/* 201: compound labels, the reservation of member 1 seen by member 2's
|
||||
* cluster-wide check, shared and host allocations (3.1, 3.2, 4.1, 6.1,
|
||||
* 6.2). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-201 session 1");
|
||||
IN_ORDER(b, "\n requirement compound (2 members)\n",
|
||||
"\n requirement compound > member 1/2 tasks 1..1 requests "
|
||||
"license=1\n",
|
||||
"\n cluster-wide: license total 2 in-use 0 reserved 0 "
|
||||
"requested 1 passed\n",
|
||||
"\n member 1/2 satisfied\n",
|
||||
"\n requirement compound > member 2/2 tasks 2..2 requests "
|
||||
"mem=1024.000000 (1.00 KB)\n",
|
||||
"\n cluster-wide limit cluster_slots on slots: value 6 usage "
|
||||
"0 reserved 1 requested 2 passed\n",
|
||||
"\n member 2/2 satisfied\n",
|
||||
"\n compound requirement satisfied\n",
|
||||
"\n outcome: DISPATCH\n",
|
||||
"\n shared: license=1 limit charges: cluster_slots slots 3\n");
|
||||
free(b);
|
||||
|
||||
/* 202, session 1: the failing member and the members not tried
|
||||
* (3.3). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-202 session 1");
|
||||
IN_ORDER(b,
|
||||
"\n cluster-wide check failed; no placement plans tried\n",
|
||||
"\n member 1/2 not satisfied; member 2/2 not tried\n",
|
||||
"\n compound requirement not satisfied\n",
|
||||
"\n outcome: PEND\n");
|
||||
free(b);
|
||||
|
||||
/* [2]: host-scoped limit rejection; 203: selection mismatch from the
|
||||
* pre-filter (5.2, 5.3, 5.5). */
|
||||
b = block_of(r.out, "[trace] job 0-0-0-32768[2] session 1");
|
||||
IN_ORDER(b,
|
||||
"\n host w01 rejected (per-host check): limit exceeded\n"
|
||||
" limit host_slots (h=w01) on slots: value 3 usage 3 "
|
||||
"reserved 0 requested 1\n"
|
||||
" host w02 accepted\n");
|
||||
free(b);
|
||||
b = block_of(r.out, "[trace] job 0-0-0-203 session 1");
|
||||
IN_ORDER(b,
|
||||
"\n host w01 rejected (pre-filter): selection mismatch; "
|
||||
"per-host check not performed\n"
|
||||
" host w02 accepted\n"
|
||||
" position 1 filled by w02\n");
|
||||
free(b);
|
||||
|
||||
/* 7.9, 7.11: `info trace` for the array prints each element's trace,
|
||||
* for one element only that one, and an unknown job is reported. */
|
||||
IN_ORDER(mark,
|
||||
"\n[trace] job 0-0-0-32768[1] session 1 position 3 level "
|
||||
"full\n",
|
||||
"\n[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
||||
"full\n",
|
||||
"\n[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
||||
"full\n",
|
||||
"\n[sim] info trace: job 0-0-0-999: unknown job\n");
|
||||
|
||||
/* Session 2: 202 again; a shortfall row beside a limit row on w02 and
|
||||
* plan 2 committed for member 1 (5.4, 5.5, 4.5). */
|
||||
s2 = strstr(r.out, "[trace] job 0-0-0-202 session 2");
|
||||
TEST_ASSERT_NOT_NULL(s2);
|
||||
TEST_ASSERT_TRUE(follows_its_echo(r.out, s2));
|
||||
b = block_of(s2, "[trace] job 0-0-0-202 session 2");
|
||||
IN_ORDER(b,
|
||||
"\n host w02 rejected (per-host check): resource "
|
||||
"shortfall, limit exceeded\n"
|
||||
" slots total 4 in-use 1 reserved 0 requested 4\n"
|
||||
" limit host_slots (h=w02) on slots: value 3 usage 1 "
|
||||
"reserved 0 requested 4\n",
|
||||
"\n plan 2 committed; no further plans tried\n",
|
||||
"\n member 1/2 satisfied\n",
|
||||
"\n cluster-wide limit cluster_slots on slots: value 6 usage "
|
||||
"1 reserved 4 requested 1 passed\n",
|
||||
"\n compound requirement satisfied\n");
|
||||
TEST_ASSERT_NULL(strstr(b, "limit=host_slots total"));
|
||||
free(b);
|
||||
|
||||
/* 7.8, 7.9: after both elements finished, their traces remain. */
|
||||
IN_ORDER(s2,
|
||||
"\n[trace] job 0-0-0-32768[1] session 1 position 3 level "
|
||||
"full\n",
|
||||
"\n[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
||||
"full\n");
|
||||
|
||||
/* 8.1: the file holds the inline traces (not `info trace` output),
|
||||
* byte for byte. */
|
||||
traces = collect_traces(r.out, NULL, true, &tlen);
|
||||
assert_file_equals(&r, traces, tlen);
|
||||
free(traces);
|
||||
run_done(&r);
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
UNITY_BEGIN();
|
||||
RUN_TEST(test_step_mode_full_all_jobs_and_file_copy);
|
||||
RUN_TEST(test_continue_breakpoint_summary_and_info_trace);
|
||||
RUN_TEST(test_compound_array_selection_and_info_trace);
|
||||
return UNITY_END();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue