Rongsong Shen
7aef6b85c9
repl_u64_str_eq() compared the packed jobid's raw decimal string with the filter text, so a breakpoint on the dashed jobid the REPL prints never fired. The filter is now parsed once when the breakpoint is set (repl_parse_jobsel_arg, moved to repl_fmt.c) and matched with repl_jobsel_matches: dashed ids, X[k], X[a-b] and raw decimal ids all work, a bare array-aligned id also matches its elements, and an unparseable value is rejected instead of creating a dead breakpoint. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
675 lines
29 KiB
C
675 lines
29 KiB
C
/*
|
|
* test_repl_trace_e2e.c - scheduler-sim-decision-trace spec, task 6.1
|
|
* (design.md "Testing Strategy" -> "Integration Tests" item 2, "System
|
|
* Flows", "TraceRenderer"): end-to-end decision-trace scenarios.
|
|
*
|
|
* Each scenario runs the real, built xce_scheduler binary
|
|
* (XCE_SCHEDULER_BIN) in interactive simulation mode with a scripted REPL
|
|
* session on stdin, the way test_repl_trace_baseline.c does, and checks the
|
|
* console transcript (and the trace file copy) for the expected trace
|
|
* content and its placement. Checks look for key lines and their order,
|
|
* never one golden blob, except the file-copy check, which is byte for
|
|
* byte. Nothing depends on the wall clock: the replays' times are fixed and
|
|
* the traces print only replay times.
|
|
*
|
|
* Replays (test_repl_trace_e2e_fixture.c):
|
|
* - contention.replay, the task 1.1 baseline fixture
|
|
* (fixtures/repl_trace_baseline/, read only): two 4-slot workers, a
|
|
* 2-unit cluster license, a cluster-wide limit of 6 slots, a host-scoped
|
|
* limit of 75% of each host's slots, a parallel job (101), license jobs
|
|
* (102..104), an alternative-requirement job (105) and slot jobs
|
|
* (106, 107). In its session 5, 105's first alternative fills two
|
|
* positions of a 4-position plan and then releases them (rollback).
|
|
* - extra.replay, generated per run, for what the baseline lacks: a
|
|
* satisfied compound requirement (201), a compound requirement that
|
|
* fails at its first member (202), a job array with two elements
|
|
* (0-0-0-32768[1] and [2]) and a host-selection mismatch (203 selects
|
|
* mem>20000, which only w02 has).
|
|
*
|
|
* `break DECISION jobid=105` (a raw packed decimal id) names 0-0-0-105;
|
|
* since bug-1926 the dashed form 0-0-0-105 names it too.
|
|
*
|
|
* Scenarios:
|
|
* 1. step mode, all jobs, level full, file copy (baseline replay, five
|
|
* sessions): every section of Requirements 2-6, each trace right after
|
|
* its echo line, and the file equal to the console traces.
|
|
* 2. continue mode with a breakpoint (baseline replay): traces up to the
|
|
* stopping decision only, `info trace` at the stop, a selection change
|
|
* at the breakpoint applying from the next session, summary level,
|
|
* traced rounds kept out of collapsed ranges, `info trace` after the
|
|
* job finished and after `set units raw`.
|
|
* 3. selected jobs incl. a job array (extra replay): compound labels,
|
|
* selection mismatch, host-limit rejection, shortfall next to a limit,
|
|
* `info trace` for an array, an element and an unknown job, and the
|
|
* file equal to the inline console traces.
|
|
*/
|
|
#include <stdbool.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "test_repl_trace_e2e.h"
|
|
|
|
void setUp(void) { return; }
|
|
void tearDown(void) { return; }
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* Transcript helpers */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
/* Asserts the needles occur in `text` in the given order, each after the
|
|
* end of the previous one; returns the end of the last match. */
|
|
static const char* assert_in_order(const char* text, const char* const* nd,
|
|
size_t n)
|
|
{
|
|
const char* at = text;
|
|
size_t i;
|
|
|
|
for (i = 0; i < n; i++) {
|
|
const char* hit = strstr(at, nd[i]);
|
|
|
|
if (hit == NULL) {
|
|
char msg[1024];
|
|
|
|
snprintf(msg, sizeof(msg), "missing (or out of order): %s",
|
|
nd[i]);
|
|
TEST_FAIL_MESSAGE(msg);
|
|
}
|
|
/* A needle's closing newline may open the next needle's line. */
|
|
at = hit + strlen(nd[i]);
|
|
if ((at > hit) && (at[-1] == '\n')) {
|
|
at--;
|
|
}
|
|
}
|
|
return at;
|
|
}
|
|
|
|
#define IN_ORDER(text, ...) \
|
|
do { \
|
|
static const char* const nd_[] = { __VA_ARGS__ }; \
|
|
(void)assert_in_order((text), nd_, sizeof(nd_) / sizeof(nd_[0])); \
|
|
} while (0)
|
|
|
|
/* Pointer to the line containing `needle` (start of line), or fails. */
|
|
static const char* line_of(const char* text, const char* needle)
|
|
{
|
|
const char* hit = strstr(text, needle);
|
|
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(hit, needle);
|
|
while ((hit > text) && (hit[-1] != '\n')) {
|
|
hit--;
|
|
}
|
|
return hit;
|
|
}
|
|
|
|
/* True when the line at `p` continues a trace block: trace lines after the
|
|
* header are indented, and a truncation marker starts with "...". */
|
|
static bool is_trace_body(const char* p)
|
|
{
|
|
return (p[0] == ' ') || (strncmp(p, "...", 3) == 0);
|
|
}
|
|
|
|
static const char* next_line(const char* p)
|
|
{
|
|
const char* nl = strchr(p, '\n');
|
|
|
|
return (nl != NULL) ? nl + 1 : p + strlen(p);
|
|
}
|
|
|
|
/* Copy of the whole trace block whose header line contains `header`. */
|
|
static char* block_of(const char* text, const char* header)
|
|
{
|
|
const char* start = line_of(text, header);
|
|
const char* end = next_line(start);
|
|
char* out;
|
|
|
|
while ((*end != '\0') && is_trace_body(end)) {
|
|
end = next_line(end);
|
|
}
|
|
out = strndup(start, (size_t)(end - start));
|
|
TEST_ASSERT_NOT_NULL(out);
|
|
return out;
|
|
}
|
|
|
|
/* True when the line before `line` is the echo line of a decision for the
|
|
* job the trace header at `line` names: "[sim] <OP> jobid=<id>\n" before
|
|
* "[trace] job <id> ...". */
|
|
static bool follows_its_echo(const char* text, const char* line)
|
|
{
|
|
const char* prev;
|
|
const char* id = line + strlen("[trace] job ");
|
|
const char* id_end = strstr(id, " ");
|
|
const char* eq;
|
|
|
|
if ((line == text) || (id_end == NULL)) {
|
|
return false;
|
|
}
|
|
prev = line - 1; /* the '\n' ending the previous line */
|
|
while ((prev > text) && (prev[-1] != '\n')) {
|
|
prev--;
|
|
}
|
|
if (strncmp(prev, "[sim] ", 6) != 0) {
|
|
return false;
|
|
}
|
|
eq = strstr(prev, " jobid=");
|
|
if ((eq == NULL) || (eq > line)) {
|
|
return false;
|
|
}
|
|
eq += strlen(" jobid=");
|
|
return (strncmp(eq, id, (size_t)(id_end - id)) == 0)
|
|
&& (eq[id_end - id] == '\n');
|
|
}
|
|
|
|
/* Concatenation of the trace blocks of `text` (up to `stop`, or the end
|
|
* when NULL). With `inline_only`, only blocks printed right after their
|
|
* decision's echo line, i.e. not `info trace` output. */
|
|
static char* collect_traces(const char* text, const char* stop,
|
|
bool inline_only, long* len)
|
|
{
|
|
size_t cap = strlen(text) + 1;
|
|
char* out = (char*)malloc(cap);
|
|
size_t n = 0;
|
|
const char* p = text;
|
|
|
|
TEST_ASSERT_NOT_NULL(out);
|
|
while ((*p != '\0') && ((stop == NULL) || (p < stop))) {
|
|
if ((strncmp(p, "[trace] job ", 12) == 0)
|
|
&& (!inline_only || follows_its_echo(text, p))) {
|
|
const char* end = next_line(p);
|
|
|
|
while ((*end != '\0') && is_trace_body(end)) {
|
|
end = next_line(end);
|
|
}
|
|
memcpy(out + n, p, (size_t)(end - p));
|
|
n += (size_t)(end - p);
|
|
p = end;
|
|
continue;
|
|
}
|
|
p = next_line(p);
|
|
}
|
|
out[n] = '\0';
|
|
*len = (long)n;
|
|
return out;
|
|
}
|
|
|
|
/* Every inline trace header before `stop` (NULL = end) follows its echo
|
|
* line (7.1, 7.3). `info trace` output is excluded by the caller choosing
|
|
* `stop` before the first `info trace`. */
|
|
static void assert_every_trace_follows_echo(const char* text,
|
|
const char* stop)
|
|
{
|
|
const char* p = text;
|
|
int n = 0;
|
|
|
|
while ((p = strstr(p, "[trace] job ")) != NULL) {
|
|
if ((stop != NULL) && (p >= stop)) {
|
|
break;
|
|
}
|
|
if ((p == text) || (p[-1] == '\n')) {
|
|
char msg[256];
|
|
|
|
snprintf(msg, sizeof(msg), "trace not after its echo line: %.80s",
|
|
p);
|
|
TEST_ASSERT_TRUE_MESSAGE(follows_its_echo(text, p), msg);
|
|
n++;
|
|
}
|
|
p++;
|
|
}
|
|
TEST_ASSERT_TRUE_MESSAGE(n > 0, "expected at least one inline trace");
|
|
}
|
|
|
|
/* The trace file of run `r` equals `expect` byte for byte (8.1). */
|
|
static void assert_file_equals(const run_t* r, const char* expect, long len)
|
|
{
|
|
char path[PATH_MAX + 32];
|
|
long flen;
|
|
char* file;
|
|
|
|
snprintf(path, sizeof(path), "%s/%s", r->dir, TRACE_FILE);
|
|
file = slurp(path, &flen);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(file, "the trace file must exist");
|
|
TEST_ASSERT_TRUE_MESSAGE(len > 0, "console traces must not be empty");
|
|
TEST_ASSERT_EQUAL_INT64_MESSAGE(len, flen,
|
|
"trace file length must equal the "
|
|
"console traces");
|
|
TEST_ASSERT_EQUAL_MEMORY_MESSAGE(expect, file, (size_t)len,
|
|
"trace file must equal the console "
|
|
"traces byte for byte");
|
|
free(file);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* 1. Step mode, all jobs, full level, file copy */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_step_mode_full_all_jobs_and_file_copy(void)
|
|
{
|
|
run_t r;
|
|
char* b;
|
|
char* traces;
|
|
long tlen;
|
|
|
|
run_repl(&r, BASE_REPLAY,
|
|
"trace on all level full file " TRACE_FILE "\n"
|
|
"step\nstep\nstep\nstep\nstep\nquit\n");
|
|
|
|
/* 1.1, 2.1, 7.1: every job of session 1 in pick order, each trace
|
|
* right after its echo line. */
|
|
assert_every_trace_follows_echo(r.out, NULL);
|
|
IN_ORDER(r.out,
|
|
"[sim] DISPATCH jobid=0-0-0-101\n"
|
|
"[trace] job 0-0-0-101 session 1 position 1 level full\n",
|
|
"[sim] DISPATCH jobid=0-0-0-102\n"
|
|
"[trace] job 0-0-0-102 session 1 position 2 level full\n",
|
|
"[trace] job 0-0-0-103 session 1 position 3 level full\n",
|
|
"[sim] PEND jobid=0-0-0-104\n"
|
|
"[trace] job 0-0-0-104 session 1 position 4 level full\n",
|
|
"[trace] job 0-0-0-105 session 1 position 5 level full\n",
|
|
"[trace] job 0-0-0-106 session 1 position 6 level full\n",
|
|
"[trace] job 0-0-0-107 session 1 position 7 level full\n",
|
|
"[trace] job 0-0-0-104 session 2 position 1 level full\n",
|
|
"[trace] job 0-0-0-105 session 5 position 2 level full\n",
|
|
"[trace] job 0-0-0-106 session 5 position 3 level full\n");
|
|
|
|
/* 101: selection, host-scoped limit rejections, plans, positions,
|
|
* already-claimed, commit, dispatch with limit charges (7.6 order). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-101 session 1");
|
|
IN_ORDER(b,
|
|
"\n selection: policy builtin_fcfs queue normal (priority 1)"
|
|
" job priority 0 queued 1970-01-01T00:18:21Z\n",
|
|
"\n cluster-wide limits: cluster_slots; host-scoped limits are "
|
|
"reported with each host's verdict\n",
|
|
"\n requirement single tasks 2..4 requests mem=1024.000000 "
|
|
"(1.00 KB)\n",
|
|
"\n cluster-wide limit cluster_slots on slots: value 6 usage 0"
|
|
" reserved 0 requested 4 passed\n",
|
|
"\n cluster-wide check passed\n",
|
|
"\n plan 1 hosts 1 tasks per position [4]\n",
|
|
"\n position 1 (4 tasks)\n",
|
|
"\n host w01 rejected (per-host check): limit exceeded\n"
|
|
" limit host_slots (h=w01) on slots: value 3 usage 0 "
|
|
"reserved 0 requested 4\n",
|
|
"\n host w02 rejected (per-host check): limit exceeded\n",
|
|
"\n position 1 not filled: no host accepted\n",
|
|
"\n plan 1 failed: position 1 could not be filled\n",
|
|
"\n plan 2 hosts 2 tasks per position [2, 2]\n",
|
|
"\n host w01 accepted\n",
|
|
"\n position 1 filled by w01\n",
|
|
"\n host w01 skipped: already claimed by an earlier "
|
|
"position\n",
|
|
"\n position 2 filled by w02\n",
|
|
"\n plan 2 succeeded\n",
|
|
"\n plan 2 committed; no further plans tried\n",
|
|
"\n plans: 2 generated, 2 tried, 0 discarded, 0 not tried\n",
|
|
"\n single requirement satisfied\n",
|
|
"\n outcome: DISPATCH\n",
|
|
"\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit charges: "
|
|
"host_slots (h=w01) slots 2\n",
|
|
"\n host w02: mem=2048.000000 (2.00 KB) slots=2 limit charges: "
|
|
"host_slots (h=w02) slots 2\n",
|
|
"\n shared: none limit charges: cluster_slots slots 4\n");
|
|
/* 5.5: a limit hit is never also a resource shortfall. */
|
|
TEST_ASSERT_NULL(strstr(b, "resource shortfall"));
|
|
free(b);
|
|
|
|
/* 102: a shared resource row, shared allocation and its limit charge
|
|
* kept apart (4.1, 6.2, 6.3). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-102 session 1");
|
|
IN_ORDER(b,
|
|
"\n cluster-wide: license total 2 in-use 0 reserved 0 "
|
|
"requested 1 passed\n",
|
|
"\n cluster-wide limit cluster_slots on slots: value 6 usage 4"
|
|
" reserved 0 requested 1 passed\n",
|
|
"\n host w01: slots=1 limit charges: host_slots (h=w01) "
|
|
"slots 1\n",
|
|
"\n shared: license=1 limit charges: cluster_slots slots 1\n");
|
|
free(b);
|
|
|
|
/* 104: cluster-wide shortfall and cluster-wide limit rejection, no
|
|
* plans, pend reasons (4.1, 4.2, 4.8, 6.4). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-104 session 1");
|
|
IN_ORDER(b,
|
|
"\n cluster-wide: license total 2 in-use 2 reserved 0 "
|
|
"requested 1 failed\n",
|
|
"\n cluster-wide limit cluster_slots on slots: value 6 usage 6"
|
|
" reserved 0 requested 1 failed\n",
|
|
"\n cluster-wide check failed; no placement plans tried\n",
|
|
"\n single requirement not satisfied\n",
|
|
"\n outcome: PEND\n",
|
|
"\n main reason: none\n",
|
|
"\n resource license: NO_RESOURCE\n",
|
|
"\n resource limit=cluster_slots: NO_RESOURCE\n");
|
|
TEST_ASSERT_NULL(strstr(b, "plan 1"));
|
|
free(b);
|
|
/* 4.2: the cluster-wide limit is never shown as a resource row. */
|
|
TEST_ASSERT_NULL(strstr(r.out, "cluster-wide: limit="));
|
|
|
|
/* 105, session 1: alternative labels and every alternative's failure
|
|
* reasons (3.1, 3.2, 3.5). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-105 session 1");
|
|
IN_ORDER(b,
|
|
"\n requirement alternative 1/2 (compound, 1 member)\n",
|
|
"\n requirement alternative 1/2 > member 1/1 tasks 4..4 "
|
|
"requests mem=12288.000000 (12.00 KB)\n",
|
|
"\n member 1/1 not satisfied\n",
|
|
"\n failure reasons: NO_RESOURCE x1 (resources: "
|
|
"limit=cluster_slots)\n",
|
|
"\n alternative 1/2 not satisfied\n",
|
|
"\n requirement alternative 2/2 (compound, 1 member)\n",
|
|
"\n failure reasons: NO_RESOURCE x1 (resources: "
|
|
"limit=cluster_slots)\n",
|
|
"\n alternative 2/2 not satisfied\n",
|
|
"\n no alternative chosen: none of the 2 alternatives was "
|
|
"satisfied\n",
|
|
"\n outcome: PEND\n", "\n main reason: none\n");
|
|
free(b);
|
|
|
|
/* 105, session 5: per-host shortfall rows next to limit rows, the
|
|
* rollback, no plan placed, then the chosen alternative (3.4, 4.6,
|
|
* 5.4, 5.5, 5.8). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-105 session 5");
|
|
IN_ORDER(b,
|
|
"\n host w01 rejected (per-host check): resource "
|
|
"shortfall, limit exceeded\n"
|
|
" mem total 16384.000000 (16.00 KB) in-use 0.000000 "
|
|
"(0.00 B) reserved 0.000000 (0.00 B) requested 49152.000000 "
|
|
"(48.00 KB)\n"
|
|
" slots total 4 in-use 1 reserved 0 requested 4\n"
|
|
" limit host_slots (h=w01) on slots: value 3 usage 1 "
|
|
"reserved 0 requested 4\n",
|
|
"\n plan 4 hosts 4 tasks per position [1, 1, 1, 1]\n",
|
|
"\n position 1 filled by w01\n",
|
|
"\n position 2 filled by w02\n",
|
|
"\n position 3 not filled: no host accepted\n"
|
|
" released hosts: w01, w02\n"
|
|
" plan 4 failed: position 3 could not be filled\n",
|
|
"\n plans: 4 generated, 4 tried, 0 discarded, 0 not tried; "
|
|
"no plan could be placed\n",
|
|
"\n alternative 1/2 not satisfied\n",
|
|
"\n plan 1 committed; no further plans tried\n",
|
|
"\n alternative 2/2 satisfied\n"
|
|
" alternative 2/2 chosen; no alternatives after it\n"
|
|
" outcome: DISPATCH\n"
|
|
" host w01: mem=2048.000000 (2.00 KB) slots=1 limit charges: "
|
|
"host_slots (h=w01) slots 1\n");
|
|
TEST_ASSERT_NULL(strstr(b, "limit=host_slots total"));
|
|
free(b);
|
|
|
|
/* 8.1: the file holds exactly the console's traces. */
|
|
traces = collect_traces(r.out, NULL, false, &tlen);
|
|
assert_file_equals(&r, traces, tlen);
|
|
free(traces);
|
|
run_done(&r);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* 2. Continue mode, breakpoint, summary level, info trace */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_continue_breakpoint_summary_and_info_trace(void)
|
|
{
|
|
run_t r;
|
|
const char* hit;
|
|
const char* after;
|
|
char* b;
|
|
|
|
run_repl(&r, BASE_REPLAY,
|
|
"trace on all level full\n"
|
|
"break DECISION jobid=105 type=PEND\n"
|
|
"continue\n"
|
|
"info trace 0-0-0-105\n"
|
|
"info trace 0-0-0-106\n"
|
|
"trace on job 0-0-0-105 0-0-0-106 level summary\n"
|
|
"delete all\n"
|
|
"continue\n"
|
|
"info trace 0-0-0-101\n"
|
|
"set units raw\n"
|
|
"info trace 0-0-0-101\n"
|
|
"info trace 0-0-0-105\n"
|
|
"quit\n");
|
|
|
|
/* 7.5: at the stop, the round's traces up to and including the
|
|
* stopping decision are printed, and nothing later. */
|
|
hit = line_of(r.out, "[sim] hit breakpoint 1 @ decision: PEND "
|
|
"jobid=0-0-0-105");
|
|
assert_every_trace_follows_echo(r.out, hit);
|
|
IN_ORDER(r.out, "[trace] job 0-0-0-101 session 1 position 1",
|
|
"[trace] job 0-0-0-104 session 1 position 4",
|
|
"[sim] PEND jobid=0-0-0-105\n"
|
|
"[trace] job 0-0-0-105 session 1 position 5 level full\n",
|
|
"[sim] hit breakpoint 1 @ decision: PEND jobid=0-0-0-105\n");
|
|
TEST_ASSERT_TRUE(strstr(r.out, "[trace] job 0-0-0-106") > hit);
|
|
|
|
/* 7.5, 7.8: `info trace` at the stop shows the stopping decision's
|
|
* trace; the job decided later in this round has none yet. */
|
|
IN_ORDER(hit, "\n[trace] job 0-0-0-105 session 1 position 5 level "
|
|
"full\n",
|
|
"\n[sim] info trace: job 0-0-0-106: no trace available (state "
|
|
"PEND)\n",
|
|
"\n[sim] trace: enabled; applies from the next session\n");
|
|
|
|
/* 1.6, 7.3: the rest of session 1 keeps the settings it started with
|
|
* (all jobs, full), echo line then trace in continue mode, then the
|
|
* round's line; 106 and 107 are traced at level full. */
|
|
after = strstr(hit, "[sim] deleted ");
|
|
TEST_ASSERT_NOT_NULL(after);
|
|
IN_ORDER(after,
|
|
"\n[sim] PEND jobid=0-0-0-106\n"
|
|
"[trace] job 0-0-0-106 session 1 position 6 level full\n",
|
|
"\n[sim] PEND jobid=0-0-0-107\n"
|
|
"[trace] job 0-0-0-107 session 1 position 7 level full\n",
|
|
"\n[sim] round 1-1 (",
|
|
/* 1.2, 1.4, 7.3, 7.4: from session 2 only 105 and 106, at
|
|
* summary level, each traced round reported on its own. */
|
|
"\n[sim] PEND jobid=0-0-0-105\n"
|
|
"[trace] job 0-0-0-105 session 2 position 2 level summary\n",
|
|
"\n[sim] PEND jobid=0-0-0-106\n"
|
|
"[trace] job 0-0-0-106 session 2 position 3 level summary\n",
|
|
"\n[sim] round 2-2 (",
|
|
"[trace] job 0-0-0-105 session 3", "\n[sim] round 3-3 (",
|
|
"[trace] job 0-0-0-105 session 4", "\n[sim] round 4-4 (",
|
|
"\n[sim] DISPATCH jobid=0-0-0-105\n"
|
|
"[trace] job 0-0-0-105 session 5 position 2 level summary\n",
|
|
"\n[sim] DISPATCH jobid=0-0-0-106\n"
|
|
"[trace] job 0-0-0-106 session 5 position 3 level summary\n",
|
|
"\n[sim] round 5-5 (",
|
|
/* untraced rounds still collapse */
|
|
"\n[sim] round 6-7 (2 round(s)",
|
|
"\n[sim] replay complete.");
|
|
TEST_ASSERT_NULL(strstr(r.out, "[sim] round 2-4"));
|
|
TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-104 session 2"));
|
|
TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-107 session 2"));
|
|
|
|
/* 5.8, 5.9: summary level counts verdicts per position, names each
|
|
* limit hit, keeps the rollback, and lists no host lines. */
|
|
b = block_of(r.out, "[trace] job 0-0-0-105 session 5");
|
|
IN_ORDER(b,
|
|
"\n plan 1 hosts 1 tasks per position [4]\n"
|
|
" position 1 (4 tasks)\n"
|
|
" accepted none; rejected: selection mismatch 0, resource "
|
|
"shortfall 2, limit exceeded 2 (host_slots (h=w01) 1, "
|
|
"host_slots (h=w02) 1), already claimed 0, not evaluated "
|
|
"(internal error) 0\n",
|
|
"\n position 3 (1 task)\n"
|
|
" accepted none; rejected: selection mismatch 0, resource "
|
|
"shortfall 0, limit exceeded 0, already claimed 2, not evaluated "
|
|
"(internal error) 0\n"
|
|
" released hosts: w01, w02\n",
|
|
"\n alternative 2/2 chosen; no alternatives after it\n");
|
|
TEST_ASSERT_NULL(strstr(b, " host w01 "));
|
|
free(b);
|
|
b = block_of(r.out, "[trace] job 0-0-0-106 session 5");
|
|
IN_ORDER(b,
|
|
"\n accepted w02; rejected: selection mismatch 0, resource "
|
|
"shortfall 0, limit exceeded 1 (host_slots (h=w01) 1), already "
|
|
"claimed 0, not evaluated (internal error) 0\n",
|
|
"\n host w02: mem=1024.000000 (1.00 KB) slots=2 limit "
|
|
"charges: host_slots (h=w02) slots 2\n");
|
|
free(b);
|
|
|
|
/* 7.8: after the replay (101 finished at 5500), `info trace` re-prints
|
|
* its stored full trace, then again with the raw units mode. 105's
|
|
* latest trace is the session 5 summary one. */
|
|
after = strstr(r.out, "[sim] simulation finished");
|
|
TEST_ASSERT_NOT_NULL(after);
|
|
IN_ORDER(after,
|
|
"\n[trace] job 0-0-0-101 session 1 position 1 level full\n",
|
|
"\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit "
|
|
"charges: host_slots (h=w01) slots 2\n",
|
|
"\n[sim] units set to raw\n"
|
|
"[trace] job 0-0-0-101 session 1 position 1 level full\n",
|
|
"\n requirement single tasks 2..4 requests mem=1024.000000\n",
|
|
"\n host w01: mem=2048.000000 slots=2 limit charges: "
|
|
"host_slots (h=w01) slots 2\n",
|
|
"\n[trace] job 0-0-0-105 session 5 position 2 level "
|
|
"summary\n",
|
|
"\n requirement alternative 1/2 > member 1/1 tasks 4..4 "
|
|
"requests mem=12288.000000\n");
|
|
run_done(&r);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* 3. Selected jobs, compound, array, info trace on an array */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_compound_array_selection_and_info_trace(void)
|
|
{
|
|
run_t r;
|
|
const char* mark;
|
|
const char* s2;
|
|
char* b;
|
|
char* traces;
|
|
long tlen;
|
|
|
|
run_repl(&r, EXTRA_REPLAY,
|
|
"trace on job 0-0-0-201 0-0-0-202 0-0-0-32768 0-0-0-203 "
|
|
"level full file " TRACE_FILE "\n"
|
|
"step\n"
|
|
"trace status\n"
|
|
"info trace 0-0-0-32768\n"
|
|
"info trace 0-0-0-32768[2]\n"
|
|
"info trace 0-0-0-999\n"
|
|
"step\n"
|
|
"info trace 0-0-0-32768\n"
|
|
"quit\n");
|
|
|
|
mark = line_of(r.out, "[sim] trace status");
|
|
assert_every_trace_follows_echo(r.out, mark);
|
|
/* 1.2: the bare array id selects both elements. */
|
|
IN_ORDER(r.out,
|
|
"[sim] DISPATCH jobid=0-0-0-201\n"
|
|
"[trace] job 0-0-0-201 session 1 position 1 level full\n",
|
|
"[sim] PEND jobid=0-0-0-202\n"
|
|
"[trace] job 0-0-0-202 session 1 position 2 level full\n",
|
|
"[sim] DISPATCH jobid=0-0-0-32768[1]\n"
|
|
"[trace] job 0-0-0-32768[1] session 1 position 3 level "
|
|
"full\n",
|
|
"[sim] DISPATCH jobid=0-0-0-32768[2]\n"
|
|
"[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
|
"full\n",
|
|
"[sim] DISPATCH jobid=0-0-0-203\n"
|
|
"[trace] job 0-0-0-203 session 1 position 5 level full\n");
|
|
|
|
/* 201: compound labels, the reservation of member 1 seen by member 2's
|
|
* cluster-wide check, shared and host allocations (3.1, 3.2, 4.1, 6.1,
|
|
* 6.2). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-201 session 1");
|
|
IN_ORDER(b, "\n requirement compound (2 members)\n",
|
|
"\n requirement compound > member 1/2 tasks 1..1 requests "
|
|
"license=1\n",
|
|
"\n cluster-wide: license total 2 in-use 0 reserved 0 "
|
|
"requested 1 passed\n",
|
|
"\n member 1/2 satisfied\n",
|
|
"\n requirement compound > member 2/2 tasks 2..2 requests "
|
|
"mem=1024.000000 (1.00 KB)\n",
|
|
"\n cluster-wide limit cluster_slots on slots: value 6 usage "
|
|
"0 reserved 1 requested 2 passed\n",
|
|
"\n member 2/2 satisfied\n",
|
|
"\n compound requirement satisfied\n",
|
|
"\n outcome: DISPATCH\n",
|
|
"\n shared: license=1 limit charges: cluster_slots slots 3\n");
|
|
free(b);
|
|
|
|
/* 202, session 1: the failing member and the members not tried
|
|
* (3.3). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-202 session 1");
|
|
IN_ORDER(b,
|
|
"\n cluster-wide check failed; no placement plans tried\n",
|
|
"\n member 1/2 not satisfied; member 2/2 not tried\n",
|
|
"\n compound requirement not satisfied\n",
|
|
"\n outcome: PEND\n");
|
|
free(b);
|
|
|
|
/* [2]: host-scoped limit rejection; 203: selection mismatch from the
|
|
* pre-filter (5.2, 5.3, 5.5). */
|
|
b = block_of(r.out, "[trace] job 0-0-0-32768[2] session 1");
|
|
IN_ORDER(b,
|
|
"\n host w01 rejected (per-host check): limit exceeded\n"
|
|
" limit host_slots (h=w01) on slots: value 3 usage 3 "
|
|
"reserved 0 requested 1\n"
|
|
" host w02 accepted\n");
|
|
free(b);
|
|
b = block_of(r.out, "[trace] job 0-0-0-203 session 1");
|
|
IN_ORDER(b,
|
|
"\n host w01 rejected (pre-filter): selection mismatch; "
|
|
"per-host check not performed\n"
|
|
" host w02 accepted\n"
|
|
" position 1 filled by w02\n");
|
|
free(b);
|
|
|
|
/* 7.9, 7.11: `info trace` for the array prints each element's trace,
|
|
* for one element only that one, and an unknown job is reported. */
|
|
IN_ORDER(mark,
|
|
"\n[trace] job 0-0-0-32768[1] session 1 position 3 level "
|
|
"full\n",
|
|
"\n[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
|
"full\n",
|
|
"\n[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
|
"full\n",
|
|
"\n[sim] info trace: job 0-0-0-999: unknown job\n");
|
|
|
|
/* Session 2: 202 again; a shortfall row beside a limit row on w02 and
|
|
* plan 2 committed for member 1 (5.4, 5.5, 4.5). */
|
|
s2 = strstr(r.out, "[trace] job 0-0-0-202 session 2");
|
|
TEST_ASSERT_NOT_NULL(s2);
|
|
TEST_ASSERT_TRUE(follows_its_echo(r.out, s2));
|
|
b = block_of(s2, "[trace] job 0-0-0-202 session 2");
|
|
IN_ORDER(b,
|
|
"\n host w02 rejected (per-host check): resource "
|
|
"shortfall, limit exceeded\n"
|
|
" slots total 4 in-use 1 reserved 0 requested 4\n"
|
|
" limit host_slots (h=w02) on slots: value 3 usage 1 "
|
|
"reserved 0 requested 4\n",
|
|
"\n plan 2 committed; no further plans tried\n",
|
|
"\n member 1/2 satisfied\n",
|
|
"\n cluster-wide limit cluster_slots on slots: value 6 usage "
|
|
"1 reserved 4 requested 1 passed\n",
|
|
"\n compound requirement satisfied\n");
|
|
TEST_ASSERT_NULL(strstr(b, "limit=host_slots total"));
|
|
free(b);
|
|
|
|
/* 7.8, 7.9: after both elements finished, their traces remain. */
|
|
IN_ORDER(s2,
|
|
"\n[trace] job 0-0-0-32768[1] session 1 position 3 level "
|
|
"full\n",
|
|
"\n[trace] job 0-0-0-32768[2] session 1 position 4 level "
|
|
"full\n");
|
|
|
|
/* 8.1: the file holds the inline traces (not `info trace` output),
|
|
* byte for byte. */
|
|
traces = collect_traces(r.out, NULL, true, &tlen);
|
|
assert_file_equals(&r, traces, tlen);
|
|
free(traces);
|
|
run_done(&r);
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_step_mode_full_all_jobs_and_file_copy);
|
|
RUN_TEST(test_continue_breakpoint_summary_and_info_trace);
|
|
RUN_TEST(test_compound_array_selection_and_info_trace);
|
|
return UNITY_END();
|
|
}
|