/* * test_repl_trace_e2e.c - scheduler-sim-decision-trace spec, task 6.1 * (design.md "Testing Strategy" -> "Integration Tests" item 2, "System * Flows", "TraceRenderer"): end-to-end decision-trace scenarios. * * Each scenario runs the real, built xce_scheduler binary * (XCE_SCHEDULER_BIN) in interactive simulation mode with a scripted REPL * session on stdin, the way test_repl_trace_baseline.c does, and checks the * console transcript (and the trace file copy) for the expected trace * content and its placement. Checks look for key lines and their order, * never one golden blob, except the file-copy check, which is byte for * byte. Nothing depends on the wall clock: the replays' times are fixed and * the traces print only replay times. * * Replays (test_repl_trace_e2e_fixture.c): * - contention.replay, the task 1.1 baseline fixture * (fixtures/repl_trace_baseline/, read only): two 4-slot workers, a * 2-unit cluster license, a cluster-wide limit of 6 slots, a host-scoped * limit of 75% of each host's slots, a parallel job (101), license jobs * (102..104), an alternative-requirement job (105) and slot jobs * (106, 107). In its session 5, 105's first alternative fills two * positions of a 4-position plan and then releases them (rollback). * - extra.replay, generated per run, for what the baseline lacks: a * satisfied compound requirement (201), a compound requirement that * fails at its first member (202), a job array with two elements * (0-0-0-32768[1] and [2]) and a host-selection mismatch (203 selects * mem>20000, which only w02 has). * * `break DECISION jobid=105` (a raw packed decimal id) names 0-0-0-105; * since bug-1926 the dashed form 0-0-0-105 names it too. * * Scenarios: * 1. step mode, all jobs, level full, file copy (baseline replay, five * sessions): every section of Requirements 2-6, each trace right after * its echo line, and the file equal to the console traces. * 2. continue mode with a breakpoint (baseline replay): traces up to the * stopping decision only, `info trace` at the stop, a selection change * at the breakpoint applying from the next session, summary level, * traced rounds kept out of collapsed ranges, `info trace` after the * job finished and after `set units raw`. * 3. selected jobs incl. a job array (extra replay): compound labels, * selection mismatch, host-limit rejection, shortfall next to a limit, * `info trace` for an array, an element and an unknown job, and the * file equal to the inline console traces. */ #include #include #include #include #include #include "test_repl_trace_e2e.h" void setUp(void) { return; } void tearDown(void) { return; } /* ---------------------------------------------------------------------- */ /* Transcript helpers */ /* ---------------------------------------------------------------------- */ /* Asserts the needles occur in `text` in the given order, each after the * end of the previous one; returns the end of the last match. */ static const char* assert_in_order(const char* text, const char* const* nd, size_t n) { const char* at = text; size_t i; for (i = 0; i < n; i++) { const char* hit = strstr(at, nd[i]); if (hit == NULL) { char msg[1024]; snprintf(msg, sizeof(msg), "missing (or out of order): %s", nd[i]); TEST_FAIL_MESSAGE(msg); } /* A needle's closing newline may open the next needle's line. */ at = hit + strlen(nd[i]); if ((at > hit) && (at[-1] == '\n')) { at--; } } return at; } #define IN_ORDER(text, ...) \ do { \ static const char* const nd_[] = { __VA_ARGS__ }; \ (void)assert_in_order((text), nd_, sizeof(nd_) / sizeof(nd_[0])); \ } while (0) /* Pointer to the line containing `needle` (start of line), or fails. */ static const char* line_of(const char* text, const char* needle) { const char* hit = strstr(text, needle); TEST_ASSERT_NOT_NULL_MESSAGE(hit, needle); while ((hit > text) && (hit[-1] != '\n')) { hit--; } return hit; } /* True when the line at `p` continues a trace block: trace lines after the * header are indented, and a truncation marker starts with "...". */ static bool is_trace_body(const char* p) { return (p[0] == ' ') || (strncmp(p, "...", 3) == 0); } static const char* next_line(const char* p) { const char* nl = strchr(p, '\n'); return (nl != NULL) ? nl + 1 : p + strlen(p); } /* Copy of the whole trace block whose header line contains `header`. */ static char* block_of(const char* text, const char* header) { const char* start = line_of(text, header); const char* end = next_line(start); char* out; while ((*end != '\0') && is_trace_body(end)) { end = next_line(end); } out = strndup(start, (size_t)(end - start)); TEST_ASSERT_NOT_NULL(out); return out; } /* True when the line before `line` is the echo line of a decision for the * job the trace header at `line` names: "[sim] jobid=\n" before * "[trace] job ...". */ static bool follows_its_echo(const char* text, const char* line) { const char* prev; const char* id = line + strlen("[trace] job "); const char* id_end = strstr(id, " "); const char* eq; if ((line == text) || (id_end == NULL)) { return false; } prev = line - 1; /* the '\n' ending the previous line */ while ((prev > text) && (prev[-1] != '\n')) { prev--; } if (strncmp(prev, "[sim] ", 6) != 0) { return false; } eq = strstr(prev, " jobid="); if ((eq == NULL) || (eq > line)) { return false; } eq += strlen(" jobid="); return (strncmp(eq, id, (size_t)(id_end - id)) == 0) && (eq[id_end - id] == '\n'); } /* Concatenation of the trace blocks of `text` (up to `stop`, or the end * when NULL). With `inline_only`, only blocks printed right after their * decision's echo line, i.e. not `info trace` output. */ static char* collect_traces(const char* text, const char* stop, bool inline_only, long* len) { size_t cap = strlen(text) + 1; char* out = (char*)malloc(cap); size_t n = 0; const char* p = text; TEST_ASSERT_NOT_NULL(out); while ((*p != '\0') && ((stop == NULL) || (p < stop))) { if ((strncmp(p, "[trace] job ", 12) == 0) && (!inline_only || follows_its_echo(text, p))) { const char* end = next_line(p); while ((*end != '\0') && is_trace_body(end)) { end = next_line(end); } memcpy(out + n, p, (size_t)(end - p)); n += (size_t)(end - p); p = end; continue; } p = next_line(p); } out[n] = '\0'; *len = (long)n; return out; } /* Every inline trace header before `stop` (NULL = end) follows its echo * line (7.1, 7.3). `info trace` output is excluded by the caller choosing * `stop` before the first `info trace`. */ static void assert_every_trace_follows_echo(const char* text, const char* stop) { const char* p = text; int n = 0; while ((p = strstr(p, "[trace] job ")) != NULL) { if ((stop != NULL) && (p >= stop)) { break; } if ((p == text) || (p[-1] == '\n')) { char msg[256]; snprintf(msg, sizeof(msg), "trace not after its echo line: %.80s", p); TEST_ASSERT_TRUE_MESSAGE(follows_its_echo(text, p), msg); n++; } p++; } TEST_ASSERT_TRUE_MESSAGE(n > 0, "expected at least one inline trace"); } /* The trace file of run `r` equals `expect` byte for byte (8.1). */ static void assert_file_equals(const run_t* r, const char* expect, long len) { char path[PATH_MAX + 32]; long flen; char* file; snprintf(path, sizeof(path), "%s/%s", r->dir, TRACE_FILE); file = slurp(path, &flen); TEST_ASSERT_NOT_NULL_MESSAGE(file, "the trace file must exist"); TEST_ASSERT_TRUE_MESSAGE(len > 0, "console traces must not be empty"); TEST_ASSERT_EQUAL_INT64_MESSAGE(len, flen, "trace file length must equal the " "console traces"); TEST_ASSERT_EQUAL_MEMORY_MESSAGE(expect, file, (size_t)len, "trace file must equal the console " "traces byte for byte"); free(file); } /* ---------------------------------------------------------------------- */ /* 1. Step mode, all jobs, full level, file copy */ /* ---------------------------------------------------------------------- */ static void test_step_mode_full_all_jobs_and_file_copy(void) { run_t r; char* b; char* traces; long tlen; run_repl(&r, BASE_REPLAY, "trace on all level full file " TRACE_FILE "\n" "step\nstep\nstep\nstep\nstep\nquit\n"); /* 1.1, 2.1, 7.1: every job of session 1 in pick order, each trace * right after its echo line. */ assert_every_trace_follows_echo(r.out, NULL); IN_ORDER(r.out, "[sim] DISPATCH jobid=0-0-0-101\n" "[trace] job 0-0-0-101 session 1 position 1 level full\n", "[sim] DISPATCH jobid=0-0-0-102\n" "[trace] job 0-0-0-102 session 1 position 2 level full\n", "[trace] job 0-0-0-103 session 1 position 3 level full\n", "[sim] PEND jobid=0-0-0-104\n" "[trace] job 0-0-0-104 session 1 position 4 level full\n", "[trace] job 0-0-0-105 session 1 position 5 level full\n", "[trace] job 0-0-0-106 session 1 position 6 level full\n", "[trace] job 0-0-0-107 session 1 position 7 level full\n", "[trace] job 0-0-0-104 session 2 position 1 level full\n", "[trace] job 0-0-0-105 session 5 position 2 level full\n", "[trace] job 0-0-0-106 session 5 position 3 level full\n"); /* 101: selection, host-scoped limit rejections, plans, positions, * already-claimed, commit, dispatch with limit charges (7.6 order). */ b = block_of(r.out, "[trace] job 0-0-0-101 session 1"); IN_ORDER(b, "\n selection: policy builtin_fcfs queue normal (priority 1)" " job priority 0 queued 1970-01-01T00:18:21Z\n", "\n cluster-wide limits: cluster_slots; host-scoped limits are " "reported with each host's verdict\n", "\n requirement single tasks 2..4 requests mem=1024.000000 " "(1.00 KB)\n", "\n cluster-wide limit cluster_slots on slots: value 6 usage 0" " reserved 0 requested 4 passed\n", "\n cluster-wide check passed\n", "\n plan 1 hosts 1 tasks per position [4]\n", "\n position 1 (4 tasks)\n", "\n host w01 rejected (per-host check): limit exceeded\n" " limit host_slots (h=w01) on slots: value 3 usage 0 " "reserved 0 requested 4\n", "\n host w02 rejected (per-host check): limit exceeded\n", "\n position 1 not filled: no host accepted\n", "\n plan 1 failed: position 1 could not be filled\n", "\n plan 2 hosts 2 tasks per position [2, 2]\n", "\n host w01 accepted\n", "\n position 1 filled by w01\n", "\n host w01 skipped: already claimed by an earlier " "position\n", "\n position 2 filled by w02\n", "\n plan 2 succeeded\n", "\n plan 2 committed; no further plans tried\n", "\n plans: 2 generated, 2 tried, 0 discarded, 0 not tried\n", "\n single requirement satisfied\n", "\n outcome: DISPATCH\n", "\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit charges: " "host_slots (h=w01) slots 2\n", "\n host w02: mem=2048.000000 (2.00 KB) slots=2 limit charges: " "host_slots (h=w02) slots 2\n", "\n shared: none limit charges: cluster_slots slots 4\n"); /* 5.5: a limit hit is never also a resource shortfall. */ TEST_ASSERT_NULL(strstr(b, "resource shortfall")); free(b); /* 102: a shared resource row, shared allocation and its limit charge * kept apart (4.1, 6.2, 6.3). */ b = block_of(r.out, "[trace] job 0-0-0-102 session 1"); IN_ORDER(b, "\n cluster-wide: license total 2 in-use 0 reserved 0 " "requested 1 passed\n", "\n cluster-wide limit cluster_slots on slots: value 6 usage 4" " reserved 0 requested 1 passed\n", "\n host w01: slots=1 limit charges: host_slots (h=w01) " "slots 1\n", "\n shared: license=1 limit charges: cluster_slots slots 1\n"); free(b); /* 104: cluster-wide shortfall and cluster-wide limit rejection, no * plans, pend reasons (4.1, 4.2, 4.8, 6.4). */ b = block_of(r.out, "[trace] job 0-0-0-104 session 1"); IN_ORDER(b, "\n cluster-wide: license total 2 in-use 2 reserved 0 " "requested 1 failed\n", "\n cluster-wide limit cluster_slots on slots: value 6 usage 6" " reserved 0 requested 1 failed\n", "\n cluster-wide check failed; no placement plans tried\n", "\n single requirement not satisfied\n", "\n outcome: PEND\n", "\n main reason: none\n", "\n resource license: NO_RESOURCE\n", "\n resource limit=cluster_slots: NO_RESOURCE\n"); TEST_ASSERT_NULL(strstr(b, "plan 1")); free(b); /* 4.2: the cluster-wide limit is never shown as a resource row. */ TEST_ASSERT_NULL(strstr(r.out, "cluster-wide: limit=")); /* 105, session 1: alternative labels and every alternative's failure * reasons (3.1, 3.2, 3.5). */ b = block_of(r.out, "[trace] job 0-0-0-105 session 1"); IN_ORDER(b, "\n requirement alternative 1/2 (compound, 1 member)\n", "\n requirement alternative 1/2 > member 1/1 tasks 4..4 " "requests mem=12288.000000 (12.00 KB)\n", "\n member 1/1 not satisfied\n", "\n failure reasons: NO_RESOURCE x1 (resources: " "limit=cluster_slots)\n", "\n alternative 1/2 not satisfied\n", "\n requirement alternative 2/2 (compound, 1 member)\n", "\n failure reasons: NO_RESOURCE x1 (resources: " "limit=cluster_slots)\n", "\n alternative 2/2 not satisfied\n", "\n no alternative chosen: none of the 2 alternatives was " "satisfied\n", "\n outcome: PEND\n", "\n main reason: none\n"); free(b); /* 105, session 5: per-host shortfall rows next to limit rows, the * rollback, no plan placed, then the chosen alternative (3.4, 4.6, * 5.4, 5.5, 5.8). */ b = block_of(r.out, "[trace] job 0-0-0-105 session 5"); IN_ORDER(b, "\n host w01 rejected (per-host check): resource " "shortfall, limit exceeded\n" " mem total 16384.000000 (16.00 KB) in-use 0.000000 " "(0.00 B) reserved 0.000000 (0.00 B) requested 49152.000000 " "(48.00 KB)\n" " slots total 4 in-use 1 reserved 0 requested 4\n" " limit host_slots (h=w01) on slots: value 3 usage 1 " "reserved 0 requested 4\n", "\n plan 4 hosts 4 tasks per position [1, 1, 1, 1]\n", "\n position 1 filled by w01\n", "\n position 2 filled by w02\n", "\n position 3 not filled: no host accepted\n" " released hosts: w01, w02\n" " plan 4 failed: position 3 could not be filled\n", "\n plans: 4 generated, 4 tried, 0 discarded, 0 not tried; " "no plan could be placed\n", "\n alternative 1/2 not satisfied\n", "\n plan 1 committed; no further plans tried\n", "\n alternative 2/2 satisfied\n" " alternative 2/2 chosen; no alternatives after it\n" " outcome: DISPATCH\n" " host w01: mem=2048.000000 (2.00 KB) slots=1 limit charges: " "host_slots (h=w01) slots 1\n"); TEST_ASSERT_NULL(strstr(b, "limit=host_slots total")); free(b); /* 8.1: the file holds exactly the console's traces. */ traces = collect_traces(r.out, NULL, false, &tlen); assert_file_equals(&r, traces, tlen); free(traces); run_done(&r); } /* ---------------------------------------------------------------------- */ /* 2. Continue mode, breakpoint, summary level, info trace */ /* ---------------------------------------------------------------------- */ static void test_continue_breakpoint_summary_and_info_trace(void) { run_t r; const char* hit; const char* after; char* b; run_repl(&r, BASE_REPLAY, "trace on all level full\n" "break DECISION jobid=105 type=PEND\n" "continue\n" "info trace 0-0-0-105\n" "info trace 0-0-0-106\n" "trace on job 0-0-0-105 0-0-0-106 level summary\n" "delete all\n" "continue\n" "info trace 0-0-0-101\n" "set units raw\n" "info trace 0-0-0-101\n" "info trace 0-0-0-105\n" "quit\n"); /* 7.5: at the stop, the round's traces up to and including the * stopping decision are printed, and nothing later. */ hit = line_of(r.out, "[sim] hit breakpoint 1 @ decision: PEND " "jobid=0-0-0-105"); assert_every_trace_follows_echo(r.out, hit); IN_ORDER(r.out, "[trace] job 0-0-0-101 session 1 position 1", "[trace] job 0-0-0-104 session 1 position 4", "[sim] PEND jobid=0-0-0-105\n" "[trace] job 0-0-0-105 session 1 position 5 level full\n", "[sim] hit breakpoint 1 @ decision: PEND jobid=0-0-0-105\n"); TEST_ASSERT_TRUE(strstr(r.out, "[trace] job 0-0-0-106") > hit); /* 7.5, 7.8: `info trace` at the stop shows the stopping decision's * trace; the job decided later in this round has none yet. */ IN_ORDER(hit, "\n[trace] job 0-0-0-105 session 1 position 5 level " "full\n", "\n[sim] info trace: job 0-0-0-106: no trace available (state " "PEND)\n", "\n[sim] trace: enabled; applies from the next session\n"); /* 1.6, 7.3: the rest of session 1 keeps the settings it started with * (all jobs, full), echo line then trace in continue mode, then the * round's line; 106 and 107 are traced at level full. */ after = strstr(hit, "[sim] deleted "); TEST_ASSERT_NOT_NULL(after); IN_ORDER(after, "\n[sim] PEND jobid=0-0-0-106\n" "[trace] job 0-0-0-106 session 1 position 6 level full\n", "\n[sim] PEND jobid=0-0-0-107\n" "[trace] job 0-0-0-107 session 1 position 7 level full\n", "\n[sim] round 1-1 (", /* 1.2, 1.4, 7.3, 7.4: from session 2 only 105 and 106, at * summary level, each traced round reported on its own. */ "\n[sim] PEND jobid=0-0-0-105\n" "[trace] job 0-0-0-105 session 2 position 2 level summary\n", "\n[sim] PEND jobid=0-0-0-106\n" "[trace] job 0-0-0-106 session 2 position 3 level summary\n", "\n[sim] round 2-2 (", "[trace] job 0-0-0-105 session 3", "\n[sim] round 3-3 (", "[trace] job 0-0-0-105 session 4", "\n[sim] round 4-4 (", "\n[sim] DISPATCH jobid=0-0-0-105\n" "[trace] job 0-0-0-105 session 5 position 2 level summary\n", "\n[sim] DISPATCH jobid=0-0-0-106\n" "[trace] job 0-0-0-106 session 5 position 3 level summary\n", "\n[sim] round 5-5 (", /* untraced rounds still collapse */ "\n[sim] round 6-7 (2 round(s)", "\n[sim] replay complete."); TEST_ASSERT_NULL(strstr(r.out, "[sim] round 2-4")); TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-104 session 2")); TEST_ASSERT_NULL(strstr(r.out, "[trace] job 0-0-0-107 session 2")); /* 5.8, 5.9: summary level counts verdicts per position, names each * limit hit, keeps the rollback, and lists no host lines. */ b = block_of(r.out, "[trace] job 0-0-0-105 session 5"); IN_ORDER(b, "\n plan 1 hosts 1 tasks per position [4]\n" " position 1 (4 tasks)\n" " accepted none; rejected: selection mismatch 0, resource " "shortfall 2, limit exceeded 2 (host_slots (h=w01) 1, " "host_slots (h=w02) 1), already claimed 0, not evaluated " "(internal error) 0\n", "\n position 3 (1 task)\n" " accepted none; rejected: selection mismatch 0, resource " "shortfall 0, limit exceeded 0, already claimed 2, not evaluated " "(internal error) 0\n" " released hosts: w01, w02\n", "\n alternative 2/2 chosen; no alternatives after it\n"); TEST_ASSERT_NULL(strstr(b, " host w01 ")); free(b); b = block_of(r.out, "[trace] job 0-0-0-106 session 5"); IN_ORDER(b, "\n accepted w02; rejected: selection mismatch 0, resource " "shortfall 0, limit exceeded 1 (host_slots (h=w01) 1), already " "claimed 0, not evaluated (internal error) 0\n", "\n host w02: mem=1024.000000 (1.00 KB) slots=2 limit " "charges: host_slots (h=w02) slots 2\n"); free(b); /* 7.8: after the replay (101 finished at 5500), `info trace` re-prints * its stored full trace, then again with the raw units mode. 105's * latest trace is the session 5 summary one. */ after = strstr(r.out, "[sim] simulation finished"); TEST_ASSERT_NOT_NULL(after); IN_ORDER(after, "\n[trace] job 0-0-0-101 session 1 position 1 level full\n", "\n host w01: mem=2048.000000 (2.00 KB) slots=2 limit " "charges: host_slots (h=w01) slots 2\n", "\n[sim] units set to raw\n" "[trace] job 0-0-0-101 session 1 position 1 level full\n", "\n requirement single tasks 2..4 requests mem=1024.000000\n", "\n host w01: mem=2048.000000 slots=2 limit charges: " "host_slots (h=w01) slots 2\n", "\n[trace] job 0-0-0-105 session 5 position 2 level " "summary\n", "\n requirement alternative 1/2 > member 1/1 tasks 4..4 " "requests mem=12288.000000\n"); run_done(&r); } /* ---------------------------------------------------------------------- */ /* 3. Selected jobs, compound, array, info trace on an array */ /* ---------------------------------------------------------------------- */ static void test_compound_array_selection_and_info_trace(void) { run_t r; const char* mark; const char* s2; char* b; char* traces; long tlen; run_repl(&r, EXTRA_REPLAY, "trace on job 0-0-0-201 0-0-0-202 0-0-0-32768 0-0-0-203 " "level full file " TRACE_FILE "\n" "step\n" "trace status\n" "info trace 0-0-0-32768\n" "info trace 0-0-0-32768[2]\n" "info trace 0-0-0-999\n" "step\n" "info trace 0-0-0-32768\n" "quit\n"); mark = line_of(r.out, "[sim] trace status"); assert_every_trace_follows_echo(r.out, mark); /* 1.2: the bare array id selects both elements. */ IN_ORDER(r.out, "[sim] DISPATCH jobid=0-0-0-201\n" "[trace] job 0-0-0-201 session 1 position 1 level full\n", "[sim] PEND jobid=0-0-0-202\n" "[trace] job 0-0-0-202 session 1 position 2 level full\n", "[sim] DISPATCH jobid=0-0-0-32768[1]\n" "[trace] job 0-0-0-32768[1] session 1 position 3 level " "full\n", "[sim] DISPATCH jobid=0-0-0-32768[2]\n" "[trace] job 0-0-0-32768[2] session 1 position 4 level " "full\n", "[sim] DISPATCH jobid=0-0-0-203\n" "[trace] job 0-0-0-203 session 1 position 5 level full\n"); /* 201: compound labels, the reservation of member 1 seen by member 2's * cluster-wide check, shared and host allocations (3.1, 3.2, 4.1, 6.1, * 6.2). */ b = block_of(r.out, "[trace] job 0-0-0-201 session 1"); IN_ORDER(b, "\n requirement compound (2 members)\n", "\n requirement compound > member 1/2 tasks 1..1 requests " "license=1\n", "\n cluster-wide: license total 2 in-use 0 reserved 0 " "requested 1 passed\n", "\n member 1/2 satisfied\n", "\n requirement compound > member 2/2 tasks 2..2 requests " "mem=1024.000000 (1.00 KB)\n", "\n cluster-wide limit cluster_slots on slots: value 6 usage " "0 reserved 1 requested 2 passed\n", "\n member 2/2 satisfied\n", "\n compound requirement satisfied\n", "\n outcome: DISPATCH\n", "\n shared: license=1 limit charges: cluster_slots slots 3\n"); free(b); /* 202, session 1: the failing member and the members not tried * (3.3). */ b = block_of(r.out, "[trace] job 0-0-0-202 session 1"); IN_ORDER(b, "\n cluster-wide check failed; no placement plans tried\n", "\n member 1/2 not satisfied; member 2/2 not tried\n", "\n compound requirement not satisfied\n", "\n outcome: PEND\n"); free(b); /* [2]: host-scoped limit rejection; 203: selection mismatch from the * pre-filter (5.2, 5.3, 5.5). */ b = block_of(r.out, "[trace] job 0-0-0-32768[2] session 1"); IN_ORDER(b, "\n host w01 rejected (per-host check): limit exceeded\n" " limit host_slots (h=w01) on slots: value 3 usage 3 " "reserved 0 requested 1\n" " host w02 accepted\n"); free(b); b = block_of(r.out, "[trace] job 0-0-0-203 session 1"); IN_ORDER(b, "\n host w01 rejected (pre-filter): selection mismatch; " "per-host check not performed\n" " host w02 accepted\n" " position 1 filled by w02\n"); free(b); /* 7.9, 7.11: `info trace` for the array prints each element's trace, * for one element only that one, and an unknown job is reported. */ IN_ORDER(mark, "\n[trace] job 0-0-0-32768[1] session 1 position 3 level " "full\n", "\n[trace] job 0-0-0-32768[2] session 1 position 4 level " "full\n", "\n[trace] job 0-0-0-32768[2] session 1 position 4 level " "full\n", "\n[sim] info trace: job 0-0-0-999: unknown job\n"); /* Session 2: 202 again; a shortfall row beside a limit row on w02 and * plan 2 committed for member 1 (5.4, 5.5, 4.5). */ s2 = strstr(r.out, "[trace] job 0-0-0-202 session 2"); TEST_ASSERT_NOT_NULL(s2); TEST_ASSERT_TRUE(follows_its_echo(r.out, s2)); b = block_of(s2, "[trace] job 0-0-0-202 session 2"); IN_ORDER(b, "\n host w02 rejected (per-host check): resource " "shortfall, limit exceeded\n" " slots total 4 in-use 1 reserved 0 requested 4\n" " limit host_slots (h=w02) on slots: value 3 usage 1 " "reserved 0 requested 4\n", "\n plan 2 committed; no further plans tried\n", "\n member 1/2 satisfied\n", "\n cluster-wide limit cluster_slots on slots: value 6 usage " "1 reserved 4 requested 1 passed\n", "\n compound requirement satisfied\n"); TEST_ASSERT_NULL(strstr(b, "limit=host_slots total")); free(b); /* 7.8, 7.9: after both elements finished, their traces remain. */ IN_ORDER(s2, "\n[trace] job 0-0-0-32768[1] session 1 position 3 level " "full\n", "\n[trace] job 0-0-0-32768[2] session 1 position 4 level " "full\n"); /* 8.1: the file holds the inline traces (not `info trace` output), * byte for byte. */ traces = collect_traces(r.out, NULL, true, &tlen); assert_file_equals(&r, traces, tlen); free(traces); run_done(&r); } int main(void) { UNITY_BEGIN(); RUN_TEST(test_step_mode_full_all_jobs_and_file_copy); RUN_TEST(test_continue_breakpoint_summary_and_info_trace); RUN_TEST(test_compound_array_selection_and_info_trace); return UNITY_END(); }