feat(scheduler-sim-decision-trace): store the latest trace per job and serve info trace

Task 4.2: a new trace replaces a job's earlier one; above 64 MiB the
oldest traces are evicted (never the newest) and marked discarded until
a newer capture; info trace renders a job's or an array's stored traces
in the current units mode, or reports discarded / no trace with the
job's state / unknown job.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-08 13:50:02 +00:00
parent fe445e9453
commit 73416e3e91
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
6 changed files with 1130 additions and 1 deletions

View file

@ -2735,3 +2735,24 @@ sched_trace_render_test = executable(
test('scheduler trace render test',
sched_trace_render_test, suite: 'scheduler',
env: ['ASAN_OPTIONS=detect_leaks=0'])
# scheduler-sim-decision-trace task 4.2: the trace store and `info trace`
# (design.md "TraceStore"; Requirements 1.7, 7.8-7.11, 9.5, 9.6).
# Replacement, eviction at 64 MiB that never drops the newest record, the
# three no-trace replies and arrays mixing stored and discarded elements.
# Links the record model, the renderer, the shared REPL formatting module
# and the store; the job table is a fixture kvmgr with sched_get_job() and
# repl_job_state_name() defined by the test.
sched_trace_store_test = executable(
'sched_trace_store_test',
['test_trace_store.c', '../src/sched_trace.c',
'../src/sched_trace_scan.c', '../src/repl_trace_render.c',
'../src/repl_trace_render_outcome.c', '../src/repl_fmt.c',
'../src/repl_trace_store.c'],
include_directories:[incdir],
dependencies: [scheduler_test_deps],
install: false)
test('scheduler trace store test',
sched_trace_store_test, suite: 'scheduler',
env: ['ASAN_OPTIONS=detect_leaks=0'])

View file

@ -0,0 +1,567 @@
/*
* test_trace_store.c - the REPL trace store and `info trace`
* (scheduler-sim-decision-trace spec, task 4.2; design.md "TraceStore").
*
* Links the record model (src/sched_trace.c, src/sched_trace_scan.c), the
* renderer (src/repl_trace_render*.c), the shared REPL formatting module
* (src/repl_fmt.c) and the store (src/repl_trace_store.c). The job table is
* a fixture: a real KVMGR_KEY_INT kvmgr in sim.core.jobs, with
* sched_get_job() and repl_job_state_name() defined here in place of
* core.c and repl_info.c.
*
* Covers:
* - a new trace replaces the stored one (7.10);
* - eviction at 64 MiB drops the oldest first and never the newest, even
* when the newest alone is over the bound (9.5);
* - a newer capture clears a discarded mark;
* - the three replies without a stored trace: discarded (9.6), no trace
* with the job's state, unknown job (7.11);
* - `info trace` renders at the captured level in the units mode current
* when it is issued (7.8);
* - an array mixing stored and discarded elements, in index order, and
* an array with no stored trace listing its elements' states (7.9).
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
#include "kvmgr.h"
#include "repl_internal.h"
#include "sched/trace.h"
#include "sim.h"
#include "utils.h"
#define MIB ((size_t)1024 * 1024)
#define PLAIN_JOB MAKE_JOBID(1, 2, 1000, 0x0, (uint64_t)42)
#define PLAIN_JOB2 MAKE_JOBID(1, 2, 1000, 0x0, (uint64_t)43)
#define UNKNOWN_JOB MAKE_JOBID(1, 2, 1000, 0x0, (uint64_t)99)
#define ARRAY_SEQ ((uint64_t)32768 * 5)
#define ARRAY_BASE MAKE_JOBID(1, 2, 1000, 0x0, ARRAY_SEQ)
#define ELEM(k) \
MAKE_JOBID(1, 2, 1000, TAG_JOB_ARRAY, ARRAY_SEQ | (uint64_t)(k))
#define ARRAY2_SEQ ((uint64_t)32768 * 6)
#define ARRAY2_BASE MAKE_JOBID(1, 2, 1000, 0x0, ARRAY2_SEQ)
#define ELEM2(k) \
MAKE_JOBID(1, 2, 1000, TAG_JOB_ARRAY, ARRAY2_SEQ | (uint64_t)(k))
static xce_sched_sim_driver_t g_sim;
static xce_sched_repl_state_t g_rs;
/* ---- fixture job table -------------------------------------------------- */
xce_sched_job_t* sched_get_job(xce_sched_core_t* core, uint64_t jobid)
{
xce_sched_job_t* job = NULL;
if (key_find(core->jobs, (void*)jobid, (void**)&job) <= 0) {
return NULL;
}
return job;
}
const char* repl_job_state_name(uint32_t state)
{
if (JOB_IS_RUN(state)) {
return "RUN";
}
if (JOB_IS_EXIT(state)) {
return "EXIT";
}
if (JOB_IS_DONE(state)) {
return "DONE";
}
if (JOB_IS_PEND(state)) {
return "PEND";
}
return "UNKNOWN";
}
static void add_job(uint64_t jobid, uint32_t state)
{
xce_sched_job_t* j = (xce_sched_job_t*)calloc(1, sizeof(*j));
TEST_ASSERT_NOT_NULL(j);
j->jobid = jobid;
j->state = state;
TEST_ASSERT_TRUE(key_add(g_sim.core.jobs, (void*)jobid, j) >= 0);
}
void setUp(void)
{
static bool made = false;
if (!made) {
kvmgr_params_t p = { .key_type = KVMGR_KEY_INT,
.nslots = 59,
.delta = 16,
.datakiller = NULL };
memset(&g_sim, 0, sizeof(g_sim));
g_sim.core.jobs = makeInstance(&xkvmgr_type, &p);
TEST_ASSERT_NOT_NULL(g_sim.core.jobs);
add_job(PLAIN_JOB, JOB_STAT_PEND);
add_job(PLAIN_JOB2, JOB_STAT_RUN);
add_job(ELEM(1), JOB_STAT_PEND);
add_job(ELEM(2), JOB_STAT_RUN);
add_job(ELEM(3), JOB_STAT_PEND);
add_job(ELEM(4), JOB_STAT_DONE);
add_job(ELEM2(2), JOB_STAT_RUN);
add_job(ELEM2(1), JOB_STAT_PEND);
made = true;
}
memset(&g_rs, 0, sizeof(g_rs));
g_rs.units_mode = XCE_REPL_UNITS_BOTH;
g_sim.repl_state = &g_rs;
}
void tearDown(void)
{
repl_trace_store_fini(&g_rs.trace);
g_sim.repl_state = NULL;
}
/* ---- records ------------------------------------------------------------ */
/* A reconstructed dispatch of 8 GiB mem on n01, plus `pad` bytes of arena so
* the record's size can be set for the memory bound. `session` tells two
* records of one job apart in the rendering. */
static xce_sched_trace_rec_t* new_rec(uint64_t jobid, uint64_t session,
xce_sched_trace_level_t level,
size_t pad)
{
xce_sched_trace_ctl_t ctl;
xce_sched_trace_rec_t* rec;
sched_trace_host_out_t* h;
sched_trace_alloc_row_t row;
memset(&ctl, 0, sizeof(ctl));
ctl.all = true;
ctl.level = level;
ctl.line_bound = 1000;
ctl.session_no = session;
ctl.position = 1;
rec = sched_trace_rec_new(&ctl, jobid, XCE_TRACE_REC_RECONSTRUCTED);
TEST_ASSERT_NOT_NULL(rec);
memset(&row, 0, sizeof(row));
row.res = sched_trace_arena_strdup(rec, "mem");
row.value = 8.0 * 1073741824.0;
rec->outcome = SCHED_TRACE_OUTCOME_DISPATCH;
rec->dispatch.nhosts = 1;
rec->dispatch.hosts = (sched_trace_host_out_t*)sched_trace_arena_alloc(
rec, sizeof(sched_trace_host_out_t));
h = &rec->dispatch.hosts[0];
h->host = sched_trace_arena_strdup(rec, "n01");
h->nres = 1;
h->res = (sched_trace_alloc_row_t*)sched_trace_arena_memdup(
rec, &row, sizeof(row));
if (pad > 0) {
TEST_ASSERT_NOT_NULL(sched_trace_arena_alloc(rec, pad));
}
TEST_ASSERT_FALSE(rec->incomplete);
return rec;
}
static char* render(const xce_sched_trace_rec_t* rec,
xce_repl_units_mode_t units)
{
char* buf = NULL;
size_t len = 0;
FILE* f = open_memstream(&buf, &len);
TEST_ASSERT_NOT_NULL(f);
repl_trace_render(rec, units, f, NULL);
fclose(f);
return buf;
}
static char* fmt_id(uint64_t jobid)
{
static char bufs[8][64];
static int next = 0;
char* b = bufs[next++ % 8];
fmt_jobid(b, 64, jobid);
return b;
}
/* ---- running `info trace` ---------------------------------------------- */
/* Whitespace split into REPL tokens (repl.c's repl_split_tokens is not
* linked here). */
static int split(const char* line, char tokens[][XCE_REPL_TOKEN_LEN])
{
int n = 0;
const char* p = line;
while ((*p != '\0') && (n < XCE_REPL_MAX_TOKENS)) {
size_t k = 0;
while ((*p == ' ') || (*p == '\t')) {
p++;
}
if (*p == '\0') {
break;
}
while ((*p != '\0') && (*p != ' ') && (*p != '\t')) {
if (k < (XCE_REPL_TOKEN_LEN - 1)) {
tokens[n][k++] = *p;
}
p++;
}
tokens[n++][k] = '\0';
}
return n;
}
/* Runs `info trace <args>` with stdout redirected to a temp file and
* returns what it printed (caller frees). */
static char* info_trace(const char* args)
{
char line[512];
char tokens[XCE_REPL_MAX_TOKENS][XCE_REPL_TOKEN_LEN];
int ntok;
char tmp[] = "/tmp/xce_trace_store_stdout_XXXXXX";
int fd = mkstemp(tmp);
int saved;
off_t len;
char* buf;
TEST_ASSERT_TRUE(fd >= 0);
snprintf(line, sizeof(line), "info trace %s", args);
ntok = split(line, tokens);
fflush(stdout);
saved = dup(STDOUT_FILENO);
TEST_ASSERT_TRUE(saved >= 0);
TEST_ASSERT_TRUE(dup2(fd, STDOUT_FILENO) >= 0);
TEST_ASSERT_FALSE(repl_cmd_info_trace(&g_sim, tokens, ntok));
fflush(stdout);
dup2(saved, STDOUT_FILENO);
close(saved);
len = lseek(fd, 0, SEEK_CUR);
TEST_ASSERT_TRUE(len >= 0);
buf = (char*)calloc(1, (size_t)len + 1);
TEST_ASSERT_NOT_NULL(buf);
TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0);
if (len > 0) {
TEST_ASSERT_TRUE(read(fd, buf, (size_t)len) == len);
}
close(fd);
unlink(tmp);
return buf;
}
static void assert_text(const char* want, const char* got)
{
if (strcmp(want, got) != 0) {
fprintf(stderr, "---- want ----\n%s---- got ----\n%s----\n", want,
got);
}
TEST_ASSERT_EQUAL_STRING(want, got);
}
static void assert_info(const char* args, const char* want)
{
char* got = info_trace(args);
assert_text(want, got);
free(got);
}
static char* discarded_line(uint64_t jobid)
{
static char b[4][160];
static int next = 0;
char* p = b[next++ % 4];
snprintf(p, 160, "[sim] info trace: job %s: trace discarded (memory "
"bound)\n",
fmt_id(jobid));
return p;
}
/* ---- tests ---------------------------------------------------------------- */
/* 7.10: a new trace for a job replaces the earlier one; the store holds
* only the new record's bytes. */
static void test_put_replaces_earlier_trace(void)
{
xce_sched_trace_rec_t* r1 = new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 0);
xce_sched_trace_rec_t* r2 = new_rec(PLAIN_JOB, 2, XCE_TRACE_LEVEL_FULL, 0);
size_t b1 = sched_trace_rec_bytes(r1);
size_t b2 = sched_trace_rec_bytes(r2);
char* want;
repl_trace_store_put(&g_rs.trace, r1);
TEST_ASSERT_EQUAL_size_t(b1, repl_trace_store_bytes(&g_rs.trace));
want = render(r2, XCE_REPL_UNITS_BOTH);
repl_trace_store_put(&g_rs.trace, r2); /* frees r1 */
TEST_ASSERT_EQUAL_size_t(b2, repl_trace_store_bytes(&g_rs.trace));
assert_info(fmt_id(PLAIN_JOB), want);
TEST_ASSERT_NOT_NULL(strstr(want, "session 2"));
free(want);
}
/* 7.8: rendered at the captured level, in the units mode current when
* `info trace` is issued. */
static void test_info_uses_current_units_and_captured_level(void)
{
xce_sched_trace_rec_t* r
= new_rec(PLAIN_JOB, 3, XCE_TRACE_LEVEL_SUMMARY, 0);
char* raw = render(r, XCE_REPL_UNITS_RAW);
char* human = render(r, XCE_REPL_UNITS_HUMAN);
TEST_ASSERT_TRUE(strcmp(raw, human) != 0);
TEST_ASSERT_NOT_NULL(strstr(raw, "level summary"));
repl_trace_store_put(&g_rs.trace, r);
g_rs.units_mode = XCE_REPL_UNITS_RAW;
assert_info(fmt_id(PLAIN_JOB), raw);
g_rs.units_mode = XCE_REPL_UNITS_HUMAN;
assert_info(fmt_id(PLAIN_JOB), human);
free(raw);
free(human);
}
/* 9.5: at 64 MiB the oldest records go first; each evicted job is marked
* discarded (9.6). */
static void test_eviction_drops_oldest_first(void)
{
xce_sched_trace_rec_t* a
= new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 24 * MIB);
xce_sched_trace_rec_t* b
= new_rec(PLAIN_JOB2, 1, XCE_TRACE_LEVEL_FULL, 24 * MIB);
xce_sched_trace_rec_t* c
= new_rec(ELEM(1), 1, XCE_TRACE_LEVEL_FULL, 24 * MIB);
size_t bc = sched_trace_rec_bytes(b) + sched_trace_rec_bytes(c);
char* want_c;
repl_trace_store_put(&g_rs.trace, a);
repl_trace_store_put(&g_rs.trace, b);
want_c = render(c, XCE_REPL_UNITS_BOTH);
repl_trace_store_put(&g_rs.trace, c);
TEST_ASSERT_EQUAL_size_t(bc, repl_trace_store_bytes(&g_rs.trace));
TEST_ASSERT_TRUE(repl_trace_store_bytes(&g_rs.trace)
<= XCE_REPL_TRACE_STORE_BOUND);
assert_info(fmt_id(PLAIN_JOB), discarded_line(PLAIN_JOB));
assert_info(fmt_id(ELEM(1)), want_c);
free(want_c);
}
/* 9.5: the newest record stays even when it alone is over the bound. */
static void test_eviction_never_drops_newest(void)
{
xce_sched_trace_rec_t* a
= new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 8 * MIB);
xce_sched_trace_rec_t* big
= new_rec(PLAIN_JOB2, 1, XCE_TRACE_LEVEL_FULL, 70 * MIB);
size_t bbig = sched_trace_rec_bytes(big);
char* want;
TEST_ASSERT_TRUE(bbig > XCE_REPL_TRACE_STORE_BOUND);
repl_trace_store_put(&g_rs.trace, a);
want = render(big, XCE_REPL_UNITS_BOTH);
repl_trace_store_put(&g_rs.trace, big);
TEST_ASSERT_EQUAL_size_t(bbig, repl_trace_store_bytes(&g_rs.trace));
assert_info(fmt_id(PLAIN_JOB2), want);
assert_info(fmt_id(PLAIN_JOB), discarded_line(PLAIN_JOB));
free(want);
}
/* A newer capture of a discarded job clears its mark. */
static void test_newer_capture_clears_discarded(void)
{
xce_sched_trace_rec_t* a
= new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 40 * MIB);
xce_sched_trace_rec_t* b
= new_rec(PLAIN_JOB2, 1, XCE_TRACE_LEVEL_FULL, 40 * MIB);
xce_sched_trace_rec_t* a2 = new_rec(PLAIN_JOB, 5, XCE_TRACE_LEVEL_FULL, 0);
char* want;
repl_trace_store_put(&g_rs.trace, a);
repl_trace_store_put(&g_rs.trace, b); /* evicts a */
assert_info(fmt_id(PLAIN_JOB), discarded_line(PLAIN_JOB));
want = render(a2, XCE_REPL_UNITS_BOTH);
repl_trace_store_put(&g_rs.trace, a2);
assert_info(fmt_id(PLAIN_JOB), want);
free(want);
}
/* 7.11: no stored trace and none discarded: the job's state, or unknown. */
static void test_no_trace_replies(void)
{
char want[256];
snprintf(want, sizeof(want),
"[sim] info trace: job %s: no trace available (state PEND)\n",
fmt_id(PLAIN_JOB));
assert_info(fmt_id(PLAIN_JOB), want);
snprintf(want, sizeof(want),
"[sim] info trace: job %s: no trace available (state RUN)\n",
fmt_id(ELEM(2)));
assert_info(fmt_id(ELEM(2)), want);
snprintf(want, sizeof(want), "[sim] info trace: job %s: unknown job\n",
fmt_id(UNKNOWN_JOB));
assert_info(fmt_id(UNKNOWN_JOB), want);
}
/* 7.8: a trace stays available after its job left the job table. */
static void test_trace_of_removed_job(void)
{
xce_sched_trace_rec_t* r = new_rec(UNKNOWN_JOB, 4, XCE_TRACE_LEVEL_FULL, 0);
char* want = render(r, XCE_REPL_UNITS_BOTH);
repl_trace_store_put(&g_rs.trace, r);
assert_info(fmt_id(UNKNOWN_JOB), want);
free(want);
}
/* 7.9: an array mixing stored and discarded elements, in index order;
* elements with neither are not listed. */
static void test_array_mixed_stored_and_discarded(void)
{
xce_sched_trace_rec_t* e2
= new_rec(ELEM(2), 1, XCE_TRACE_LEVEL_FULL, 40 * MIB);
xce_sched_trace_rec_t* e3 = new_rec(ELEM(3), 1, XCE_TRACE_LEVEL_FULL, 0);
xce_sched_trace_rec_t* e1
= new_rec(ELEM(1), 1, XCE_TRACE_LEVEL_SUMMARY, 30 * MIB);
char* r1;
char* r3;
char* want = NULL;
size_t n = 0;
FILE* f;
r3 = render(e3, XCE_REPL_UNITS_BOTH);
r1 = render(e1, XCE_REPL_UNITS_BOTH);
repl_trace_store_put(&g_rs.trace, e2);
repl_trace_store_put(&g_rs.trace, e3);
repl_trace_store_put(&g_rs.trace, e1); /* evicts e2 */
f = open_memstream(&want, &n);
fputs(r1, f);
fputs(discarded_line(ELEM(2)), f);
fputs(r3, f);
fclose(f);
assert_info(fmt_id(ARRAY_BASE), want);
free(want);
/* an element range selects within the array */
want = NULL;
f = open_memstream(&want, &n);
fputs(discarded_line(ELEM(2)), f);
fputs(r3, f);
fclose(f);
{
char sel[96];
snprintf(sel, sizeof(sel), "%s[2-4]", fmt_id(ARRAY_BASE));
assert_info(sel, want);
}
free(want);
/* an element by `array[k]` */
{
char sel[96];
snprintf(sel, sizeof(sel), "%s[1]", fmt_id(ARRAY_BASE));
assert_info(sel, r1);
}
free(r1);
free(r3);
}
/* 7.9: no element has a stored trace: say so and list element states in
* index order. */
static void test_array_without_traces_lists_states(void)
{
char want[512];
snprintf(want, sizeof(want),
"[sim] info trace: array %s: no element has a stored trace\n"
" %s PEND\n"
" %s RUN\n",
fmt_id(ARRAY2_BASE), fmt_id(ELEM2(1)), fmt_id(ELEM2(2)));
assert_info(fmt_id(ARRAY2_BASE), want);
}
/* 7.9 with 9.6: only discarded elements: they are marked, then the states. */
static void test_array_only_discarded(void)
{
xce_sched_trace_rec_t* e1
= new_rec(ELEM2(1), 1, XCE_TRACE_LEVEL_FULL, 40 * MIB);
xce_sched_trace_rec_t* other
= new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 40 * MIB);
char want[768];
repl_trace_store_put(&g_rs.trace, e1);
repl_trace_store_put(&g_rs.trace, other); /* evicts e1 */
snprintf(want, sizeof(want),
"%s"
"[sim] info trace: array %s: no element has a stored trace\n"
" %s PEND\n"
" %s RUN\n",
discarded_line(ELEM2(1)), fmt_id(ARRAY2_BASE), fmt_id(ELEM2(1)),
fmt_id(ELEM2(2)));
assert_info(fmt_id(ARRAY2_BASE), want);
}
/* Usage and a malformed id leave the store untouched. */
static void test_usage_and_invalid_id(void)
{
char* got = info_trace("");
TEST_ASSERT_NOT_NULL(strstr(got, "usage: info trace"));
free(got);
got = info_trace("1-2-x-4");
TEST_ASSERT_NOT_NULL(strstr(got, "[sim] info trace: invalid job id"));
free(got);
}
/* A NULL record or store is ignored; fini empties and allows reuse. */
static void test_null_and_fini(void)
{
repl_trace_store_put(NULL, NULL);
repl_trace_store_put(&g_rs.trace, NULL);
TEST_ASSERT_EQUAL_size_t(0, repl_trace_store_bytes(&g_rs.trace));
repl_trace_store_put(&g_rs.trace,
new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 0));
repl_trace_store_fini(&g_rs.trace);
TEST_ASSERT_EQUAL_size_t(0, repl_trace_store_bytes(&g_rs.trace));
repl_trace_store_put(&g_rs.trace,
new_rec(PLAIN_JOB, 1, XCE_TRACE_LEVEL_FULL, 0));
TEST_ASSERT_TRUE(repl_trace_store_bytes(&g_rs.trace) > 0);
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_put_replaces_earlier_trace);
RUN_TEST(test_info_uses_current_units_and_captured_level);
RUN_TEST(test_eviction_drops_oldest_first);
RUN_TEST(test_eviction_never_drops_newest);
RUN_TEST(test_newer_capture_clears_discarded);
RUN_TEST(test_no_trace_replies);
RUN_TEST(test_trace_of_removed_job);
RUN_TEST(test_array_mixed_stored_and_discarded);
RUN_TEST(test_array_without_traces_lists_states);
RUN_TEST(test_array_only_discarded);
RUN_TEST(test_usage_and_invalid_id);
RUN_TEST(test_null_and_fini);
return UNITY_END();
}