lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(xce-harness-component): relocate xce_test into standalone harness component

Browse source 
Move xce_test's 15 production sources + headers from src/controller/testsuite/
into a new src/harness/ component with its own meson.build, and strip every
now-dangling controller-side reference (11 controller_unit_test_src entries,
4 standalone test-binary blocks, 11 controller_test.c extern/RUN_TEST lines)
so the controller build is never left broken.

Fixes a broken relative-symlink bridge (xce_test_controller_link) added to
keep two controller-embedded tests (test_test_cli.c, test_xcetest_process_entry.c)
finding the relocated xce_test binary until task 2.1 moves them too.

Discovered and documented in design.md: four controller-domain regression
tests call functions directly from four of xce_test's own files, requiring
controller_unit_test to permanently recompile them from src/harness/ — a
real dependency neither the original discovery nor design audit caught.

controller_unit_test: 1228 Tests 0 Failures 0 Ignored (verified via the
Unity binary's own printed summary line, not just meson's aggregate count).

Task 1.1 of xce-harness-component.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
Rongsong Shen 2026-09-14 05:08:15 +00:00
parent b8ef82c1b2
commit cf55d691de
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
36 changed files with 550 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

219
src/harness/configcompare.c Normal file
View file

@ -0,0 +1,219 @@
#define _GNU_SOURCE
#include "configcompare.h"
#include <errno.h>
#include <inttypes.h>
#include <openssl/sha.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
/* Bound on a member's config bytes this tool will read -- mirrors
* persist.c's PERSIST_CONFIG_BYTES_MAX_SIZE / config_repl.h's CONF_MAX_SIZE
* (1 MiB) exactly. Duplicated as its own local constant rather than
* including either header, for the same reason the resolve-then-read logic
* below is duplicated -- see read_member_config_sha256()'s doc comment. */
#define CONFIGCOMPARE_CONFIG_MAX_SIZE (1u * 1024u * 1024u)
/* Resolve `<conf_dir>/conf/current` -> `<conf_dir>/conf/<versioned file>`,
* read its raw bytes, and hash them into `out_sha`.
*
* DELIBERATE DUPLICATION -- this is the FOURTH independent copy of this
* exact resolve-then-read-then-hash logic in this codebase:
* 1. config_repl.c's conf_read_current_filename() + conf_read_raw()
* (the production implementation -- config replication).
* 2. config_repl.c's local_config_sha256() (activation-fence hash check;
* an inline copy for the same "no cross-file call needed here"
* reason as #4 below).
* 3. persist.c's read_active_config_bytes() (checkpoint config-bytes
* embedding; see its own doc comment for the full "why duplicate,
* not call" rationale -- avoiding a persist.c -> config_repl.c
* layering dependency).
* 4. This function. configcompare.c is compiled into the standalone
* xce_test CLI (meson.build's xce_test_src), which links ONLY
* libxce_dep / librtti_dep / struct2json_dep / readline_dep /
* history_dep / ncurses_dep / tinylog_dep / zlib_dep /
* c_capnproto_dep -- deliberately NOT config_repl.c, persist.c,
* cluster.c, or any other heavy controller object file. This is a
* lightweight, dependency-light offline CLI tool family
* (walcompare/snapdiff/snapverify all share the same constraint; no
* live RPC to any cluster member, consistent with every other tool
* in this family -- design.md "Regression Tooling"). Adding a
* config_repl.c dependency to link in one function would violate
* that boundary, so the ~25-line resolve+read is duplicated inline
* here instead, exactly as persist.c's read_active_config_bytes()
* already chose to do for the identical reason.
*
* Returns 0 with `out_sha` filled on success. Returns a negative errno on
* any failure to resolve or read: -ENOENT (no conf/current pointer, or it
* names a versioned file that is not present -- benign "not provisioned
* yet" as well as genuine absence, both surfaced identically to the
* caller as "unreadable"), -EINVAL (malformed pointer file: empty,
* embedded '/', wrong prefix, wrong hex length), -EIO (short
* read/zero-byte versioned file), -EFBIG (versioned file exceeds
* CONFIGCOMPARE_CONFIG_MAX_SIZE), or -ENOMEM. This is the caller's
* "unreadable/absent" signal -- distinct from and never conflated with a
* mismatched hash. */
static int read_member_config_sha256(const char* conf_dir, uint8_t out_sha[32])
{
if (!conf_dir || !*conf_dir)
return -EINVAL;
char head_path[4160];
snprintf(head_path, sizeof(head_path), "%s/conf/current", conf_dir);
FILE* hf = fopen(head_path, "r");
if (hf == NULL)
return -ENOENT; /* no config pushed to this member yet */
struct stat hst;
if (fstat(fileno(hf), &hst) == 0 && hst.st_size == 0) {
fclose(hf);
return -ENOENT; /* interrupted-write pointer file -- treat as absent */
}
char fname[256];
if (fgets(fname, (int)sizeof(fname), hf) == NULL) {
fclose(hf);
return -EIO;
}
fclose(hf);
fname[strcspn(fname, "\n")] = '\0';
if (strchr(fname, '/') != NULL)
return -EINVAL; /* corrupted pointer file -- embedded path separator */
if (strncmp(fname, "cluster.bin.", 12) != 0)
return -EINVAL;
const char* hex = fname + 12;
if (strlen(hex) != 12 || strspn(hex, "0123456789abcdef") != 12)
return -EINVAL;
char cfg_path[4416];
snprintf(cfg_path, sizeof(cfg_path), "%s/conf/%s", conf_dir, fname);
FILE* cf = fopen(cfg_path, "rb");
if (cf == NULL)
return -ENOENT; /* pointer references a file that is not there */
if (fseek(cf, 0, SEEK_END) != 0) { fclose(cf); return -EIO; }
long sz = ftell(cf);
if (sz < 0) { fclose(cf); return -EIO; }
if (fseek(cf, 0, SEEK_SET) != 0) { fclose(cf); return -EIO; }
if (sz == 0) {
fclose(cf);
return -EIO; /* zero-byte versioned config file: genuine corruption */
}
if ((size_t)sz > CONFIGCOMPARE_CONFIG_MAX_SIZE) {
fclose(cf);
return -EFBIG;
}
uint8_t* buf = (uint8_t*)malloc((size_t)sz);
if (buf == NULL) {
fclose(cf);
return -ENOMEM;
}
if (fread(buf, 1, (size_t)sz, cf) != (size_t)sz) {
free(buf);
fclose(cf);
return -EIO;
}
fclose(cf);
SHA256(buf, (size_t)sz, out_sha);
free(buf);
return 0;
}
static const char* sha256_hex(const uint8_t sha[32], char out[65])
{
static const char digits[] = "0123456789abcdef";
for (int i = 0; i < 32; i++) {
out[i * 2] = digits[(sha[i] >> 4) & 0xF];
out[i * 2 + 1] = digits[sha[i] & 0xF];
}
out[64] = '\0';
return out;
}
int configcompare_paths(const char* const* conf_paths, int n, FILE* out,
configcompare_report_t* rep)
{
configcompare_report_t local;
memset(&local, 0, sizeof(local));
local.first_divergent_index = -1;
if (!conf_paths || n < 2) {
if (rep)
*rep = local;
if (out)
fprintf(out,
"configcompare: nothing to compare (need >= 2 members, "
"got %d)\n", n);
return -EINVAL;
}
if (n > CONFIGCOMPARE_MAX_MEMBERS) {
if (rep)
*rep = local;
if (out)
fprintf(out, "configcompare: too many members (%d > max %d)\n", n,
CONFIGCOMPARE_MAX_MEMBERS);
return -EINVAL;
}
local.n = n;
int first_ok = -1;
char hexbuf[65];
for (int i = 0; i < n; i++) {
int rc = read_member_config_sha256(conf_paths[i], local.sha256[i]);
local.read_rc[i] = rc;
if (rc != 0) {
if (out)
fprintf(out,
"configcompare: member %d (%s): UNREADABLE: %s\n", i,
conf_paths[i] ? conf_paths[i] : "(null)",
strerror(-rc));
if (rep)
*rep = local;
/* Short-circuit on the first unreadable member, mirroring
* snapdiff_paths's early-return-on-open-failure style: a
* member we could not read is not evidence of matching OR
* mismatching configuration, so there is no point continuing
* to accumulate a verdict that will be discarded anyway. */
return rc;
}
if (out)
fprintf(out, "configcompare: member %d (%s): sha256=%s\n", i,
conf_paths[i], sha256_hex(local.sha256[i], hexbuf));
if (first_ok < 0) {
first_ok = i;
} else if (local.first_divergent_index < 0 &&
memcmp(local.sha256[i], local.sha256[first_ok], 32) != 0) {
local.first_divergent_index = i;
}
}
local.all_present = 1;
local.all_identical = (local.first_divergent_index < 0);
if (rep)
*rep = local;
if (!local.all_identical) {
if (out)
fprintf(out,
"configcompare: DIVERGENT -- member %d differs from "
"member %d\n",
local.first_divergent_index, first_ok);
return 1;
}
if (out)
fprintf(out, "configcompare: IDENTICAL across %d members\n", n);
return 0;
}

105
src/harness/confirm.c Normal file
View file

@ -0,0 +1,105 @@
/*
* confirm.c -- destructive-command confirmation gate. See confirm.h for
* the full contract, return-value semantics, and design rationale
* (xce-test-cli-ux spec, task 2.1; Requirements 9.1-9.4, 9.6-9.9).
*
* IMPLEMENTATION NOTES
*
* - confirm_destructive() checks confirm_is_skip_set() and
* xcetest_is_interactive() BEFORE touching stdin or stderr at all --
* both are pure, side-effect-free reads, so ordering between them
* doesn't matter, but checking them first is what makes the "proceed
* without printing or reading anything" postcondition (9.4, 9.7) hold.
*
* - The prompt read is exactly one fgets() call on stdin, not a loop.
* This is deliberate and is the crux of Requirement 9.9: glibc's
* buffered stdio does NOT internally retry a read() that returns
* -1/EINTR (it propagates the failure up as fgets() returning NULL
* with errno left at EINTR) -- so as long as this function itself
* never loops back to retry after seeing EINTR, a real Ctrl-C during
* the prompt reliably falls straight through to the decline path. A
* naive wrapper that did `while (fgets(...) == NULL) { if (errno ==
* EINTR) continue; ... }` would defeat this entirely; no such loop
* exists here. (This assumes test.c's main() installs SIGINT via
* xcetest_install_sigint() -- SA_RESTART cleared -- per task 1.3; see
* confirm.h's precondition note.)
*
* - errno is reset to 0 immediately before the fgets() call. fgets()
* does not promise to clear a stale errno on success, and NULL is the
* only signal we get to even look at errno, so a stale nonzero errno
* from some earlier, unrelated libc call could otherwise be
* misread as this call's own EINTR.
*/
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include "confirm.h"
static int g_confirm_skip = 0; /* 0 == confirmation required (default at
* process start); mirrors
* g_continue_on_error's exact style
* (test.c). */
void confirm_set_skip(int skip) { g_confirm_skip = skip ? 1 : 0; }
int confirm_is_skip_set(void) { return g_confirm_skip; }
/* Accepted spellings: "y" or "yes", case-insensitive, tolerant of
* incidental leading/trailing whitespace (including the trailing
* newline fgets() leaves in place). Anything else -- "n", "no", empty,
* garbage -- is NOT affirmative; confirm_destructive() maps every
* non-affirmative case to the same "not run" outcome (see confirm.h's
* two-valued return contract). */
static int is_affirmative(const char* raw)
{
char norm[8];
size_t i = 0, n = 0;
while (raw[i] != '\0' && isspace((unsigned char)raw[i]))
i++;
while (raw[i] != '\0' && !isspace((unsigned char)raw[i]) &&
n < sizeof(norm) - 1) {
norm[n++] = (char)tolower((unsigned char)raw[i]);
i++;
}
norm[n] = '\0';
return (strcmp(norm, "y") == 0) || (strcmp(norm, "yes") == 0);
}
int confirm_destructive(const char* action_description)
{
char buf[64];
char* line;
/* Requirement 9.6/9.7: the session-scoped toggle bypasses the prompt
* entirely, regardless of TTY state. Requirement 9.4: a non-interactive
* session bypasses it too. Neither path touches stdin/stderr. */
if (confirm_is_skip_set() || !xcetest_is_interactive())
return 1;
/* Requirement 9.8: the prompt goes to stderr, never stdout -- a
* destructive command's successful result data (if any) must stay
* uncontaminated by this diagnostic text. Deliberately NOT routed
* through libedit's readline() (see confirm.h). */
fprintf(stderr, "About to %s. Continue? [y/N] ", action_description);
fflush(stderr);
errno = 0;
line = fgets(buf, sizeof(buf), stdin);
if (line == NULL) {
/* Requirement 9.9: a real Ctrl-C during the prompt's blocking
* read surfaces as EINTR here (never retried -- see the file
* header) and is treated exactly like a decline, not a distinct
* "error" outcome. EOF (errno left at 0, or set to something
* other than EINTR by the underlying read) is likewise mapped
* to decline -- see confirm.h's return-contract note on why no
* third outcome exists. */
return 0;
}
return is_affirmative(buf) ? 1 : 0;
}

298
src/harness/flagspec.c Normal file
View file

@ -0,0 +1,298 @@
#define _GNU_SOURCE
#include "flagspec.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* True iff `token` (a raw command-line argument, e.g. "-h", "--host", or
* -- defensively -- a dash-free bare name) identifies the flag described
* by `fs`. A "--name" token is matched against fs->long_opt; a "-X" token
* is matched against fs->short_opt (single character only, so "-xy" never
* falsely matches a single-letter short_opt). */
static int flag_token_matches(const char* token, const flagspec_t* fs)
{
if (!token || !fs)
return 0;
if (token[0] == '-' && token[1] == '-')
return fs->long_opt != NULL && strcmp(token + 2, fs->long_opt) == 0;
if (token[0] == '-' && token[1] != '\0')
return fs->short_opt != 0 && token[1] == fs->short_opt &&
token[2] == '\0';
/* No leading dash at all -- not how a real getopt_long() failure
* token looks, but handled so this helper never mis-signals on
* unexpected input from a caller that skips the usual
* argv[optind - 1] convention. */
if (fs->long_opt != NULL && strcmp(token, fs->long_opt) == 0)
return 1;
return fs->short_opt != 0 && token[0] == fs->short_opt &&
token[1] == '\0';
}
int flagspec_build_opts(const flagspec_t* spec, size_t n,
struct option* out_opts, char* out_optstring,
size_t out_optstring_cap)
{
if (!spec || !out_opts || !out_optstring || n == 0 ||
out_optstring_cap == 0)
return -EINVAL;
size_t str_idx = 0;
for (size_t i = 0; i < n; i++) {
const flagspec_t* fs = &spec[i];
if (!fs->long_opt || !*fs->long_opt)
return -EINVAL;
if (fs->has_arg != no_argument && fs->has_arg != required_argument)
return -EINVAL;
out_opts[i].name = fs->long_opt;
out_opts[i].has_arg = fs->has_arg;
out_opts[i].flag = NULL;
out_opts[i].val = fs->short_opt
? (int)fs->short_opt
: (int)(FLAGSPEC_LONG_ONLY_BASE + i);
if (fs->short_opt != 0) {
size_t need = (fs->has_arg == required_argument) ? 2 : 1;
/* Leave room for the final NUL written after the loop. */
if (str_idx + need >= out_optstring_cap)
return -ENOSPC;
out_optstring[str_idx++] = fs->short_opt;
if (fs->has_arg == required_argument)
out_optstring[str_idx++] = ':';
}
}
out_optstring[str_idx] = '\0';
/* Trailing all-zero sentinel getopt_long() requires to terminate the
* table; out_opts must have room for n+1 entries (precondition). */
out_opts[n].name = NULL;
out_opts[n].has_arg = 0;
out_opts[n].flag = NULL;
out_opts[n].val = 0;
return 0;
}
/* Levenshtein edit distance between two NUL-terminated strings -- the
* minimum number of single-character insertions/deletions/substitutions
* that turns `a` into `b`. Small and self-contained: research.md confirms
* no edit-distance utility exists anywhere in the codebase yet. Standard
* two-row rolling DP, O(len(a)*len(b)) time, O(min(len(a),len(b))+1)
* space -- more than fast enough for flag-name-sized strings.
*
* Returns (size_t)-1 on allocation failure, which callers must treat as
* "not a close match" rather than a false zero-distance/exact-match
* signal. */
static size_t levenshtein(const char* a, const char* b)
{
size_t la = strlen(a), lb = strlen(b);
/* Keep the rolling row sized to the SHORTER string. */
if (lb > la) {
const char* tmp = a;
a = b;
b = tmp;
size_t t = la;
la = lb;
lb = t;
}
size_t* row = calloc(lb + 1, sizeof(size_t));
if (!row)
return (size_t)-1;
for (size_t j = 0; j <= lb; j++)
row[j] = j;
for (size_t i = 1; i <= la; i++) {
size_t prev_diag = row[0];
row[0] = i;
for (size_t j = 1; j <= lb; j++) {
size_t saved = row[j];
size_t cost = (a[i - 1] == b[j - 1]) ? 0 : 1;
size_t del = row[j] + 1;
size_t ins = row[j - 1] + 1;
size_t sub = prev_diag + cost;
size_t best = del < ins ? del : ins;
if (sub < best)
best = sub;
row[j] = best;
prev_diag = saved;
}
}
size_t result = row[lb];
free(row);
return result;
}
/* True iff `candidate` (one flag's literal spelling, e.g. "-h" or
* "--host") "closely matches" `bad_flag` for typo-suggestion purposes
* (Requirement 10.1).
*
* Threshold, a deliberate design choice (design.md/research.md do not
* mandate a specific algorithm or cutoff): floor(len(candidate)/4),
* clamped to [1,3]. A fixed absolute threshold would be wrong at both
* ends -- distance 1 from a 2-character short flag ("-h" vs "-z") is a
* near-total rewrite, while distance 1 from an 8+ character long flag is
* a single typo -- so the tolerance scales with the spelling's own
* length instead. Exact matches (distance 0) are never "close" here:
* flagspec_reject() only calls this from the already-unrecognized branch,
* where an exact-spelling candidate would be a contradiction, and this
* keeps the helper correct standalone too. */
static int is_close_match(const char* bad_flag, const char* candidate)
{
size_t dist = levenshtein(bad_flag, candidate);
if (dist == 0 || dist == (size_t)-1)
return 0;
size_t len = strlen(candidate);
size_t threshold = len / 4;
if (threshold < 1)
threshold = 1;
if (threshold > 3)
threshold = 3;
return dist <= threshold;
}
/* Writes `fs`'s display label ("-h/--host" or, for a long-only flag,
* "--json") into `buf`. */
static void format_flag_label(char* buf, size_t cap, const flagspec_t* fs)
{
if (fs->short_opt)
snprintf(buf, cap, "-%c/--%s", fs->short_opt, fs->long_opt);
else
snprintf(buf, cap, "--%s", fs->long_opt);
}
/* Returns the single flagspec_t in `spec` whose short or long spelling
* closely matches `bad_flag`, or NULL when zero or more than one entry
* qualifies -- Requirement 10.1 only promises a guess when exactly one
* flag is a plausible intended target; an ambiguous typo (two or more
* equally-plausible candidates) gets no guess at all, the same as a typo
* that matches nothing (Requirement 10.2). */
static const flagspec_t* find_typo_suggestion(const flagspec_t* spec, size_t n,
const char* bad_flag)
{
if (!spec || !bad_flag || !*bad_flag)
return NULL;
const flagspec_t* found = NULL;
for (size_t i = 0; i < n; i++) {
const flagspec_t* fs = &spec[i];
int is_close = 0;
if (fs->short_opt) {
char short_form[3] = { '-', fs->short_opt, '\0' };
if (is_close_match(bad_flag, short_form))
is_close = 1;
}
if (!is_close && fs->long_opt && *fs->long_opt) {
char long_form[128];
snprintf(long_form, sizeof(long_form), "--%s", fs->long_opt);
if (is_close_match(bad_flag, long_form))
is_close = 1;
}
if (is_close) {
if (found)
return NULL; /* ambiguous -- more than one candidate */
found = fs;
}
}
return found;
}
int flagspec_reject(const char* cmd_name, const flagspec_t* spec, size_t n,
const char* bad_flag, const char* usage_text)
{
const char* display = bad_flag ? bad_flag : "?";
const char* who = cmd_name ? cmd_name : "?";
const flagspec_t* match = NULL;
if (spec) {
for (size_t i = 0; i < n; i++) {
if (flag_token_matches(bad_flag, &spec[i])) {
match = &spec[i];
break;
}
}
}
if (match && match->has_arg == required_argument) {
fprintf(stderr, "xce_test: %s: flag '%s' requires a value\n", who,
display);
} else if (match) {
fprintf(stderr, "xce_test: %s: flag '%s' does not take a value\n",
who, display);
} else {
const flagspec_t* suggestion = find_typo_suggestion(spec, n, bad_flag);
if (suggestion) {
char label[144];
format_flag_label(label, sizeof(label), suggestion);
fprintf(stderr,
"xce_test: %s: unrecognized flag '%s' (did you mean "
"'%s'?)\n",
who, display, label);
} else {
fprintf(stderr, "xce_test: %s: unrecognized flag '%s'\n", who,
display);
}
}
if (usage_text && *usage_text) {
size_t len = strlen(usage_text);
if (len > 0 && usage_text[len - 1] == '\n')
fprintf(stderr, "usage: %s", usage_text);
else
fprintf(stderr, "usage: %s\n", usage_text);
}
return -EINVAL;
}
char* flagspec_render_help(const flagspec_t* spec, size_t n)
{
if (!spec || n == 0)
return NULL;
char* buf = NULL;
size_t size = 0;
FILE* f = open_memstream(&buf, &size);
if (!f)
return NULL;
for (size_t i = 0; i < n; i++) {
const flagspec_t* fs = &spec[i];
if (fs->short_opt)
fprintf(f, " -%c, --%s", fs->short_opt, fs->long_opt);
else
fprintf(f, " --%s", fs->long_opt);
fprintf(f, " %s", fs->required ? "required" : "optional");
if (fs->default_desc && *fs->default_desc)
fprintf(f, " (default: %s)", fs->default_desc);
fprintf(f, "\n");
}
if (fclose(f) != 0) {
free(buf);
return NULL;
}
return buf;
}

74
src/harness/include/configcompare.h Normal file
View file

@ -0,0 +1,74 @@
#pragma once
/*
* configcompare.h — cross-member active-configuration comparator
* (config-upgrade-snapshot task 8.4, Requirement 4.2).
*
* Compares each cluster member's locally-held ACTIVE configuration content
* hash, recomputed FRESH from that member's own on-disk configuration file
* (never trusted from a cached/embedded tag), and reports whether every
* member agrees.
*
* A new sibling tool to walcompare, NOT an extension of it: walcompare's
* internal model is WAL-entry-indexed (lsn -> term, pcrc) -- an awkward fit
* for a single (tag, sha256) value per member with no LSN dimension
* (design.md "Regression Tooling"). Mirrors snapdiff_paths's exact
* signature/reporting shape instead.
*
* Offline/file-based: no live RPC to any cluster member, consistent with
* every other tool in this family (walcompare, snapdiff, snapverify).
*
* Strictly read-only: opens files O_RDONLY, mutates nothing.
*/
#include <stdint.h>
#include <stdio.h>
/* Bound on the number of members a single call can compare -- generous for
* any real cluster size this codebase targets; a caller with more members
* than this is almost certainly passing a malformed argument list. */
#define CONFIGCOMPARE_MAX_MEMBERS 32
typedef struct {
int n; /* member count passed in */
int read_rc[CONFIGCOMPARE_MAX_MEMBERS]; /* 0 = resolved+hashed ok,
* else -errno; entries at
* or past a short-circuit
* failure are unset (0)
* and must be ignored --
* see all_present */
uint8_t sha256[CONFIGCOMPARE_MAX_MEMBERS][32]; /* valid only where
* read_rc[i] == 0 */
int all_present; /* 1 iff every member (0..n-1) was resolved and
* hashed successfully */
int all_identical; /* 1 iff all_present && every member's sha256
* equals the first member's */
int first_divergent_index; /* -1 if all_identical or !all_present;
* else the first index (after the
* reference, member 0) whose hash
* differs from member 0's */
} configcompare_report_t;
/* Compare the active-configuration content hash of `n` members. Each
* `conf_paths[i]` is a member's `workdir/name` directory -- the SAME
* directory persist.c's `read_active_config_bytes()` and config_repl.c's
* `local_config_sha256()` both resolve `conf/current` (and the versioned
* config file it points at) under, matching where this spec's snapshot-embed
* step also reads from. NOT the raw `conf/current` file itself -- this
* function resolves `<path>/conf/current` and `<path>/conf/<versioned-file>`
* internally, exactly like those two functions do.
*
* Prints a human-readable report to `out` (may be NULL) and fills `*rep`
* (may be NULL). Returns:
* 0 if every member's config was readable and all hashes are identical
* 1 if every member's config was readable but at least one diverges
* (see `*rep->first_divergent_index` for which)
* <0 (negative errno) if `n < 2` (nothing to compare), `n` exceeds
* CONFIGCOMPARE_MAX_MEMBERS, or any member's config could not be
* resolved/read -- mirrors snapdiff_paths's early-return-on-open-
* failure style: the first unreadable member short-circuits the scan,
* and `*rep` is still filled with whatever was learned about the
* members checked before the failure, so the report can say which
* member and why. A member that could not be read is NOT evidence of
* matching OR mismatching configuration -- it is a distinct third
* outcome from "identical" and "divergent". */
int configcompare_paths(const char* const* conf_paths, int n, FILE* out,
configcompare_report_t* rep);

101
src/harness/include/confirm.h Normal file
View file

@ -0,0 +1,101 @@
#pragma once
/*
* confirm.h -- destructive-command confirmation gate: prompts the operator
* on stderr before a destructive command runs, offers a session-scoped
* toggle to skip that prompt, and treats a Ctrl-C during the prompt the
* same as declining it (xce-test-cli-ux spec, task 2.1; Requirements
* 9.1-9.4, 9.6-9.9. Requirement 9.5 -- the five-command enumeration and
* each cmd_*'s own `--yes` bypass flag -- is task 3.3, NOT this module;
* this module is the generic, command-agnostic gate mechanism only).
*
* THE MECHANISM (see design.md's "Confirmation Gate" component and its
* "Destructive Command Confirmation Gate" flowchart):
* - confirm_destructive() is the gate itself. It internally checks, in
* order: the session-scoped skip toggle (confirm_is_skip_set()), then
* whether the top-level session is even interactive
* (xcetest_is_interactive(), see below). If either says "don't
* prompt", it returns 1 (proceed) immediately WITHOUT touching stdin
* or printing anything. Otherwise it prints a yes/no prompt to
* stderr and performs one blocking read of the answer from stdin.
* - confirm_destructive() deliberately has NO parameter for a
* per-command `--yes` bypass flag -- that flag is per-INVOCATION, not
* session state, so it cannot be folded into confirm_is_skip_set()'s
* persistent toggle without leaking into the next command. Task 3.3's
* five call sites (cmd_jobkill, cmd_shutdown, cmd_qctrl, cmd_hostctrl,
* cmd_upgradeconfig) are responsible for short-circuiting around
* confirm_destructive() entirely when their OWN --yes flag was given,
* e.g.:
* if (!opts.yes && !confirm_destructive("kill job 1234"))
* return NOT_RUN;
* (design.md's Implementation Notes describes this same three-way
* bypass -- toggle, TTY, --yes -- as one boolean expression each call
* site evaluates; confirm_destructive() itself owns the first two,
* the call site owns the third because only it knows its own flag.)
*
* RETURN CONTRACT (confirm_destructive): exactly two outcomes, matching
* design.md's Service Interface postcondition -- no third "error" case is
* defined for this function.
* 1 -- proceed: skip-toggle set, OR non-interactive session, OR the
* operator answered affirmatively ("y"/"yes", case-insensitive,
* surrounding whitespace ignored).
* 0 -- not run: the operator declined (anything else, including a bare
* Enter or unrecognized text), OR stdin hit EOF, OR the prompt's
* blocking read was interrupted by a real Ctrl-C/SIGINT (EINTR;
* Requirement 9.9 -- treated identically to a decline, never
* retried). The caller must report this as "not run", distinct
* from "failed" (Requirement 9.2/9.9).
*
* WHY THE PROMPT READ IS A PLAIN fgets(), NOT libedit's readline(): a
* y/n answer routed through readline() would pollute the session's
* command history and behave inconsistently when stdin isn't a TTY
* (design.md's Implementation Notes / research.md). It also would not
* reliably surface EINTR the way a bare blocking libc read does.
*
* WHY EINTR IS RELIABLE HERE: task 1.3 (xcetest_interrupt.h,
* xcetest_install_sigint()) replaced test.c's old `signal(SIGINT,
* sigbreak)` installation with an explicit sigaction() call with
* SA_RESTART cleared, once, at process startup. That is a PRECONDITION
* of this module, not something confirm_destructive() re-checks on every
* call -- see design.md's Service Interface preconditions.
*/
/* Prompts the operator (on stderr) to confirm a destructive action
* described by `action_description` (a short phrase completing a
* sentence like "About to <action_description>. Continue? [y/N] " --
* e.g. "kill job 1234"), unless the skip toggle is set or the session
* isn't interactive, in which case it returns 1 without printing
* anything or reading stdin. See the return contract above. */
int confirm_destructive(const char* action_description);
/* Sets/gets the session-scoped confirmation-skip toggle (Requirement
* 9.6/9.7), in the exact same style as test.c's existing g_continue_on_error
* `-e`/`+e` toggle: a plain `static int`, defaulting to 0 (confirmation
* required -- NOT skipped) at process start, flipped by `cmd_set` (task
* 3.3 wires the `set` command's new branch; this module only owns the
* flag itself). While set, confirm_destructive() proceeds without
* prompting regardless of TTY state, so a `source`d script (Requirement
* 13) can opt in to unattended destructive commands explicitly. */
void confirm_set_skip(int skip);
int confirm_is_skip_set(void);
/* Read-only accessor over test.c's private `g_interactive` flag (set once
* in main() from isatty(STDIN_FILENO), never changed after that) --
* mirrors xcetest_interrupted()'s accessor pattern over test.c's private
* `break_out` flag (xcetest_interrupt.h, task 1.3). Declared here, not in
* xcetest_interrupt.h, because confirm.c is its only consumer; g_interactive
* itself is unrelated to that header's SIGINT/EINTR concern.
*
* Per design.md's Confirmation Gate invariant, this is deliberately the
* TOP-LEVEL session's own stdin, not "is this line currently being read
* from a sourced file" -- Requirement 13.7 (a later task) depends on that
* distinction: a destructive command inside a `source`d file in an
* otherwise-interactive session must still prompt unless the skip toggle
* is set.
*
* Implemented in test.c. test.c itself is never linked into
* controller_unit_test (it pulls in libedit/cJSON/libxce/Cap'n Proto/
* OpenSSL and defines its own main() -- see xcetest_interrupt.h's header
* comment for the same reasoning applied to xcetest_interrupted()), so
* test_confirm.c supplies its own local fake definition, exactly as
* test_interruptcall.c already does for xcetest_interrupted(). */
int xcetest_is_interactive(void);

154
src/harness/include/flagspec.h Normal file
View file

@ -0,0 +1,154 @@
#pragma once
/*
* flagspec.h -- flag-descriptor table format, getopt_long-based parsing
* core, typo suggestion, and help-flag-block rendering for xce_test's
* command-line surface (xce-test-cli-ux spec, tasks 1.1/1.2; Requirements
* 2.1-2.4, 3.1, 6.1-6.4, 8.2, 10.1-10.3; the flagspec_t shape is a stable
* contract also relied on by task 3.1 (migrating every existing cmd_*
* command onto this table)).
*
* bug-1711's root cause was that xce_test's flag parsing had no shared
* helper at all: a plain getopt() loop per command, each with its own
* hand-written switch/default, drifted independently from its
* hand-authored USAGE_* string. This module gives every flag-accepting
* command ONE descriptor table to build its getopt_long() option table
* from, ONE shared rejection path (with typo suggestion) to call on any
* parse failure, and ONE renderer for the flag-listing portion of its
* help text, so the table is the single source of truth for all three.
*
* Task 1.1 declared/implemented flagspec_build_opts() and
* flagspec_reject() (without typo suggestion). THIS TASK (1.2) adds
* flagspec_render_help() and extends flagspec_reject()'s rejection
* message with a "did you mean" suggestion (Requirement 10) when exactly
* one flag in `spec` closely matches the unrecognized token.
*/
#include <getopt.h>
#include <stddef.h>
/* One flag's full description: its short/long spellings, the real-world
* concept it represents (for cross-command naming consistency, see
* design.md Requirement 6.2), whether getopt_long() expects a value, and
* documentation state consumed by later tasks (help rendering).
*
* Any two flagspec_t[] tables (i.e. any two commands) that describe the
* "same concept" (e.g. a target host, a job id) MUST use the identical
* long_opt string for it -- enforced by table review, not at runtime. */
typedef struct {
char short_opt; /* 0 if long-only */
const char* long_opt; /* stable, cross-command-consistent name;
* spelled WITHOUT leading dashes, e.g.
* "host", not "--host" */
const char* concept; /* e.g. "host", "jobid", "command-line";
* consumed by help rendering (task 1.2),
* not by flagspec_build_opts()/
* flagspec_reject() */
int has_arg; /* no_argument | required_argument, per
* <getopt.h>; this design uses no
* optional_argument flags anywhere */
int required; /* 1 if the command fails without it;
* enforcement is each command's own
* post-parse-loop check (see design.md's
* bug-1704/cmd_qctrl note), not this
* module's job */
const char* default_desc; /* human text describing the default when
* the flag is omitted, NULL if none;
* consumed by help rendering (task 1.2) */
} flagspec_t;
/* Synthetic getopt_long() `val` assigned to a long-only flag (short_opt
* == 0) at spec index i: FLAGSPEC_LONG_ONLY_BASE + i. Chosen safely above
* the full unsigned-char range (0-255) so it can never collide with a
* real short_opt value, with getopt_long()'s own -1 end-of-options
* sentinel, or with '?' (parse error). Every flagspec_t entry -- long-only
* or not -- therefore returns a value from getopt_long() that a caller's
* switch() can distinguish. */
#define FLAGSPEC_LONG_ONLY_BASE 0x100
/* Builds a getopt_long()-compatible option table and optstring from a
* flagspec_t descriptor table.
*
* `out_opts` must have room for at least n+1 entries -- this function
* appends the trailing all-zero sentinel entry getopt_long() requires to
* terminate the table at index n. `out_optstring` is filled with the
* short-option spec getopt_long()'s `optstring` argument expects: each
* flag with a nonzero short_opt contributes that character, followed by
* ':' when has_arg == required_argument; a long-only flag (short_opt ==
* 0) contributes nothing. The result is NUL-terminated.
*
* Invariant: the built optstring never begins with ':' -- this module
* deliberately does not use getopt_long()'s own leading-':' convention
* for distinguishing "missing argument" from "unrecognized option";
* flagspec_reject() makes that distinction itself by looking the
* offending token up in `spec`. The caller must set `opterr = 0` before
* its getopt_long() loop so libc's own diagnostic never doubles up with
* flagspec_reject()'s.
*
* Returns 0 on success. Returns a negative errno on failure:
* -EINVAL spec/out_opts/out_optstring is NULL, n == 0,
* out_optstring_cap == 0, or a spec entry is malformed
* (long_opt is NULL/empty, or has_arg is neither no_argument
* nor required_argument)
* -ENOSPC out_optstring_cap is too small to hold the built optstring
* (including its terminating NUL)
*
* On any failure, the contents of out_opts/out_optstring are unspecified
* -- the caller must not use them. */
int flagspec_build_opts(const flagspec_t* spec, size_t n,
struct option* out_opts, char* out_optstring,
size_t out_optstring_cap);
/* Shared rejection path for an unrecognized or malformed flag, called
* from a command's getopt_long() loop's `case '?':` arm (or right after
* the loop, for a bad_flag captured some other way).
*
* `bad_flag` is the literal command-line token that failed -- e.g. "-z"
* or "--bogus" -- ordinarily read as `argv[optind - 1]` right after
* getopt_long() returns '?'. flagspec_reject() looks `bad_flag` up
* against `spec` (matching a "-X" token by short_opt, a "--name" token by
* long_opt) to decide which case applies:
* - `bad_flag` matches no entry in `spec` ->
* "unrecognized flag" (Requirement 2.1)
* - `bad_flag` matches an entry with has_arg == required_argument ->
* "flag ... requires a value" (Requirement 2.2)
* - `bad_flag` matches an entry that takes no argument (getopt_long()
* still returned '?', e.g. "--flag=value" given for a no_argument
* long flag) -> "flag ... does not take a value"
* Either way, it prints the offending flag and `usage_text` (Requirement
* 2.4) to stderr only (Requirement 3.1 -- never stdout) and returns a
* negative value; it never partially executes the command.
*
* Typo suggestion (Requirement 10): when `bad_flag` matches no entry in
* `spec` (the "unrecognized flag" case above) and exactly one entry's
* short or long spelling closely matches it by edit distance (see
* flagspec.c's is_close_match()), the rejection message names that flag
* as the likely intended one (Requirement 10.1). When zero or more than
* one entry qualifies as a close match, no guess is made (Requirement
* 10.2) -- an ambiguous typo is treated the same as a typo that matches
* nothing at all. Suggestion is never attempted for the "requires a
* value"/"does not take a value" cases, since those already identify the
* exact intended flag.
*
* `usage_text` is expected to be the command's one-line usage summary
* (e.g. "jobkill -j <jobid> [-s <signo>]"), not its full help block; it
* may be NULL to omit the usage line entirely (defensive only -- every
* real call site always has one).
*
* Returns -EINVAL (a fixed, nonzero, negative-errno result) in every
* case. */
int flagspec_reject(const char* cmd_name, const flagspec_t* spec, size_t n,
const char* bad_flag, const char* usage_text);
/* Renders the flag-listing portion of a command's help text (Requirement
* 8.2) directly from `spec`: one line per flag, each stating its short
* form (when it has one), its long form, whether it is required or
* optional, and its default-value description (when `default_desc` is
* non-NULL). This is the flag-block ONLY -- the surrounding description,
* example, and related-command prose (Requirement 8.1/8.3/8.4) stay
* hand-authored, per design.md's "Flag-descriptor table as the single
* source of truth" decision.
*
* Returns a heap-allocated, NUL-terminated string the caller must free().
* Returns NULL if `spec` is NULL or n == 0, or if the underlying
* allocation fails; it never returns a partially-written buffer. */
char* flagspec_render_help(const flagspec_t* spec, size_t n);

326
src/harness/include/interruptcall.h Normal file
View file

@ -0,0 +1,326 @@
#pragma once
/*
* interruptcall.h -- xcetest_run_interruptible(): a detached-worker-thread
* wrapper that lets a command make a blocking xce_* RPC call cancellably
* from the operator's Ctrl-C, without ever touching libxce's own blocking
* primitive (xce-test-cli-ux spec, task 1.4; Requirements 1.1, 1.2, 1.3,
* 1.4).
*
* ROOT CAUSE this module addresses: every blocking xce_* call
* (src/lib/xceapi.c) waits on pthread_cond_timedwait() with a fixed
* 10-second deadline (cbuf_deadline_10s/cbuf_timedget, src/lib/mylock.c) --
* a wait a signal-setting flag alone cannot interrupt (glibc's condvar
* wait retries internally). Modifying that shared libxce primitive is out
* of this spec's boundary (research.md's Architecture Pattern Evaluation:
* "Modify libxce's cbuf_timedget to poll a cancellation token" -- Rejected,
* out of boundary). The fix is entirely client-side: run the existing
* blocking call, UNMODIFIED, on a detached worker thread; have the calling
* thread poll xcetest_interrupted() (xcetest_interrupt.h, task 1.3) on a
* short interval instead of blocking on the call itself; on interrupt,
* abandon the wait and return immediately, leaving the worker thread to
* finish (or hit its own 10s deadline) on its own -- satisfying "leave the
* connection intact" (Requirement 1.5) and the design's "cancelling a
* client-side wait does not need to cancel the server-side request" note.
*
* THE HARD PART this module exists to get right: the worker thread and the
* calling thread both have a legitimate reason to touch the heap-allocated
* call state backing one call (the worker to run fn()/store its result;
* the caller to read that result) -- but at most ONE of them may free it,
* and it must be resolved without a lock the caller would have to block
* on (that would defeat the whole point of "abandon immediately"). See
* design.md's "Heap-Block Ownership Handoff" state diagram and
* research.md's "Decision: Interruptible RPC via detached worker thread
* with CAS-based ownership handoff" for the full rationale, including why
* pthread_cancel() was rejected (cancellation-safety of
* pthread_cond_timedwait()'s internal state is not established).
*
* PER-CONNECTION LOCK REGISTRY (task 1.5, Requirements 1.1, 1.5): an
* abandoned worker thread from task 1.4's mechanism above keeps running
* against whatever connection state `ctx` closes over. If the operator's
* very next command reuses that SAME connection, its own call would race
* the still-running abandoned worker for that connection's single-
* outstanding-reply state -- not a rare edge case, but the direct, likely
* consequence of Ctrl-C's own purpose (cancel, then immediately try
* something else). xcetest_run_interruptible() closes this by acquiring
* a per-connection lock -- looked up or lazily created in a small,
* process-lifetime registry keyed by whatever `xcetest_set_active_
* connection()` was last called with -- BEFORE spawning the worker, and
* holding it for the call's full lifetime, including through an
* abandonment (ownership of "release the lock" passes to the worker in
* that case, exactly mirroring the heap-block CAS handoff above). Lock
* acquisition is itself interruptible (the same break_out poll loop), so
* a second call waiting on a connection a prior abandoned call still
* holds can itself be Ctrl-C'd out of that wait, before ever acquiring
* the lock or spawning a worker of its own. See design.md's "Interruptible
* RPC Call and Ctrl-C Abandonment" sequence diagram and research.md's
* "Decision: Per-connection lock serializes overlapping calls on the
* Interruptible Call Adapter" for the full rationale.
*
* WHY THE LOCK IS A POSIX SEMAPHORE (sem_t), NOT A MUTEX: the handoff
* above is a genuine CROSS-THREAD release -- the REPL/caller thread
* acquires the lock before spawning the worker, but on the abandonment
* path it is the still-running WORKER thread that later releases it,
* once fn() finishes. A pthread_mutex_t (of ANY type, including the
* default PTHREAD_MUTEX_NORMAL) is the wrong primitive for this: POSIX
* defines pthread_mutex_t in terms of a single owning thread, and
* classifies pthread_mutex_unlock() from a non-owning thread as
* undefined behavior for every mutex type -- PTHREAD_MUTEX_ERRORCHECK/
* _RECURSIVE are simply *required* to detect and report that UB (hence
* the EPERM an earlier version of this code hit immediately), while
* NORMAL/DEFAULT are merely *permitted* not to detect it. POSIX does NOT
* separately declare cross-thread unlock well-defined for NORMAL; it
* just does not mandate an error return for it. A pthread_mutex_t-based
* fix that swaps ERRORCHECK for the default type therefore only trades a
* loud crash for a silent reliance on non-portable, POSIX-classified-UB
* behavior -- something that happens to hold on this project's actual
* target (glibc/NPTL's PTHREAD_MUTEX_NORMAL has had no ownership
* tracking for decades, and XCE's Nix-hermetic build is glibc/Linux-only)
* but is the wrong thing to build on when a fully-defined alternative
* exists at the same code complexity.
*
* sem_t has NO ownership concept at all: sem_wait()/sem_trywait()/
* sem_post() may be called from any thread regardless of which thread
* called which, with fully-defined POSIX semantics for exactly this
* cross-thread signal pattern. Each registry entry is therefore a binary
* semaphore (sem_init(&sem, 0, 1)) used as a mutual-exclusion lock:
* sem_trywait() takes the place of pthread_mutex_trylock() in the
* interruptible poll loop below (EAGAIN where trylock would have
* returned EBUSY), and sem_post() takes the place of
* pthread_mutex_unlock() at both release sites -- with no UB caveat on
* either.
*
* WHY THE REGISTRY IS A SINGLY-LINKED LIST, NOT A GROWABLE ARRAY: a
* sem_t* handed out by this registry may be cached inside a call block
* that a still-running abandoned worker thread holds onto indefinitely
* (up to ~10s, per the heap-block ownership handoff above). A growable
* array's realloc() can MOVE every previously-returned element the
* moment a later, unrelated connection is first seen -- silently
* invalidating that worker's pointer into stale/reused memory. A linked
* list's nodes are individually allocated and, per this registry's own
* "entries are never removed" contract, never freed or moved for the
* rest of the process's life, so a pointer into it stays valid forever
* once handed out.
*
* CONCURRENCY OF THE REGISTRY ITSELF: per design.md's Implementation
* Notes, "the registry's lookup/insert step needs no synchronization of
* its own -- a new registry entry is only ever created from the single
* REPL thread that issues commands" (detached worker threads never
* initiate a new top-level call; they only run the `fn` they were given
* and release the lock/free the block they were handed). The SEMAPHORE
* each entry owns, once handed out, is of course safe for concurrent
* wait/post from different threads -- that safety is sem_t's own job,
* not the registry's.
*/
#include <stdatomic.h>
/* A blocking call this module runs on a detached worker thread. `ctx` is
* whatever heap-owned state `fn` needs -- see xcetest_run_interruptible()'s
* preconditions below for the one hard rule about what `fn` may touch. */
typedef int (*xcetest_blocking_fn)(void* ctx);
/* The heap-allocated call block's lifecycle, resolved by exactly one
* atomic compare-and-swap regardless of which side "wins":
* RUNNING -- fn() has not yet returned, and no one has decided
* to abandon the call yet. Initial state.
* DONE_UNCLAIMED -- fn() returned and the worker's CAS from RUNNING to
* this state succeeded (i.e. the caller had not yet
* abandoned it). The CALLER is left owning the
* block: it reads rc and frees it.
* ABANDONED -- the caller's CAS from RUNNING to this state
* succeeded (i.e. it observed an interrupt before
* fn() finished). The WORKER is left owning the
* block: once fn() eventually returns, its own CAS
* attempt (RUNNING -> DONE_UNCLAIMED) fails against
* this state, and it frees the block itself instead.
* See design.md's "Heap-Block Ownership Handoff" stateDiagram-v2 -- this
* enum's three values are exactly its three states. */
typedef enum {
XCT_RUNNING,
XCT_DONE_UNCLAIMED,
XCT_ABANDONED
} xcetest_call_state_t;
/*
* Sets the connection-identity key that xcetest_run_interruptible() below
* reads INTERNALLY when a new call starts, to look up (or lazily create)
* that connection's entry in the per-connection lock registry (task 1.5,
* see the file header above). This is a simple settable-elsewhere,
* read-internally piece of process state -- deliberately mirroring how
* break_out/xcetest_interrupted() already works -- NOT a parameter added
* to xcetest_run_interruptible()'s own signature, which stays exactly
* `(fn, ctx)` per this task's design constraint.
*
* SEAM FOR A FUTURE CALLER (task 3.2): production code (test.c's cmd_*
* functions) does not exist inside this module and has no notion of
* "the currently active command_ctx_t's connection pointer" that this
* header can reach into directly -- test.c itself is never linked into
* this binary's unit tests (see this file's own header and
* xcetest_interrupt.h's). The intended real wiring is: before invoking
* xcetest_run_interruptible() for a blocking RPC, a cmd_* function (or a
* shared dispatch point above all of them) calls
* xcetest_set_active_connection(<that connection's own stable pointer,
* e.g. the command_ctx_t's connection field>). Any stable, comparable-by-
* identity pointer works as `conn_key` -- this module never dereferences
* it, only compares it by pointer equality against the registry's keys.
* Until task 3.2 wires that in, xce_test's real binary calls this
* exactly once, from wherever main()/the dispatcher sets up each
* command's active connection context.
*
* Concurrency: like break_out, callable only from the single REPL thread
* that issues top-level commands -- see the file header's "CONCURRENCY
* OF THE REGISTRY ITSELF" note. Passing the same key across unrelated
* connections would incorrectly serialize them against each other;
* passing a fresh/different key per connection is what keeps unrelated
* connections independent (Requirement 1.5, "leave ... its underlying
* connection intact").
*/
void xcetest_set_active_connection(void* conn_key);
/*
* Runs `fn(ctx)` on a detached worker thread while THIS (the calling)
* thread polls xcetest_interrupted() every ~50ms (design.md's Interruptible
* Call Adapter component), so a Ctrl-C observed during the wait abandons
* it immediately instead of blocking for fn()'s own, possibly much
* longer, duration.
*
* Before spawning that worker, this function first acquires -- also
* interruptibly, via the same ~50ms xcetest_interrupted() poll, but
* using sem_trywait() instead of a blocking sem_wait() -- the
* per-connection lock for whichever connection key was last passed to
* xcetest_set_active_connection() (task 1.5; see the file header's "PER-
* CONNECTION LOCK REGISTRY" section above). This serializes this call
* against any other in-flight or abandoned-but-still-running call on the
* same connection, so that connection's single-outstanding-reply state
* is never touched by two threads at once.
*
* Preconditions:
* - `fn` is non-NULL.
* - `fn` touches only what `ctx` itself heap-owns -- NEVER memory on
* this function's caller's own stack frame. This is the precondition
* that makes the -EINTR postcondition below safe: once ownership of
* `ctx` has passed to the still-running worker, the caller's own
* stack frame may have long since unwound.
*
* Postconditions:
* - Returns fn()'s own return value when fn() finishes before this
* thread's abandon-CAS wins the race (whether or not
* xcetest_interrupted() ever became true during the wait) -- see the
* header comment's "near-simultaneous" case below. On this path, the
* per-connection lock is released by this (the caller's) thread,
* after reading rc and freeing the block.
* - Returns -EINTR when this thread observes an interrupt WHILE
* WAITING FOR THE PER-CONNECTION LOCK -- i.e. before ever acquiring
* it. No worker is spawned for this call at all in this case, and
* nothing needs releasing, since this call never held the lock to
* begin with.
* - Returns -EINTR when this thread observes an interrupt AFTER
* acquiring the lock and spawning the worker, and its own CAS
* (RUNNING -> ABANDONED) wins the race. On this return, the caller
* MUST NOT touch `ctx` again -- ownership of both the call block AND
* the per-connection lock has passed to the still-running worker
* thread, which will finish fn(), free the call block, and release
* the lock on its own, asynchronously, with no further signal back
* to this caller.
* - Returns -EINVAL if `fn` is NULL (fn is never invoked in this case;
* no lock is acquired).
* - Returns a negative errno from the per-connection lock registry
* lookup (e.g. -ENOMEM), from sem_trywait() itself for an
* unexpected error other than EAGAIN, or from pthread_attr_init()/
* pthread_create() if the worker thread could not be set up or
* spawned at all; in every one of these cases fn() is never invoked,
* any call block already allocated is freed before returning, and
* the per-connection lock -- if this call had already acquired it --
* is released before returning.
*
* The near-simultaneous race (this thread notices the interrupt in the
* same window the worker finishes fn()) is resolved by the SAME single
* CAS deciding both sides at once: whichever of {this thread's
* RUNNING->ABANDONED attempt, the worker's RUNNING->DONE_UNCLAIMED
* attempt} executes first wins, and the loser observes that the state is
* no longer RUNNING and falls back to the OTHER side's outcome -- a
* caller who loses the abandon-CAS falls through to the normal
* completion path (reads rc, frees the block, releases the lock, returns
* rc) instead of returning -EINTR; a worker who loses the done-CAS frees
* the block and releases the lock itself instead of leaving them for the
* caller. Exactly one free() of the block, and exactly one release of
* the lock, are therefore reachable on every possible interleaving --
* never zero, never two.
*/
int xcetest_run_interruptible(xcetest_blocking_fn fn, void* ctx);
/*
* xcetest_wait_connection_idle (xce-test-cli-ux task 3.2 remediation round
* 1, bug-1795) -- blocks the calling thread until no worker thread
* (running or abandoned-but-still-running) is active on `conn_key`, per
* this module's own "at most one worker thread per connection" invariant,
* then returns immediately.
*
* WHY THIS EXISTS: xcetest_run_interruptible()'s per-connection lock
* registry above enforces that invariant BETWEEN calls on the same
* connection -- a second call on a connection an abandoned worker still
* holds waits (interruptibly) for that worker to finish. It does NOT by
* itself protect connection TEARDOWN: test.c's leave_ctx() (called by
* `up`/`quit`) tears down the connection object out from under an
* abandoned-but-still-running worker with no wait/join of its own,
* because leave_ctx() never calls xcetest_run_interruptible() -- it is
* connection teardown, not a new RPC. xcecli_wakeup() (called by
* leave_ctx() just before this function, best-effort) nudges that worker
* to unwind faster than its own ~10s deadline, but nothing forced
* leave_ctx() to actually wait for it -- a genuine, exploitable
* use-after-free window (any abandoned worker still reading/writing the
* connection object while leave_ctx()'s xcecli_stop()/xcecli_ctx_fini()
* frees it). This function closes that gap by reusing the SAME
* per-connection semaphore registry xcetest_run_interruptible() already
* builds and maintains, rather than inventing a second mechanism.
*
* CONTRACT: looks up (or lazily creates, via the exact same internal
* lookup xcetest_run_interruptible() itself uses) the sem_t for
* `conn_key`, does a PLAIN BLOCKING sem_wait() on it (deliberately NOT
* the interruptible sem_trywait()-poll variant used inside
* xcetest_run_interruptible() -- this is a narrow, safety-critical wait
* where correctness matters more than Ctrl-C responsiveness; it should
* resolve quickly in the ordinary case since the caller's own
* xcecli_wakeup() call already nudges the worker to finish sooner), then
* immediately sem_post()s it back before returning. It does NOT hold the
* semaphore locked on return -- the semantic this function provides is
* "prove that, at this instant, no worker is active on this connection,"
* not "hold this connection locked for some ongoing operation of the
* caller's own." A caller that needs the latter must acquire and hold the
* lock itself (which only xcetest_run_interruptible() currently does).
*
* A transient EINTR from sem_wait() itself (a signal delivered while
* blocked, distinct from `errno`-based break_out signal state) is
* retried internally rather than surfaced -- this function's whole
* purpose is to make the wait unconditional, not skippable.
*
* Safe to call with any `conn_key` previously passed to
* xcetest_set_active_connection() (or, for that matter, any key never
* seen before -- the lazy-create path simply creates a fresh, immediately
* uncontended semaphore for it and returns right away). Must be called
* from the same single REPL thread this module's other entry points are
* documented to run on (see this header's "CONCURRENCY OF THE REGISTRY
* ITSELF" note) -- it is not itself safe to call concurrently from
* multiple threads, though the semaphore it waits on is of course safe to
* be concurrently sem_post()'d by a worker thread while this function
* waits on it.
*/
void xcetest_wait_connection_idle(void* conn_key);
#ifdef XCE_TESTING
/*
* Test-only instrumentation, NOT part of the production API and never
* even declared unless XCE_TESTING is defined (i.e. never visible to the
* real xce_test binary): incremented (via atomic_fetch_add) by the worker
* thread immediately after it frees an ABANDONED call block.
*
* Exists solely so test_interruptcall.c's forced-abandonment test can
* confirm the abandoned worker's asynchronous cleanup genuinely completed
* before the test process exits, rather than merely hoping a fixed sleep
* was long enough -- a still-running detached worker racing a leak
* detector's exit-time check could otherwise either mask a real bug or
* (more likely, if the implementation above is correct) flag a false
* leak on an allocation that was always going to be freed, just not yet.
*/
extern _Atomic int xcetest_test_abandoned_free_count;
#endif

81
src/harness/include/joblist_display.h Normal file
View file

@ -0,0 +1,81 @@
#ifndef JOBLIST_DISPLAY_H
#define JOBLIST_DISPLAY_H
/* joblist_display.h -- xce-test-cli-ux task 2.5 remediation (bug-1711 fixes)
*
* Extracted from test.c's worker_status_str()/job_state_str()/
* format_job_time(): pure, dependency-light display-formatting helpers, with
* NO networking, REPL, or global-session-state dependency (job_state_str's
* only state is a function-local `static` scratch buffer, used solely to
* render its own return value -- no cross-call or cross-session state).
* This isolation is what makes them unit-testable outside of test.c, which
* cannot be white-box #include'd into a Unity test (thousands of lines, its
* own main(), heavy libedit/network dependencies) -- same rationale as
* resval_typeresolve.c's extraction from set_resval() (resource-value-
* verification task 1.1) and misc.c's update_host_runusage()/
* aggregate_job_runusage() (bug-1735).
*
* test.c calls these directly at their existing call sites (cmd_workerlist,
* joblist_query, print_jobinfo) -- no behavior change, extraction only.
*/
#include <stddef.h> /* size_t */
#include <stdint.h> /* uint32_t */
#include <time.h> /* time_t */
/* Render a worker's status bitmask (xce_worker_t.status, XCE_WORKER_* bits
* in xce/types.h) as a short display string for cmd_workerlist's Status
* column.
*
* Priority order (first match wins, unchanged from the pre-extraction
* logic): CLOSED > UNAVAIL > LOCKEDU > LOCKEDW > BUSY > (status == 0) OK.
* Admin-set CLOSED intentionally takes priority over the dynamic UNAVAIL
* bit: a host closed via hostctrl that later disconnects would otherwise
* print only "UNAVAIL", masking the admin bit this check exists to verify
* (config-upgrade-snapshot soak, task 8.6).
*
* Any other nonzero bit combination not covered above returns "UNKNOWN"
* (Requirement 11.1 -- bug-1711's fix; previously misspelled "UNKOWN").
*
* Returns a string literal -- never a caller-owned buffer, safe to use
* across calls with no ownership/lifetime concerns. */
char* worker_status_str(uint32_t status);
/* Render a job's state bitmask (xce_job_t.state, JOB_STAT_* bits in
* xce/types.h) as a short display string for cmd_joblist's State column
* (joblist_query) and print_jobinfo's "state:" line.
*
* Priority order (first match wins, unchanged from the pre-extraction
* logic): PEND (+PSUSP) > RUN (+USUSP/+SSUSP/+EXIT) > EXIT > DONE > ERROR >
* UNKNOWN.
*
* Requirement 11.7 (bug-1711): a job that is simultaneously RUN and EXIT
* (e.g. a worker-reported FAIL received without a prior FINISH;
* apply_job_fail_transition ORs in EXIT|PERR without clearing RUN) is
* reported via a raw-state fallback "RUN(0x%x)", consistent with the
* fallback print_jobinfo already applies for that same ambiguity via its
* separate state_raw line -- checked AFTER USUSP/SSUSP in the same
* else-if chain, so it never masks either of those.
*
* Returns a string literal for every branch except the RUN+EXIT fallback,
* which is rendered into a function-local `static` scratch buffer valid
* until the next call (the same lifetime contract test.c relied on before
* extraction). */
char* job_state_str(int state);
/* Format a job time_t field (epoch seconds) using ctrlstatus_fmt.h's
* ctrlstatus_format_time_ms human-readable formatter (which takes epoch
* milliseconds) into `buf` (at least 24 bytes -- see
* ctrlstatus_format_time_ms's own size contract).
*
* A job time of 0 means the corresponding event has not happened yet (e.g.
* an unstarted job's start_time/finish_time) -- returns "-" rather than a
* misleading 1970-01-01 epoch date (Requirement 11.5). Also returns "-" if
* the underlying formatter itself fails (e.g. `buf`/`bufsize` too small).
*
* Returns `buf` on success (nonzero `t`), or a string literal ("-") for the
* zero/failure cases -- callers must not assume the return value is always
* `buf`. */
const char* format_job_time(time_t t, char* buf, size_t bufsize);
#endif /* JOBLIST_DISPLAY_H */

68
src/harness/include/join_worker_args.h Normal file
View file

@ -0,0 +1,68 @@
#ifndef JOIN_WORKER_ARGS_H
#define JOIN_WORKER_ARGS_H
/* join_worker_args.h -- xce-test-cli-ux task 2.6 remediation (bug-1557 fix)
*
* Extracted from test.c's cmd_join_worker(): the pure argument-parsing/
* defaulting block that populates a hostattrs_t (xce/types.h) from argc/argv,
* including the bug-1557 maxmem default fix. This block runs entirely before
* cmd_join_worker's only network call (xcecli_join()) and touches no
* active_ctx, REPL state, or any global beyond the standard `optind` reset
* every other cmd_ handler in this file already performs -- the same
* isolation rationale as joblist_display.c's extraction (task 2.5) and
* resval_typeresolve.c's (resource-value-verification task 1.1).
*
* test.c's cmd_join_worker() calls this directly in place of its old inline
* parsing block, then proceeds to xcecli_join() exactly as before -- no
* behavior change to the network-call phase, extraction only.
*/
#include <stddef.h> /* size_t */
#include "flagspec.h" /* flagspec_t */
#include "xce/types.h" /* hostattrs_t */
/* join's flagspec_t table, exposed so test.c's `join` command_t entry
* (worker context, cmd_join_worker) can wire it into the Help Renderer's
* generated flag block (xce-test-cli-ux task 3.5, Requirement 8.2, 12.2) --
* the same table join_worker_parse_attrs() itself parses against, so the
* rendered help and the actual parsing/defaulting behavior can never drift
* apart.
*
* JOIN_FLAGS_COUNT is a compile-time literal (not a `const size_t`
* variable) specifically so test.c's file-scope command_t table
* initializers -- which the C standard requires to be constant
* expressions -- can use it directly for a `.flagspec_count` field; an
* extern variable's value is only known at link time and is not a
* constant expression. Declaring `join_flags[]` here with this explicit
* bound, then defining it in join_worker_args.c with a plain `[]` and a
* 6-entry initializer, gets a free safety net: the two declarations of
* the same array must agree on size, so a future edit that adds/removes a
* flag without updating JOIN_FLAGS_COUNT fails the build immediately
* ("conflicting types") instead of silently reporting a wrong count.
* `JOIN_NFLAGS` (join_worker_args.c) stays an internal alias of this same
* constant, used to size join_worker_parse_attrs()'s own getopt_long()
* option table -- unchanged behavior, just no longer computed via
* sizeof(). */
#define JOIN_FLAGS_COUNT 6
extern const flagspec_t join_flags[JOIN_FLAGS_COUNT];
/* Parse join's CLI flags ("-f cpufactor -c maxcpus -m maxmem -s maxswap
* -t maxtmp -d ndisks") into `*attrs`, first defaulting every field
* (including the bug-1557-corrected maxmem default of 64 * 1024 * 1024
* bytes -- see the in-body comment in join_worker_args.c for the full
* bytes-vs-MB history) and then overriding any field whose flag was given.
*
* Resets `optind` to 1 before parsing, matching this file's own convention
* for repeated in-process command invocations (see cmd_jobfinish's comment
* on the same convention).
*
* Returns 0 on success. Returns -EINVAL on an unrecognized flag or a NULL
* `attrs`, leaving `*attrs` fully defaulted (and possibly partially
* overridden by whichever flags were already parsed before the
* unrecognized one) -- callers must not use `*attrs` after a nonzero
* return. On a parse error, this function itself prints the same usage
* string cmd_join_worker printed before extraction. */
int join_worker_parse_attrs(int argc, char* argv[], hostattrs_t* attrs);
#endif /* JOIN_WORKER_ARGS_H */

54
src/harness/include/listfmt_json.h Normal file
View file

@ -0,0 +1,54 @@
#pragma once
/*
* listfmt_json.h -- shared cJSON array/object plumbing for xce_test's
* listing commands' JSON output mode (xce-test-cli-ux spec, task 2.2;
* Requirements 5.1, 5.3).
*
* This module owns ONLY the JSON array/object lifecycle: start an array,
* append a new row object to it, add string/number fields to a row, then
* print + free the whole array. It does NOT know about workerlist's/
* joblist's/reslist's own field sets -- each command's existing row
* struct already exists and is stable (design.md's Simplification lens),
* so this library introduces no generic "record"/"row" abstraction beyond
* the bare cJSON object handle itself. Wiring this into cmd_workerlist/
* cmd_joblist/cmd_reslist (their field mapping, their --json flag) is
* task 3.4, not this one.
*
* cJSON is already linked into xce_test (struct2json_dep, per
* research.md) and already used for job exdata blobs (test.c's
* build_job_envs()) -- this module reuses that exact same cJSON API
* surface, not a different one.
*/
#include <cjson/cJSON.h>
/* Creates a new, empty JSON array meant to hold one object per output
* row. The caller passes the returned array to listfmt_json_new_row()
* once per row, then to listfmt_json_finish() exactly once when done.
* Returns NULL on allocation failure. */
cJSON* listfmt_json_begin(void);
/* Adds a string field `key`: `val` to `row`. `val` is copied in full,
* regardless of its length -- no field is ever truncated or dropped for
* being "too long" (Requirement 5.3). A NULL `val` is stored as an
* explicit JSON null rather than silently omitted or crashing. No-op if
* `row` or `key` is NULL. */
void listfmt_json_add_string(cJSON* row, const char* key, const char* val);
/* Adds a numeric field `key`: `val` to `row`. No-op if `row` or `key` is
* NULL. */
void listfmt_json_add_number(cJSON* row, const char* key, double val);
/* Creates a new, empty JSON object, appends it to `array`, and returns
* that object so the caller can then call listfmt_json_add_string()/
* listfmt_json_add_number() on it for each of this row's fields. Returns
* NULL (leaving `array` unmodified) if `array` is NULL or the new object
* could not be allocated. */
cJSON* listfmt_json_new_row(cJSON* array);
/* Prints `array` to stdout as JSON -- command-RESULT data, per
* Requirement 3's stdout/stderr discipline, so it is written to stdout
* only, never stderr -- and then frees the entire cJSON structure
* (`array` and every row/field it holds). `array` must not be
* referenced again after this call. No-op if `array` is NULL. */
void listfmt_json_finish(cJSON* array);

113
src/harness/include/listrow_json.h Normal file
View file

@ -0,0 +1,113 @@
#pragma once
/*
* listrow_json.h -- pure per-command JSON-row field-mapping helpers for
* cmd_reslist/cmd_workerlist/joblist_query's new --json output mode
* (xce-test-cli-ux task 3.4, Requirements 5.1, 5.2, 5.3).
*
* Each function here takes one already-fetched record's fields --
* already extracted as plain scalars by the caller, no xce/types.h struct
* dependency, no RPC, no networking, no global session state -- and
* populates one JSON row object (from listfmt_json_new_row(), task 2.2)
* via listfmt_json.h's add_string()/add_number() primitives, mapping
* EVERY column the corresponding command's human-readable table prints
* (Requirement 5.1) with no truncation regardless of value length
* (Requirement 5.3).
*
* Extracted out of test.c (rather than inlined in cmd_reslist/
* cmd_workerlist/joblist_query alongside their RPC calls) specifically so
* this field-mapping logic is unit-testable in isolation with plain
* literal inputs -- test.c itself cannot be white-box #include'd into a
* Unity test (thousands of lines, its own main(), heavy libedit/network
* dependencies), and cmd_reslist/joblist_query's fetch+print bodies were
* already found "genuinely untestable without a live cluster" for the
* SAME reason during task 2.5's review. This mirrors joblist_display.c's
* extraction precedent (task 2.5) for worker_status_str()/job_state_str()/
* format_job_time() -- pure, dependency-light display formatting pulled
* out specifically to get real automated coverage.
*
* The caller (test.c) remains responsible for: calling xcecli_reslist()/
* xcecli_workerlist()/xcecli_joblist(), iterating the result, calling
* listfmt_json_begin()/listfmt_json_new_row() around each call here, and
* listfmt_json_finish() exactly once when done -- this module never
* touches the array lifecycle itself (listfmt_json.h already owns that,
* task 2.2, and is not modified by this task).
*/
#include <stdint.h>
#include <time.h>
#include <cjson/cJSON.h>
/* Maps cmd_reslist's table columns -- Name, Flags, Interval, Description,
* see test.c's printf("%-10s%10s%10d %-64s\n", ...) row -- onto `row`.
*
* `flags` is emitted as a JSON NUMBER under the key "flags" (the raw
* xce_resource_t.flags value), not the table's "0x%x"-formatted hex
* string (Requirement 11.4's hex prefix is a display-only convention for
* the human-readable table; the underlying integer value is identical
* either way) -- a number already carries the same information the
* table's Flags column shows, so nothing is lost relative to the table
* (Requirement 5.1), and a script consuming the JSON output gets a
* directly-usable integer instead of having to re-parse a "0x..." string.
*
* No-op if `row` is NULL (mirrors listfmt_json_add_string/number's own
* NULL-row no-op contract). */
void reslist_json_row(cJSON* row, const char* name, int flags, int interval,
const char* desc);
/* Maps cmd_workerlist's table columns -- HostName, Status, HostType,
* HostModel, CPUfactor, MaxCPU, MaxMem, MaxSwap, MaxTmp, NDisks -- onto
* `row`.
*
* Unlike the table (which humanizes MaxMem/MaxSwap/MaxTmp via
* format_bytes_human() for display, e.g. "64.0 MB"), this emits their RAW
* numeric byte values (`maxmem`/`maxswap`/`maxtmp`, matching hostattrs_t's
* own `double` field types in xce/types.h) as JSON numbers under keys
* "maxmem"/"maxswap"/"maxtmp" -- the same underlying value the table's
* humanized string represents, but directly script-consumable without
* re-parsing a unit suffix.
*
* `status_str` is the caller's worker_status_str(workers[i]->status)
* result (joblist_display.h) -- passed in rather than recomputed here, so
* this module has no dependency on xce/types.h's XCE_WORKER_* bit
* definitions.
*
* `cpufactor` is `double` (not truncated to float/int) so a genuine
* fractional value -- not just a power-of-two -- round-trips exactly
* through the JSON text (see test_listrow_json.c's fractional round-trip
* test, closing the gap task 2.2's Implementation Notes flagged: its own
* tests only exercised power-of-two doubles).
*
* No-op if `row` is NULL. */
void workerlist_json_row(cJSON* row, const char* name, const char* status_str,
const char* hosttype, const char* hostmodel,
double cpufactor, int maxcpus, double maxmem,
double maxswap, double maxtmp, int ndisks);
/* Maps joblist_query's table columns -- JobId, User, Queue, State,
* SubmitTime, StartTime, FinishTime, MinTasks, MaxTasks, Command -- onto
* `row`.
*
* `jobid_str` is the caller's fmt_jobid(...) result and `state_str` the
* caller's job_state_str(jobs[i]->state) result (joblist_display.h) --
* both passed in rather than recomputed here, keeping this module free of
* test.c's own fmt_jobid()/RPC dependencies. `state_str` is stored
* verbatim (including job_state_str()'s Requirement-11.7 raw-state
* fallback text for an ambiguous RUN+EXIT state, e.g. "RUN(0x38)"), so no
* information the table's State column carries is lost.
*
* Unlike the table (which formats SubmitTime/StartTime/FinishTime as
* human-readable text via format_job_time(), Requirement 11.5), this
* emits their RAW epoch-second values as JSON numbers under keys
* "submit_time"/"start_time"/"finish_time" -- 0 meaning "has not happened
* yet", the same convention format_job_time() itself uses for its "-"
* display -- the same underlying value the table's formatted string
* represents, directly script-consumable without re-parsing a date
* string.
*
* No-op if `row` is NULL. */
void joblist_json_row(cJSON* row, const char* jobid_str, const char* user,
const char* queue, const char* state_str,
time_t submit_time, time_t start_time,
time_t finish_time, uint32_t mintasks,
uint32_t maxtasks, const char* cmdline);

39
src/harness/include/rcfile.h Normal file
View file

@ -0,0 +1,39 @@
#pragma once
/*
* rcfile.h -- xce_test startup rc-file path resolution and line-loading
* (xce-test-rc-file spec, tasks 1.1/1.2).
*
* A standalone, dependency-light module following the same pattern as
* configcompare.c/h -- no dependency on test.c internals, linked into both
* the xce_test executable and controller_unit_test.
*
* Task 1.1 implemented path resolution (rcfile_resolve_path()). Task 1.2
* adds line-loading (rcfile_load()) with a pluggable runner callback.
* Wiring rcfile_load() into xce_test's own startup/main() is a later task
* in the same spec (2.2) and is out of scope here -- this module has no
* knowledge of xce_test commands, contexts, or client state.
*/
/* Resolves the rc-file path per the XDG Base Directory Specification:
* "$XDG_CONFIG_HOME/xce_test/rc" when XDG_CONFIG_HOME is set and non-empty,
* else "$HOME/.config/xce_test/rc" when HOME is set and non-empty. Returns
* a heap string the caller must free(), or NULL if neither variable yields
* a usable base directory. */
char* rcfile_resolve_path(void);
/* Opens `path` and calls `runner(line)` once per line (trailing newline
* stripped), stopping early if `runner` returns nonzero. Lines are read via
* getline() with no length cap -- a line is never truncated or split
* regardless of length.
*
* Returns:
* 0 - path is NULL, the file does not exist (ENOENT),
* the file could not be opened for another reason
* (a warning is printed to stderr; non-fatal), or
* every line ran and every runner call returned 0.
* nonzero (== runner's) - the runner requested an abort; no further lines
* are read.
*
* Preconditions: `runner` is non-NULL when `path` is non-NULL.
* Postconditions: the file handle is always closed before returning. */
int rcfile_load(const char* path, int (*runner)(char* line));

53
src/harness/include/resval_typeresolve.h Normal file
View file

@ -0,0 +1,53 @@
#ifndef RESVAL_TYPERESOLVE_H
#define RESVAL_TYPERESOLVE_H
/* resval_typeresolve.h -- resource-value-verification task 1.1
*
* Extracted from test.c's set_resval(): pure resource-name -> wire-type
* resolution logic, with NO networking, REPL, or global-state dependency
* (unlike set_resval() itself, which also allocates xce_val_t and tags
* .host via get_setting("myname") -- that stays in test.c since it needs
* REPL state). This isolation is what makes the logic unit-testable outside
* of test.c, which cannot be white-box #include'd into a Unity test (4298
* lines, its own main(), heavy libedit/network dependencies).
*
* The resolved type is a Value_u_which tag (controller.capnp.h) -- e.g.
* Value_u_fvalue for a numeric resource, Value_u_svalue for a string one.
*/
#include "controller.capnp.h" /* enum Value_u_which */
/* Sentinel: not a valid Value_u_which value. Used both as:
* - the RETURN value when `name` is not present in the known-resource
* table and no override applies;
* - the `type_override` INPUT value meaning "no override requested".
*/
#define RESVAL_TYPE_UNKNOWN (-1)
/* Resolve `name` to its Value_u_which wire type.
*
* `type_override`: pass a valid Value_u_which value (Value_u_fvalue,
* Value_u_svalue, ...) to request that type for a name not in the known-
* resource table, or RESVAL_TYPE_UNKNOWN for no override.
*
* Returns:
* - the table's type, for any of the 14 currently-known resource names
* (r15s, r1m, slots, ut, mem, maxcpus, ndisks, cpuf, maxswap, maxtmp,
* type, model, verilog, dyna) -- `type_override` is ignored for a known
* name, exactly as set_resval() behaves today.
* - `type_override`, for any name NOT in the table, when `type_override`
* is not RESVAL_TYPE_UNKNOWN (Requirement 1.1's `-t` escape hatch).
* - Value_u_fvalue (numeric), for any name NOT in the table when no
* override was requested (Requirement 1.1's default fallback).
* - RESVAL_TYPE_UNKNOWN, only for `name` == NULL.
*
* *** TASK 2.1 STATUS (fallback implemented) ***
* Task 1.1 was a byte-for-byte extraction of set_resval()'s existing
* resinfo[] table lookup, with the unrecognized-name fallback left
* unimplemented (RED tests in test_resval_typeresolve.c). Task 2.1 adds the
* numeric-by-default fallback and `type_override` handling described above
* -- behavior for all 14 known names remains byte-for-byte unchanged.
*/
int resval_resolve_type(const char* name, int type_override);
#endif /* RESVAL_TYPERESOLVE_H */

43
src/harness/include/snapdiff.h Normal file
View file

@ -0,0 +1,43 @@
#pragma once
/*
* snapdiff.h — byte-level comparator for two checkpoint files
* (bug-1389 triage: OPCode_snapshotTransfer chunk-content corruption).
*
* Compares a locally-generated source checkpoint (snap-*.capnp) against a
* received one (snap-*.bin, ha_handle_snapshot_chunk) byte-for-byte, and
* reports the first divergent offset plus whether it lands exactly on an
* OPCode_snapshotTransfer chunk boundary (SNAPSHOT_CHUNK_SIZE, config_repl.c)
* — the decisive signal separating "chunk transport corrupted this specific
* chunk" from "the two files were never going to match" (different content
* entirely, unrelated to chunking).
*
* Strictly read-only: opens both files O_RDONLY, mutates nothing.
*/
#include <stdio.h>
#include <sys/types.h>
/* Mirrors config_repl.c's SNAPSHOT_CHUNK_SIZE (the OPCode_snapshotTransfer
* wire chunk size) — kept in sync manually; both are compile-time constants
* for the same wire protocol, not expected to change independently. */
#define SNAPDIFF_CHUNK_SIZE 65536
typedef struct {
off_t size1;
off_t size2;
int size_mismatch; /* 1 if size1 != size2 (diff_offset unset in that case) */
int identical; /* 1 if same size and byte-for-byte identical */
off_t diff_offset; /* -1 if identical or size_mismatch; else first
* differing byte offset */
int at_chunk_boundary; /* 1 if diff_offset is the first byte of a
* SNAPDIFF_CHUNK_SIZE-aligned chunk, i.e.
* diff_offset % SNAPDIFF_CHUNK_SIZE == 0 */
off_t chunk_index; /* diff_offset / SNAPDIFF_CHUNK_SIZE, valid only
* when diff_offset >= 0 */
} snapdiff_report_t;
/* Compare `path1` (source of truth) against `path2` (candidate, e.g. a
* received copy). Prints a human-readable report to `out` (may be NULL) and
* fills `*rep` (may be NULL). Returns 0 if identical, 1 if they differ (see
* `*rep` for detail), or a negative errno on open/fstat failure. */
int snapdiff_paths(const char* path1, const char* path2, FILE* out,
snapdiff_report_t* rep);

78
src/harness/include/snapverify.h Normal file
View file

@ -0,0 +1,78 @@
#pragma once
/*
* snapverify.h — read-only checkpoint-file structural inspector/validator
* (bug-1389 triage: OPCode_snapshotTransfer chunk-content corruption;
* extended by config-upgrade-snapshot task 8.4, Requirement 4.2, for the
* SNAPSHOT_FORMAT_VERSION=2 host + config-bytes sections and to decode --
* not merely report the presence of -- the header's cfgref tag/hash).
*
* Parses the on-disk checkpoint layout persist.c's load_snapshot() /
* load_snapshot_entities() use (design.md "Checkpoint File Layout") WITHOUT
* restoring any job/queue/host into a live cluster_t — a pure structural
* walk, so it needs no jobmgr/qmgr/workermgr context and is safe to run
* standalone against a snap-*.capnp (locally generated) or snap-*.bin
* (received via ha_handle_snapshot_chunk, ha.c) file on disk. This tool
* validates STRUCTURE only, never content: job/queue/host frame payloads
* and the config-bytes section are walked past via their length prefix,
* never capnp-decoded or interpreted.
*
* Strictly read-only: opens O_RDONLY, mutates nothing.
*/
#include <stdint.h>
#include <stdio.h>
#include <sys/types.h>
typedef struct {
uint64_t checkpoint_lsn;
int has_cfgref; /* config-ref tail present in the header
* (header[16] != 0) -- semantic presence,
* distinct from cfgref_tag/cfgref_sha below
* being decoded */
uint64_t cfgref_tag; /* header cfgref tag, decoded whenever the
* header extends to SNAP_HEADER_MAX bytes,
* REGARDLESS of has_cfgref -- meaningful
* only when has_cfgref is also 1 */
uint8_t cfgref_sha[32]; /* header cfgref sha256, same decode-
* unconditionally-but-gate-meaning-on-
* has_cfgref discipline as cfgref_tag */
int has_entity_section; /* fmt + job_count present past the header;
* 0 = legacy position-only checkpoint */
uint32_t format_version;
uint32_t job_count; /* as declared in the file (0 if legacy) */
uint32_t jobs_ok; /* job frames structurally walked before any failure */
uint32_t queue_count; /* as declared in the file (0 if legacy, or if the
* job-frame walk failed before reaching it) */
uint32_t queues_ok; /* queue frames structurally walked before any failure */
uint32_t host_count; /* as declared in the file (0 if legacy, or if an
* earlier section's walk failed before reaching
* it) */
uint32_t hosts_ok; /* host frames structurally walked before any failure */
int has_config_bytes_section; /* 1 iff the trailing config-bytes
* section's 4-byte length field
* itself was present and readable
* (i.e. the file extends past the
* host section at all) -- TRUE even
* when that length decodes to 0, and
* FALSE (not a failure) when the
* file simply ends right after the
* host section */
uint32_t config_bytes_len; /* decoded length of the config-bytes
* section; valid only when
* has_config_bytes_section is 1 */
off_t file_size;
off_t consumed; /* bytes structurally accounted for */
int ok; /* 1 = well-formed, consumed == file_size */
off_t fail_offset; /* -1 if ok; else the byte offset of the field that
* failed the structural check */
char fail_reason[160];
} snapverify_report_t;
/* Parse+validate `path` (a snap-*.capnp or snap-*.bin checkpoint file).
* Prints a human-readable report to `out` (may be NULL to suppress) and
* fills `*rep` (may be NULL). Returns 0 if the file is structurally
* well-formed to EOF, or the same rc class persist_load_snapshot() would
* return for the equivalent defect (-EINVAL bad magic / unrecognized
* format_version, -EIO short read or a length field that doesn't fit the
* remaining bytes, -errno on open/fstat failure) — so a "persist_load_snapshot
* failed rc=..." log line correlates directly against this tool's verdict. */
int snapverify_path(const char* path, FILE* out, snapverify_report_t* rep);

105
src/harness/include/walcompare.h Normal file
View file

@ -0,0 +1,105 @@
#pragma once
/*
* walcompare.h — cross-member WAL committed-prefix divergence detector
* (ha-term-safety R6.1/R6.2/R6.3).
*
* Reuses the canonical waldump.c WAL decoder (via waldump_path_visit) to index
* each member's log by `(lsn -> term, pcrc)`, then asserts that every member's
* COMMITTED prefix is identical by `(term, pcrc)` — not merely by LSN range.
*
* The decisive signal for bug-851: two members can report the same committed
* LSN over a contiguous, CRC-intact WAL yet hold DIFFERENT entries at the same
* LSN (same lsn, different term and/or payload). Comparing LSN ranges alone
* masks that; comparing `(term, pcrc)` per LSN surfaces it (R6.2).
*
* Strictly read-only — opens member WAL dirs O_RDONLY, mutates nothing. Safe to
* run against a live cluster's data dirs.
*/
#include <stdint.h>
#include <stdio.h>
/* A single member's WAL source and its committed boundary. */
typedef struct {
const char* path; /* WAL dir / data dir / seg-*.wal (waldump rules) */
uint64_t committed_lsn; /* highest committed LSN for this member; 0 =
* "derive" (use the shared boundary, see below) */
} walcompare_member_t;
/* Identity of the first divergent entry, filled on a WALCMP_DIVERGENT result. */
typedef struct {
uint64_t lsn; /* the diverging LSN within the committed prefix */
int n_members; /* members compared at this LSN */
uint64_t member_term[16]; /* per-member matched term at lsn */
uint32_t member_pcrc[16]; /* per-member payload CRC at lsn */
int member_present[16]; /* 1 if the member holds this LSN, else 0 */
int committed; /* 1 = divergence within the committed prefix
* (lsn <= committed boundary, a safety
* violation); 0 = uncommitted tail divergence */
} walcompare_divergence_t;
/* Result codes. */
enum {
WALCMP_OK = 0, /* all members' committed prefixes are identical */
WALCMP_DIVERGENT = 1, /* a committed-prefix divergence was found */
/* negative values are -errno (a load/usage error: missing WAL, bad args) */
};
/* Compare `n` members' committed WAL prefixes by `(lsn -> term, pcrc)`.
*
* Committed boundary: each member with `committed_lsn == 0` adopts the SHARED
* boundary = the minimum recovered tip (highest CRC-intact LSN) across all
* members, i.e. the largest prefix every member can be expected to hold. A
* member with an explicit `committed_lsn` uses that. An LSN in `[1..boundary]`
* is compared only against the members whose own boundary covers it; a member
* that is legitimately AHEAD (longer log) or behind the shared boundary is not
* flagged for the uncommitted tail it doesn't share (R6.1 is the committed
* prefix, not the full log).
*
* Per-member checkpoint floor (raft-ha-core-safety task 5.10a / ha-entity-
* snapshots task 7.1): a member that caught up via a full-checkpoint snapshot
* legitimately holds no raw WAL entries at or below its own installed
* checkpoint LSN — that range was restored via entity content, not WAL
* replay. Each member's floor is read independently from its own `snap/`
* sibling directory (0 if none installed, the unchanged pre-existing
* behavior); an LSN below a member's own floor is excluded from that
* member's comparison the same way an LSN above its committed_lsn already
* is — never flagged MISSING/divergent. This is a PER-MEMBER floor, never a
* shared/uniform value (unlike committed_lsn, which the caller may force to
* one shared value across all members).
*
* Legacy-zero term (R1.8) is mapped to WAL_LEGACY_TERM for matching, so two
* pre-feature entries are not spuriously divergent.
*
* On the FIRST divergent LSN, fills `*div` (when non-NULL) and prints a report
* to `out`, then returns WALCMP_DIVERGENT. Returns WALCMP_OK on identical
* committed prefixes, or a negative errno on a load/usage error. `out` may be
* NULL to suppress the report.
*/
int walcompare(const walcompare_member_t* members, int n, FILE* out,
walcompare_divergence_t* div);
/* Same comparison as walcompare(), but ALSO emits a machine-checkable,
* per-member report over the full log: one
*
* WALCMP member=<i> lsn=<L> term=<T> pcrc=<8hex> status=<present|MISSING|beyond> committed=<0|1>
*
* line per (member, LSN) for every LSN in [1..max-recovered-tip], where
* `committed=1` iff the LSN is within the committed boundary. Each divergence
* additionally emits a
*
* WALCMP_DIVERGENCE lsn=<L> class=<committed|uncommitted> compared=<c> present=<p>
*
* classification line. A `committed` divergence is fatal (verdict
* WALCMP_DIVERGENT, `div->committed = 1`); an `uncommitted`-tail divergence
* (above the committed boundary) is reported but NON-fatal — the verdict stays
* WALCMP_OK when the committed prefix is identical. Field order is fixed and
* the tokens are stable so the harness can grep them. `out` must be non-NULL.
*
* Backward-compatible: walcompare() keeps its original concise output (the
* `== walcompare ==` header, the human divergence block, and the final
* `walcompare: OK` verdict line) and additionally emits the machine
* classification at a committed divergence; only walcompare_report() prints the
* full per-LSN table and scans the uncommitted tail.
*/
int walcompare_report(const walcompare_member_t* members, int n, FILE* out,
walcompare_divergence_t* div);

76
src/harness/include/waldump.h Normal file
View file

@ -0,0 +1,76 @@
#pragma once
/*
* waldump.h — read-only WAL segment inspector for failover/durability triage.
*
* Decodes the on-disk WAL (`wal_entry_hdr_t` + payload, see wal_types.h),
* lists every physical entry, decodes JOB_SUBMIT jobids via the Cap'n Proto
* Job schema, and — the decisive signal for the committed-job-loss bug —
* flags LSN gaps (a missing prefix, or any non-contiguous hole) and reports
* the CRC-chain "recovered tip" a real recovery would stop at.
*
* Strictly read-only: opens segments O_RDONLY, applies no entries, mutates no
* persist_t state. Safe to run against a live node's data dir.
*/
#include <stdint.h>
#include <stdio.h>
/* Aggregate result of scanning one path (a seg file, a wal/ dir, or a node
* data dir). All counts are over the physically-present, CRC-valid entries. */
typedef struct {
uint64_t first_lsn; /* lowest LSN seen (0 = no entries) */
uint64_t last_lsn; /* highest LSN seen */
uint64_t recovered_tip; /* highest LSN reachable via an intact CRC chain */
uint32_t n_entries; /* total entries scanned */
uint32_t n_submits; /* WAL_JOB_SUBMIT entries */
uint32_t n_gaps; /* LSN discontinuities (missing prefix counts as 1) */
uint32_t n_segments; /* segment files scanned */
uint32_t n_dup_submits; /* JOB_SUBMITs with same jobid as the immediately-
* preceding entry at an adjacent LSN (bug-845) */
uint32_t n_dup_identical; /* of n_dup_submits, those whose payload is
* byte-identical to the prior entry */
int crc_broken; /* nonzero if a CRC/chain break truncated a segment */
int leading_gap_explained; /* nonzero if the leading (prefix) gap's
* upper LSN is covered by a sibling
* data/snap/snap-<hex>.capnp file — i.e.
* normal checkpoint-triggered WAL trim
* (persist.c trim_wal_segments), not
* genuine loss. 0 if there was no leading
* gap, or no covering snapshot was found. */
} waldump_stats_t;
/* One physically-present, CRC-valid WAL entry, surfaced to a visitor so a
* caller (e.g. walcompare) can index the log by `(lsn -> term, pcrc)` without
* re-implementing the segment/CRC parse. `pcrc` is the payload-only CRC32 (0
* for an empty payload) — the same fingerprint waldump prints, isolating the
* blob from the header so a same-LSN content divergence is detectable. */
typedef struct {
uint64_t lsn;
uint64_t term; /* raw on-disk term (legacy 0 kept verbatim) */
uint32_t type; /* WAL entry type (wal_types.h) */
uint32_t payload_len; /* payload byte count */
uint32_t pcrc; /* CRC32 of payload only (0 if payload_len == 0) */
} waldump_entry_t;
/* Per-entry visitor. Invoked once per CRC-valid entry in ascending scan order
* (before the gap line is printed). Return 0 to continue, non-zero to stop the
* scan early. `ctx` is the caller's opaque pointer. */
typedef int (*waldump_visit_fn)(void* ctx, const waldump_entry_t* e);
/* Dump every WAL entry under `path` to `out` (human-readable) and fill
* `*stats` when non-NULL. `path` may be:
* - a single seg-*.wal file,
* - a directory holding seg-*.wal files (a wal/ dir), or
* - a node/data dir whose wal/ subdir (data/wal or wal) holds the segments.
* Returns 0 on success, or a negative errno on a hard I/O error (e.g. the
* path does not exist / no segments found). Gaps and CRC breaks are reported
* through `out`/`stats`, NOT via the return code. */
int waldump_path(const char* path, FILE* out, waldump_stats_t* stats);
/* Same as waldump_path, but also invokes `visit(ctx, &entry)` for every
* CRC-valid entry so a caller can collect the `(lsn -> term, pcrc)` index.
* `visit` may be NULL (then identical to waldump_path). `out` may be a sink
* the caller does not care about (e.g. a /dev/null FILE) — the human dump is
* still produced. The single canonical WAL decoder is reused; no second
* segment parser. */
int waldump_path_visit(const char* path, FILE* out, waldump_stats_t* stats,
waldump_visit_fn visit, void* ctx);

84
src/harness/include/xcetest_interrupt.h Normal file
View file

@ -0,0 +1,84 @@
#pragma once
/*
* xcetest_interrupt.h -- tiny, storage-detail-hiding surface over xce_test's
* process-wide SIGINT handling (xce-test-cli-ux spec, task 1.3;
* Requirement 9.9).
*
* ROOT CAUSE this task fixes: test.c's main() used to install its SIGINT
* handler via the bare libc call `signal(SIGINT, sigbreak);`. glibc's
* signal() uses BSD semantics, which install the handler with SA_RESTART
* SET by default -- a slow syscall (read(), etc.) that is interrupted by
* the signal is transparently RESTARTED by the kernel instead of
* returning EINTR to the caller. That silently defeats Requirement 9.9:
* a later task's confirmation-prompt read (confirm.c) needs a real Ctrl-C
* to make its blocking read() return immediately with EINTR, not keep
* waiting for input as if nothing happened. See research.md's "Decision:
* Confirmation prompt's Ctrl-C-as-decline requires SA_RESTART cleared on
* SIGINT" for the full rationale and alternatives considered.
*
* THE FIX: xcetest_install_sigint() below installs the given handler via
* sigaction() with sa_flags = 0 (i.e. explicitly WITHOUT SA_RESTART) and
* an empty sa_mask. test.c's main() now calls
* `xcetest_install_sigint(sigbreak);` instead of the old signal() call.
*
* WHY THIS LIVES IN A HEADER, AS `static inline`, RATHER THAN AS A
* NORMAL EXPORTED FUNCTION DEFINED IN test.c: test.c pulls in a large
* dependency graph (libedit, cJSON, libxce, Cap'n Proto, OpenSSL) that
* this project's Unity-based `controller_unit_test` binary does not
* link, and test.c itself defines main() -- both make it impractical to
* link test.c into a small, fast, signal-level unit test. Defining the
* installer here means test.c's main() and this task's test
* (test_xcetest_interrupt.c) call the EXACT SAME code, not two
* hand-copies that could silently drift apart -- the test genuinely
* proves what main() actually installs.
*
* xcetest_interrupted() is a small read-only accessor over test.c's
* PRIVATE `static int break_out` flag (still declared/defined in test.c,
* unmodified in shape or storage -- only this accessor is new), so code
* outside test.c -- the Interruptible Call Adapter, `interruptcall.c`,
* task 1.4 -- can poll whether a SIGINT has been observed without
* depending on break_out's storage details.
*
* CONTRACT (task 1.4's implementer: read this before use):
* - Returns nonzero once sigbreak() has run (i.e. a SIGINT was
* delivered) since break_out was last 0; returns 0 otherwise.
* - This is a PURE READ -- calling it does NOT clear break_out as a
* side effect. It is safe to call any number of times from a polling
* loop (e.g. "poll break_out every fifty milliseconds", per
* design.md's Interruptible Call Adapter) without consuming the
* interrupt.
* - No reset/clear accessor is exposed yet. test.c itself already
* resets break_out to 0 at one existing call site (cmd_hello's
* repeat loop, immediately before that loop begins) by writing the
* static directly, since it has that access. If task 1.4's design
* needs an interruptible operation to start from a guaranteed-clear
* flag and it runs from OUTSIDE test.c, it will need a new
* xcetest_clear_interrupted()-style accessor -- none exists yet
* because no task through 1.3 needed one; do not assume one is
* silently available.
*/
#include <signal.h>
#include <string.h>
/* Installs `handler` as the process's SIGINT disposition via sigaction(),
* with sa_flags = 0 (i.e. explicitly WITHOUT SA_RESTART) and an empty
* sa_mask (no other signal is additionally blocked while `handler`
* runs). This is the one and only installation path for xce_test's
* SIGINT handling -- see the file header above for why a blocking slow
* syscall must observe EINTR rather than a silent restart. */
static inline void xcetest_install_sigint(void (*handler)(int))
{
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
sa.sa_handler = handler;
sa.sa_flags = 0; /* explicitly WITHOUT SA_RESTART */
sigemptyset(&sa.sa_mask);
sigaction(SIGINT, &sa, NULL);
}
/* See "CONTRACT" in the file header above. Implemented in test.c, over
* its private `static int break_out`. */
int xcetest_interrupted(void);

354
src/harness/interruptcall.c Normal file
View file

@ -0,0 +1,354 @@
#include "interruptcall.h"
#include <assert.h>
#include <errno.h>
#include <pthread.h>
#include <semaphore.h>
#include <stdlib.h>
#include <time.h>
#include "xcetest_interrupt.h"
/* design.md's Interruptible Call Adapter component: poll interval is a
* fixed 50ms, per the component's own Implementation Notes ("poll
* break_out every fifty milliseconds"). Fast enough that an operator's
* Ctrl-C feels immediate; slow enough not to spin the calling thread. */
#define XCETEST_POLL_INTERVAL_NS (50L * 1000L * 1000L) /* 50ms */
/* The heap-allocated call block backing one xcetest_run_interruptible()
* call. `state` is the single atomic word whose CAS transitions decide
* who frees this block -- see interruptcall.h's xcetest_call_state_t
* comment and xcetest_run_interruptible()'s own header comment for the
* full protocol. `lock` is the per-connection lock (task 1.5) this call
* acquired before spawning its worker; whichever side -- caller or
* worker -- ends up owning the block also ends up owning the job of
* releasing `lock`, per the same protocol. It is a sem_t*, not a
* pthread_mutex_t* -- see interruptcall.h's "WHY THE LOCK IS A POSIX
* SEMAPHORE" section for why: the release below is a genuine cross-
* thread handoff, which sem_t (unlike pthread_mutex_t, of any type)
* defines cleanly. */
typedef struct {
xcetest_blocking_fn fn;
void* ctx;
int rc;
sem_t* lock;
_Atomic xcetest_call_state_t state;
} xcetest_call_block_t;
#ifdef XCE_TESTING
_Atomic int xcetest_test_abandoned_free_count = 0;
#endif
/* --- Per-connection lock registry (task 1.5) ---
*
* See interruptcall.h's "PER-CONNECTION LOCK REGISTRY" and "WHY THE LOCK
* IS A POSIX SEMAPHORE" file-header sections for the full rationale,
* including why this is a singly-linked list (pointer stability for a
* sem_t* a still-running abandoned worker may hold onto indefinitely)
* rather than a growable array, and why each entry is a sem_t rather
* than a pthread_mutex_t.
*/
typedef struct xcetest_conn_lock_node {
void* key;
sem_t sem;
struct xcetest_conn_lock_node* next;
} xcetest_conn_lock_node_t;
/* Only ever read/written from the single REPL thread that issues
* top-level commands -- see interruptcall.h's "CONCURRENCY OF THE
* REGISTRY ITSELF" note. Neither variable needs its own lock. */
static xcetest_conn_lock_node_t* g_conn_lock_registry = NULL;
static void* g_active_connection = NULL;
void xcetest_set_active_connection(void* conn_key)
{
g_active_connection = conn_key;
}
/* Looks up (by pointer equality) or lazily creates the lock entry for
* `key` in the process-lifetime registry. Returns NULL only if the new
* node's allocation fails. Nodes are prepended to the list and NEVER
* freed or moved for the rest of the process's life, so a pointer this
* function has already returned to a caller stays valid forever,
* regardless of how many later, unrelated connections are subsequently
* registered. */
static sem_t* xcetest_conn_lock_for(void* key)
{
xcetest_conn_lock_node_t* node;
for (node = g_conn_lock_registry; node; node = node->next) {
if (node->key == key)
return &node->sem;
}
node = malloc(sizeof(*node));
if (!node)
return NULL;
node->key = key;
/* A binary semaphore (initial value 1), used as a mutual-exclusion
* lock, NOT a pthread_mutex_t: this design's own ownership-handoff
* protocol (see xcetest_run_interruptible()'s and
* xcetest_worker_main()'s comments) REQUIRES a cross-thread release
* on the abandonment path -- the calling thread acquires the lock,
* but the still-running WORKER thread is the one that releases it
* once fn() finishes, since ownership of "release the lock" passed
* to the worker at the same moment ownership of the call block did.
* A pthread_mutex_t (ERRORCHECK/_RECURSIVE, or even the DEFAULT/
* NORMAL type) is the wrong primitive for this: POSIX classifies an
* unlock from a non-owning thread as undefined behavior for every
* mutex type -- ERRORCHECK/_RECURSIVE are simply *required* to
* detect and report it (which is why an earlier version of this code
* hit an immediate EPERM here), while NORMAL/DEFAULT are merely
* *permitted* not to detect it, not declared well-defined for it.
* sem_t has no ownership concept at all -- sem_wait()/sem_trywait()/
* sem_post() are fully defined regardless of which thread calls
* which -- so it is the textbook-correct, portable primitive for
* this exact cross-thread signal, with no UB reliance whatsoever.
* pshared=0 since this registry is intra-process only. */
sem_init(&node->sem, 0, 1);
node->next = g_conn_lock_registry;
g_conn_lock_registry = node;
return &node->sem;
}
/* Runs on the detached worker thread. Invokes fn(ctx), stores its result,
* then resolves ownership of `block` via the RUNNING -> DONE_UNCLAIMED
* CAS: on success, the caller (still polling, or about to read the result
* of its own successful poll) owns the block from here and this function
* must not touch it again. On failure, the caller has already forced
* RUNNING -> ABANDONED and returned -EINTR without touching the block
* again -- this thread now owns it and frees it. */
static void* xcetest_worker_main(void* arg)
{
xcetest_call_block_t* block = (xcetest_call_block_t*)arg;
xcetest_call_state_t expected;
sem_t* lock = block->lock;
block->rc = block->fn(block->ctx);
expected = XCT_RUNNING;
if (atomic_compare_exchange_strong(&block->state, &expected,
XCT_DONE_UNCLAIMED)) {
/* Won the CAS: the caller still owns the block AND the
* per-connection lock. Do not touch either again from this
* thread. */
return NULL;
}
/* CAS failed. The caller is the only other party that ever writes
* block->state, and the only value it ever writes is XCT_ABANDONED --
* so `expected` (and the block's actual state) can only be
* XCT_ABANDONED here. The caller returned -EINTR without touching
* the block or the lock again; this thread now owns both. Free the
* block (and record the test-only counter) BEFORE releasing the
* lock, so that any other thread waiting on this same connection's
* lock only ever observes it become available once this cleanup has
* already fully happened. */
free(block);
#ifdef XCE_TESTING
atomic_fetch_add(&xcetest_test_abandoned_free_count, 1);
#endif
{
/* sem_post(), unlike pthread_mutex_unlock(), is well-defined from
* any thread regardless of which thread called sem_wait()/
* sem_trywait() -- see interruptcall.h's "WHY THE LOCK IS A
* POSIX SEMAPHORE" section. This cross-thread release is exactly
* the case that primitive was chosen for. */
int post_rc = sem_post(lock);
(void)post_rc; /* checked via assert() below, never inside it --
* an assert() argument's side effects vanish
* under NDEBUG, which must never apply to the
* post call itself. NOTE: this project's
* `release` build profile compiles with -DNDEBUG
* (unlike asan/debug), so this assert() -- like
* every other one in this file -- compiles out
* entirely there; it is a debug/asan-only safety
* net, not a guarantee in a release-profile
* xce_test binary. */
assert(post_rc == 0);
}
return NULL;
}
/* Sleeps one poll interval. A short nanosleep() legitimately interrupted
* early by a delivered signal (e.g. SIGINT, once xcetest_interrupt.h's
* SA_RESTART-cleared installation is in effect) just means the next
* poll-loop iteration happens slightly sooner than 50ms -- harmless, so
* the return value is deliberately ignored. */
static void xcetest_sleep_poll_interval(void)
{
struct timespec ts;
ts.tv_sec = 0;
ts.tv_nsec = XCETEST_POLL_INTERVAL_NS;
nanosleep(&ts, NULL);
}
int xcetest_run_interruptible(xcetest_blocking_fn fn, void* ctx)
{
xcetest_call_block_t* block;
sem_t* lock;
pthread_attr_t attr;
pthread_t worker;
int rc;
if (!fn)
return -EINVAL;
lock = xcetest_conn_lock_for(g_active_connection);
if (!lock)
return -ENOMEM;
/* Interruptible lock acquisition (task 1.5): poll sem_trywait() on
* the same interval used for the completion-wait poll below,
* checking xcetest_interrupted() between attempts. A plain
* sem_wait() would block uninterruptibly, defeating the whole point
* of letting a second call on this same connection be Ctrl-C'd out
* of waiting for a prior abandoned call to release it.
* sem_trywait() returns -1/EAGAIN when the semaphore is currently
* at 0 (held) -- the direct equivalent of pthread_mutex_trylock()'s
* EBUSY for this poll loop's purposes. If interrupted here, this
* call never acquires the lock and never spawns a worker -- there
* is nothing to release. */
for (;;) {
if (sem_trywait(lock) == 0)
break; /* acquired */
if (errno != EAGAIN)
return -errno; /* unexpected sem_trywait() error; nothing held */
if (xcetest_interrupted())
return -EINTR; /* never acquired; nothing to release */
xcetest_sleep_poll_interval();
}
block = malloc(sizeof(*block));
if (!block) {
rc = sem_post(lock);
(void)rc;
assert(rc == 0);
return -ENOMEM;
}
block->fn = fn;
block->ctx = ctx;
block->rc = 0;
block->lock = lock;
atomic_init(&block->state, XCT_RUNNING);
rc = pthread_attr_init(&attr);
if (rc != 0) {
free(block);
{
/* NOTE: this assert(), like every other one in this file,
* compiles out under -DNDEBUG (this project's `release`
* build profile) -- a documented debug/asan-only safety net,
* not a release-build guarantee. */
int post_rc = sem_post(lock);
(void)post_rc;
assert(post_rc == 0);
}
return -rc;
}
/* Detached: no one ever joins this thread. That is precisely why
* ownership of `block` (and of `lock`) must be resolved by the CAS
* protocol below rather than by a join -- an abandoned caller
* returns without waiting for this thread at all. */
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
rc = pthread_create(&worker, &attr, xcetest_worker_main, block);
pthread_attr_destroy(&attr);
if (rc != 0) {
/* The worker never started, so fn() was never invoked and this
* thread unambiguously still owns the block, and the lock, by
* itself. */
free(block);
{
/* NDEBUG caveat: see the pthread_attr_init() failure branch
* above. */
int post_rc = sem_post(lock);
(void)post_rc;
assert(post_rc == 0);
}
return -rc;
}
for (;;) {
xcetest_call_state_t state = atomic_load(&block->state);
if (state == XCT_DONE_UNCLAIMED) {
int result = block->rc;
free(block);
{
/* NDEBUG caveat: see the pthread_attr_init() failure
* branch above. */
int post_rc = sem_post(lock);
(void)post_rc;
assert(post_rc == 0);
}
return result;
}
if (xcetest_interrupted()) {
xcetest_call_state_t expected = XCT_RUNNING;
if (atomic_compare_exchange_strong(&block->state, &expected,
XCT_ABANDONED)) {
/* Won the CAS: the worker (still running fn()) will see
* ABANDONED via its own failed CAS once fn() returns,
* free the block, and release `lock` itself. Must not
* touch `block` or `lock` again from here. */
return -EINTR;
}
/* CAS failed: the worker is the only other party that ever
* writes block->state, and the only value it ever writes is
* XCT_DONE_UNCLAIMED -- so the block's actual state can only
* be XCT_DONE_UNCLAIMED here. The worker won the race right
* as the interrupt was noticed; fall through (via the loop's
* next iteration) to the normal completion path above
* instead of returning -EINTR, since the call genuinely
* finished. */
continue;
}
xcetest_sleep_poll_interval();
}
}
/* See interruptcall.h for the full contract. Reuses xcetest_conn_lock_for()
* -- the SAME lazy-lookup/create used by xcetest_run_interruptible() above
* -- rather than a parallel registry, so this genuinely observes the same
* per-connection semaphore any in-flight or abandoned worker on `conn_key`
* is holding. */
void xcetest_wait_connection_idle(void* conn_key)
{
sem_t* lock;
int rc;
lock = xcetest_conn_lock_for(conn_key);
if (!lock) {
/* Allocation failure in the lazy-create path: this key has never
* been registered, so by definition no worker could be holding a
* lock for it via this registry. Nothing to wait for. */
return;
}
/* Plain BLOCKING wait, deliberately not the interruptible
* sem_trywait()-poll loop used above -- see interruptcall.h's
* "CONTRACT" section on this function. A signal delivered while
* blocked here can legitimately interrupt sem_wait() with EINTR even
* though nothing else ever posts spuriously; retry rather than treat
* that as "acquired" or bail out early, since either would defeat
* this function's entire purpose (proving the wait genuinely
* completed before the caller proceeds to tear the connection down). */
do {
rc = sem_wait(lock);
} while ((rc != 0) && (errno == EINTR));
assert(rc == 0); /* NDEBUG caveat: see xcetest_worker_main()'s comment. */
/* Immediately release: the semantic is "prove idle right now," not
* "hold the connection locked." */
rc = sem_post(lock);
(void)rc;
assert(rc == 0);
}

99
src/harness/joblist_display.c Normal file
View file

@ -0,0 +1,99 @@
/* joblist_display.c -- xce-test-cli-ux task 2.5 remediation (bug-1711 fixes)
*
* See joblist_display.h for the full rationale. Verbatim extraction of
* test.c's worker_status_str()/job_state_str()/format_job_time() -- no
* behavior change from the pre-extraction versions.
*/
#include "joblist_display.h"
#include <stdio.h> /* snprintf */
#include "ctrlstatus_fmt.h" /* ctrlstatus_format_time_ms */
#include "xce/types.h" /* XCE_WORKER_*, JOB_STAT_* bit definitions */
char* worker_status_str(uint32_t status)
{
/* Admin-set CLOSED takes priority over the dynamic UNAVAIL bit: the
* config-upgrade-snapshot soak (task 8.6) needs to assert a host's
* administratively-set status specifically, and a host closed via
* hostctrl that later disconnects (e.g. its xce_test `worker`/`join`
* connection exits after the script ends) would otherwise print only
* "UNAVAIL", masking the admin bit this check exists to verify. */
if ((status & XCE_WORKER_CLOSED) != 0) {
return "CLOSED";
} else if ((status & XCE_WORKER_UNAVAIL) != 0) {
return "UNAVAIL";
} else if ((status & XCE_WORKER_LOCKEDU) != 0) {
return "LOCKU";
} else if ((status & XCE_WORKER_LOCKEDW) != 0) {
return "LOCKW";
} else if ((status & XCE_WORKER_BUSY) != 0) {
return "BUSY";
} else if (status == 0) {
return "OK";
}
return "UNKNOWN";
}
char* job_state_str(int state)
{
/* bug-1711 (Requirement 11.7): raw-state fallback mirroring
* print_jobinfo's existing state_raw guard against the same ambiguity
* (see print_jobinfo's comment above its "state_raw: 0x%x" print) -- a
* job that is simultaneously RUN and EXIT (e.g. a worker-reported FAIL
* received without a prior FINISH; apply_job_fail_transition ORs in
* EXIT|PERR without clearing RUN) would otherwise silently print as
* plain "RUN" via the if/else-if priority ordering below, hiding the
* EXIT bit. Unlike print_jobinfo (which always has room for a separate
* state_raw line), job_state_str's result is a single cmd_joblist table
* cell, so the raw state is folded into the same string only for this
* specific ambiguous case. */
static char ambiguous_buf[32];
if ((state & JOB_STAT_PEND) != 0) {
if ((state & JOB_STAT_PSUSP) != 0) {
return "PSUSP";
}
return "PEND";
} else if ((state & JOB_STAT_RUN) != 0) {
if ((state & JOB_STAT_USUSP) != 0) {
return "USUSP";
} else if ((state & JOB_STAT_SSUSP) != 0) {
return "SSUSP";
} else if ((state & JOB_STAT_EXIT) != 0) {
snprintf(ambiguous_buf, sizeof(ambiguous_buf), "RUN(0x%x)", state);
return ambiguous_buf;
}
return "RUN";
} else if ((state & JOB_STAT_EXIT) != 0) {
return "EXIT";
} else if ((state & JOB_STAT_DONE) != 0) {
return "DONE";
} else if ((state & JOB_STAT_ERR) != 0) {
return "ERROR";
}
return "UNKNOWN";
}
/* bug-1711 (Requirement 11.5): format a job time_t field (epoch seconds) using
* ctrlstatus_fmt.h's ctrlstatus_format_time_ms human-readable formatter
* (which takes epoch milliseconds), instead of printing the raw epoch-seconds
* value directly as cmd_joblist did before. A job time of 0 means the
* corresponding event has not happened yet (e.g. an unstarted job's
* start_time/finish_time) -- print "-" rather than a misleading 1970-01-01
* epoch date. */
const char* format_job_time(time_t t, char* buf, size_t bufsize)
{
if (t == 0) {
return "-";
}
if (ctrlstatus_format_time_ms((uint64_t)t * 1000u, buf, bufsize) == NULL) {
return "-";
}
return buf;
}

110
src/harness/join_worker_args.c Normal file
View file

@ -0,0 +1,110 @@
/* join_worker_args.c -- xce-test-cli-ux task 2.6 remediation (bug-1557 fix)
*
* See join_worker_args.h for the full extraction rationale. Verbatim
* extraction of test.c's cmd_join_worker() parsing/defaulting block -- no
* behavior change from the pre-extraction version.
*/
#include "join_worker_args.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h> /* getopt_long, optarg, optind */
#include "flagspec.h" /* shared flagspec_t descriptor-table flag parsing +
* uniform rejection (xce-test-cli-ux tasks 1.1/1.2),
* migrated onto by task 3.1 */
/* Every flag's existing short letter, accepted value, and default are
* unchanged by this migration (Requirement 6.3) -- only a long-form
* equivalent and uniform rejection of an unrecognized/malformed flag are
* added. Long names reuse the same concept spellings used elsewhere in
* xce_test for the identical real-world concepts (e.g. "host"/"jobid" in
* test.c's own migrated commands); the ones here (cpufactor, maxcpus,
* maxmem, maxswap, maxtmp, ndisks) are unique to `join` itself. */
const flagspec_t join_flags[] = {
{ 'f', "cpufactor", "cpufactor", required_argument, 0, "1.0" },
{ 'c', "maxcpus", "maxcpus", required_argument, 0, "4" },
{ 'm', "maxmem", "maxmem", required_argument, 0,
"64 * 1024 * 1024 (64 MB in bytes)" },
{ 's', "maxswap", "maxswap", required_argument, 0, "1024" },
{ 't', "maxtmp", "maxtmp", required_argument, 0, "2048" },
{ 'd', "ndisks", "ndisks", required_argument, 0, "1" },
};
/* JOIN_NFLAGS is an internal alias of the header's JOIN_FLAGS_COUNT (a
* compile-time literal, not sizeof()-derived) -- see join_worker_args.h's
* doc comment for why. The array declaration above and the header's
* `extern const flagspec_t join_flags[JOIN_FLAGS_COUNT];` must agree on
* size, so this stays a hard build failure, not a silent drift, if a
* future edit changes the entry count here without updating the header. */
#define JOIN_NFLAGS JOIN_FLAGS_COUNT
int join_worker_parse_attrs(int argc, char* argv[], hostattrs_t* attrs)
{
int opt;
struct option long_opts[JOIN_NFLAGS + 1];
char optstring[16];
if (attrs == NULL) {
return -EINVAL;
}
if (flagspec_build_opts(join_flags, JOIN_NFLAGS, long_opts, optstring,
sizeof(optstring))
< 0) {
fprintf(stderr, "xce_test: join: internal error building flag table\n");
return -EINVAL;
}
/* bug-1557: maxmem is bytes-canonical (see resources.yaml, hardware_
* monitor.c's total_memory) -- this default used to read plain "64"
* back when maxmem was MB-scale (64 MB); an earlier session converted
* the codebase's canonical unit to bytes but missed this test-tool
* default, leaving it at a nonsensical 64 BYTES. Expressed here as
* 64 * 1024 * 1024 (64MB in bytes) to preserve the original intent,
* matching this file's own byte-constant convention (see test.c's
* main() `logsize = 64 * 1024 * 1024`). */
attrs->cpufactor = 1.0;
attrs->maxcpus = 4;
attrs->maxmem = 64 * 1024 * 1024;
attrs->maxswap = 1024;
attrs->maxtmp = 2048;
attrs->ndisks = 1;
attrs->hostmodel = "Intel64";
attrs->hosttype = "LINUX64";
optind = 1;
opterr = 0;
while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL))
!= -1) {
switch (opt) {
case 'f':
attrs->cpufactor = atof(optarg);
break;
case 'c':
attrs->maxcpus = atoi(optarg);
break;
case 'm':
attrs->maxmem = atoi(optarg);
break;
case 's':
attrs->maxswap = atoi(optarg);
break;
case 't':
attrs->maxtmp = atoi(optarg);
break;
case 'd':
attrs->ndisks = atoi(optarg);
break;
case '?':
return flagspec_reject(
"join", join_flags, JOIN_NFLAGS, argv[optind - 1],
"join [-f <cpufactor>] [-c <maxcpus>] [-m <maxmem>] "
"[-s <maxswap>] [-t <maxtmp>] [-d <ndisks>]");
}
}
return 0;
}

59
src/harness/listfmt_json.c Normal file
View file

@ -0,0 +1,59 @@
#include "listfmt_json.h"
#include <stdio.h>
cJSON* listfmt_json_begin(void)
{
return cJSON_CreateArray();
}
void listfmt_json_add_string(cJSON* row, const char* key, const char* val)
{
if (!row || !key)
return;
if (val)
cJSON_AddStringToObject(row, key, val);
else
cJSON_AddNullToObject(row, key);
}
void listfmt_json_add_number(cJSON* row, const char* key, double val)
{
if (!row || !key)
return;
cJSON_AddNumberToObject(row, key, val);
}
cJSON* listfmt_json_new_row(cJSON* array)
{
if (!array)
return NULL;
cJSON* row = cJSON_CreateObject();
if (!row)
return NULL;
if (!cJSON_AddItemToArray(array, row)) {
cJSON_Delete(row);
return NULL;
}
return row;
}
void listfmt_json_finish(cJSON* array)
{
if (!array)
return;
char* text = cJSON_Print(array);
if (text) {
fputs(text, stdout);
fputc('\n', stdout);
cJSON_free(text);
}
cJSON_Delete(array);
}

62
src/harness/listrow_json.c Normal file
View file

@ -0,0 +1,62 @@
/* listrow_json.c -- xce-test-cli-ux task 3.4 (Requirements 5.1, 5.2, 5.3)
*
* See listrow_json.h for the full rationale. Pure field-mapping only: each
* function below just calls listfmt_json_add_string()/add_number() (task
* 2.2, unmodified) once per table column.
*/
#include "listrow_json.h"
#include "listfmt_json.h"
void reslist_json_row(cJSON* row, const char* name, int flags, int interval,
const char* desc)
{
if (!row)
return;
listfmt_json_add_string(row, "name", name);
listfmt_json_add_number(row, "flags", flags);
listfmt_json_add_number(row, "interval", interval);
listfmt_json_add_string(row, "description", desc);
}
void workerlist_json_row(cJSON* row, const char* name, const char* status_str,
const char* hosttype, const char* hostmodel,
double cpufactor, int maxcpus, double maxmem,
double maxswap, double maxtmp, int ndisks)
{
if (!row)
return;
listfmt_json_add_string(row, "hostname", name);
listfmt_json_add_string(row, "status", status_str);
listfmt_json_add_string(row, "hosttype", hosttype);
listfmt_json_add_string(row, "hostmodel", hostmodel);
listfmt_json_add_number(row, "cpufactor", cpufactor);
listfmt_json_add_number(row, "maxcpu", maxcpus);
listfmt_json_add_number(row, "maxmem", maxmem);
listfmt_json_add_number(row, "maxswap", maxswap);
listfmt_json_add_number(row, "maxtmp", maxtmp);
listfmt_json_add_number(row, "ndisks", ndisks);
}
void joblist_json_row(cJSON* row, const char* jobid_str, const char* user,
const char* queue, const char* state_str,
time_t submit_time, time_t start_time,
time_t finish_time, uint32_t mintasks,
uint32_t maxtasks, const char* cmdline)
{
if (!row)
return;
listfmt_json_add_string(row, "jobid", jobid_str);
listfmt_json_add_string(row, "user", user);
listfmt_json_add_string(row, "queue", queue);
listfmt_json_add_string(row, "state", state_str);
listfmt_json_add_number(row, "submit_time", (double)submit_time);
listfmt_json_add_number(row, "start_time", (double)start_time);
listfmt_json_add_number(row, "finish_time", (double)finish_time);
listfmt_json_add_number(row, "mintasks", mintasks);
listfmt_json_add_number(row, "maxtasks", maxtasks);
listfmt_json_add_string(row, "command", cmdline);
}

56
src/harness/meson.build Normal file
View file

@ -0,0 +1,56 @@
# TEMPORARY (xce-harness-component task 1.1): '../controller/testsuite' is
# only here so the bare `#include "ctrlstatus_fmt.h"` in test.c/
# joblist_display.c keeps resolving -- ctrlstatus_fmt.h itself still
# physically lives at src/controller/testsuite/ctrlstatus_fmt.h pending
# task 1.2, which promotes it to src/include/xce/ctrlstatus_fmt.h and
# repoints these two #include lines at it. Remove this entry when task 1.2
# lands (it will make it redundant, since src/include is already on this
# path via '../include').
incdir = include_directories('.', '../include', '../lib', 'include',
'../controller/testsuite')
zlib_dep = dependency('zlib')
# BSD-licensed readline replacement (scheduler-core spec, task 12.1 review
# finding): GNU readline is GPL; xce_test's REPL only needs readline()/
# add_history(), both provided by libedit's editline/readline.h emulation
# header under a permissive license. Single dependency, no separate
# ncurses/history split (libedit's own .pc pulls ncurses in privately).
libedit_dep = dependency('libedit')
ssl_dep = dependency('openssl') # also needed by configcompare.c (SHA256())
xce_test_src = [
'test.c',
'waldump.c',
'walcompare.c',
'snapverify.c',
'snapdiff.c',
'configcompare.c',
'resval_typeresolve.c',
'joblist_display.c',
'join_worker_args.c',
'rcfile.c',
'flagspec.c',
'interruptcall.c',
'confirm.c',
'listfmt_json.c',
'listrow_json.c',
]
xce_test_deps = [
libxce_dep,
librtti_dep,
struct2json_dep,
libedit_dep,
tinylog_dep,
zlib_dep, # waldump: crc32 WAL validation
c_capnproto_dep, # waldump: decode JOB_SUBMIT jobid
ssl_dep, # configcompare: SHA256() active-config hashing
thread_dep, # interruptcall.c: xcetest_run_interruptible() detached worker (task 1.4)
]
xce_test = executable('xce_test',
xce_test_src,
include_directories: [incdir],
dependencies: xce_test_deps,
install: true)

80
src/harness/rcfile.c Normal file
View file

@ -0,0 +1,80 @@
#define _GNU_SOURCE
#include "rcfile.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Suffixes appended to the resolved base directory to form the rc-file
* path -- kept as two separate constants (rather than one shared tail)
* since the XDG_CONFIG_HOME branch is already rooted at a "config home"
* directory while the HOME fallback still needs the "/.config" segment. */
#define RCFILE_XDG_SUFFIX "/xce_test/rc"
#define RCFILE_HOME_SUFFIX "/.config/xce_test/rc"
char* rcfile_resolve_path(void)
{
const char* xdg_config_home = getenv("XDG_CONFIG_HOME");
if (xdg_config_home && *xdg_config_home) {
size_t len = strlen(xdg_config_home) + strlen(RCFILE_XDG_SUFFIX) + 1;
char* path = (char*)malloc(len);
if (!path)
return NULL;
snprintf(path, len, "%s%s", xdg_config_home, RCFILE_XDG_SUFFIX);
return path;
}
const char* home = getenv("HOME");
if (home && *home) {
size_t len = strlen(home) + strlen(RCFILE_HOME_SUFFIX) + 1;
char* path = (char*)malloc(len);
if (!path)
return NULL;
snprintf(path, len, "%s%s", home, RCFILE_HOME_SUFFIX);
return path;
}
return NULL;
}
int rcfile_load(const char* path, int (*runner)(char* line))
{
if (!path)
return 0;
FILE* f = fopen(path, "r");
if (!f) {
if (errno == ENOENT)
return 0; /* missing rc file: silent no-op success */
fprintf(stderr, "xce_test: warning: could not open rc file %s: %s\n",
path, strerror(errno));
return 0; /* present but unreadable: non-fatal warning */
}
int rc = 0;
char* line = NULL;
size_t cap = 0;
ssize_t n;
while ((n = getline(&line, &cap, f)) != -1) {
if (n > 0 && line[n - 1] == '\n')
line[n - 1] = '\0';
rc = runner(line);
if (rc != 0)
break;
}
/* fopen() succeeds on a directory on this platform -- the failure only
* surfaces on the first read (getline() returns -1 with errno==EISDIR
* and ferror(f) set, feof(f) NOT set). Treat that the same as an open
* failure: one non-fatal stderr warning, no runner calls attributed to
* it (rc is still 0 here since no line was ever successfully read). */
if (rc == 0 && ferror(f)) {
fprintf(stderr, "xce_test: warning: could not read rc file %s: %s\n",
path, strerror(errno));
}
free(line);
fclose(f);
return rc;
}

57
src/harness/resval_typeresolve.c Normal file
View file

@ -0,0 +1,57 @@
/* resval_typeresolve.c -- resource-value-verification task 1.1
*
* See resval_typeresolve.h for the full rationale and TASK 1.1 STATUS note.
*/
#include <string.h>
#include "resval_typeresolve.h"
int resval_resolve_type(const char* name, int type_override)
{
/* Verbatim extraction of set_resval()'s resinfo[] table (test.c),
* unchanged: type/model are Value_u_svalue; everything else here is
* Value_u_fvalue. */
static const struct {
const char* name;
int valtype;
} resinfo[] = {
{ "r15s", Value_u_fvalue },
{ "r1m", Value_u_fvalue },
{ "slots", Value_u_fvalue },
{ "ut", Value_u_fvalue },
{ "mem", Value_u_fvalue },
{ "maxcpus", Value_u_fvalue },
{ "ndisks", Value_u_fvalue },
{ "cpuf", Value_u_fvalue },
{ "maxswap", Value_u_fvalue },
{ "maxtmp", Value_u_fvalue },
{ "type", Value_u_svalue },
{ "model", Value_u_svalue },
{ "verilog", Value_u_fvalue },
{ "dyna", Value_u_fvalue },
{ NULL, 0 }
};
int i;
if (name == NULL) {
return RESVAL_TYPE_UNKNOWN;
}
for (i = 0; resinfo[i].name != NULL; i++) {
if (strcmp(resinfo[i].name, name) == 0) {
/* Known name: exact existing behavior, type_override ignored
* (Requirement 1.4). */
return resinfo[i].valtype;
}
}
/* task 2.1: unrecognized name -- Requirement 1.1's fallback. Default to
* numeric (Value_u_fvalue) unless the caller passed an explicit
* type_override (any valid Value_u_which tag other than the
* RESVAL_TYPE_UNKNOWN sentinel), in which case honor the override. */
if (type_override != RESVAL_TYPE_UNKNOWN) {
return type_override;
}
return Value_u_fvalue;
}

113
src/harness/snapdiff.c Normal file
View file

@ -0,0 +1,113 @@
#define _GNU_SOURCE
#include "snapdiff.h"
#include <errno.h>
#include <fcntl.h>
#include <stdint.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#define SNAPDIFF_BUF_SIZE 65536
int snapdiff_paths(const char* path1, const char* path2, FILE* out,
snapdiff_report_t* rep)
{
if (!path1 || !path2)
return -EINVAL;
snapdiff_report_t local;
memset(&local, 0, sizeof(local));
local.diff_offset = -1;
int fd1 = open(path1, O_RDONLY);
if (fd1 < 0) {
int saved = errno;
if (out)
fprintf(out, "snapdiff: %s: %s\n", path1, strerror(saved));
return -saved;
}
int fd2 = open(path2, O_RDONLY);
if (fd2 < 0) {
int saved = errno;
close(fd1);
if (out)
fprintf(out, "snapdiff: %s: %s\n", path2, strerror(saved));
return -saved;
}
struct stat st1, st2;
if (fstat(fd1, &st1) != 0 || fstat(fd2, &st2) != 0) {
int saved = errno;
close(fd1);
close(fd2);
if (out)
fprintf(out, "snapdiff: fstat failed: %s\n", strerror(saved));
return -saved;
}
local.size1 = st1.st_size;
local.size2 = st2.st_size;
if (st1.st_size != st2.st_size) {
local.size_mismatch = 1;
close(fd1);
close(fd2);
if (rep)
*rep = local;
if (out)
fprintf(out,
"snapdiff: %s (%jd bytes) vs %s (%jd bytes): SIZE MISMATCH\n",
path1, (intmax_t)local.size1, path2, (intmax_t)local.size2);
return 1;
}
uint8_t buf1[SNAPDIFF_BUF_SIZE];
uint8_t buf2[SNAPDIFF_BUF_SIZE];
off_t off = 0;
int found_diff = 0;
for (;;) {
ssize_t n1 = read(fd1, buf1, sizeof(buf1));
ssize_t n2 = read(fd2, buf2, sizeof(buf2));
if (n1 <= 0 || n2 <= 0)
break;
ssize_t n = (n1 < n2) ? n1 : n2;
for (ssize_t i = 0; i < n; i++) {
if (buf1[i] != buf2[i]) {
local.diff_offset = off + (off_t)i;
found_diff = 1;
break;
}
}
if (found_diff)
break;
off += (off_t)n;
}
close(fd1);
close(fd2);
if (!found_diff) {
local.identical = 1;
if (rep)
*rep = local;
if (out)
fprintf(out, "snapdiff: %s vs %s: IDENTICAL (%jd bytes)\n", path1,
path2, (intmax_t)local.size1);
return 0;
}
local.chunk_index = local.diff_offset / SNAPDIFF_CHUNK_SIZE;
local.at_chunk_boundary = (local.diff_offset % SNAPDIFF_CHUNK_SIZE == 0);
if (rep)
*rep = local;
if (out)
fprintf(out,
"snapdiff: %s vs %s: DIFFER at offset %jd (chunk %jd%s)\n",
path1, path2, (intmax_t)local.diff_offset,
(intmax_t)local.chunk_index,
local.at_chunk_boundary
? " -- exactly at a chunk boundary, first byte of chunk"
: " -- NOT at a chunk boundary, mid-chunk");
return 1;
}

417
src/harness/snapverify.c Normal file
View file

@ -0,0 +1,417 @@
#define _GNU_SOURCE
#include "snapverify.h"
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
/* Mirrors persist.c's SNAPSHOT_FORMAT_VERSION / header layout exactly — see
* persist.c write_snapshot() (~line 1758) and load_snapshot() (~line 2325).
* config-upgrade-snapshot task 1 bumped persist.c's SNAPSHOT_FORMAT_VERSION
* 1->2 (host/queue admin-state frames migrated to capnp, Req 1.9/5.7) --
* this MUST track that bump, or this tool hard-rejects every real
* checkpoint the current codebase writes (task 8.4 finding). */
#define SNAP_FORMAT_VERSION 2u
#define SNAP_HEADER_MAX (16 + 1 + 8 + 32) /* magic+lsn + optional cfgref tail */
static void fail(snapverify_report_t* rep, int rc, off_t off, const char* why)
{
rep->ok = 0;
rep->fail_offset = off;
snprintf(rep->fail_reason, sizeof(rep->fail_reason), "%s", why);
(void)rc;
}
int snapverify_path(const char* path, FILE* out, snapverify_report_t* rep)
{
if (!path)
return -EINVAL;
snapverify_report_t local;
memset(&local, 0, sizeof(local));
local.ok = 1;
local.fail_offset = -1;
int fd = open(path, O_RDONLY);
if (fd < 0) {
int saved = errno;
fail(&local, -saved, -1, strerror(saved));
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s\n", path, strerror(saved));
return -saved;
}
struct stat st;
if (fstat(fd, &st) != 0) {
int saved = errno;
close(fd);
fail(&local, -saved, -1, strerror(saved));
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: fstat: %s\n", path, strerror(saved));
return -saved;
}
local.file_size = st.st_size;
char header[SNAP_HEADER_MAX];
ssize_t n = read(fd, header, sizeof(header));
if (n < 16) {
close(fd);
fail(&local, -EIO, 0, "short read on header (need >= 16 bytes)");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
return -EIO;
}
if (memcmp(header, "XCESNAP\0", 8) != 0) {
close(fd);
fail(&local, -EINVAL, 0, "bad magic (not XCESNAP)");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
return -EINVAL;
}
memcpy(&local.checkpoint_lsn, header + 8, 8);
if (n >= SNAP_HEADER_MAX) {
/* Decode the tag/hash UNCONDITIONALLY whenever the header extends
* this far, regardless of the presence byte -- so a caller can
* inspect the raw bytes even in edge cases. has_cfgref keeps its
* existing meaning (whether those bytes are semantically
* meaningful) untouched. */
memcpy(&local.cfgref_tag, header + 17, 8);
memcpy(local.cfgref_sha, header + 25, 32);
if (header[16] != 0)
local.has_cfgref = 1;
}
off_t consumed = (off_t)n;
if (st.st_size <= consumed) {
/* Legacy, position-only checkpoint — not a defect (design.md). */
close(fd);
local.consumed = consumed;
if (rep)
*rep = local;
if (out)
fprintf(out,
"snapverify: %s: OK (legacy position-only, lsn=%" PRIu64
", size=%jd)\n",
path, local.checkpoint_lsn, (intmax_t)local.file_size);
return 0;
}
uint32_t fmt = 0;
if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) {
close(fd);
fail(&local, -EIO, consumed, "short read on format_version");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
local.fail_reason, (intmax_t)consumed);
return -EIO;
}
consumed += (off_t)sizeof(fmt);
local.has_entity_section = 1;
local.format_version = fmt;
if (fmt != SNAP_FORMAT_VERSION) {
close(fd);
fail(&local, -EINVAL, consumed - (off_t)sizeof(fmt),
"unrecognized format_version");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (got %u, expected %u, offset %jd)\n",
path, local.fail_reason, fmt, SNAP_FORMAT_VERSION,
(intmax_t)local.fail_offset);
return -EINVAL;
}
uint32_t job_count = 0;
if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) {
close(fd);
fail(&local, -EIO, consumed, "short read on job_count");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
local.fail_reason, (intmax_t)consumed);
return -EIO;
}
consumed += (off_t)sizeof(job_count);
local.job_count = job_count;
for (uint32_t i = 0; i < job_count; i++) {
off_t len_off = consumed;
uint32_t plen = 0;
if (read(fd, &plen, sizeof(plen)) != (ssize_t)sizeof(plen)) {
close(fd);
fail(&local, -EIO, len_off, "short read on job payload_len");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: job %u: %s (offset %jd)\n", path,
i, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
consumed += (off_t)sizeof(plen);
if (plen == 0 || (off_t)plen > st.st_size - consumed) {
close(fd);
char why[160];
snprintf(why, sizeof(why),
"bad payload_len=%u (remaining=%jd)", plen,
(intmax_t)(st.st_size - consumed));
fail(&local, -EIO, len_off, why);
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: job %u: %s (length field at "
"offset %jd)\n",
path, i, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
if (lseek(fd, plen, SEEK_CUR) < 0) {
close(fd);
fail(&local, -EIO, consumed, "seek past job payload failed");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: job %u: %s\n", path, i,
local.fail_reason);
return -EIO;
}
consumed += (off_t)plen;
local.jobs_ok++;
}
uint32_t queue_count = 0;
if (read(fd, &queue_count, sizeof(queue_count)) !=
(ssize_t)sizeof(queue_count)) {
close(fd);
fail(&local, -EIO, consumed, "short read on queue_count");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
local.fail_reason, (intmax_t)consumed);
return -EIO;
}
consumed += (off_t)sizeof(queue_count);
local.queue_count = queue_count;
for (uint32_t i = 0; i < queue_count; i++) {
off_t len_off = consumed;
uint32_t plen = 0;
if (read(fd, &plen, sizeof(plen)) != (ssize_t)sizeof(plen)) {
close(fd);
fail(&local, -EIO, len_off, "short read on queue payload_len");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: queue %u: %s (offset %jd)\n",
path, i, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
consumed += (off_t)sizeof(plen);
if (plen == 0 || (off_t)plen > st.st_size - consumed) {
close(fd);
char why[160];
snprintf(why, sizeof(why),
"bad payload_len=%u (remaining=%jd)", plen,
(intmax_t)(st.st_size - consumed));
fail(&local, -EIO, len_off, why);
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: queue %u: %s (length field at "
"offset %jd)\n",
path, i, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
if (lseek(fd, plen, SEEK_CUR) < 0) {
close(fd);
fail(&local, -EIO, consumed, "seek past queue payload failed");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: queue %u: %s\n", path, i,
local.fail_reason);
return -EIO;
}
consumed += (off_t)plen;
local.queues_ok++;
}
/* Host section (config-upgrade-snapshot task 4/8.4, SNAPSHOT_FORMAT_
* VERSION=2): new after the queue section (design.md "Checkpoint File
* Layout", persist.c load_snapshot_entities()/write_snapshot()). Same
* [payload_len][bytes] per-frame structural shape as the job/queue
* loops above -- this tool validates structure only, never decodes the
* capnp payload. */
uint32_t host_count = 0;
if (read(fd, &host_count, sizeof(host_count)) !=
(ssize_t)sizeof(host_count)) {
close(fd);
fail(&local, -EIO, consumed, "short read on host_count");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
local.fail_reason, (intmax_t)consumed);
return -EIO;
}
consumed += (off_t)sizeof(host_count);
local.host_count = host_count;
for (uint32_t i = 0; i < host_count; i++) {
off_t len_off = consumed;
uint32_t plen = 0;
if (read(fd, &plen, sizeof(plen)) != (ssize_t)sizeof(plen)) {
close(fd);
fail(&local, -EIO, len_off, "short read on host payload_len");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: host %u: %s (offset %jd)\n",
path, i, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
consumed += (off_t)sizeof(plen);
if (plen == 0 || (off_t)plen > st.st_size - consumed) {
close(fd);
char why[160];
snprintf(why, sizeof(why),
"bad payload_len=%u (remaining=%jd)", plen,
(intmax_t)(st.st_size - consumed));
fail(&local, -EIO, len_off, why);
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: host %u: %s (length field at "
"offset %jd)\n",
path, i, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
if (lseek(fd, plen, SEEK_CUR) < 0) {
close(fd);
fail(&local, -EIO, consumed, "seek past host payload failed");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: host %u: %s\n", path, i,
local.fail_reason);
return -EIO;
}
consumed += (off_t)plen;
local.hosts_ok++;
}
/* Config-bytes section (config-upgrade-snapshot task 2.3/8.4): a
* single flat length-prefixed blob appended LAST, after the host
* section -- NOT a [count][frame...] shape like job/queue/host above,
* since there is exactly one active configuration, never a set
* (persist.c write_snapshot()). The length field itself may be
* legitimately ABSENT (a file that ends right after the host section
* is still a valid, if slightly older, fmt=2-consistent shape) --
* distinguish that from the length field being present-but-zero
* (persist.c explicitly writes an empty-but-present section when no
* local config bytes exist yet) and from a genuinely truncated length
* field (a short, non-zero partial read -- real corruption). */
uint32_t cfg_bytes_len = 0;
ssize_t cfg_len_n = read(fd, &cfg_bytes_len, sizeof(cfg_bytes_len));
if (cfg_len_n == 0) {
/* Clean EOF right after the host section -- no config-bytes
* section present at all. Not a defect. */
local.has_config_bytes_section = 0;
} else if (cfg_len_n != (ssize_t)sizeof(cfg_bytes_len)) {
close(fd);
fail(&local, -EIO, consumed, "short read on config_bytes_len");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (offset %jd)\n", path,
local.fail_reason, (intmax_t)consumed);
return -EIO;
} else {
off_t len_off = consumed;
consumed += (off_t)sizeof(cfg_bytes_len);
local.has_config_bytes_section = 1;
local.config_bytes_len = cfg_bytes_len;
if ((off_t)cfg_bytes_len > st.st_size - consumed) {
close(fd);
char why[160];
snprintf(why, sizeof(why),
"bad config_bytes_len=%u (remaining=%jd)", cfg_bytes_len,
(intmax_t)(st.st_size - consumed));
fail(&local, -EIO, len_off, why);
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s (length field at offset "
"%jd)\n",
path, local.fail_reason, (intmax_t)len_off);
return -EIO;
}
if (cfg_bytes_len > 0) {
if (lseek(fd, cfg_bytes_len, SEEK_CUR) < 0) {
close(fd);
fail(&local, -EIO, consumed,
"seek past config bytes failed");
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s\n", path,
local.fail_reason);
return -EIO;
}
consumed += (off_t)cfg_bytes_len;
}
}
close(fd);
local.consumed = consumed;
if (consumed != st.st_size) {
char why[160];
snprintf(why, sizeof(why),
"trailing garbage: consumed=%jd but file_size=%jd",
(intmax_t)consumed, (intmax_t)st.st_size);
fail(&local, -EIO, consumed, why);
if (rep)
*rep = local;
if (out)
fprintf(out, "snapverify: %s: %s\n", path, local.fail_reason);
return -EIO;
}
if (rep)
*rep = local;
if (out)
fprintf(out,
"snapverify: %s: OK (lsn=%" PRIu64 " fmt=%u jobs=%u queues=%u "
"hosts=%u cfgref_tag=%016" PRIx64 " cfg_bytes_len=%u "
"has_cfg_bytes=%d size=%jd)\n",
path, local.checkpoint_lsn, local.format_version,
local.job_count, local.queue_count, local.host_count,
local.cfgref_tag, local.config_bytes_len,
local.has_config_bytes_section, (intmax_t)local.file_size);
return 0;
}

6996
src/harness/test.c Normal file
View file

File diff suppressed because it is too large Load diff

515
src/harness/walcompare.c Normal file
View file

@ -0,0 +1,515 @@
#define _GNU_SOURCE
#include "walcompare.h"
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "persist.h" /* WAL_LEGACY_TERM */
#include "waldump.h"
#define WALCMP_MAX_MEMBERS 16
/* One member's collected log, indexed by LSN. `slot[lsn]` is valid when
* `present[lsn]` is set. LSNs are 1-based; index 0 is unused. */
typedef struct {
uint64_t recovered_tip; /* highest CRC-intact LSN held (waldump tip) */
uint64_t committed_lsn; /* effective committed boundary for this member */
/* raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: the
* highest checkpoint LSN this member has installed (0 = never installed
* a snapshot -- expect raw WAL from LSN 1, the pre-existing behavior,
* unchanged). A member that caught up via a full-checkpoint snapshot
* legitimately holds NO raw WAL entries at or below its own floor_lsn --
* that range was restored via entity content, not WAL replay -- so it
* must never be flagged MISSING there. Read once per member from its own
* snap/ directory (scan_snapshot_floor); never uniform across members
* (unlike committed_lsn, which the caller may force to one shared value). */
uint64_t floor_lsn;
uint64_t cap; /* allocated index size (LSN range [0..cap-1]) */
uint64_t* term; /* raw on-disk term per LSN (0 = legacy / absent) */
uint32_t* pcrc; /* payload CRC per LSN */
uint8_t* present; /* 1 if this LSN is held */
} member_log_t;
/* Visitor context: collect entries into a member_log_t. */
typedef struct {
member_log_t* log;
int oom; /* set on an allocation failure mid-scan */
} collect_ctx_t;
/* Map a raw on-disk term to its matching term: legacy 0 -> WAL_LEGACY_TERM
* (R1.8) so two pre-feature entries are not spuriously divergent. */
static uint64_t match_term(uint64_t raw)
{
return raw == 0 ? WAL_LEGACY_TERM : raw;
}
/* Grow the per-LSN index so `lsn` is addressable. Returns 0 or -ENOMEM. */
static int member_log_ensure(member_log_t* m, uint64_t lsn)
{
if (lsn < m->cap)
return 0;
uint64_t newcap = m->cap ? m->cap : 64;
while (newcap <= lsn)
newcap *= 2;
uint64_t* nt = realloc(m->term, newcap * sizeof(*nt));
uint32_t* np = realloc(m->pcrc, newcap * sizeof(*np));
uint8_t* ne = realloc(m->present, newcap * sizeof(*ne));
if (!nt || !np || !ne) {
/* realloc may have partially succeeded; keep whatever grew so the
* eventual free() is safe, but report OOM. */
if (nt)
m->term = nt;
if (np)
m->pcrc = np;
if (ne)
m->present = ne;
return -ENOMEM;
}
/* Zero the newly added tail. */
memset(nt + m->cap, 0, (newcap - m->cap) * sizeof(*nt));
memset(np + m->cap, 0, (newcap - m->cap) * sizeof(*np));
memset(ne + m->cap, 0, (newcap - m->cap) * sizeof(*ne));
m->term = nt;
m->pcrc = np;
m->present = ne;
m->cap = newcap;
return 0;
}
static int collect_visit(void* ctx, const waldump_entry_t* e)
{
collect_ctx_t* cc = ctx;
member_log_t* m = cc->log;
if (e->lsn == 0)
return 0; /* LSN 0 is never a real committed entry */
if (member_log_ensure(m, e->lsn) != 0) {
cc->oom = 1;
return 1; /* stop the scan */
}
m->term[e->lsn] = e->term;
m->pcrc[e->lsn] = e->pcrc;
m->present[e->lsn] = 1;
return 0;
}
static void member_log_free(member_log_t* m)
{
free(m->term);
free(m->pcrc);
free(m->present);
m->term = NULL;
m->pcrc = NULL;
m->present = NULL;
m->cap = 0;
}
/* Scan one directory for `snap-<16-hex-digit-lsn>.{capnp,bin}` files (the
* on-disk naming write_snapshot()/legacy checkpoints use — persist.c) and
* return the highest embedded LSN found, or 0 if the directory doesn't exist
* or holds no matching file. A bare 16-hex-digit match is required (skips
* `.tmp` in-progress files and anything else) so this never mis-parses an
* unrelated entry as a checkpoint LSN. */
static uint64_t scan_snap_dir_floor(const char* dir)
{
DIR* d = opendir(dir);
if (!d)
return 0; /* no such dir (no snapshot ever installed here) — floor 0 */
uint64_t max_lsn = 0;
struct dirent* de;
while ((de = readdir(d)) != NULL) {
unsigned long long lsn = 0;
char ext[8];
/* "snap-XXXXXXXXXXXXXXXX.capnp" / ".bin" — %16llx would also accept
* fewer than 16 digits, so additionally check the literal width via
* strlen on the numeric run; simplest robust check: require the
* filename to be exactly "snap-" + 16 hex chars + "." + ext. */
size_t len = strlen(de->d_name);
if (len < 5 + 16 + 1 + 3 || strncmp(de->d_name, "snap-", 5) != 0)
continue;
int matched = sscanf(de->d_name + 5, "%16llx.%7s", &lsn, ext);
if (matched != 2)
continue;
if (strcmp(ext, "capnp") != 0 && strcmp(ext, "bin") != 0)
continue;
/* Confirm the hex run is EXACTLY 16 chars (reject a short-digit name
* like "snap-1.capnp" that sscanf's %16llx would still happily take). */
int hexlen = 0;
while (hexlen < 16 && isxdigit((unsigned char)de->d_name[5 + hexlen]))
hexlen++;
if (hexlen != 16 || de->d_name[5 + 16] != '.')
continue;
if ((uint64_t)lsn > max_lsn)
max_lsn = (uint64_t)lsn;
}
closedir(d);
return max_lsn;
}
/* Find this member's own checkpoint floor (raft-ha-core-safety task 5.10a /
* ha-entity-snapshots task 7.1). `src->path` may be either a "data" dir
* containing sibling `wal/`+`snap/` subdirs (the soak harness's convention:
* `$TOP/controller/ctrl-N/data`), or the WAL dir itself passed directly (seen
* in unit-test call sites, e.g. `persist_t.wal_path`) — in the latter case the
* sibling snap dir sits at the same parent level with `wal` replaced by
* `snap`. Try both; a nonexistent candidate just yields floor 0 (harmless). */
static uint64_t member_snapshot_floor(const char* path)
{
if (!path || !*path)
return 0;
char candidate[4160];
uint64_t floor1 = 0, floor2 = 0;
/* Candidate 1: <path>/snap (path is a "data" parent dir). */
snprintf(candidate, sizeof(candidate), "%s/snap", path);
floor1 = scan_snap_dir_floor(candidate);
/* Candidate 2: sibling of <path> with a trailing "/wal" (or "wal")
* component replaced by "/snap" (path is the WAL dir itself). */
size_t plen = strlen(path);
const char* base = strrchr(path, '/');
const char* leaf = base ? base + 1 : path;
if (strcmp(leaf, "wal") == 0) {
size_t prefix_len = (size_t)(leaf - path);
if (prefix_len + 4 < sizeof(candidate)) {
memcpy(candidate, path, prefix_len);
snprintf(candidate + prefix_len, sizeof(candidate) - prefix_len, "snap");
floor2 = scan_snap_dir_floor(candidate);
}
}
(void)plen;
return floor1 > floor2 ? floor1 : floor2;
}
/* Load one member's WAL into `*m`. Returns 0 or a negative errno. */
static int load_member(const walcompare_member_t* src, member_log_t* m, FILE* out)
{
memset(m, 0, sizeof(*m));
waldump_stats_t st;
memset(&st, 0, sizeof(st));
collect_ctx_t cc = { .log = m, .oom = 0 };
/* Send the human dump to the caller's `out` (or a discard sink). The
* visitor harvests the (lsn -> term, pcrc) index from the same decode. */
FILE* dump_sink = out ? out : fopen("/dev/null", "w");
int own_sink = (out == NULL);
if (!dump_sink)
return -errno;
int rc = waldump_path_visit(src->path, dump_sink, &st, collect_visit, &cc);
if (own_sink)
fclose(dump_sink);
if (rc != 0) {
member_log_free(m);
return rc; /* hard load error (missing path / no segments) */
}
if (cc.oom) {
member_log_free(m);
return -ENOMEM;
}
m->recovered_tip = st.recovered_tip;
/* An explicit committed_lsn wins; 0 means "derive from the shared
* boundary" (filled by the caller after all members are loaded). */
m->committed_lsn = src->committed_lsn;
/* Always PER-MEMBER, never uniform/shared (unlike committed_lsn) — a
* member's own checkpoint floor has nothing to do with any other
* member's state. */
m->floor_lsn = member_snapshot_floor(src->path);
return 0;
}
/* Loaded cluster state shared by walcompare() and walcompare_report(). */
typedef struct {
member_log_t logs[WALCMP_MAX_MEMBERS];
int n;
int loaded; /* members successfully loaded (for cleanup) */
uint64_t boundary; /* highest committed LSN compared (committed edge) */
uint64_t max_tip; /* highest recovered tip across members (tail edge) */
} wc_cluster_t;
/* Stable status token for one member at one LSN. */
static const char* wc_status(const member_log_t* m, uint64_t lsn)
{
if ((lsn < m->cap) && m->present[lsn])
return "present";
if (lsn <= m->floor_lsn)
return "compacted"; /* below this member's OWN checkpoint floor —
* legitimately absent (entity-restored, not
* WAL-held), never a divergence signal */
if (lsn <= m->committed_lsn)
return "MISSING"; /* a committed entry this member SHOULD hold (R4.2) */
return "beyond"; /* legitimately not held (above its committed prefix) */
}
/* Load all members and compute the committed boundary + tail edge. On a load
* error, frees what loaded, reports to `out`, returns a negative errno. */
static int wc_load(const walcompare_member_t* members, int n, FILE* out,
wc_cluster_t* cl)
{
memset(cl, 0, sizeof(*cl));
cl->n = n;
for (int i = 0; i < n; i++) {
int rc = load_member(&members[i], &cl->logs[i], out);
if (rc != 0) {
if (out)
fprintf(out, "walcompare: cannot load member %d ('%s'): %s\n", i,
members[i].path ? members[i].path : "(null)",
strerror(-rc));
for (int j = 0; j < cl->loaded; j++)
member_log_free(&cl->logs[j]);
return rc;
}
cl->loaded = i + 1;
}
/* Shared committed boundary = the minimum recovered tip across members:
* the largest committed prefix every member can be expected to hold. A
* member with an explicit committed_lsn keeps its own. */
uint64_t shared = cl->logs[0].recovered_tip;
for (int i = 1; i < n; i++)
if (cl->logs[i].recovered_tip < shared)
shared = cl->logs[i].recovered_tip;
for (int i = 0; i < n; i++) {
if (cl->logs[i].committed_lsn == 0)
cl->logs[i].committed_lsn = shared;
if (cl->logs[i].committed_lsn > cl->boundary)
cl->boundary = cl->logs[i].committed_lsn;
if (cl->logs[i].recovered_tip > cl->max_tip)
cl->max_tip = cl->logs[i].recovered_tip;
}
return 0;
}
static void wc_unload(wc_cluster_t* cl)
{
for (int i = 0; i < cl->loaded; i++)
member_log_free(&cl->logs[i]);
}
/* One machine line for member `i` at `lsn`. Fixed field order, stable tokens. */
static void wc_emit_member(const wc_cluster_t* cl, int i, uint64_t lsn,
int committed, FILE* out)
{
const member_log_t* m = &cl->logs[i];
int holds = (lsn < m->cap) && m->present[lsn];
fprintf(out,
"WALCMP member=%d lsn=%" PRIu64 " term=%" PRIu64
" pcrc=%08x status=%s committed=%d\n",
i, lsn, holds ? match_term(m->term[lsn]) : 0,
holds ? m->pcrc[lsn] : 0u, wc_status(m, lsn), committed);
}
/* Full per-member machine table over [1..max_tip]; committed=1 within the
* committed boundary, 0 in the uncommitted tail. */
static void wc_emit_table(const wc_cluster_t* cl, FILE* out)
{
for (uint64_t lsn = 1; lsn <= cl->max_tip; lsn++) {
int committed = (lsn <= cl->boundary) ? 1 : 0;
for (int i = 0; i < cl->n; i++)
wc_emit_member(cl, i, lsn, committed, out);
}
}
/* Emit the classification + per-member identity for ONE diverging LSN. */
static void wc_emit_divergence(const wc_cluster_t* cl, uint64_t lsn,
int committed, int compared, int present,
FILE* out)
{
fprintf(out,
"WALCMP_DIVERGENCE lsn=%" PRIu64 " class=%s compared=%d present=%d\n",
lsn, committed ? "committed" : "uncommitted", compared, present);
for (int i = 0; i < cl->n; i++)
wc_emit_member(cl, i, lsn, committed, out);
}
/* Compare one committed LSN. Fills `div` (committed=1) and prints the human +
* machine report when divergent. Returns 1 if divergent, else 0. */
static int wc_compare_committed(const wc_cluster_t* cl,
const walcompare_member_t* members, uint64_t lsn,
FILE* out, walcompare_divergence_t* div)
{
int have = 0, present = 0, have_ref = 0, divergent = 0;
uint64_t ref_term = 0;
uint32_t ref_pcrc = 0;
for (int i = 0; i < cl->n; i++) {
if (lsn > cl->logs[i].committed_lsn)
continue; /* beyond member i's committed prefix */
if (lsn <= cl->logs[i].floor_lsn)
continue; /* below member i's OWN checkpoint floor — legitimately
* compacted (entity-restored), never a divergence */
have++;
const member_log_t* m = &cl->logs[i];
int holds = (lsn < m->cap) && m->present[lsn];
if (!holds) {
divergent = 1; /* member should hold this committed LSN but doesn't */
continue;
}
present++;
uint64_t mt = match_term(m->term[lsn]);
uint32_t mp = m->pcrc[lsn];
if (!have_ref) {
ref_term = mt;
ref_pcrc = mp;
have_ref = 1;
} else if (mt != ref_term || mp != ref_pcrc) {
divergent = 1;
}
}
if (have < 2 || !divergent)
return 0;
if (div) {
memset(div, 0, sizeof(*div));
div->lsn = lsn;
div->n_members = cl->n;
div->committed = 1;
for (int i = 0; i < cl->n && i < 16; i++) {
const member_log_t* m = &cl->logs[i];
int holds = (lsn <= m->committed_lsn) && (lsn > m->floor_lsn) &&
(lsn < m->cap) && m->present[lsn];
div->member_present[i] = holds ? 1 : 0;
div->member_term[i] = holds ? match_term(m->term[lsn]) : 0;
div->member_pcrc[i] = holds ? m->pcrc[lsn] : 0;
}
}
if (out) {
fprintf(out,
"*** DIVERGENCE at committed LSN %" PRIu64
" (compared %d, present %d) ***\n",
lsn, have, present);
for (int i = 0; i < cl->n; i++) {
if (lsn > cl->logs[i].committed_lsn) {
fprintf(out,
" member %d ('%s'): LSN beyond its committed prefix"
" (committed=%" PRIu64 ")\n",
i, members[i].path, cl->logs[i].committed_lsn);
continue;
}
if (lsn <= cl->logs[i].floor_lsn) {
fprintf(out,
" member %d ('%s'): LSN below its own checkpoint floor"
" (floor=%" PRIu64 ") — legitimately compacted\n",
i, members[i].path, cl->logs[i].floor_lsn);
continue;
}
int holds = (lsn < cl->logs[i].cap) && cl->logs[i].present[lsn];
if (holds)
fprintf(out, " member %d ('%s'): term=%" PRIu64 " pcrc=%08x\n",
i, members[i].path, match_term(cl->logs[i].term[lsn]),
cl->logs[i].pcrc[lsn]);
else
fprintf(out, " member %d ('%s'): MISSING committed entry\n", i,
members[i].path);
}
wc_emit_divergence(cl, lsn, 1, have, present, out);
}
return 1;
}
/* Scan the uncommitted tail (boundary < lsn <= max_tip): emit a NON-fatal
* class=uncommitted classification for any LSN where >=2 members hold the entry
* but disagree on (term, pcrc). Never changes the verdict. */
static void wc_scan_uncommitted(const wc_cluster_t* cl, FILE* out)
{
for (uint64_t lsn = cl->boundary + 1; lsn <= cl->max_tip; lsn++) {
int present = 0, have_ref = 0, divergent = 0;
uint64_t ref_term = 0;
uint32_t ref_pcrc = 0;
for (int i = 0; i < cl->n; i++) {
const member_log_t* m = &cl->logs[i];
int holds = (lsn < m->cap) && m->present[lsn];
if (!holds)
continue;
present++;
uint64_t mt = match_term(m->term[lsn]);
uint32_t mp = m->pcrc[lsn];
if (!have_ref) {
ref_term = mt;
ref_pcrc = mp;
have_ref = 1;
} else if (mt != ref_term || mp != ref_pcrc) {
divergent = 1;
}
}
if (present < 2 || !divergent)
continue;
wc_emit_divergence(cl, lsn, 0, present, present, out);
}
}
/* Shared driver for walcompare() (full_report=0) and walcompare_report()
* (full_report=1). */
static int wc_run(const walcompare_member_t* members, int n, FILE* out,
walcompare_divergence_t* div, int full_report)
{
if (!members || n < 2)
return -EINVAL;
if (n > WALCMP_MAX_MEMBERS)
return -E2BIG;
wc_cluster_t cl;
int rc = wc_load(members, n, out, &cl);
if (rc != 0)
return rc;
if (out)
fprintf(out,
"== walcompare: %d members, committed boundary up to LSN %" PRIu64
" ==\n",
n, cl.boundary);
if (full_report && out)
wc_emit_table(&cl, out);
/* Walk every committed LSN. At each LSN, compare the (term, pcrc) of the
* members whose own committed boundary covers it and that hold the entry.
* A member that does not hold a committed LSN it SHOULD is a committed-
* prefix divergence: a committed entry is missing (R4.2 / R6.4). */
rc = WALCMP_OK;
for (uint64_t lsn = 1; lsn <= cl.boundary; lsn++) {
if (wc_compare_committed(&cl, members, lsn, out, div)) {
rc = WALCMP_DIVERGENT; /* first committed divergence is the verdict */
goto done;
}
}
/* The committed prefix is identical. In full-report mode also classify any
* uncommitted-tail divergence (informational, never fatal). */
if (full_report && out)
wc_scan_uncommitted(&cl, out);
if (out)
fprintf(out, "walcompare: OK — committed prefixes identical\n");
done:
wc_unload(&cl);
return rc;
}
int walcompare(const walcompare_member_t* members, int n, FILE* out,
walcompare_divergence_t* div)
{
return wc_run(members, n, out, div, 0);
}
int walcompare_report(const walcompare_member_t* members, int n, FILE* out,
walcompare_divergence_t* div)
{
return wc_run(members, n, out, div, 1);
}

626
src/harness/waldump.c Normal file
View file

@ -0,0 +1,626 @@
#define _GNU_SOURCE
#include "waldump.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <zlib.h>
#include "config.h" /* WAL_HDR_CRC_SIZE, WAL_CRC_NONE, MAX_WAL_PAYLOAD_BYTES */
#include "controller.capnp.h" /* Job decode for WAL_JOB_SUBMIT */
#include "utils.h" /* fmt_jobid — human-readable JobId formatting */
#include "wal_types.h"
#define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
#define MAX_SEGS 4096
/* ------------------------------------------------------------------ */
/* Small helpers */
/* ------------------------------------------------------------------ */
static const char* wal_type_name(uint32_t t)
{
switch (t) {
case WAL_JOB_SUBMIT: return "JOB_SUBMIT";
case WAL_JOB_ARRAY_SUBMIT: return "JOB_ARRAY_SUBMIT"; /* job-array-single-
* wal-record task 6.1: name the type so the HA soak's WAL-record-count
* assertion (exactly one JOB_ARRAY_SUBMIT per array submission,
* regardless of array size) can grep an exact type name instead of
* the generic "UNKNOWN" bucket, which would also catch any future,
* genuinely-unrecognized type. */
case WAL_JOB_START: return "JOB_START";
case WAL_JOB_FINISH: return "JOB_FINISH";
case WAL_JOB_FAIL: return "JOB_FAIL";
case WAL_JOB_STOP: return "JOB_STOP";
case WAL_JOB_RESUME: return "JOB_RESUME";
case WAL_JOB_UPDATE: return "JOB_UPDATE";
case WAL_JOB_RUSAGE: return "JOB_RUSAGE";
case WAL_JOB_CKPT: return "JOB_CKPT";
case WAL_JOB_PREASON: return "JOB_PREASON";
case WAL_QUEUE_CTRL: return "QUEUE_CTRL";
case WAL_WORKER_CTRL: return "WORKER_CTRL";
case WAL_NOOP: return "NOOP";
case WAL_CONFIG_MARKER: return "CONFIG_MARKER";
case WAL_CHECKPOINT: return "CHECKPOINT";
default: return "UNKNOWN";
}
}
/* Decoded fields of a WAL_JOB_SUBMIT Cap'n Proto payload (debug view). */
typedef struct {
uint64_t jobid;
uint64_t subtime; /* job_submit stamps this with time(NULL) per submit */
uint32_t state;
} submit_info_t;
/* Decode a WAL_JOB_SUBMIT payload. Mirrors restore_job() (job.c) but reads a
* few fields only, zero-alloc. Returns 0 on success, -1 if undecodable.
* subtime is the decisive bug-845 signal: two entries of the SAME job written
* in one submit share a subtime (a literal double-append); two separate
* submits (e.g. a failover retry) would differ. */
static int decode_submit_info(const uint8_t* payload, uint32_t len,
submit_info_t* out)
{
if (!payload || len == 0)
return -1;
struct capn cctx;
if (capn_init_mem(&cctx, payload, len, 0) != 0)
return -1;
Job_ptr jp;
jp.p = capn_getp(capn_root(&cctx), 0, 1);
if (jp.p.type == CAPN_NULL) {
capn_free(&cctx);
return -1;
}
struct Job cj;
memset(&cj, 0, sizeof(cj));
read_Job(&cj, jp);
out->jobid = cj.jobid;
out->subtime = cj.subtime;
out->state = cj.state;
capn_free(&cctx);
return 0;
}
/* True for the WAL types whose payload is the flat, non-capnp
* wal_jobstate_t {jobid, state, _pad} (wal_types.h) — every worker-reported
* job-state fact (START/FINISH/FAIL/STOP/RESUME/UPDATE/RUSAGE/CKPT/PREASON).
* Added for raft-apply-decoupling task 7.2: the kill-mid-fact-stream live
* regression needs to verify, per jobid, which fact WAL entries actually
* committed on each node — before this, waldump only decoded jobid for
* WAL_JOB_SUBMIT, leaving every fact entry printed as "jobid=?". */
static int wal_type_is_jobstate(uint32_t type)
{
switch (type) {
case WAL_JOB_START:
case WAL_JOB_FINISH:
case WAL_JOB_FAIL:
case WAL_JOB_STOP:
case WAL_JOB_RESUME:
case WAL_JOB_UPDATE:
case WAL_JOB_RUSAGE:
case WAL_JOB_CKPT:
case WAL_JOB_PREASON:
return 1;
default:
return 0;
}
}
/* Decode a wal_jobstate_t payload. Native-endian flat struct (single-node WAL
* — see wal_types.h), so this is a bounds-checked copy, not a parser. Returns
* 0 on success, -1 if the payload is too short to hold the struct. */
static int decode_jobstate_info(const uint8_t* payload, uint32_t len,
wal_jobstate_t* out)
{
if (!payload || len < sizeof(*out))
return -1;
memcpy(out, payload, sizeof(*out));
return 0;
}
/* Decode a WAL_CONFIG_MARKER payload -- fixed 40-byte form (8-byte
* big-endian version_tag + 32-byte raw sha256), never capnp-encoded
* (config_repl.c's config_marker_encode/config_marker_decode). Hand-decoded
* here rather than linking config_repl.c, which would pull controller-
* internal dependencies into xce_test's link graph (meson.build's
* xce_test_src is deliberately limited to read-only dump/compare tools).
* Mirrors config_marker_decode's byte layout exactly. */
static int decode_config_marker_info(const uint8_t* payload, uint32_t len,
uint64_t* out_tag, uint8_t sha256_out[32])
{
if (!payload || len < WAL_CONFIG_MARKER_PAYLOAD_LEN)
return -1;
uint64_t tag = 0;
for (int i = 0; i < 8; i++)
tag = (tag << 8) | payload[i];
*out_tag = tag;
memcpy(sha256_out, payload + 8, 32);
return 0;
}
/* Highest LSN among sibling data/snap/snap-<hex>.capnp files next to
* `wal_dir` (persist.c hardcodes both as siblings under a node's data/,
* e.g. .../data/wal and .../data/snap). A leading WAL gap up to this LSN is
* explained by normal checkpoint-triggered trim (persist.c's
* trim_wal_segments only removes segments below a replication-safe floor,
* never a segment a standby still needs to catch up) rather than genuine
* loss -- without this check, EVERY long-running node's WAL (which will
* have been checkpointed at least once) prints the same alarming
* "(committed prefix missing!)" wording a real, unexplained loss would,
* making the message a standing false alarm. Returns 0 if `wal_dir` isn't
* named "wal", or no sibling snap dir/files exist (unknown -- callers keep
* today's un-downgraded wording, the conservative default). */
static uint64_t highest_snapshot_lsn(const char* wal_dir)
{
char snap_dir[4096];
snprintf(snap_dir, sizeof(snap_dir), "%s", wal_dir);
char* slash = strrchr(snap_dir, '/');
if (!slash || strcmp(slash + 1, "wal") != 0)
return 0;
snprintf(slash + 1, sizeof(snap_dir) - (size_t)(slash + 1 - snap_dir),
"snap");
DIR* d = opendir(snap_dir);
if (!d)
return 0;
uint64_t max_lsn = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL) {
uint64_t lsn;
if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) == 1 &&
lsn > max_lsn)
max_lsn = lsn;
}
closedir(d);
return max_lsn;
}
/* True if `dir` directly contains at least one seg-*.wal file. */
static int dir_has_segments(const char* dir)
{
DIR* d = opendir(dir);
if (!d)
return 0;
int found = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL) {
size_t n = strlen(e->d_name);
if (n > 4 && strncmp(e->d_name, "seg-", 4) == 0 &&
strcmp(e->d_name + n - 4, ".wal") == 0) {
found = 1;
break;
}
}
closedir(d);
return found;
}
/* Resolve `path` (seg file / wal dir / node-or-data dir) to the wal dir that
* holds the seg-*.wal files. Writes it to `out`. Returns 0, or -errno. When
* `path` is itself a single seg file, sets *single_seg to that file path. */
static int resolve_wal_dir(const char* path, char* out, size_t outsz,
char* single_seg, size_t segsz)
{
struct stat st;
if (stat(path, &st) != 0)
return -errno;
single_seg[0] = '\0';
if (S_ISREG(st.st_mode)) {
/* Only a seg-*.wal file is a WAL segment. Reject anything else (e.g. a
* config blob like conf/cluster.bin) instead of parsing it as a WAL and
* silently reporting 0 entries. */
const char* base = strrchr(path, '/');
base = base ? base + 1 : path;
size_t bl = strlen(base);
if (strncmp(base, "seg-", 4) != 0 || bl < 5 ||
strcmp(base + bl - 4, ".wal") != 0)
return -EINVAL;
snprintf(single_seg, segsz, "%s", path);
/* wal dir = dirname(path); only used for display */
snprintf(out, outsz, "%s", path);
char* slash = strrchr(out, '/');
if (slash)
*slash = '\0';
return 0;
}
if (!S_ISDIR(st.st_mode))
return -ENOTDIR;
/* A directory: try it directly, then data/wal, then wal/. */
if (dir_has_segments(path)) {
snprintf(out, outsz, "%s", path);
return 0;
}
char cand[4096];
snprintf(cand, sizeof(cand), "%s/data/wal", path);
if (dir_has_segments(cand)) {
snprintf(out, outsz, "%s", cand);
return 0;
}
snprintf(cand, sizeof(cand), "%s/wal", path);
if (dir_has_segments(cand)) {
snprintf(out, outsz, "%s", cand);
return 0;
}
return -ENOENT; /* no segments anywhere under path */
}
/* Parse the start LSN out of a seg file basename ("seg-<hex16>.wal"). */
static int parse_seg_start(const char* name, uint64_t* lsn)
{
return sscanf(name, "seg-%" SCNx64 ".wal", lsn) == 1 ? 0 : -1;
}
/* Collect + sort segment start LSNs under wal_dir. Returns count, or -errno. */
static int collect_segments(const char* wal_dir, uint64_t* seg_lsns, int max)
{
DIR* d = opendir(wal_dir);
if (!d)
return -errno;
int n = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL && n < max) {
uint64_t s;
if (parse_seg_start(e->d_name, &s) == 0)
seg_lsns[n++] = s;
}
closedir(d);
for (int i = 1; i < n; i++) { /* insertion sort — segment count is tiny */
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) {
seg_lsns[j + 1] = seg_lsns[j];
j--;
}
seg_lsns[j + 1] = key;
}
return n;
}
/* ------------------------------------------------------------------ */
/* Per-segment scan: prints entries, validates the CRC chain, detects */
/* LSN gaps. Carries the chain CRC and prev LSN across segment calls. */
/* ------------------------------------------------------------------ */
static void scan_segment(const char* seg_path, FILE* out,
uint32_t* last_crc, uint64_t* prev_lsn,
uint64_t* prev_submit_jobid, uint32_t* prev_submit_pcrc,
waldump_stats_t* st, uint64_t snap_lsn,
waldump_visit_fn visit, void* visit_ctx,
int* visit_stop)
{
int fd = open(seg_path, O_RDONLY);
if (fd < 0) {
fprintf(out, " !! cannot open %s: %s\n", seg_path, strerror(errno));
return;
}
uint32_t off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
if (hn == 0)
break; /* clean EOF */
if (hn < (ssize_t)sizeof(hdr))
break; /* truncated / pre-allocated zeros */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break; /* pre-allocated zero region = clean end */
/* Bound payload before trusting it (corruption guard). */
if (hdr.payload_len > MAX_WAL_PAYLOAD_BYTES) {
fprintf(out, " !! LSN %" PRIu64 " payload_len %u exceeds max — stop\n",
hdr.lsn, hdr.payload_len);
st->crc_broken = 1;
break;
}
/* CRC chain check (identical rule to persist recovery). */
int chain_ok;
if (off == 0)
chain_ok = (hdr.prev_crc == WAL_CRC_NONE || *last_crc == WAL_CRC_NONE ||
hdr.prev_crc == *last_crc);
else
chain_ok = (hdr.prev_crc == *last_crc);
if (!chain_ok) {
fprintf(out, " !! LSN %" PRIu64 " prev_crc chain break — stop\n",
hdr.lsn);
st->crc_broken = 1;
break;
}
/* Verify entry CRC over header + payload. */
uint8_t* payload = NULL;
uint32_t want_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
if (hdr.payload_len > 0) {
payload = malloc(hdr.payload_len);
if (!payload)
break;
ssize_t pn = pread(fd, payload, hdr.payload_len,
(off_t)(off + sizeof(hdr)));
if (pn != (ssize_t)hdr.payload_len) {
free(payload);
break; /* short read — clean end */
}
want_crc = crc32(want_crc, (const Bytef*)payload, hdr.payload_len);
}
if (hdr.crc != want_crc) {
fprintf(out, " !! LSN %" PRIu64 " CRC mismatch — stop\n", hdr.lsn);
st->crc_broken = 1;
free(payload);
break;
}
/* Gap detection against the running LSN sequence. */
if (*prev_lsn == 0) {
if (hdr.lsn > 1) {
int explained = snap_lsn >= hdr.lsn - 1;
fprintf(out, " *** GAP: missing prefix LSN 1..%" PRIu64
" (%" PRIu64 " entries absent)%s ***\n",
hdr.lsn - 1, hdr.lsn - 1,
explained ? " -- covered by snapshot, expected "
"post-checkpoint trim"
: "");
st->n_gaps++;
st->leading_gap_explained = explained;
}
} else if (hdr.lsn > *prev_lsn + 1) {
fprintf(out, " *** GAP: missing LSN %" PRIu64 "..%" PRIu64 " ***\n",
*prev_lsn + 1, hdr.lsn - 1);
st->n_gaps++;
}
/* payload-only CRC — isolates the blob from the header (LSN/term/ts),
* so two entries carrying the SAME job blob share a pcrc regardless of
* their differing LSNs. Decisive for bug-845: equal pcrc on an adjacent
* same-jobid pair ⇒ a literal byte-for-byte double-append. */
uint32_t pcrc = (hdr.payload_len > 0)
? crc32(0L, (const Bytef*)payload, hdr.payload_len)
: 0;
/* Surface the CRC-valid entry to the visitor so a caller can index the
* log by (lsn -> term, pcrc) — the single canonical decoder, no second
* parser. */
if (visit) {
waldump_entry_t e = {
.lsn = hdr.lsn,
.term = hdr.term,
.type = hdr.type,
.payload_len = hdr.payload_len,
.pcrc = pcrc,
};
if (visit(visit_ctx, &e) != 0) {
if (visit_stop)
*visit_stop = 1;
free(payload);
break; /* visitor asked to stop */
}
}
/* Per-entry line, with jobid/subtime for submits. Sized to hold the
* formatted JobId (fmt_jobid, up to ~40 chars) + subtime + state. */
char info[192] = "";
int is_dup = 0;
int dup_identical = 0;
if (hdr.type == WAL_JOB_SUBMIT) {
submit_info_t si;
uint64_t jobid = 0;
if (decode_submit_info(payload, hdr.payload_len, &si) == 0) {
jobid = si.jobid;
/* Render the JobId in the same human-readable form the client
* (joblist / jobsub) prints — fmt_jobid (utils.c) — so a waldump
* jobid can be eyeballed against an `expect-job <id>` / joblist
* entry without converting the raw uint64. */
char jidStr[128];
if (fmt_jobid(jidStr, sizeof(jidStr), si.jobid) < 0)
snprintf(jidStr, sizeof(jidStr), "%" PRIu64, si.jobid);
snprintf(info, sizeof(info),
"jobid=%s subtime=%" PRIu64 " state=%u",
jidStr, si.subtime, si.state);
} else {
snprintf(info, sizeof(info), "jobid=?");
}
st->n_submits++;
/* bug-845: adjacent duplicate submit — same jobid as the previous
* entry at lsn = prev_lsn + 1. Capture identity BEFORE updating the
* carried pcrc. */
if (jobid != 0 && jobid == *prev_submit_jobid &&
hdr.lsn == *prev_lsn + 1) {
is_dup = 1;
dup_identical = (pcrc == *prev_submit_pcrc);
st->n_dup_submits++;
if (dup_identical)
st->n_dup_identical++;
}
*prev_submit_jobid = jobid;
*prev_submit_pcrc = pcrc;
} else if (wal_type_is_jobstate(hdr.type)) {
/* Worker-reported job-state fact (task 7.2): decode the flat
* {jobid, state, _pad} payload so a per-jobid committed-fact check
* (e.g. "did WAL_JOB_FINISH for jobid X actually commit on this
* node") can grep waldump output directly, the same way the
* existing WAL_JOB_SUBMIT branch above already supports
* expect-job/joblist cross-checks. */
wal_jobstate_t js;
if (decode_jobstate_info(payload, hdr.payload_len, &js) == 0) {
char jidStr[128];
if (fmt_jobid(jidStr, sizeof(jidStr), js.jobid) < 0)
snprintf(jidStr, sizeof(jidStr), "%" PRIu64, js.jobid);
snprintf(info, sizeof(info), "jobid=%s state=0x%x pad=0x%x",
jidStr, js.state, js._pad);
} else {
snprintf(info, sizeof(info), "jobid=?");
}
/* a non-submit entry breaks the adjacency for dup detection */
*prev_submit_jobid = 0;
} else if (hdr.type == WAL_CONFIG_MARKER) {
uint64_t tag = 0;
uint8_t sha[32];
if (decode_config_marker_info(payload, hdr.payload_len, &tag,
sha) == 0) {
char hex[65];
for (int i = 0; i < 32; i++)
snprintf(hex + i * 2, 3, "%02x", sha[i]);
snprintf(info, sizeof(info), "tag=%016" PRIx64 " sha256=%s",
tag, hex);
} else {
snprintf(info, sizeof(info), "tag=?");
}
*prev_submit_jobid = 0;
} else {
/* a non-submit entry breaks the adjacency for dup detection */
*prev_submit_jobid = 0;
}
fprintf(out, " %8" PRIu64 " | %5" PRIu64 " | %-11s | %5u | %08x | %s\n",
hdr.lsn, hdr.term, wal_type_name(hdr.type), hdr.payload_len,
pcrc, info);
if (is_dup)
fprintf(out, " ^^ DUP submit: payload %s the prior entry\n",
dup_identical ? "BYTE-IDENTICAL to" : "DIFFERS from");
free(payload);
/* Advance watermarks. */
if (st->first_lsn == 0 || hdr.lsn < st->first_lsn)
st->first_lsn = hdr.lsn;
if (hdr.lsn > st->last_lsn)
st->last_lsn = hdr.lsn;
if (hdr.lsn > st->recovered_tip)
st->recovered_tip = hdr.lsn;
st->n_entries++;
*prev_lsn = hdr.lsn;
*last_crc = hdr.crc;
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
/* ------------------------------------------------------------------ */
/* Public entry point */
/* ------------------------------------------------------------------ */
int waldump_path(const char* path, FILE* out, waldump_stats_t* stats)
{
return waldump_path_visit(path, out, stats, NULL, NULL);
}
int waldump_path_visit(const char* path, FILE* out, waldump_stats_t* stats,
waldump_visit_fn visit, void* ctx)
{
if (!path || !out)
return -EINVAL;
char wal_dir[4096];
char single_seg[4096];
int rc = resolve_wal_dir(path, wal_dir, sizeof(wal_dir),
single_seg, sizeof(single_seg));
if (rc != 0) {
/* Clear, actionable error instead of silently scanning a non-WAL path
* (e.g. conf/cluster.bin) and reporting "0 entries". */
fprintf(out,
"waldump: no WAL segments at '%s' (%s)\n"
" point at a node's WAL dir, e.g. "
"<deploy>/controller/ctrl-N/data/wal\n"
" (or a specific seg-*.wal file inside it)\n",
path, strerror(-rc));
return rc;
}
waldump_stats_t st;
memset(&st, 0, sizeof(st));
uint64_t seg_lsns[MAX_SEGS];
int nseg;
/* wider than wal_dir[4096] so the "/seg-<hex16>.wal" suffix never truncates */
char seg_path[4096 + 64];
if (single_seg[0] != '\0') {
/* A single seg file: parse its start for ordering display. */
const char* base = strrchr(single_seg, '/');
base = base ? base + 1 : single_seg;
uint64_t start = 0;
parse_seg_start(base, &start);
seg_lsns[0] = start;
nseg = 1;
snprintf(seg_path, sizeof(seg_path), "%s", single_seg);
/* If the dir holds OTHER segments, this single-file dump shows only a
* slice of the log. A node that recovered then caught up keeps its
* prefix in seg-...001.wal and the suffix in seg-...<current_lsn>.wal,
* so a single-file dump looks (wrongly) like missing data. Steer the
* user to the whole dir. */
uint64_t others[MAX_SEGS];
int total = collect_segments(wal_dir, others, MAX_SEGS);
if (total > 1)
fprintf(out,
"NOTE: %s holds %d segments — this dumps only %s.\n"
" Run 'waldump %s' (the directory) for the full "
"stitched 1..N log.\n",
wal_dir, total, base, wal_dir);
} else {
nseg = collect_segments(wal_dir, seg_lsns, MAX_SEGS);
if (nseg < 0)
return nseg;
if (nseg == 0)
return -ENOENT;
}
st.n_segments = (uint32_t)nseg;
uint64_t snap_lsn = highest_snapshot_lsn(wal_dir);
fprintf(out, "== WAL: %s (%d segment%s) ==\n", wal_dir, nseg,
nseg == 1 ? "" : "s");
fprintf(out, " %8s | %5s | %-11s | %5s | %8s | %s\n",
"LSN", "TERM", "TYPE", "PLEN", "PCRC", "INFO");
uint32_t last_crc = WAL_CRC_NONE;
uint64_t prev_lsn = 0;
uint64_t prev_submit_jobid = 0;
uint32_t prev_submit_pcrc = 0;
int visit_stop = 0;
for (int i = 0; i < nseg && !visit_stop; i++) {
if (single_seg[0] == '\0')
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, wal_dir,
seg_lsns[i]);
scan_segment(seg_path, out, &last_crc, &prev_lsn,
&prev_submit_jobid, &prev_submit_pcrc, &st, snap_lsn,
visit, ctx, &visit_stop);
}
fprintf(out, "-- summary --\n");
fprintf(out, " segments: %u entries: %u submits: %u\n",
st.n_segments, st.n_entries, st.n_submits);
fprintf(out, " lsn range: %" PRIu64 "..%" PRIu64
" recovered tip: %" PRIu64 "\n",
st.first_lsn, st.last_lsn, st.recovered_tip);
fprintf(out, " gaps: %u%s crc chain: %s\n", st.n_gaps,
(st.n_gaps && st.first_lsn > 1)
? (st.leading_gap_explained
? " (explained by checkpoint/snapshot -- not a loss)"
: " (committed prefix missing!)")
: "",
st.crc_broken ? "BROKEN" : "intact");
if (st.n_dup_submits > 0)
fprintf(out, " duplicate submits: %u (byte-identical: %u) — bug-845\n",
st.n_dup_submits, st.n_dup_identical);
if (stats)
*stats = st;
return 0;
}