#include #include #include #include #include /* PATH_MAX -- cmd_source's realpath() canonicalization * buffer (xce-test-cli-ux task 2.4) */ #include #include #include #include #include #include #include #include /* libedit (BSD) -- provides readline()/ * add_history(), same as GNU readline's * split readline.h/history.h pair */ #include #include #include /* SHA256() — cmd_upgradeconfig hashes the config blob */ #include "client.h" #include "config.h" /* XCE_MAXNAME_LEN */ #include "controller.capnp.h" #include "ctrlstatus_fmt.h" /* ctrlstatus_format_time_ms (task 6.3) */ #include "delay.h" #include "flagspec.h" /* shared flagspec_t descriptor-table flag parsing, * rejection (with typo suggestion), and help * rendering (xce-test-cli-ux tasks 1.1/1.2); every * flag-accepting command below migrates onto this * (task 3.1) */ #include "idgen.h" #include "preason.h" #include "walcompare.h" /* cross-member committed-prefix divergence (cmd_walcompare) */ #include "waldump.h" /* offline WAL inspector (cmd_waldump) */ #include "snapdiff.h" /* byte-level checkpoint-file comparator (cmd_snapdiff) */ #include "snapverify.h" /* offline checkpoint-file structural validator (cmd_snapverify) */ #include "configcompare.h" /* cross-member active-config comparator (cmd_configcompare) */ #include "confirm.h" /* destructive-command confirmation gate (xce-test-cli-ux task 2.1) */ #include "joblist_display.h" /* worker_status_str/job_state_str/format_job_time * (xce-test-cli-ux task 2.5 remediation) */ #include "join_worker_args.h" /* join_worker_parse_attrs (xce-test-cli-ux * task 2.6 remediation, bug-1557) */ #include "rcfile.h" /* startup rc-file path resolution + loading (xce-test-rc-file task 2.2) */ #include "resval_typeresolve.h" /* resval_resolve_type (resource-value-verification task 2.1) */ #include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */ #include "xce/types.h" #include "xcetest_interrupt.h" /* xcetest_install_sigint, xcetest_interrupted (xce-test-cli-ux task 1.3) */ typedef int (*cmd_func_t)(int, char*[]); typedef struct { char* name; cmd_func_t func; char* doc; const char* usage; } command_t; typedef struct { char* key; char* value; char* defval; } setting_t; typedef struct _command_ctx { struct _command_ctx* parent; char* prompt; setting_t* settings; command_t* commands; xcecli_ctx_t ctx; } command_ctx_t; typedef struct { uint64_t jobid; int pid; time_t ts; } run_job_t; typedef struct { uint64_t jobid; uint32_t taskid; int clientidx; int pid; time_t ts; } run_task_t; typedef struct { char* host; int ntasks; } host_task_t; typedef struct { char* PATH; } sub_envs_t; typedef struct { sub_envs_t sub_env_vars; } exdata_t; #define MAX_RUN_JOBS (20) #define MAX_TASK_HOSTS (5) #define MAX_RUN_TASKS (50) #define RUSAGE_UPDATE_INTERVAL (60) #define MAX_LINE_LEN (40960) #define S2J_JSON_SET_var_string_ELEMENT(to_json, from_struct, _element) \ if (((from_struct) != NULL) && (((from_struct)->_element) != NULL)) { \ cJSON_AddStringToObject((to_json), #_element, \ (from_struct)->_element); \ } static run_job_t run_jobs[MAX_RUN_JOBS]; static run_task_t run_tasks[MAX_RUN_TASKS]; static int myreq_handler(xce_request_t*, void*); extern int cmd_help(int, char*[]); extern int cmd_up(int, char*[]); extern int cmd_worker(int, char*[]); extern int cmd_scheduler(int, char*[]); extern int cmd_client(int, char*[]); extern int cmd_quit(int, char*[]); extern int cmd_hello(int, char*[]); extern int cmd_shutdown(int, char*[]); extern int cmd_set(int, char*[]); extern int cmd_reslist(int, char*[]); extern int cmd_join_worker(int, char*[]); extern int cmd_jobstart(int, char*[]); extern int cmd_jobfail(int, char*[]); extern int cmd_join_sched(int, char*[]); extern int cmd_workerlist(int, char*[]); extern int cmd_joblist(int, char*[]); extern int cmd_jobrun(int, char*[]); extern int cmd_jobsub(int, char*[]); extern int cmd_getconf(int, char*[]); extern int cmd_jobfinish(int, char*[]); extern int cmd_resupdate(int, char*[]); extern int cmd_resval(int, char*[]); extern int cmd_jobkill(int, char*[]); extern int cmd_jobckpt(int, char*[]); extern int cmd_qctrl(int, char*[]); extern int cmd_hostctrl(int, char*[]); extern int cmd_upgradeconfig(int, char*[]); extern int cmd_runtasks(int, char*[]); extern int cmd_setpend(int, char*[]); extern int cmd_jobinfo(int, char*[]); extern int cmd_echo(int, char*[]); extern int cmd_expect_jobs(int, char*[]); extern int cmd_expect_job(int, char*[]); extern int cmd_wait_primary(int, char*[]); extern int cmd_ctrlstatus(int, char*[]); extern int cmd_waldump(int, char*[]); extern int cmd_walcompare(int, char*[]); extern int cmd_snapverify(int, char*[]); extern int cmd_snapdiff(int, char*[]); extern int cmd_configcompare(int, char*[]); extern int cmd_source(int, char*[]); static int worker_house_keeping(void*); /* Session verdict / loop state (xce-test-e2e-primitives task 1.1). * * g_session_failed set to 1 the first time any command in the session * fails; drives the process exit code in main(). * g_continue_on_error when set (via `set continue-on-error`), a failing * command does not stop the read loop; the session still * exits non-zero at the end. * g_last_status status of the most recently executed command * (0 == success, non-zero == failure). Future tasks * expose this as the `$?` auto-variable. */ static int g_session_failed = 0; static int g_continue_on_error = 0; static int g_last_status = 0; /* Interactive (TTY) sessions are a REPL: a failed command prints its error and * the loop CONTINUES — never abort the read loop on a typo. The abort-on-first- * failure + non-zero-exit verdict is a SCRIPTED/piped-mode concept only. Set * from isatty(stdin) in main. */ static int g_interactive = 0; /* Auto-variable state for the fixed `$jobid` / `$jobcount` substitution * (xce-test-e2e-primitives task 3.1, Req 3.1/3.3/3.5/3.6). * * g_last_jobid most recently acknowledged JobId as a string; empty ("") * means unset. Populated later by cmd_jobsub (task 4.1). * g_last_jobcount most recent job count; -1 means unset. Populated by * cmd_joblist on a successful `joblist` (task 5.1, Req 3.4); * a failed joblist leaves it untouched so no fake count is * advertised. * * Until a command sets one, it stays unset — which is exactly what makes an * unset `$jobid`/`$jobcount` reference fail the command (Req 3.6). */ static char g_last_jobid[XCE_MAXNAME_LEN] = { 0 }; static int g_last_jobcount = -1; static idgen_ctx_t idctx; delaymgr_t delaymgr; static setting_t settings[] = { /* HA controller list: comma-separated "name=host:port" or "host:port" * entries (same syntax as the controller --peers flag), enabling the * test client to fail over across controller candidates. */ { (char*)"masters", (char*)NULL, (char*)"localhost:9990" }, { (char*)"secret", (char*)NULL, (char*)"keepSecret" }, { (char*)"myname", (char*)NULL, (char*)"localhost" }, { (char*)NULL, (char*)NULL, (char*)NULL } }; /* Detailed per-command `help ` text (design: keep these OUT of the * command_t table initializers below so the tables stay readable). Each * constant's shape: first line = command name + flag/positional syntax, * then one indented line per flag/positional explaining what it does and * whether it's required, then (where useful) an `example:` line. Derived * from actually reading each cmd_* handler's getopt() string and switch * cases (or its positional argv[] handling) -- not guessed from the name. * * Commands that are identical (same func, same semantics) across more than * one of the 4 command tables share ONE constant here, referenced from every * table, so the text can't drift out of sync between contexts. */ static const char USAGE_HELP[] = "help []\n" " optional command name; prints its detailed usage instead of\n" " the one-line summary\n" " With no argument, lists every command in the current context with its\n" " one-line summary (unaffected by this detailed-usage text).\n" "example: help resval\n"; static const char USAGE_QUIT[] = "quit\n" " (no arguments)\n" " Leaves the current context (if any) and exits xce_test.\n"; static const char USAGE_ECHO[] = "echo []\n" " zero or more words, printed space-joined followed by a\n" " newline (always succeeds)\n" " Useful to print the $jobid / $jobcount / $? auto-variables in scripts.\n" "example: echo job is $jobid\n"; static const char USAGE_UP[] = "..\n" " (no arguments)\n" " Leaves the current context and returns to its parent, closing the\n" " underlying client connection for this context.\n"; static const char USAGE_WALDUMP[] = "waldump [more...]\n" " one or more paths: a seg-*.wal file, a wal/ dir, or a\n" " node/data dir\n" " Offline, read-only WAL inspector: lists every entry, decodes\n" " JOB_SUBMIT jobids, and flags LSN gaps and a CRC-broken tail.\n" " Returns nonzero if any path fails to load or any gap/CRC break is\n" " found.\n" "example: waldump /var/xce/ctrl-1/wal\n"; static const char USAGE_WALCOMPARE[] = "walcompare [--report] [--committed ] [more...]\n" " --report also emit a full per-member (lsn,term,pcrc,status)\n" " machine-checkable table, not just the verdict\n" " --committed use LSN N as the shared committed boundary instead\n" " of deriving it as the minimum CRC-intact tip across\n" " members\n" " 2-16 member paths (seg-*.wal file, wal/ dir, or\n" " node/data dir)\n" " Returns 0 if all members' committed WAL prefixes are identical,\n" " nonzero on divergence or load error.\n" "example: walcompare --report /var/xce/ctrl-1/wal /var/xce/ctrl-2/wal\n"; static const char USAGE_SNAPVERIFY[] = "snapverify [ ...]\n" " one or more snap-*.capnp or snap-*.bin checkpoint files\n" " Offline structural validator: walks the same layout\n" " persist_load_snapshot() parses, without restoring anything. Reports\n" " the byte offset of the first structural defect and how many frames\n" " parsed cleanly before it.\n" " Returns nonzero if any file fails to verify.\n" "example: snapverify /var/xce/ctrl-1/snap/snap-000123.capnp\n"; static const char USAGE_SNAPDIFF[] = "snapdiff \n" " the primary's own locally generated checkpoint\n" " (snap-*.capnp)\n" " the copy a standby assembled from the wire\n" " (snap-*.bin)\n" " Byte-level comparator; reports the first differing offset and\n" " whether it lands on an OPCode_snapshotTransfer chunk boundary.\n" " Returns 0 if identical, nonzero otherwise.\n" "example: snapdiff snap-000123.capnp snap-000123.bin\n"; static const char USAGE_CONFIGCOMPARE[] = "configcompare [ ...]\n" " a member's workdir/name conf-root directory (NOT a raw\n" " config file path) -- the dir under which conf/current\n" " resolves\n" " Recomputes each member's active-config sha256 fresh from its own\n" " on-disk bytes and reports whether all members match.\n" " Returns 0 if all members are present and identical, nonzero if any\n" " member is unreadable or divergent.\n" "example: configcompare /var/xce/ctrl-1 /var/xce/ctrl-2 /var/xce/ctrl-3\n"; static const char USAGE_SOURCE[] = "source \n" " path to a file containing xce_test commands, one per line\n" " (required)\n" " Reads and executes each line of in order, exactly as if it\n" " had been typed at the current prompt -- including a nested `source`\n" " command, so one script file can compose others. A file that does\n" " not exist or cannot be read is rejected with an explicit error and\n" " no line of it is executed; a file that would source itself,\n" " directly or transitively, is rejected as a cycle instead of\n" " hanging. A failing line inside the file follows the session's\n" " current -e/+e/continue-on-error setting (see `help set`), exactly\n" " as it does for lines typed at the top-level prompt.\n" "example: source /path/to/script.xt\n"; static const char USAGE_WORKER[] = "worker\n" " (no arguments)\n" " Enters the worker context: connects to the controller cluster as a\n" " worker client and starts the local delay-manager housekeeping loop.\n"; static const char USAGE_SCHEDULER[] = "scheduler\n" " (no arguments)\n" " Enters the scheduler context: connects to the controller cluster as\n" " a scheduler client.\n"; static const char USAGE_CLIENT[] = "client\n" " (no arguments)\n" " Enters the client context: connects to the controller cluster as a\n" " regular API client (job submission, resource/worker queries, etc.).\n"; static const char USAGE_SET[] = "set\n" "set \n" "set -e | +e | continue-on-error\n" " (no args) list all settings in the current context\n" " (current value + default)\n" " set setting (e.g. masters, secret,\n" " myname) to \n" " -e abort the read loop on the first failing\n" " command (errexit)\n" " +e | continue-on-error keep the read loop running after a\n" " failing command\n" "example: set masters ctrl1=localhost:9991,ctrl2=localhost:9992\n"; static const char USAGE_JOIN_WORKER[] = "join [-f ] [-c ] [-m ] [-s ]\n" " [-t ] [-d ]\n" " -f relative CPU speed factor (default 1.0)\n" " -c max CPU count (default 4)\n" " -m max memory in bytes (default 64)\n" " -s max swap in bytes (default 1024)\n" " -t max tmp space in bytes (default 2048)\n" " -d number of local disks (default 1)\n" " Joins the cluster as a worker (hosttype LINUX64, hostmodel Intel64).\n" "example: join -c 8 -m 16384\n"; static const char USAGE_JOBSTART[] = "jstart -j [-p ]\n" " -j jobid to report a synthetic job-start fact for\n" " (required)\n" " -p pid to report (default: this xce_test process's own\n" " pid)\n" "example: jstart -j 12345 -p 4242\n"; static const char USAGE_JOBFINISH[] = "jfinish -j [-c ]\n" " -j jobid to report a synthetic job-finish fact for\n" " (required)\n" " -c exit code to report (default 0)\n" "example: jfinish -j 12345 -c 0\n"; static const char USAGE_JOBFAIL[] = "jfail -j [-c ] [-s ]\n" " -j jobid to report a synthetic job-fail fact for\n" " (required)\n" " -c failure code to report (default 0)\n" " -s exec-stage code to report (default\n" " JOB_EXEC_STAGE_POST_EXEC)\n" "example: jfail -j 12345 -c 1\n"; static const char USAGE_RESUPDATE[] = "resupdate -n -v [-t ]\n" " [-n -v [-t ] ...]\n" " -n resource name to update; any name the cluster has\n" " registered is accepted (e.g. slots, mem, cpuf,\n" " maxswap, maxtmp, maxcpus, ndisks, type, model,\n" " dynres1, dynres2, r1m, r15s, r15m, pg, ls, it, swp,\n" " io, tmp); may repeat for multiple resources\n" " -v value for the immediately preceding -n (numeric for\n" " most resources, string for type/model)\n" " -t optional: force the wire type for the\n" " immediately preceding -n/-v pair. Only meaningful for\n" " a resource name NOT already known to this tool -- a\n" " known name (e.g. mem, type) always keeps its own\n" " fixed type, and -t is ignored for it. An unrecognized\n" " name defaults to numeric when -t is omitted.\n" " Order matters: pair each -n with its -v (and optional -t) right\n" " after it.\n" "example: resupdate -n slots -v 4 -n mem -v 8192\n" "example: resupdate -n r15m -v 1.5\n" "example: resupdate -n mylabel -v foo -t string\n"; static const char USAGE_HELLO[] = "hello [-c ] [-m ]\n" " -c number of times to send the hello (default 1)\n" " -m greeting text to send (default \"Greeting from xce\n" " client\")\n" "example: hello -c 3 -m \"ping\"\n"; static const char USAGE_RESLIST[] = "reslist\n" " (no arguments)\n" " Prints all cluster-wide resources (name, flags, interval,\n" " description).\n"; static const char USAGE_WORKERLIST[] = "workerlist\n" " (no arguments)\n" " Prints all joined workers (host, status, host type/model, cpu\n" " factor, max cpu/mem/swap/tmp, ndisks).\n"; static const char USAGE_QCTRL[] = "qctrl -q [-a] [-i] [-o] [-c]\n" " -q queue name to control (required)\n" " -a set queue ACTIVE (accepts dispatch)\n" " -i set queue INACTIVE (rejects dispatch)\n" " -o set queue OPEN (accepts new submissions)\n" " -c set queue CLOSE (rejects new submissions)\n" "example: qctrl -q normal -i\n"; static const char USAGE_HOSTCTRL[] = "hostctrl -h (-o|-c)\n" " -h host to administratively control (required)\n" " -o open the host (allow scheduling)\n" " -c close the host (admin-close; shows CLOSED in\n" " workerlist)\n" "example: hostctrl -h node01 -c\n"; static const char USAGE_UPGRADECONFIG[] = "upgradeconfig \n" " path to a capnp-encoded ClusterConfig blob (same\n" " format `xceadmin config build` produces and\n" " deploy-ha.sh installs)\n" " Computes its SHA-256 and submits it as a configuration upgrade via\n" " OP_upgradeConfig.\n" "example: upgradeconfig /var/xce/cluster.bin\n"; static const char USAGE_JOBLIST[] = "joblist\n" " (no arguments)\n" " Prints all jobs in the cluster (JobId, user, queue, state, times,\n" " ntasks, command) and captures the count into $jobcount.\n"; static const char USAGE_EXPECT_JOBS[] = "expect-jobs \n" " non-negative integer; asserts the current job count equals N\n" " Fails (nonzero) on a malformed N, a failed joblist query, or a count\n" " mismatch.\n" "example: expect-jobs 3\n"; static const char USAGE_EXPECT_JOB[] = "expect-job \n" " a JobId string (as printed by joblist / captured into\n" " $jobid); asserts a job with this id is present in the\n" " current job list\n" " Fails (nonzero) on a missing argument, a failed joblist query, or if\n" " the job is not found.\n" "example: expect-job $jobid\n"; static const char USAGE_WAIT_PRIMARY[] = "wait-primary []\n" " optional max wait in milliseconds (default 30000);\n" " must be a non-negative integer\n" " Blocks, polling every 200 ms, until the AUTHORITATIVE controller\n" " status probe reaches the serving primary (a standby/hint answer does\n" " not count), or the timeout elapses.\n" " Returns 0 once a primary answers, nonzero (ETIMEDOUT) on timeout.\n" "example: wait-primary 10000\n"; static const char USAGE_CTRLSTATUS[] = "ctrlstatus\n" " (no arguments)\n" " Prints the responding controller's identity/role, cluster name,\n" " peer topology, config version/checksum, and HA health (term,\n" " committedLsn, appliedLsn, fenced) from a single any-node status\n" " read.\n"; static const char USAGE_SHUTDOWN[] = "shutdown\n" " (no arguments)\n" " Shuts down the xce controller and, on success, leaves the client\n" " context (same as running `..`).\n"; static const char USAGE_JOBRUN[] = "jobrun -j [-m [:]] [-m ...]\n" " -j jobid to dispatch (required, must be > 0)\n" " -m [:] host to run tasks on, optional :ntasks count\n" " (default 1); may be repeated up to 5 times\n" "example: jobrun -j 12345 -m node01:2 -m node02\n"; static const char USAGE_JOBSUB[] = "jobsub -u -q -c [-R ]\n" " [-n [,]] [-x ] [-a ]\n" " -u submitting user (required)\n" " -q target queue (required)\n" " -c command line to run (required)\n" " -R resource requirement expression (default\n" " \"type == any\")\n" " -n [,] task count, or a min,max range (default 1)\n" " -x raw exdata JSON string (overrides the built-in\n" " PATH env)\n" " -a job array size (default 1)\n" " On success, captures the submitted JobId into the $jobid\n" " auto-variable.\n" "example: jobsub -u alice -q normal -c \"sleep 10\" -n 1,4\n"; static const char USAGE_RESVAL[] = "resval -n [-m ]\n" " -n resource name to query (required)\n" " -m host to query (required unless the resource is a\n" " shared/cluster-wide resource, not per-host)\n" "example: resval -n slots -m myhost\n"; static const char USAGE_JOBKILL[] = "jobkill -j [-s ]\n" " -j jobid to signal (required)\n" " -s signal number to send (default SIGTERM)\n" "example: jobkill -j 12345 -s 9\n"; static const char USAGE_JOBCKPT[] = "jobckpt -j [-k] [-m ]\n" " -j jobid to checkpoint (required)\n" " -k kill the job immediately after checkpointing\n" " -m checkpoint method name (default \"default\")\n" "example: jobckpt -j 12345 -m default\n"; static const char USAGE_JOBINFO[] = "jobinfo -j [-j ...]\n" " -j jobid to show info for; may be repeated to show\n" " several jobs in one call\n" "example: jobinfo -j 12345\n"; static const char USAGE_TASKRUN[] = "taskrun [-j ] [-m [:]]... [-u ] [-c ]\n" " [-x ] [-d ] [-t ]\n" " -j jobid this task run belongs to\n" " -m [:] host to run on, optional :ntasks count\n" " (default 1); may be repeated up to 5 times\n" " -u exec user, applied to every host\n" " -c command line, applied to every host\n" " -x exdata string, applied to every host\n" " -d exec home/cwd directory, applied to every host\n" " -t launch type (default \"mpi\" if the total task\n" " count across all -m hosts is > 1, else a\n" " single-task default)\n" "example: taskrun -j 12345 -m node01:2 -c \"hostname\" -u alice\n"; static const char USAGE_JOIN_SCHED[] = "join\n" " (no arguments)\n" " Joins the cluster as a scheduler.\n"; static const char USAGE_GETCONF[] = "getconf\n" " (no arguments)\n" " Prints the full cluster configuration (name, resources, queues,\n" " workers, user/host groups, configs) known to the scheduler.\n"; static const char USAGE_SETPEND[] = "setpend -j -p [-p ...]\n" " -j jobid to set the pend reason for (required)\n" " -p pend reason spec, may repeat: `` sets the main\n" " reason; `:` adds a host-based reason;\n" " `::` adds a resource-based reason\n" "example: setpend -j 12345 -p 5\n"; static command_t top_commands[] = { { "help", cmd_help, "Display this text", USAGE_HELP }, { "worker", cmd_worker, "In worker context", USAGE_WORKER }, { "scheduler", cmd_scheduler, "In scheduler context", USAGE_SCHEDULER }, { "client", cmd_client, "In client context", USAGE_CLIENT }, { "set", cmd_set, "Show/Set settings", USAGE_SET }, { "echo", cmd_echo, "Print a message", USAGE_ECHO }, { "waldump", cmd_waldump, "Inspect WAL segments", USAGE_WALDUMP }, { "walcompare", cmd_walcompare, "Diff member committed WAL prefixes", USAGE_WALCOMPARE }, { "snapverify", cmd_snapverify, "Verify checkpoint file structure", USAGE_SNAPVERIFY }, { "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files", USAGE_SNAPDIFF }, { "configcompare", cmd_configcompare, "Compare active config hash across members", USAGE_CONFIGCOMPARE }, { "source", cmd_source, "Run commands from a file", USAGE_SOURCE }, { "quit", cmd_quit, "Quit xce test", USAGE_QUIT }, { (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL } }; static command_ctx_t topctx = { .parent = NULL, .prompt = "xce", .settings = settings, .commands = top_commands }; static command_t worker_commands[] = { { "..", cmd_up, "Goto upper context", USAGE_UP }, { "help", cmd_help, "Display this text", USAGE_HELP }, { "quit", cmd_quit, "Quit xce test", USAGE_QUIT }, { "join", cmd_join_worker, "Join the cluster", USAGE_JOIN_WORKER }, { "jstart", cmd_jobstart, "tell controller jobstart", USAGE_JOBSTART }, { "jfinish", cmd_jobfinish, "tell controller jobfinish", USAGE_JOBFINISH }, { "jfail", cmd_jobfail, "tell controller jobfail", USAGE_JOBFAIL }, { "resupdate", cmd_resupdate, "update res vals", USAGE_RESUPDATE }, { "echo", cmd_echo, "Print a message", USAGE_ECHO }, { "source", cmd_source, "Run commands from a file", USAGE_SOURCE }, { (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL } }; static command_ctx_t workerctx = { .parent = &topctx, .prompt = "worker", .settings = settings, .commands = worker_commands }; static command_t client_commands[] = { { "..", cmd_up, "Goto upper context", USAGE_UP }, { "hello", cmd_hello, "Say hello to xce controller", USAGE_HELLO }, { "reslist", cmd_reslist, "Display cluster resources", USAGE_RESLIST }, { "workerlist", cmd_workerlist, "list workers", USAGE_WORKERLIST }, { "qctrl", cmd_qctrl, "queue control", USAGE_QCTRL }, { "hostctrl", cmd_hostctrl, "host control", USAGE_HOSTCTRL }, { "upgradeconfig", cmd_upgradeconfig, "submit a config upgrade (cluster.bin file)", USAGE_UPGRADECONFIG }, { "joblist", cmd_joblist, "list jobs", USAGE_JOBLIST }, { "expect-jobs", cmd_expect_jobs, "assert current job count", USAGE_EXPECT_JOBS }, { "expect-job", cmd_expect_job, "assert a JobId is present", USAGE_EXPECT_JOB }, { "wait-primary", cmd_wait_primary, "wait until a primary serves [timeout_ms]", USAGE_WAIT_PRIMARY }, { "ctrlstatus", cmd_ctrlstatus, "show controller/cluster status", USAGE_CTRLSTATUS }, { "waldump", cmd_waldump, "Inspect WAL segments", USAGE_WALDUMP }, { "walcompare", cmd_walcompare, "Diff member committed WAL prefixes", USAGE_WALCOMPARE }, { "snapverify", cmd_snapverify, "Verify checkpoint file structure", USAGE_SNAPVERIFY }, { "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files", USAGE_SNAPDIFF }, { "configcompare", cmd_configcompare, "Compare active config hash across members", USAGE_CONFIGCOMPARE }, { "shutdown", cmd_shutdown, "shutdown xce controller", USAGE_SHUTDOWN }, { "jobrun", cmd_jobrun, "run a job", USAGE_JOBRUN }, { "jobsub", cmd_jobsub, "submit a job", USAGE_JOBSUB }, { "jobkill", cmd_jobkill, "signal a job", USAGE_JOBKILL }, { "jobckpt", cmd_jobckpt, "checkpoint a job", USAGE_JOBCKPT }, { "jobinfo", cmd_jobinfo, "job information", USAGE_JOBINFO }, { "resval", cmd_resval, "get value of res", USAGE_RESVAL }, { "taskrun", cmd_runtasks, "run tasks", USAGE_TASKRUN }, { "help", cmd_help, "Display this text", USAGE_HELP }, { "echo", cmd_echo, "Print a message", USAGE_ECHO }, { "source", cmd_source, "Run commands from a file", USAGE_SOURCE }, { "quit", cmd_quit, "Quit xce test", USAGE_QUIT }, { (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL } }; static command_ctx_t clientctx = { .parent = &topctx, .prompt = "client", .settings = settings, .commands = client_commands }; static command_t scheduler_commands[] = { { "..", cmd_up, "Goto upper context", USAGE_UP }, { "help", cmd_help, "Display this text", USAGE_HELP }, { "quit", cmd_quit, "Quit xce test", USAGE_QUIT }, { "join", cmd_join_sched, "Join the cluster", USAGE_JOIN_SCHED }, { "getconf", cmd_getconf, "Get conf of cluster", USAGE_GETCONF }, { "setpend", cmd_setpend, "set job pend reason", USAGE_SETPEND }, { "echo", cmd_echo, "Print a message", USAGE_ECHO }, { "source", cmd_source, "Run commands from a file", USAGE_SOURCE }, { (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL } }; static command_ctx_t schedulerctx = { .parent = &topctx, .prompt = "scheduler", .settings = settings, .commands = scheduler_commands }; static command_ctx_t* active_ctx = &topctx; static char prompt[256]; static int break_out = 0; /* xce-test-cli-ux task 1.6, Requirement 4.3: describes the in-session * `help' command as the way to list/learn available commands, in addition * to argp's own auto-generated --help/-?/-V flag documentation below. */ static char doc[] = "xce_test -- interactive/scripted test client for the controller, " "worker, and scheduler\v" "Once a session is running, type `help' at the prompt to list the " "commands available in the current context, or `help ' for " "details on a specific one."; static char args_doc[] = ""; /* build-time-versioning task 3.2: no longer a compile-time initializer -- * argp reads this by pointer when --version/-V is processed, and * xce_compose_version() (utils.h) is a runtime function, not a constant * expression, so it cannot initialize a file-scope global. main() assigns * this to a static buffer, computed via xce_compose_version(), before * argp_parse() runs. See utils.h's NOTE FOR TASK 3.2 above MK_VERSION. */ const char* argp_program_version = NULL; const char* argp_program_bug_address = BUG_EMAIL; /* Long-option key (above ASCII to avoid collisions with short opts) -- * mirrors src/controller/main.c's OPT_PEERS/OPT_CONF_KEEP_N idiom. */ #define OPT_NORC 256 struct arguments { int verbose; int norc; }; static struct argp_option options[] = { { "verbose", 'v', 0, 0, "Produce verbose logs" }, { "norc", OPT_NORC, 0, 0, "Skip loading the rc file for this invocation" }, /* xce-test-cli-ux task 1.6, Requirement 4.1: argp's OWN built-in help * alias is `-?` (via --help), NOT `-h` -- glibc argp does not add `-h` * automatically. `-h` is hidden from the rendered help/usage text * (OPTION_HIDDEN) so it doesn't duplicate the auto-generated * "-?, --help" entry there; it is still a fully functional alias, * handled explicitly in parse_opt() below via argp_state_help(). */ { 0, 'h', 0, OPTION_HIDDEN, 0 }, { 0 } }; static error_t parse_opt(int key, char* arg, struct argp_state* state) { struct arguments* arguments = state->input; switch (key) { case 'v': arguments->verbose++; break; case OPT_NORC: arguments->norc = 1; break; case 'h': /* ARGP_HELP_STD_HELP includes ARGP_HELP_EXIT_OK, so this prints * the same help text as --help/-? and exits(0) -- it does not * return. */ argp_state_help(state, stdout, ARGP_HELP_STD_HELP); break; default: return ARGP_ERR_UNKNOWN; } return 0; } static struct argp argp = { options, parse_opt, args_doc, doc }; void sigbreak(int signo) { break_out = 1; } /* xce-test-cli-ux task 1.3, Requirement 9.9: read-only accessor over * `break_out` for code outside this file (interruptcall.c, task 1.4) to * poll -- see xcetest_interrupt.h's file-header comment for the full * contract (pure read, no reset side effect, break_out's own reset stays * test.c-private). */ int xcetest_interrupted(void) { return break_out; } /* xce-test-cli-ux task 2.1, Requirement 9.4/9.6/13.7: read-only accessor * over `g_interactive` (this file's top-level-session TTY flag, set once * in main() from isatty(STDIN_FILENO) and never changed after that) for * code outside this file (confirm.c) to consult -- mirrors * xcetest_interrupted()'s accessor pattern immediately above. Pure read; * declared in confirm.h rather than xcetest_interrupt.h since confirm.c * is its only consumer and g_interactive is unrelated to that header's * SIGINT/EINTR concern. Deliberately the TOP-LEVEL session's stdin, not * "is this line from a sourced file" -- that distinction is Requirement * 13.7, a later task; this accessor's contract must already be correct * for it, per design.md's Confirmation Gate invariant. */ int xcetest_is_interactive(void) { return g_interactive; } static char* get_setting(char* key) { int i; for (i = 0; active_ctx->settings[i].key; i++) { if (strcmp(key, active_ctx->settings[i].key) != 0) { continue; } if (active_ctx->settings[i].value != NULL) { return active_ctx->settings[i].value; } return active_ctx->settings[i].defval; } printf("Key `%s' is not in settings\n", key); return NULL; } static xce_client_config_t config; /* Holds the parsed HA controller list. Static so the hostname[] buffers it * owns outlive enter_ctx(): xcecli_ctx_init() stores the config pointer (it * does not copy masters[]), so config.masters[i] must reference storage that * persists for the lifetime of the client context. */ static xce_cluster_config_t master_cfg; int enter_ctx(command_ctx_t* ctx) { char* masters = get_setting("masters"); int i; /* Parse the controller HA peer list with the shared libxce parser — the * same "name=host:port,..."/"host:port,..." syntax used by the controller * --peers flag — so the test client fails over across candidates exactly * like the worker/scheduler/apiserver do. */ memset(&master_cfg, 0, sizeof(master_cfg)); if ((masters == NULL) || (xce_parse_peers(masters, &master_cfg) != 0) || (master_cfg.candidates_count < 1)) { fprintf(stderr, "enter_ctx: invalid 'masters' setting `%s'\n", masters ? masters : "(null)"); return -EINVAL; } config.nmasters = master_cfg.candidates_count; if (config.nmasters > N_MAX_MASTERS) { fprintf(stderr, "enter_ctx: %d masters configured, using first %d " "(N_MAX_MASTERS)\n", config.nmasters, N_MAX_MASTERS); config.nmasters = N_MAX_MASTERS; } for (i = 0; i < config.nmasters; i++) { config.masters[i] = master_cfg.candidates[i].hostname; config.master_ports[i] = master_cfg.candidates[i].port; } config.from = get_setting("myname"); config.key = get_setting("secret"); xcecli_ctx_init(&(ctx->ctx), &config); xcecli_set_handler(ctx->ctx, &myreq_handler, NULL); xcecli_start(ctx->ctx); return 0; } int leave_ctx(command_ctx_t* ctx) { if (ctx == &topctx) { return 0; } if (ctx == &workerctx) { delaymgr_fini(&delaymgr); } xcecli_wakeup(active_ctx->ctx, (int)-EINTR); xcecli_stop(ctx->ctx); xcecli_ctx_fini(ctx->ctx); return 0; } static int fill_prompt(command_ctx_t* ctx, char* buf) { if (ctx->parent != NULL) { fill_prompt(ctx->parent, buf); strcat(buf, "/"); } strcat(buf, ctx->prompt); return strlen(buf); } command_t* find_command(char* name) { int i; for (i = 0; active_ctx->commands[i].name; i++) { if (strcmp(active_ctx->commands[i].name, name) == 0) { return &(active_ctx->commands[i]); } } return NULL; } /* xce-test-cli-ux task 2.3, Requirement 7: command-name tab completion. * * libedit API verification (design.md's flagged P0 risk, research.md's * "libedit completion API may not perfectly mirror GNU readline" risk): * confirmed by reading the ACTUAL linked headers in this build environment * (this project's `src/flake.nix`-pinned libedit, pkg-config name * "libedit", providing `editline/readline.h` -- the same header already * included at the top of this file) plus a standalone pty-driven probe * program built and run against this exact library. Findings: * * - The GNU-readline-style completion API IS present verbatim: * `rl_completion_func_t* rl_attempted_completion_function` (signature * `char** (*)(const char*, int, int)`, exactly matching this task's * `xcetest_complete` shape from design.md) and * `char** rl_completion_matches(const char*, rl_compentry_func_t*)` * (matches on a `char* (*)(const char*, int)` generator, exactly * `xcetest_complete_generator`'s shape). No lower-level `` * fallback was needed. * - `rl_attempted_completion_over` exists and suppresses libedit's * default filename-completion fallback -- used below so an empty * match list in the active context yields NO completion rather than * local filenames. * - Single/multiple-match UI behavior is entirely the library's: a sole * match is completed in full (plus a trailing space) with ZERO extra * code (7.2); multiple matches with no further common prefix ring the * bell on the first Tab and print the candidate list (input line left * untouched -- confirmed via a raw pty capture) on a SECOND, * consecutive Tap of Tab -- standard GNU-readline/bash double-Tab * convention (7.3), not something this code implements. This * implementation's own responsibility is narrowly the context-scoped * candidate lookup below. * - `rl_variable_bind("show-all-if-ambiguous", "on")` was probed and * returns failure (1) against this libedit build -- that variable is * not implemented here, so it is not used; the stock double-Tab * behavior above is what ships. */ /* xcetest_complete_generator - the readline "compentry" generator: called * repeatedly with state==0 on the first call (resets the scan) and * state!=0 on each subsequent call, returning one more `strdup()`ed match * per call (readline/libedit frees each returned string) and NULL once * exhausted. Iterates active_ctx->commands[] -- the CURRENTLY active * context at the moment Tab is pressed, read fresh on every call rather * than captured at registration time -- so completion always tracks * `worker`/`scheduler`/`client`/`..` context switches for the life of the * session (Requirement 7.1/7.4: only the active context's commands are * ever offered). Plain prefix match against `text`, same convention as * find_command()'s exact-match lookup just above. */ static char* xcetest_complete_generator(const char* text, int state) { static int idx; static size_t len; if (state == 0) { idx = 0; len = strlen(text); } while (active_ctx->commands[idx].name != NULL) { const char* name = active_ctx->commands[idx].name; idx++; if (strncmp(name, text, len) == 0) { return strdup(name); } } return NULL; } /* xcetest_complete - the readline "attempted completion" entry point, * registered as rl_attempted_completion_function only when g_interactive * (main(), below). `start`/`end` are byte offsets of the word being * completed within rl_line_buffer; `start == 0` means the FIRST word on * the line, i.e. the command name itself. Per this task's explicit * boundary (command-NAME completion only -- no flag-value completion of * jobids/hostnames/resource names), every other word position (start != 0) * returns NULL and leaves rl_attempted_completion_over at its default so * libedit's normal fallback applies there, unchanged from today's * zero-customization behavior. For the command-name position, this always * sets rl_attempted_completion_over so a context with zero matching * commands offers NOTHING (never falls back to filename completion). */ static char** xcetest_complete(const char* text, int start, int end) { (void)end; if (start != 0) { return NULL; } rl_attempted_completion_over = 1; return rl_completion_matches(text, xcetest_complete_generator); } /* Execute a command line. */ int execute_line(char* line) { int rc; command_t* command; int argc; char** argv; argc = parse_argv(line, &argv); if (argc < 0) { SET_ERROR(-argc); perror("execute_line"); return -1; } if ((argv == NULL) || (argv[0] == NULL)) { return 0; } command = find_command(argv[0]); if (!command) { fprintf(stderr, "%s: No such command for xce test.\n", argv[0]); free_argv(argv, argc); return (-1); } rc = (command->func)(argc, argv); free_argv(argv, argc); return rc; } /* Substitute the fixed auto-variables `$?`, `$jobid`, `$jobcount` on a raw line * before tokenizing (xce-test-e2e-primitives task 3.1, Req 3.1/3.3/3.5/3.6). * * Only WHOLE-TOKEN occurrences are recognized: a `$name` must be bounded by the * start of the line / whitespace on its left and whitespace / end-of-line on its * right. A `$name` glued to other non-whitespace (e.g. `x$jobid`, `$jobidx`) is * NOT a recognized auto-variable and is copied through verbatim. No other `$...` * syntax is recognized: an unknown whole token such as `$other` is also copied * verbatim (it is simply not one of the three names). * * Replacements: * $? -> decimal of g_last_status (always available). * $jobid -> g_last_jobid if set (non-empty); unset => command failure. * $jobcount -> decimal of g_last_jobcount if set (>=0); unset => failure. * * Writes the expanded line into `out` (capacity `outsz`, always NUL-terminated * on success). Returns 0 on success, -EINVAL on an unset auto-var reference, * -ENOSPC if the expansion would overflow `out`. On any error `out` content is * unspecified and the caller must fail the command rather than run it. */ static int substitute_autovars(const char* line, char* out, size_t outsz) { const char* p = line; size_t o = 0; if ((line == NULL) || (out == NULL) || (outsz == 0)) { return -EINVAL; } while (*p != '\0') { /* A `$` starts a candidate auto-var only when it begins a token, i.e. * the previous char is the line start or whitespace. */ int at_token_start = (p == line) || (*(p - 1) == ' ') || (*(p - 1) == '\t'); if ((*p == '$') && at_token_start) { const char* name = p + 1; const char* end = name; const char* repl = NULL; char numbuf[32]; /* Scan to the token end (whitespace or NUL). */ while ((*end != '\0') && (*end != ' ') && (*end != '\t')) { end++; } size_t namelen = (size_t)(end - name); if ((namelen == 1) && (name[0] == '?')) { snprintf(numbuf, sizeof(numbuf), "%d", g_last_status); repl = numbuf; } else if ((namelen == 5) && (strncmp(name, "jobid", 5) == 0)) { if (g_last_jobid[0] == '\0') { fprintf(stderr, "autovar: $jobid referenced before set\n"); return -EINVAL; } repl = g_last_jobid; } else if ((namelen == 8) && (strncmp(name, "jobcount", 8) == 0)) { if (g_last_jobcount < 0) { fprintf(stderr, "autovar: $jobcount referenced before set\n"); return -EINVAL; } snprintf(numbuf, sizeof(numbuf), "%d", g_last_jobcount); repl = numbuf; } if (repl != NULL) { size_t rlen = strlen(repl); if ((o + rlen) >= outsz) { return -ENOSPC; } memcpy(out + o, repl, rlen); o += rlen; p = end; /* consume the whole `$name` token */ continue; } /* Not a recognized auto-var: copy the whole token verbatim. */ while (p < end) { if ((o + 1) >= outsz) { return -ENOSPC; } out[o++] = *p++; } continue; } if ((o + 1) >= outsz) { return -ENOSPC; } out[o++] = *p++; } out[o] = '\0'; return 0; } /* Main-loop wrapper around execute_line that implements the session verdict * (xce-test-e2e-primitives task 1.1): * - blank / whitespace-only lines and `#`-comment lines are ignored; * - normal lines run via execute_line; g_last_status / g_session_failed are * updated from the result; * - in default (abort) mode a failure stops the loop; under * g_continue_on_error the loop keeps running but the failure is recorded. * * Returns 0 to keep the read loop running, non-zero to request that the loop * stop (abort-on-failure). The leading `!` shell-escape and `$`-substitution * are intentionally handled in later tasks (2.1 / 3.1); this function is the * single chokepoint where they will be added. */ int preprocess_and_run(char* line) { char* p = line; int rc; /* Skip leading whitespace to classify the line. */ while ((*p == ' ') || (*p == '\t')) { p++; } /* Ignore blank / whitespace-only lines and `#` comments. */ if ((*p == '\0') || (*p == '#')) { return 0; } /* Shell-escape: a line whose first non-whitespace char is `!` runs the * remainder via system(). The shell command's exit status becomes * g_last_status (the `$?` auto-variable); a non-zero status is a command * failure subject to the same abort-vs-continue rules as execute_line. The * remainder is passed raw to the shell (no parse_argv / auto-var * substitution — that is task 3.1). (xce-test-e2e-primitives task 2.1) */ if (*p == '!') { const char* rest = p + 1; int sysrc = system(rest); int status; if (sysrc == -1) { /* system() itself failed to run the shell — treat as failure. */ status = 127; } else if (WIFEXITED(sysrc)) { status = WEXITSTATUS(sysrc); } else { /* Signaled / abnormal termination — non-zero failure status. */ status = 1; } g_last_status = status; if (status != 0) { g_session_failed = 1; if (!g_continue_on_error && !g_interactive) { return 1; /* abort: stop the read loop (scripted mode only) */ } } else { add_history(line); } return 0; } /* Substitute the fixed auto-variables (`$?`, `$jobid`, `$jobcount`) before * tokenizing (xce-test-e2e-primitives task 3.1). An unset `$jobid` / * `$jobcount` reference (Req 3.6) — or an overflow — fails the command: * we do NOT run execute_line, and the result is treated exactly like a * failing command (g_last_status non-zero, g_session_failed, abort vs * continue). The leading `!` shell-escape above is intentionally NOT * substituted: that line goes raw to system() and the shell owns `$?`. */ { char expanded[MAX_LINE_LEN]; int subrc = substitute_autovars(p, expanded, sizeof(expanded)); if (subrc < 0) { g_last_status = subrc; g_session_failed = 1; if (!g_continue_on_error && !g_interactive) { return 1; /* abort: stop the read loop (scripted mode only) */ } return 0; } rc = execute_line(expanded); } g_last_status = (rc < 0) ? (rc) : 0; if (rc < 0) { SET_ERROR(-rc); g_session_failed = 1; if (!g_continue_on_error) { return 1; /* errexit (-e): stop the read loop on first failure */ } } else { add_history(line); } return 0; } int cmd_help(int argc, char* argv[]) { int i; int rc = 0; if (argc > 1) { char* cmd = argv[1]; for (i = 0; active_ctx->commands[i].name != NULL; i++) { if (strcmp(argv[1], active_ctx->commands[i].name) == 0) { /* `help ` prints the detailed per-command usage (flags, * positionals, example) -- distinct from the compact one-line * `doc` the bare `help` listing below still uses unchanged. * Every table entry is expected to carry a non-NULL `usage` * now; the `doc` fallback stays only as a defensive guard * against a future entry that forgets to set it. */ if (active_ctx->commands[i].usage != NULL) { fputs(active_ctx->commands[i].usage, stdout); } else { printf("%s\t\t%s.\n", active_ctx->commands[i].name, active_ctx->commands[i].doc); } return 0; } } printf("No commands matchs `%s'. Possibilities are:\n", cmd); rc = -1; } for (i = 0; active_ctx->commands[i].name != NULL; i++) { printf("\t%-12s\t%s\n", active_ctx->commands[i].name, active_ctx->commands[i].doc); } return rc; } /* echo — print the remainder of the line (argv[1..] joined by * single spaces) followed by a newline and succeed (xce-test-e2e-primitives * task 1.2, Req 6.3). `echo` with no argument prints an empty line and still * succeeds. Returns 0 always so it never marks the session failed. */ int cmd_echo(int argc, char* argv[]) { int i; for (i = 1; i < argc; i++) { if (i > 1) { putchar(' '); } fputs(argv[i], stdout); } putchar('\n'); return 0; } /* waldump [ ...] — offline WAL inspector for failover/durability * triage. Each path may be a seg-*.wal file, a wal/ dir, or a node/data dir. * Read-only: lists every entry, decodes JOB_SUBMIT jobids, and flags LSN gaps * (a missing committed prefix is THE failover job-loss signal) plus the CRC * recovered tip. Returns nonzero if any path failed or any gap was found, so * it doubles as an assertion in scripted E2E runs. */ int cmd_waldump(int argc, char* argv[]) { if (argc < 2) { fprintf(stderr, "usage: waldump [more...]\n"); return -1; } int rc = 0; for (int i = 1; i < argc; i++) { waldump_stats_t st; int r = waldump_path(argv[i], stdout, &st); if (r != 0) { fprintf(stderr, "waldump: %s: %s\n", argv[i], strerror(-r)); rc = -1; continue; } if (st.n_gaps > 0 || st.crc_broken) rc = -1; /* surface the defect as a nonzero command verdict */ } return rc; } /* walcompare [--report] [--committed N] [more...] — diff * member WAL committed prefixes by (lsn -> term, pcrc) and FAIL (nonzero) on any * committed-prefix divergence (ha-term-safety R6.1/R6.2/R6.3), reusing the * waldump decoder. Each member path may be a seg-*.wal file, a wal/ dir, or a * node/data dir (same resolution as waldump). With no --committed, the shared * committed boundary = the minimum CRC-intact tip across members. The diverging * LSN and per-member (term, pcrc) identities are reported. With --report, also * emit a machine-checkable per-member `(lsn, term, pcrc, status)` table and a * committed-vs-uncommitted classification for each divergence (task 1.1). Exit * 0 = identical, nonzero = divergence or load error — so it doubles as a * scripted assertion. */ /* Both flags are long-only (no short letter ever existed for either -- * cmd_walcompare was the one command using a fully hand-rolled long-flag * parser, per xce-test-cli-ux task 3.1's explicit scope). Their synthetic * getopt_long() `val`s are FLAGSPEC_LONG_ONLY_BASE + * (flagspec.h) -- the enum below names those indices so the switch below * doesn't rely on bare magic numbers. */ enum { WALCOMPARE_OPT_REPORT = 0, WALCOMPARE_OPT_COMMITTED = 1 }; static const flagspec_t walcompare_flags[] = { { 0, "report", "report", no_argument, 0, NULL }, { 0, "committed", "committed", required_argument, 0, "derive the shared boundary from members' CRC-intact tips" }, }; #define WALCOMPARE_NFLAGS \ (sizeof(walcompare_flags) / sizeof(walcompare_flags[0])) int cmd_walcompare(int argc, char* argv[]) { uint64_t committed = 0; /* 0 = derive the shared boundary */ int report = 0; /* --report => full per-member machine table */ int opt; struct option long_opts[WALCOMPARE_NFLAGS + 1]; char optstring[8]; char posix_optstring[sizeof(optstring) + 1]; optind = 1; opterr = 0; if (flagspec_build_opts(walcompare_flags, WALCOMPARE_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: walcompare: internal error building flag table\n"); return -EINVAL; } /* Leading '+' (GNU getopt extension): stop at the first non-option * argument instead of permuting argv -- matches the pre-migration * hand-rolled parser's exact behavior (flags recognized only before the * positional member paths; both flags here are long-only, so the built * optstring is empty and this is simply "+"). */ posix_optstring[0] = '+'; strcpy(posix_optstring + 1, optstring); while ((opt = getopt_long(argc, argv, posix_optstring, long_opts, NULL)) != -1) { switch (opt) { case FLAGSPEC_LONG_ONLY_BASE + WALCOMPARE_OPT_REPORT: report = 1; break; case FLAGSPEC_LONG_ONLY_BASE + WALCOMPARE_OPT_COMMITTED: { char* end = NULL; unsigned long v = strtoul(optarg, &end, 10); if (end == optarg || *end != '\0') { fprintf(stderr, "walcompare: bad --committed LSN '%s'\n", optarg); return -1; } committed = (uint64_t)v; break; } case '?': return flagspec_reject( "walcompare", walcompare_flags, WALCOMPARE_NFLAGS, argv[optind - 1], "walcompare [--report] [--committed ] " " [more...]"); } } int n = argc - optind; if (n < 2) { fprintf(stderr, "usage: walcompare [--report] [--committed N] " " [more...]\n"); return -1; } if (n > 16) { fprintf(stderr, "walcompare: at most 16 members\n"); return -1; } walcompare_member_t members[16]; for (int i = 0; i < n; i++) { members[i].path = argv[optind + i]; members[i].committed_lsn = committed; /* 0 => derive shared boundary */ } walcompare_divergence_t div; int rc = report ? walcompare_report(members, n, stdout, &div) : walcompare(members, n, stdout, &div); if (rc == WALCMP_OK) return 0; if (rc == WALCMP_DIVERGENT) { fprintf(stderr, "walcompare: FAIL — committed-prefix divergence at LSN %" PRIu64 "\n", div.lsn); return -1; /* nonzero session verdict */ } fprintf(stderr, "walcompare: error: %s\n", strerror(-rc)); return -1; } /* snapverify [ ...] — offline checkpoint-file * structural validator for force-snapshot triage (bug-1389: OPCode_ * snapshotTransfer chunk-content corruption). Each path is a snap-*.capnp * (locally generated) or snap-*.bin (received via ha_handle_snapshot_chunk) * file. Read-only: walks the same layout persist_load_snapshot() parses * without restoring anything, and on the first structural defect reports the * exact byte offset and how many job/queue frames parsed cleanly before it — * the diagnostic a bare "persist_load_snapshot failed rc=-5" log line * doesn't give you. Returns nonzero if any path failed to verify. */ int cmd_snapverify(int argc, char* argv[]) { if (argc < 2) { fprintf(stderr, "usage: snapverify [more...]\n"); return -1; } int rc = 0; for (int i = 1; i < argc; i++) { snapverify_report_t rep; int r = snapverify_path(argv[i], stdout, &rep); if (r != 0) rc = -1; } return rc; } /* snapdiff — byte-level checkpoint-file * comparator for force-snapshot triage (bug-1389). `source_file` is the * primary's own locally generated checkpoint (snap-*.capnp); `received_file` * is the copy a standby assembled from the wire (snap-*.bin). Reports the * first differing byte offset and whether it lands exactly on an * OPCode_snapshotTransfer chunk boundary — the decisive signal for "the * transport corrupted one chunk" vs "these were never the same content". * Returns 0 if identical, nonzero otherwise. */ int cmd_snapdiff(int argc, char* argv[]) { if (argc != 3) { fprintf(stderr, "usage: snapdiff \n"); return -1; } snapdiff_report_t rep; int rc = snapdiff_paths(argv[1], argv[2], stdout, &rep); return (rc == 0) ? 0 : -1; } /* configcompare [more...] — cross-member * active-configuration comparator (config-upgrade-snapshot task 8.4, Req * 4.2). Each path is a member's `workdir/name` conf-root directory (the * same directory persist.c's read_active_config_bytes() and config_repl.c's * local_config_sha256() resolve `conf/current` under) — NOT a raw config * file path. Recomputes each member's active-config sha256 fresh from its * own on-disk bytes and reports whether all members match. Returns 0 if * all members are present and identical, nonzero if any member is * unreadable or divergent. */ int cmd_configcompare(int argc, char* argv[]) { if (argc < 3) { fprintf(stderr, "usage: configcompare [more...]\n"); return -1; } configcompare_report_t rep; int rc = configcompare_paths((const char* const*)&argv[1], argc - 1, stdout, &rep); return (rc == 0) ? 0 : -1; } /* -------------------------------------------------------------------- * source -- in-session script sourcing (xce-test-cli-ux task 2.4, * Requirements 13.1-13.6; 13.7's TTY-condition clarification relating * `source` to the Confirmation Gate is a separate, later validation task * and is NOT implemented here). * * Reuses rcfile_load() (rcfile.c/h, xce-test-rc-file spec) UNCHANGED as the * actual line-reading/dispatch loop, passing preprocess_and_run as its * runner -- the same per-line command dispatcher the interactive REPL loop * and the startup rc-file loader already use. rcfile.c itself is not * touched: its own missing-file-is-silent-success policy is correct for * its one other caller (the startup rc file) but wrong for an explicit * user command, so cmd_source layers its own missing-file check (access()) * and cycle detection (a small in-flight canonical-path stack) on top, * entirely in this file -- per research.md's "source built directly on * rcfile_load" decision. * * Cycle detection: source_push_path()/source_pop_path() maintain a static * LIFO stack of realpath()-canonicalized in-flight source paths. * source_push_path() scans the stack for the same canonical path already * in flight (a real cycle -- including through symlinks or a differently- * spelled relative path, since realpath() runs before the scan) and * returns -ELOOP without pushing if found; otherwise it pushes and returns * 0. cmd_source calls source_pop_path() on EVERY exit path once the push * succeeded (success, or rcfile_load returning nonzero because a line * inside the file failed under -e/abort-on-failure semantics) -- this is * the "path stack popped exactly once per successful push, on every exit * path" invariant design.md requires. When source_push_path() itself * fails (cycle detected), nothing was pushed, so cmd_source returns * immediately WITHOUT calling source_pop_path() -- there is nothing to * pop. * * Abort/continue-on-error composition (Req 13.3): a failing line inside * the sourced file makes preprocess_and_run (the runner) return 1 once * g_continue_on_error is false, which is exactly rcfile_load's own * "stop on nonzero runner return" mechanism -- no extra code is needed * here for that part; it falls out of reusing rcfile_load() unchanged. * cmd_source's own return value then translates rcfile_load's result * (0 == every line ran without requesting an abort) into this codebase's * negative-errno failure convention (nonzero -> a generic -1 failure, * mirroring cmd_configcompare's/cmd_waldump's own "rc == 0 ? 0 : -1" * pattern) rather than forwarding rcfile_load's raw positive "stop" signal * verbatim: forwarding it unchanged would be silently treated as SUCCESS * by an enclosing preprocess_and_run call (only rc<0 counts as failure * there), which would incorrectly let an enclosing script's later lines * keep running past a `source` that itself aborted under -e, and would * break that composition for nested sourcing (source A containing * "source B", where B aborts). Translating to a negative value here makes * a failed `source` compose correctly at every nesting depth, exactly * like any other failing command. * * Nested sourcing (Req 13.4): "source" is registered in every command * table (see the four command_t[] tables below); a line inside a sourced * file that itself reads "source " is dispatched by * preprocess_and_run/execute_line exactly like any other line, so nested * sourcing falls out of dispatch being generic. The path stack is a * single process-wide stack (not scoped per active context), so a cycle * is caught regardless of which context each nesting level was sourced * from. */ #define SOURCE_MAX_DEPTH 64 static char* g_source_stack[SOURCE_MAX_DEPTH]; static int g_source_depth = 0; /* Returns 0 and pushes `canonical_path` if it is not already in flight, or * -ELOOP without pushing if it is (a cycle -- the same canonicalized path * sourcing itself, directly or transitively). Returns -ENOMEM if the stack * is exhausted (SOURCE_MAX_DEPTH levels of DISTINCT, non-cyclic nested * sourcing) or a heap copy of the path could not be made -- either way, * nothing is pushed, so the caller must not call source_pop_path(). */ static int source_push_path(const char* canonical_path) { int i; for (i = 0; i < g_source_depth; i++) { if (strcmp(g_source_stack[i], canonical_path) == 0) { return -ELOOP; } } if (g_source_depth >= SOURCE_MAX_DEPTH) { return -ENOMEM; } g_source_stack[g_source_depth] = strdup(canonical_path); if (g_source_stack[g_source_depth] == NULL) { return -ENOMEM; } g_source_depth++; return 0; } /* Pops the most recently pushed path (LIFO). A no-op if the stack is * already empty (defensive; every real call site only calls this after a * source_push_path() call that itself returned 0). */ static void source_pop_path(void) { if (g_source_depth <= 0) { return; } g_source_depth--; free(g_source_stack[g_source_depth]); g_source_stack[g_source_depth] = NULL; } /* source -- reads and executes each line of as if it had * been typed at the current prompt, in order (Req 13.1). A missing or * unreadable file is rejected with an explicit stderr error BEFORE * rcfile_load() is ever called, bypassing its silent-missing-file * behavior entirely (Req 13.2) -- rcfile.c itself is unmodified. */ int cmd_source(int argc, char* argv[]) { char canonical[PATH_MAX]; int rc; if (argc != 2) { fprintf(stderr, "usage: source \n"); return -EINVAL; } if (access(argv[1], R_OK) != 0) { fprintf(stderr, "source: %s: %s\n", argv[1], strerror(errno)); return -errno; } if (realpath(argv[1], canonical) == NULL) { fprintf(stderr, "source: %s: %s\n", argv[1], strerror(errno)); return -errno; } rc = source_push_path(canonical); if (rc == -ELOOP) { fprintf(stderr, "source: %s: cycle detected (file sources itself, directly " "or transitively)\n", canonical); return rc; } else if (rc != 0) { fprintf(stderr, "source: %s: %s\n", canonical, strerror(-rc)); return rc; } rc = rcfile_load(argv[1], preprocess_and_run); source_pop_path(); /* popped on EVERY exit path past the push above */ return (rc == 0) ? 0 : -1; } int cmd_up(int argc, char* argv[]) { if (active_ctx->parent != NULL) { int rc; if ((rc = leave_ctx(active_ctx)) < 0) { return rc; } active_ctx = active_ctx->parent; } return 0; } int cmd_worker(int argc, char* argv[]) { int rc; if ((rc = enter_ctx(&workerctx)) < 0) { return rc; } active_ctx = &workerctx; delaymgr_init(&delaymgr, 2000 /* precision_ms */, NULL); delaymgr_start(&delaymgr); delay_task(&delaymgr, 5, -1, (delay_task_entry_fn_t)worker_house_keeping, &workerctx); return 0; } int cmd_scheduler(int argc, char* argv[]) { int rc; if ((rc = enter_ctx(&schedulerctx)) < 0) { return rc; } active_ctx = &schedulerctx; return 0; } int cmd_client(int argc, char* argv[]) { int rc; if ((rc = enter_ctx(&clientctx)) < 0) { return rc; } active_ctx = &clientctx; return 0; } static int done = 0; int cmd_quit(int argc, char* argv[]) { leave_ctx(active_ctx); done = 1; return 0; } static const flagspec_t hello_flags[] = { { 'c', "count", "count", required_argument, 0, "1" }, { 'm', "message", "message", required_argument, 0, "\"Greeting from xce client\"" }, }; #define HELLO_NFLAGS (sizeof(hello_flags) / sizeof(hello_flags[0])) int cmd_hello(int argc, char* argv[]) { int rc = 0; int opt; char* msg = "Greeting from xce client"; int count = 1; struct option long_opts[HELLO_NFLAGS + 1]; char optstring[32]; optind = 1; opterr = 0; if (flagspec_build_opts(hello_flags, HELLO_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: hello: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'c': count = atoi(optarg); break; case 'm': msg = optarg; break; case '?': return flagspec_reject("hello", hello_flags, HELLO_NFLAGS, argv[optind - 1], "hello [-c ] [-m ]"); } } break_out = 0; while ((!break_out) && (count > 0)) { rc = xcecli_hello(active_ctx->ctx, msg); count--; } return rc; } int cmd_shutdown(int argc, char* argv[]) { int rc; rc = xcecli_shutdown(active_ctx->ctx); /* leave client context */ if (rc >= 0) { execute_line(".."); } return rc; } int cmd_reslist(int argc, char* argv[]) { int rc, i; xce_resource_t** res; rc = xcecli_reslist(active_ctx->ctx, &res); if (rc < 0) { SET_ERROR(-rc); perror("xce_reslist"); return rc; } /* Note the extra literal space before the Description specifier: unlike * cmd_workerlist's all-left-justified text columns (which self-separate * via their own trailing padding), Interval stays right-justified here, * so without an explicit separator a short Interval value would run * directly into the now-left-justified Description text. */ printf("%-10s%10s%10s %-64s\n", "Name", "Flags", "Interval", "Description"); for (i = 0; i < rc; i++) { /* bug-1711 (Requirement 11.4): prefix as hex ("0x%x"), matching this * file's own state_raw convention (print_jobinfo), instead of an * unlabeled bare %x that is ambiguous vs decimal. */ char flags_buf[16]; snprintf(flags_buf, sizeof(flags_buf), "0x%x", res[i]->flags); printf("%-10s%10s%10d %-64s\n", res[i]->name, flags_buf, res[i]->interval, res[i]->desc); } free_Resource_list(rc, res); return 0; } /* worker_status_str() -- extracted to joblist_display.c/.h (xce-test-cli-ux * task 2.5 remediation): pure, dependency-light display formatting, no * networking/REPL/global-state coupling, so it can be unit-tested outside * of this file (see test_joblist_display.c), mirroring set_resval()'s own * extraction to resval_typeresolve.c below. Call sites are unchanged -- * they now resolve through joblist_display.h's declaration. */ /* forward decl: defined near cmd_resval, below, which also uses it */ static void format_bytes_human(double bytes, char* buf, size_t bufsize); int cmd_workerlist(int argc, char* argv[]) { int rc, i; xce_worker_t** workers; rc = xcecli_workerlist(active_ctx->ctx, &workers); if (rc < 0) { SET_ERROR(-rc); perror("xce_workerlist"); return rc; } printf("%-20s%-8s%-25s%-25s%12s%12s%12s%12s%12s%12s\n", "HostName", "Status", "HostType", "HostModel", "CPUfactor", "MaxCPU", "MaxMem", "MaxSwap", "MaxTmp", "NDisks"); for (i = 0; i < rc; i++) { /* MaxMem/MaxSwap/MaxTmp are stored/wired as raw bytes (see * is_bytes_valued_resource) -- humanize for display only, every * other column stays as-is. */ char maxmem_buf[64]; char maxswap_buf[64]; char maxtmp_buf[64]; format_bytes_human(workers[i]->attrs->maxmem, maxmem_buf, sizeof(maxmem_buf)); format_bytes_human(workers[i]->attrs->maxswap, maxswap_buf, sizeof(maxswap_buf)); format_bytes_human(workers[i]->attrs->maxtmp, maxtmp_buf, sizeof(maxtmp_buf)); printf("%-20s%-8s%-25s%-25s%12f%12d%12s%12s%12s%12d\n", workers[i]->name, worker_status_str(workers[i]->status), workers[i]->attrs->hosttype, workers[i]->attrs->hostmodel, workers[i]->attrs->cpufactor, workers[i]->attrs->maxcpus, maxmem_buf, maxswap_buf, maxtmp_buf, workers[i]->attrs->ndisks); } free_Worker_list(rc, workers); return 0; } /* job_state_str()/format_job_time() -- extracted to joblist_display.c/.h * (xce-test-cli-ux task 2.5 remediation), same rationale as * worker_status_str() above: pure, dependency-light display formatting, * unit-tested outside of this file in test_joblist_display.c. Call sites * (below, and print_jobinfo's "state:" line) are unchanged. */ /* Shared, print-free joblist core for `cmd_joblist` and the `expect-jobs` / * `expect-job` assertions (xce-test-e2e-primitives task 5.1, design.md:81, * 116-117). * * Performs the single `xcecli_joblist` call (through the failover-aware client * context) and returns the current job count (>= 0), or a negative errno-style * value on failure. When `find_jobid != NULL`, `*contains` is set to 1 if a job * whose `fmt_jobid(...)` string equals `find_jobid` is present, else 0; the * `fmt_jobid` representation matches the one `cmd_joblist` prints and the one * `cmd_jobsub` captures into `$jobid` (task 4.1), so `expect-job $jobid` lines * up. When `print_table` is set, the human-readable table is printed (the * `cmd_joblist` path). The job array is always freed internally. * * On error this returns the negative rc WITHOUT setting any auto-variable, so a * failed joblist never advertises a fake `$jobcount` (Req 3.6). */ static int joblist_query(const char* find_jobid, int* contains, int print_table, int quiet) { int rc, i; xce_job_t** jobs; if (contains != NULL) { *contains = 0; } rc = xcecli_joblist(active_ctx->ctx, &jobs); if (rc < 0) { SET_ERROR(-rc); /* `quiet` callers (e.g. the wait-primary poll, which probes repeatedly * and EXPECTS failure until a primary serves) suppress the per-attempt * error print so a normal failover wait does not spam the console. */ if (!quiet) { perror("xce_joblist"); } return rc; } if (print_table) { printf("%-26s%-10s%-10s%-8s%-24s%-24s%-24s%12s%12s%30s\n", "JobId", "User", "Queue", "State", "SubmitTime", "StartTime", "FinishTime", "MinTasks", "MaxTasks", "Command"); } for (i = 0; i < rc; i++) { char buf[128]; fmt_jobid(buf, sizeof(buf) - 1, jobs[i]->jobid); if ((find_jobid != NULL) && (contains != NULL) && (strcmp(buf, find_jobid) == 0)) { *contains = 1; } if (print_table) { char submit_buf[32]; char start_buf[32]; char finish_buf[32]; printf("%-26s%-10s%-10s%-8s%-24s%-24s%-24s%12d%12d%30s\n", buf, jobs[i]->submit->user, jobs[i]->submit->queue, job_state_str(jobs[i]->state), format_job_time(jobs[i]->queue_time, submit_buf, sizeof(submit_buf)), format_job_time(jobs[i]->start_time, start_buf, sizeof(start_buf)), format_job_time(jobs[i]->finish_time, finish_buf, sizeof(finish_buf)), jobs[i]->submit->mintasks, jobs[i]->submit->maxtasks, jobs[i]->submit->start_info->cmdline); } } for (i = 0; i < rc; i++) { free_Job_ptr(&(jobs[i])); } free(jobs); return rc; } int cmd_joblist(int argc, char* argv[]) { int count; count = joblist_query(NULL, NULL, 1, /*quiet=*/0); if (count < 0) { return count; } /* Capture $jobcount (Req 3.4): a successful joblist publishes the returned * job count so a later `echo $jobcount` / `expect-jobs` substitutes it. On * failure we returned above without touching g_last_jobcount, so a failed * joblist never advertises a fake count (Req 3.6). */ g_last_jobcount = count; return 0; } /* `expect-jobs N` assertion (xce-test-e2e-primitives task 5.2, Req 5.1/5.2, * design.md:116-117). * * Passes (returns 0) when the current job count equals N; otherwise prints the * expected vs actual count and returns a negative errno-style value so * preprocess_and_run records the session as failed (Req 1.1). * * A malformed argument (missing or non-numeric N) is itself an assertion * failure, not a no-op. A failed joblist query is treated as a failure too: we * cannot determine the count, so we never silently equate "query failed" with * "zero jobs" (Req 5.2 reports actual; a failed query has no trustworthy * actual). */ int cmd_expect_jobs(int argc, char* argv[]) { long expected; char* end; int count; if (argc != 2) { printf("expect-jobs \n"); return -EINVAL; } errno = 0; expected = strtol(argv[1], &end, 10); if ((errno != 0) || (end == argv[1]) || (*end != '\0') || (expected < 0)) { printf("expect-jobs: invalid count '%s' (expected a non-negative " "integer)\n", argv[1]); return -EINVAL; } /* Print-free count via the shared helper. A negative rc means the query * itself failed (e.g. no reachable master) — that is an assertion failure, * not "0 jobs". */ count = joblist_query(NULL, NULL, 0, /*quiet=*/0); if (count < 0) { printf("expect-jobs: could not determine job count (joblist query " "failed)\n"); return count; } if (count != (int)expected) { printf("expect-jobs: expected %ld, actual %d\n", expected, count); return -EINVAL; } printf("expect-jobs: ok (%d)\n", count); return 0; } /* `expect-job ` assertion (xce-test-e2e-primitives task 5.3, Req 5.3/5.4, * design.md cmd_expect_jobs / cmd_expect_job). * * Passes (returns 0) when a job whose `fmt_jobid` string equals argv[1] is * present in the current job list; otherwise prints the missing id and returns a * negative errno-style value so preprocess_and_run records the session as failed * (Req 1.1). * * `$jobid` substitution is transparent: autovar substitution (task 3.1) replaces * `$jobid` with the captured id string BEFORE this handler runs, so argv[1] is * already the literal id. The membership test in joblist_query uses the SAME * `fmt_jobid` representation that cmd_jobsub captured into `$jobid` (task 4.1), * so `expect-job $jobid` matches the joblist entry. * * A missing argument is itself an assertion failure, not a no-op. A failed * joblist query is also a failure: we cannot determine membership, so we report * the query failure distinctly rather than silently equating it with "absent". */ int cmd_expect_job(int argc, char* argv[]) { int contains; int rc; if (argc != 2) { printf("expect-job \n"); return -EINVAL; } /* Print-free membership query via the shared helper. A negative rc means the * query itself failed (e.g. no reachable master) — that is an assertion * failure, distinct from "job absent". */ rc = joblist_query(argv[1], &contains, 0, /*quiet=*/0); if (rc < 0) { printf("expect-job: could not determine membership of '%s' (joblist " "query failed)\n", argv[1]); return rc; } if (!contains) { printf("expect-job: job %s not found\n", argv[1]); return -ENOENT; } printf("expect-job: ok (%s)\n", argv[1]); return 0; } /* `wait-primary [timeout_ms]` (controller-status task 4.2, Req 9.1-9.4; * originally xce-test-e2e-primitives task 6.1; design.md "cmd_wait_primary probe * swap — State contract"). * * Blocks until the AUTHORITATIVE controller status probe reaches the serving * primary, or the timeout elapses. The probe is xcecli_ctrlstatus_primary: it * drives active_ctx + libxce master-rotation/failover and returns 0 ONLY when a * reply with responderRole == PRIMARY was obtained (it internally follows the * redirect to the primary), or a typed negative when no primary answered within * its budget (Req 9.1). A standby/hint answer is NOT success — the probe keeps * polling until the primary itself answers (this avoids a false success on a * standby's stale primary view after a primary death). * * On the first authoritative probe we print the served primary's identity and * return 0 (Req 9.2). Otherwise we sleep a short poll interval and retry, * re-checking a MONOTONIC deadline between attempts so the command returns * within a bounded time of the timeout and never hangs indefinitely when there * is no primary (Req 9.3). The deadline is derived from CLOCK_MONOTONIC, never * wall-clock, so a clock step cannot skew it. * * The timeout is argv[1] in milliseconds when supplied, else * WAIT_PRIMARY_DEFAULT_MS (Req 9.4). A malformed timeout argument is a usage * error (return < 0) rather than a silent fallback to the default. */ #define WAIT_PRIMARY_DEFAULT_MS 30000L /* default max wait when arg omitted */ #define WAIT_PRIMARY_POLL_MS 200L /* sleep between probe attempts */ #define MS_PER_SEC 1000L #define NS_PER_MS 1000000L #define US_PER_MS 1000L /* The client's configured log level (set in main from -v), so wait-primary can * temporarily raise the threshold to silence libxce's per-attempt failover INFO * spam during its poll and then RESTORE the user's chosen level. */ static char* g_active_loglevel = (char*)"info"; static long monotonic_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return ((long)ts.tv_sec * MS_PER_SEC) + (ts.tv_nsec / NS_PER_MS); } int cmd_wait_primary(int argc, char* argv[]) { long timeout_ms = WAIT_PRIMARY_DEFAULT_MS; long start_ms, elapsed_ms; if (argc > 2) { printf("wait-primary [timeout_ms]\n"); return -EINVAL; } if (argc == 2) { char* end; errno = 0; timeout_ms = strtol(argv[1], &end, 10); if ((errno != 0) || (end == argv[1]) || (*end != '\0') || (timeout_ms < 0)) { printf("wait-primary: invalid timeout '%s' (expected a " "non-negative integer in ms)\n", argv[1]); return -EINVAL; } } start_ms = monotonic_ms(); /* The poll probes repeatedly and EXPECTS failure until a primary serves, so * it must not spam the console: xcecli_ctrlstatus_primary suppresses its own * per-attempt output, and we raise the client log threshold for the duration * of the loop to silence libxce's per-attempt "xcecli failover: attempt N" * INFO logs — RESTORED to the user's level on every exit path. */ mylog_setlevel((char*)"warn"); /* Probe, then re-check the monotonic deadline between attempts. Each probe * may itself take time (libxce failover + redirect-to-primary has its own * internal retry budget); we re-read the clock after every probe returns so * the overall loop honors the deadline and returns failure in bounded time. */ for (;;) { xce_ctrlstatus_t* status = NULL; int rc = xcecli_ctrlstatus_primary(active_ctx->ctx, &status); /* Success ONLY on an authoritative reply from the serving primary * (responderRole == PRIMARY, guaranteed by xcecli_ctrlstatus_primary). * Print the served primary's identity (the responder IS the primary, so * primaryName == responderName) and free the reply (Req 9.1, 9.2). */ if (rc == 0) { const char* primary = (status != NULL) ? (NULLSTR(status->primaryName) ? status->responderName : status->primaryName) : NULL; elapsed_ms = monotonic_ms() - start_ms; mylog_setlevel(g_active_loglevel); printf("wait-primary: primary '%s' serving after %ld ms\n", NULLSTR(primary) ? "unknown" : primary, elapsed_ms); xcecli_free_ctrlstatus(status); return 0; } /* Non-authoritative / failed probe: nothing to free (the typed-failure * contract leaves *status untouched), keep polling until the deadline. */ elapsed_ms = monotonic_ms() - start_ms; if (elapsed_ms >= timeout_ms) { mylog_setlevel(g_active_loglevel); printf("wait-primary: no primary served within %ld ms (waited " "%ld ms)\n", timeout_ms, elapsed_ms); return -ETIMEDOUT; } /* Don't oversleep past the deadline: cap the poll to the time left. */ { long remaining_ms = timeout_ms - elapsed_ms; long sleep_ms = (remaining_ms < WAIT_PRIMARY_POLL_MS) ? remaining_ms : WAIT_PRIMARY_POLL_MS; if (sleep_ms > 0) { usleep((useconds_t)(sleep_ms * US_PER_MS)); } } } } /* Map a ctrl_role_t raw value (CtrlStatus.responderRole, see ha.h:60-63 and * design.md "Schema — reply payload") to a display string. PRIMARY==0, * STANDBY==1, INIT==2, ELECTING==3; anything else is reported verbatim so an * unexpected role is never silently rendered as a known one. */ static const char* ctrlstatus_role_str(uint8_t role) { switch (role) { case 0: return "primary"; case 1: return "standby"; case 2: return "initializing"; case 3: return "electing"; default: return "unknown-role"; } } /* `ctrlstatus` (controller-status task 4.1, Req 1.3/3.3/5.1/5.2/8.1/8.2/8.3, * design.md "cmd_ctrlstatus (test client) — State contract"). * * Issues the any-node status read (xcecli_ctrlstatus, is_write=false: whatever * controller answers — primary or standby hint view — rides the configured * failover/rotation) and prints the identity/health fields in human-readable * form. * * On failure (no reachable controller etc.) it prints a typed error and returns * the negative rc so the session records the command as failed and exits * non-zero (Req 8.3). On success it prints the fields and returns 0 (Req 8.2). */ int cmd_ctrlstatus(int argc, char* argv[]) { xce_ctrlstatus_t* status = NULL; int rc; int i; (void)argv; if (argc != 1) { printf("ctrlstatus\n"); return -EINVAL; } rc = xcecli_ctrlstatus(active_ctx->ctx, &status); if (rc < 0) { char ebuf[128]; ebuf[0] = '\0'; xce_strerror_r(-rc, ebuf, sizeof(ebuf)); SET_ERROR(-rc); switch (-rc) { case E_XCE_NOMASTER: fprintf(stderr, "ctrlstatus failed: no-master (%s)\n", ebuf); break; case E_XCE_NOT_READY: fprintf(stderr, "ctrlstatus failed: not-ready (%s)\n", ebuf); break; case ETIMEDOUT: fprintf(stderr, "ctrlstatus failed: timeout (%s)\n", ebuf); break; default: fprintf(stderr, "ctrlstatus failed: %s\n", ebuf); break; } return rc; } /* Current primary (Req 2.x): an empty primaryName means no elected/known * primary — render the well-defined "unknown" (Req 1.3/2.2/2.3) rather than * an empty or stale value. */ printf("Primary: %s\n", NULLSTR(status->primaryName) ? "unknown" : status->primaryName); /* Answering node + role, framed as a property of the responder and kept * DISTINCT from the authoritative Primary line above (Req 3.1/3.2/3.3): a * transient responder role (electing/initializing) must not be misread as * the cluster's primary state. */ printf("answered by %s (%s)\n", NULLSTR(status->responderName) ? "unknown" : status->responderName, ctrlstatus_role_str(status->responderRole)); printf("Cluster: %s\n", NULLSTR(status->clusterName) ? "unknown" : status->clusterName); /* Configured controller topology + count (Req 4.1/4.2). */ printf("Controllers (%u):\n", status->peerCount); for (i = 0; i < status->peers.count; i++) { xce_ctrlpeer_t* peer = status->peers.list[i]; if (peer == NULL) { continue; } printf("\t%s\t%s:%u\n", NULLSTR(peer->name) ? "?" : peer->name, NULLSTR(peer->host) ? "?" : peer->host, peer->port); } /* Config version + committed config version (Req 5.1) and the content * checksum as the hex string the controller already exposes (Req 5.2). */ printf("configVersion: %llu\n", (unsigned long long)status->configVersion); printf("committedConfigVersion: %llu\n", (unsigned long long)status->committedConfigVersion); printf("configChecksum: %s\n", NULLSTR(status->configChecksum) ? "unknown" : status->configChecksum); /* HA health / progress (Req 6.x). */ printf("term: %llu\n", (unsigned long long)status->term); printf("committedLsn: %llu\n", (unsigned long long)status->committedLsn); printf("appliedLsn: %llu\n", (unsigned long long)status->appliedLsn); printf("fenced: %s\n", status->fenced ? "true" : "false"); /* Responder snapshot timestamp (Req 11.2/11.3): render the responder's * sample-time wall clock (epoch ms) in the SAME local-time * "YYYY-MM-DD HH:MM:SS,mmm" form the controller logger emits, so it can be * compared directly against ctrl-N.log. */ { char tbuf[64]; if (ctrlstatus_format_time_ms(status->responder_time_ms, tbuf, sizeof(tbuf)) != NULL) { printf("responderTime: %s\n", tbuf); } else { printf("responderTime: unknown\n"); } } xcecli_free_ctrlstatus(status); return 0; } static void print_jobinfo(xce_job_t* job) { char idstring[128]; if (job == NULL) { return; } fmt_jobid(idstring, sizeof(idstring) - 1, job->jobid); printf("JobID: %s\n", idstring); printf("state: %s\n", job_state_str(job->state)); /* raft-apply-decoupling task 7.2: job_state_str's if/else-if chain checks * JOB_STAT_RUN before JOB_STAT_EXIT, so a job that is simultaneously RUN * and EXIT|PERR (e.g. a worker-reported FAIL received without a prior * FINISH — apply_job_fail_transition ORs in EXIT|PERR without clearing * RUN) prints as "RUN", silently hiding the EXIT/PERR bits from the * pretty string. Print the raw bitmask too so a scriptable caller (the * xce-test-ha factstream regression scenario's phantom-fact check) can * test for a specific bit's presence unambiguously instead of relying on * the display string's priority ordering. */ printf("state_raw: 0x%x\n", job->state); printf("Queue: %s\n", job->submit->queue); printf("User: %s\n", job->submit->user); printf("Slots: %d,%d\n", job->submit->mintasks, job->submit->maxtasks); printf("Command: %s\n", job->submit->start_info->cmdline); if (job->pend_reasons != NULL) { printf("Pend Reason:\n"); if (job->pend_reasons->main_reason != 0) { printf("\tmain Reason: %d\n", job->pend_reasons->main_reason); } else { if (job->pend_reasons->nhost_reasons > 0) { int i; printf("\tHost Reasons:\n"); for (i = 0; i < job->pend_reasons->nhost_reasons; i++) { printf("\t\t(%s, %d)\n", job->pend_reasons->host_based_reasons[i]->host, job->pend_reasons->host_based_reasons[i]->reason); } } if (job->pend_reasons->nres_reasons > 0) { int i; printf("\tResource Reasons:\n"); for (i = 0; i < job->pend_reasons->nres_reasons; i++) { printf("\t\t(%s, %s, %d)\n", job->pend_reasons->res_based_reasons[i]->resource, job->pend_reasons->res_based_reasons[i]->host, job->pend_reasons->res_based_reasons[i]->reason); } } } } } static const flagspec_t jobinfo_flags[] = { { 'j', "jobid", "jobid", required_argument, 1, NULL }, }; #define JOBINFO_NFLAGS (sizeof(jobinfo_flags) / sizeof(jobinfo_flags[0])) int cmd_jobinfo(int argc, char* argv[]) { int opt; struct option long_opts[JOBINFO_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; if (flagspec_build_opts(jobinfo_flags, JOBINFO_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jobinfo: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { uint64_t jobid; xce_job_t* job; int rc = 0; switch (opt) { case 'j': parse_jobid(&jobid, optarg); if (jobid > 0) { rc = xcecli_jobinfo(active_ctx->ctx, jobid, &job); if (rc < 0) { perror("xcecli_jobinfo"); continue; } print_jobinfo(job); free_Job_ptr(&job); } break; case '?': return flagspec_reject("jobinfo", jobinfo_flags, JOBINFO_NFLAGS, argv[optind - 1], "jobinfo -j [-j ...]"); } } return 0; } static int fill_job_alloc(xce_job_alloc_t* jalloc, int nhosts, host_task_t* hosts) { xce_host_alloc_t** halloc = NULL; int i; halloc = (xce_host_alloc_t**)calloc(nhosts, sizeof(xce_host_alloc_t*)); if (halloc == NULL) { return -ENOMEM; } for (i = 0; i < nhosts; i++) { xce_alloc_t* alloc; xce_alloc_t** allocs; alloc = (xce_alloc_t*)calloc(1, sizeof(*alloc)); allocs = (xce_alloc_t**)calloc(1, sizeof(*allocs)); halloc[i] = (xce_host_alloc_t*)calloc(1, sizeof(xce_host_alloc_t)); if ((halloc[i] == NULL) || (alloc == NULL) || (allocs == NULL)) { /* FIXME: handle error */ return -ENOMEM; } alloc->resname = "slots"; alloc->reserve = ""; alloc->instid = -1; alloc->value = (float)hosts[i].ntasks; alloc->from = time(NULL); alloc->duration = -1; allocs[0] = alloc; halloc[i]->nallocs = 1; halloc[i]->allocs = allocs; halloc[i]->host = hosts[i].host; } jalloc->nhosts = nhosts; jalloc->allocs = halloc; return 0; } static const flagspec_t jobrun_flags[] = { { 'm', "host", "host", required_argument, 0, "1 task; repeatable up to 5" }, { 'j', "jobid", "jobid", required_argument, 1, NULL }, }; #define JOBRUN_NFLAGS (sizeof(jobrun_flags) / sizeof(jobrun_flags[0])) int cmd_jobrun(int argc, char* argv[]) { int rc; int opt; uint64_t jobid = 0; host_task_t hosts[MAX_TASK_HOSTS]; xce_job_alloc_t jalloc; xce_job_dispatch_t jdispatch = { .njobs = 1, .allocs = (xce_job_alloc_t*[]) { &jalloc } }; int nhosts = 0; char* host; int count; struct option long_opts[JOBRUN_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; if (flagspec_build_opts(jobrun_flags, JOBRUN_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jobrun: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'm': if (nhosts >= MAX_TASK_HOSTS) { fprintf(stderr, "too many run hosts, we only support max %d hosts\n", MAX_TASK_HOSTS); return -1; } if (strchr(optarg, (int)':') != NULL) { char* p; char buf[256]; p = strchr(optarg, (int)':'); count = atoi(p + 1); memset(buf, 0x0, sizeof(buf)); strncpy(buf, optarg, (int)(p - optarg)); host = strdup(buf); } else { host = strdup(optarg); count = 1; } hosts[nhosts].host = host; hosts[nhosts].ntasks = count; nhosts++; break; case 'j': parse_jobid(&jobid, optarg); if (jobid == 0) { printf("you need to specify a positive jobid with -j"); return -1; } break; case '?': return flagspec_reject( "jobrun", jobrun_flags, JOBRUN_NFLAGS, argv[optind - 1], "jobrun -j [-m [:]] [-m ...]"); } } if (jobid == 0) { printf("no jobid specified"); return -1; } fill_job_alloc(&jalloc, nhosts, &hosts[0]); jalloc.jobid = jobid; rc = xcecli_jobdispatch(active_ctx->ctx, &jdispatch); if (rc < 0) { SET_ERROR(-rc); perror("xce_jobdispatch"); return rc; } #if 0 printf("job %s will be started on host: %s\n", argv[1], halloc.host); #endif return 0; } static int build_job_envs(xce_job_submit_t* submit) { exdata_t exdata = { .sub_env_vars = { .PATH = getenv("PATH") } }; s2j_create_json_obj(json_exdata); s2j_create_json_obj(json_envs); s2j_json_set_basic_element(json_envs, &(exdata.sub_env_vars), var_string, PATH); cJSON_AddItemToObject(json_exdata, "sub_env_vars", json_envs); submit->submit->start_info->exdata = cJSON_Print(json_exdata); cJSON_Delete(json_exdata); return 0; } static int build_job_exdata(xce_job_submit_t* submit) { build_job_envs(submit); return 0; } static const flagspec_t jobsub_flags[] = { { 'u', "user", "user", required_argument, 1, NULL }, { 'q', "queue", "queue", required_argument, 1, NULL }, { 'R', "resreq", "resreq", required_argument, 0, "\"type == any\"" }, { 'c', "command-line", "command-line", required_argument, 1, NULL }, { 'n', "ntasks", "ntasks", required_argument, 0, "1" }, { 'x', "exdata", "exdata", required_argument, 0, NULL }, { 'a', "array-size", "arraysize", required_argument, 0, "1" }, }; #define JOBSUB_NFLAGS (sizeof(jobsub_flags) / sizeof(jobsub_flags[0])) int cmd_jobsub(int argc, char* argv[]) { int rc = 0; job_submit_data_t submit; xce_job_submit_t jobsubmit = { .submit = &submit }; int opt; /* Initialize so a "rc >= 0 but *jobid never written" outcome (a non-clean * ack such as a redirect/zero-row/not-ready surfaced as a non-negative rc) * is detectable as an invalid (zero) JobId rather than printed as garbage * (bug-786). */ uint64_t jobid = 0; uint32_t min = 1, max = 1; uint32_t arsize = 1; char jobidStr[128]; struct option long_opts[JOBSUB_NFLAGS + 1]; char optstring[32]; optind = 1; opterr = 0; memset(&submit, 0x0, sizeof(submit)); submit.start_info = (job_startinfo_t*)calloc(1, sizeof(*(submit.start_info))); if (submit.start_info == NULL) { perror("calloc"); return -ENOMEM; } if (flagspec_build_opts(jobsub_flags, JOBSUB_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jobsub: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'u': submit.user = strdup(optarg); break; case 'q': submit.queue = strdup(optarg); break; case 'R': submit.resreq = strdup(optarg); break; case 'c': submit.start_info->cmdline = strdup(optarg); break; case 'x': submit.start_info->exdata = strdup(optarg); break; case 'n': if (strchr(optarg, (int)',') != NULL) { char buf[64]; char* p = strchr(optarg, (int)','); max = atoi(p + 1); memset(buf, 0x0, sizeof(buf)); strncpy(buf, optarg, (int)(p - optarg)); min = atoi(buf); if (max == 0) { printf("Incorrect setting of max tasks count, set to same " "as min task value\n"); max = min; } } else { min = atoi(optarg); max = min; } break; case 'a': arsize = atoi(optarg); if (arsize <= 0) { arsize = 1; } break; case '?': rc = flagspec_reject( "jobsub", jobsub_flags, JOBSUB_NFLAGS, argv[optind - 1], "jobsub -u -q -c [-R ] " "[-n [,]] [-x ] [-a ]"); goto cleanup; } } if (min == 0) { printf("Incorrect setting of min task count, set to 1 as default\n"); min = 1; } if (NULLSTR(submit.user)) { printf("you need to specify job user with -u\n"); rc = -EINVAL; goto cleanup; } if (NULLSTR(submit.queue)) { printf("you need to specify job queue with -q\n"); rc = -EINVAL; goto cleanup; } if (NULLSTR(submit.start_info->cmdline)) { printf("you need to specify job command with -c\n"); rc = -EINVAL; goto cleanup; } if (NULLSTR(submit.resreq)) { submit.resreq = strdup("type == any"); } submit.mintasks = min; submit.maxtasks = max; /* apiserverId=0 here let two unrelated jobs collide on the same jobid * whenever the harness spawns a fresh xce_test process within the same * wall-clock second (idgen_ctx_init resets sequence to 0 each time, * and ts/tag/seq alone were then identical for both processes' first * submission). Production apiserver instances never hit this: each * holds one process-lifetime idgen_ctx_t (clib.go's sync.Once) and is * assigned its own distinct apiserverId, so the jobid namespace is * already partitioned by config.h's bit layout. Folding this process's * pid into apiserverId gives each xce_test invocation its own slice of * that same partitioning, making a same-second cross-process collision * structurally impossible here too. */ idgen(&idctx, 0, (size_t)((uint64_t)getpid() & MAX_APISERVER_ID), arsize, &(jobsubmit.jobid)); if (NULLSTR(submit.start_info->execuser)) { submit.start_info->execuser = strdup(submit.user); } if (NULLSTR(submit.start_info->exechome)) { submit.start_info->exechome = strdup("/tmp"); } if (NULLSTR(submit.start_info->execcwd)) { submit.start_info->execcwd = strdup("/tmp"); } build_job_exdata(&jobsubmit); jobsubmit.array_size = arsize; rc = xcecli_jobsubmit(active_ctx->ctx, &jobsubmit, &jobid); cleanup: FREEUP(submit.user); FREEUP(submit.queue); FREEUP(submit.resreq); FREEUP(submit.start_info->cmdline); FREEUP(submit.start_info->exdata); FREEUP(submit.start_info); /* Report success only on a clean ack that echoes back the EXACT JobId we * submitted (bug-786, Req 7.1/7.2/7.3). xce_jobsubmit() writes *jobid only * when the reply status is >= 0; XCECLI_FAILOVER_CALL surfaces a non-clean * outcome as a negative rc (e.g. -E_XCE_NOT_READY on no-primary budget * exhaustion, -E_XCE_REDIRECT / -E_XCE_NOMASTER on a standby, -ETIMEDOUT / * -ENOTCONN on transport loss). * * The `jobid != 0` gate alone is NOT sufficient: on a failover, libxce's * handle_sub_reply() returns -1 on a non-jobid reply but LEAVES the stale * previous value in ctx->reply.u.jobid, and xceapi.c copies it out whenever * status >= 0 — so a rejected submit can surface as rc >= 0 with a NON-ZERO * stale-garbage jobid (the live soak captured `63-15--1-1015808[32751]` * while the controller logged "handle_jobsubmit fail to lock job" and never * committed it). We therefore require the returned jobid to EQUAL the JobId * we generated locally and submitted: on a genuine clean ack the controller * commits and echoes back exactly submit->jobid (handle_jobsubmit replies * reply.data.jobid = job_submit() = base_id = submit->jobid; verified * round-trip). `jobid != jobsubmit.jobid` covers BOTH the never-written * (zero) case AND the stale non-zero garbage case in one uint64 compare. */ if (rc < 0) { char ebuf[128]; ebuf[0] = '\0'; xce_strerror_r(-rc, ebuf, sizeof(ebuf)); SET_ERROR(-rc); switch (-rc) { case E_XCE_NOMASTER: fprintf(stderr, "jobsubmit failed: no-master (%s)\n", ebuf); break; case E_XCE_REDIRECT: fprintf(stderr, "jobsubmit failed: redirect (%s)\n", ebuf); break; case E_XCE_NOT_READY: fprintf(stderr, "jobsubmit failed: not-ready (%s)\n", ebuf); break; case ETIMEDOUT: fprintf(stderr, "jobsubmit failed: timeout (%s)\n", ebuf); break; default: fprintf(stderr, "jobsubmit failed: %s\n", ebuf); break; } return rc; } if (jobid != jobsubmit.jobid) { /* Non-negative rc but the returned JobId does not match the one we * submitted: not a clean ack. This rejects BOTH the never-written * (jobid == 0) case AND the stale-garbage non-zero case left in the * libxce reply union on a failover-rejected submit. Never print "ok" * with an uninitialized/garbage id (Req 7.3). */ char wantStr[128]; char gotStr[128]; fmt_jobid(wantStr, sizeof(wantStr) - 1, jobsubmit.jobid); fmt_jobid(gotStr, sizeof(gotStr) - 1, jobid); fprintf(stderr, "jobsubmit failed: no clean ack (controller did not confirm " "the submitted JobId: wanted %s, got %s)\n", wantStr, gotStr); return -E_XCE_INTERNAL; } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, array_jobid(jobid)); /* Capture $jobid (Req 3.2) using the SAME representation cmd_joblist prints * for a job's JobId column — fmt_jobid(jobs[i]->jobid), the full jobid * WITHOUT array_jobid() stripping — so a later `$jobid` substitution and * `expect-job $jobid` compare equal to the joblist entry for this job. Only * set on this clean-ack + valid-id path; never on a failure path, so an * unset $jobid still fails per Req 3.6. */ fmt_jobid(g_last_jobid, sizeof(g_last_jobid) - 1, jobid); printf("jobsubmit ok: JobId: %s\n", jobidStr); return 0; } /* * Resource names whose canonical stored/wire value is raw bytes (not a * generic schema-driven unit system -- deliberately kept as a small, * explicit, client-side list rather than a wire field, since only a * handful of resources need this and a wire-level unit hint would have * grown every Resource/ResourceVal message regardless of whether it's * used). CLI display picks the best-fit human unit adaptively * (format_bytes_human); the stored value is never converted. Covers every * memory/disk-scale built-in host attribute and load index (maxmem, mem, * tmp, maxswap, maxtmp) -- deliberately NOT maxcpus/ndisks (plain counts) * or cpuf/r1m/r15s (unitless factors/averages). */ static int is_bytes_valued_resource(const char* name) { static const char* const BYTES_VALUED_RESOURCES[] = { "maxmem", "mem", "tmp", "maxswap", "maxtmp" }; const int nres = (int)(sizeof(BYTES_VALUED_RESOURCES) / sizeof(BYTES_VALUED_RESOURCES[0])); int i; if (name == NULL) { return 0; } for (i = 0; i < nres; i++) { if (strcmp(name, BYTES_VALUED_RESOURCES[i]) == 0) { return 1; } } return 0; } /* * Render a raw byte count as a human-readable string (e.g. "30.90 GB"), * adaptively picking the smallest unit the value fits in. Only used for * display -- the stored/wire value stays raw bytes always, this never * feeds back into anything that's compared or persisted. A pure function * so it's directly unit-testable without a live client connection. */ static void format_bytes_human(double bytes, char* buf, size_t bufsize) { static const char* const UNITS[] = { "B", "KB", "MB", "GB", "TB", "PB" }; const int nunits = (int)(sizeof(UNITS) / sizeof(UNITS[0])); double val = bytes; int i = 0; while (val >= 1024.0 && i < nunits - 1) { val /= 1024.0; i++; } snprintf(buf, bufsize, "%.2f %s", val, UNITS[i]); } static const flagspec_t resval_flags[] = { { 'n', "resname", "resname", required_argument, 1, NULL }, { 'm', "host", "host", required_argument, 0, "required unless the resource is cluster-wide" }, }; #define RESVAL_NFLAGS (sizeof(resval_flags) / sizeof(resval_flags[0])) int cmd_resval(int argc, char* argv[]) { char* host = NULL; char* res = NULL; xce_resval_t** vals = NULL; int opt; int rc; struct option long_opts[RESVAL_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; if (flagspec_build_opts(resval_flags, RESVAL_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: resval: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'n': res = strdup(optarg); break; case 'm': host = strdup(optarg); break; case '?': rc = flagspec_reject("resval", resval_flags, RESVAL_NFLAGS, argv[optind - 1], "resval -n [-m ]"); FREEUP(host); FREEUP(res); return rc; } } rc = xcecli_resval(active_ctx->ctx, host, res, &vals); if (rc > 0) { int i; for (i = 0; i < rc; i++) { xce_resval_t* val = vals[i]; printf("resource %s@%s:\n", val->name, (val->host == NULL) ? "cluster" : val->host); printf("value: "); switch (val->val->which) { case Value_u_svalue: printf("%s\n", val->val->u.svalue); break; case Value_u_fvalue: if (is_bytes_valued_resource(val->name)) { char hbuf[64]; format_bytes_human(val->val->u.fvalue, hbuf, sizeof(hbuf)); printf("%f (%s)\n", val->val->u.fvalue, hbuf); } else { printf("%f\n", val->val->u.fvalue); } break; case Value_u_bvalue: printf("%s\n", (val->val->u.bvalue == 0) ? "false" : "true"); break; default: printf(" \n"); break; } free_ResourceVal_ptr(&val); } FREEUP(vals); } else { printf("no resource value found in given host\n"); } FREEUP(host); FREEUP(res); return 0; } static const flagspec_t jobkill_flags[] = { { 's', "signal", "signal", required_argument, 0, "SIGTERM" }, { 'j', "jobid", "jobid", required_argument, 1, NULL }, }; #define JOBKILL_NFLAGS (sizeof(jobkill_flags) / sizeof(jobkill_flags[0])) int cmd_jobkill(int argc, char* argv[]) { int signo; uint64_t jobid = 0; int opt; int rc; xce_jobctrl_t jobctrl; xce_signal_t siginfo; char jobidStr[64]; struct option long_opts[JOBKILL_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; signo = SIGTERM; if (flagspec_build_opts(jobkill_flags, JOBKILL_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jobkill: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 's': signo = atoi(optarg); break; case 'j': parse_jobid(&jobid, optarg); break; case '?': return flagspec_reject("jobkill", jobkill_flags, JOBKILL_NFLAGS, argv[optind - 1], "jobkill -j [-s ]"); } } if (jobid == 0) { fprintf(stderr, "you need to specify jobid with -j\n"); return -1; } jobctrl.action = JobControl_u_siginfo; jobctrl.jobid = jobid; jobctrl.u.siginfo = &siginfo; siginfo.signo = signo; siginfo.force = 1; rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl); if (rc < 0) { perror("xcecli_jobctrl"); return rc; } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, jobid); printf("Job %s signaled\n", jobidStr); return 0; } static const flagspec_t jobckpt_flags[] = { { 'k', "kill", "kill", no_argument, 0, NULL }, { 'j', "jobid", "jobid", required_argument, 1, NULL }, { 'm', "checkpoint-method", "checkpoint-method", required_argument, 0, "\"default\"" }, }; #define JOBCKPT_NFLAGS (sizeof(jobckpt_flags) / sizeof(jobckpt_flags[0])) int cmd_jobckpt(int argc, char* argv[]) { uint64_t jobid = 0; int opt; int rc; xce_jobctrl_t jobctrl; xce_ckpt_options_t ckptoption; char jobidStr[64]; struct option long_opts[JOBCKPT_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; ckptoption.kill = 0; if (flagspec_build_opts(jobckpt_flags, JOBCKPT_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jobckpt: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'k': ckptoption.kill = 1; break; case 'j': parse_jobid(&jobid, optarg); break; case 'm': ckptoption.method = optarg; break; case '?': return flagspec_reject("jobckpt", jobckpt_flags, JOBCKPT_NFLAGS, argv[optind - 1], "jobckpt -j [-k] [-m ]"); } } if (jobid == 0) { fprintf(stderr, "you need to specify jobid with -j\n"); return -1; } if (ckptoption.method == NULL) { ckptoption.method = "default"; } jobctrl.action = JobControl_u_ckptoption; jobctrl.jobid = jobid; jobctrl.u.ckptoption = &ckptoption; rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl); if (rc < 0) { perror("xcecli_jobctrl"); return rc; } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, jobid); printf("Job %s checkpointed\n", jobidStr); return 0; } static const flagspec_t qctrl_flags[] = { { 'a', "activate", "activate", no_argument, 0, NULL }, { 'i', "deactivate", "deactivate", no_argument, 0, NULL }, { 'o', "open", "open", no_argument, 0, NULL }, { 'c', "close", "close", no_argument, 0, NULL }, { 'q', "queue", "queue", required_argument, 1, NULL }, }; #define QCTRL_NFLAGS (sizeof(qctrl_flags) / sizeof(qctrl_flags[0])) int cmd_qctrl(int argc, char* argv[]) { int opt; int rc; xce_queuectrl_t queuectrl; /* bug-1704 fix: initialize to the QUEUE_DO_NONE sentinel (xce/types.h) * and reject post-loop if it is still unset -- mirrors cmd_hostctrl's * existing HOST_DO_NONE sentinel-and-reject pattern immediately below. * Before this fix, `action` had no initializer and no post-loop check, * so an invocation giving -q but none of -a/-i/-o/-c transmitted * indeterminate stack memory as the queue action. */ int action = QUEUE_DO_NONE; char* queue = NULL; struct option long_opts[QCTRL_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; if (flagspec_build_opts(qctrl_flags, QCTRL_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: qctrl: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'a': action = QUEUE_ACTIVE; break; case 'i': action = QUEUE_INACTIVE; break; case 'o': action = QUEUE_OPEN; break; case 'c': action = QUEUE_CLOSE; break; case 'q': queue = strdup(optarg); break; case '?': FREEUP(queue); return flagspec_reject("qctrl", qctrl_flags, QCTRL_NFLAGS, argv[optind - 1], "qctrl -q [-a] [-i] [-o] [-c]"); } } if (queue == NULL) { fprintf(stderr, "you need to specify queue name with -q\n"); return -1; } /* bug-1704 fix (continued): reject rather than send an indeterminate * action when none of -a/-i/-o/-c (nor their long forms) was given. */ if (action == QUEUE_DO_NONE) { fprintf(stderr, "you need to specify one of -a/--activate, -i/--deactivate, " "-o/--open, or -c/--close\n"); free(queue); return -1; } queuectrl.action = action; queuectrl.queue = queue; rc = xcecli_queuectrl(active_ctx->ctx, &queuectrl); if (rc < 0) { perror("xcecli_queuectrl"); return rc; } printf("Operation on queue %s done\n", queue); free(queue); return 0; } /* hostctrl -o|-c -h — administratively open/close a host * (xce_hostctrl_t{action, hostname} via xcecli_hostctrl). Mirrors cmd_qctrl's * exact getopt shape; only -o/-c are wired (matching xce_hostctrl_req's own * accepted action range, request.c — HOST_TAG/HOST_UNTAG exist in the enum * but are rejected -EINVAL server-side today, so no flag is offered for them * here). Used by the config-upgrade chaotic soak (task 8.6) to set a host's * admin status before/around a config upgrade and assert it survives. */ static const flagspec_t hostctrl_flags[] = { { 'o', "open", "open", no_argument, 0, NULL }, { 'c', "close", "close", no_argument, 0, NULL }, { 'h', "host", "host", required_argument, 1, NULL }, }; #define HOSTCTRL_NFLAGS (sizeof(hostctrl_flags) / sizeof(hostctrl_flags[0])) int cmd_hostctrl(int argc, char* argv[]) { int opt; int rc; xce_hostctrl_t hostctrl; int action = HOST_DO_NONE; char* hostname = NULL; struct option long_opts[HOSTCTRL_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; if (flagspec_build_opts(hostctrl_flags, HOSTCTRL_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: hostctrl: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'o': action = HOST_OPEN; break; case 'c': action = HOST_CLOSE; break; case 'h': hostname = strdup(optarg); break; case '?': FREEUP(hostname); return flagspec_reject("hostctrl", hostctrl_flags, HOSTCTRL_NFLAGS, argv[optind - 1], "hostctrl -h (-o|-c)"); } } if (hostname == NULL) { fprintf(stderr, "you need to specify host name with -h\n"); return -1; } if (action == HOST_DO_NONE) { fprintf(stderr, "you need to specify -o (open) or -c (close)\n"); free(hostname); return -1; } hostctrl.action = action; hostctrl.hostname = hostname; rc = xcecli_hostctrl(active_ctx->ctx, &hostctrl); if (rc < 0) { perror("xcecli_hostctrl"); free(hostname); return rc; } printf("Operation on host %s done\n", hostname); free(hostname); return 0; } /* upgradeconfig — submit a configuration upgrade * (config-upgrade-snapshot task 8.6). Reads the given capnp-encoded * ClusterConfig blob (the same format `xceadmin config build` produces and * deploy-ha.sh installs as cluster.bin), computes its SHA-256 (OpenSSL * SHA256(), matching config_repl.c's do_upgrade_config_with_bytes transit * verification), and drives it through xcecli_upgradeconfig -> * xce_upgradeconfig_req_raw (OP_upgradeConfig). Mirrors cmd_snapdiff's * bounds-checked file-reading style. */ int cmd_upgradeconfig(int argc, char* argv[]) { if (argc != 2) { fprintf(stderr, "usage: upgradeconfig \n"); return -1; } const char* path = argv[1]; FILE* fp = fopen(path, "rb"); if (fp == NULL) { fprintf(stderr, "upgradeconfig: cannot open %s: %s\n", path, strerror(errno)); return -1; } if (fseek(fp, 0, SEEK_END) != 0) { fprintf(stderr, "upgradeconfig: fseek failed on %s: %s\n", path, strerror(errno)); fclose(fp); return -1; } long fsz = ftell(fp); if (fsz <= 0) { fprintf(stderr, "upgradeconfig: %s is empty or ftell failed\n", path); fclose(fp); return -1; } rewind(fp); uint8_t* bytes = (uint8_t*)malloc((size_t)fsz); if (bytes == NULL) { fprintf(stderr, "upgradeconfig: out of memory reading %s\n", path); fclose(fp); return -1; } size_t nread = fread(bytes, 1, (size_t)fsz, fp); fclose(fp); if (nread != (size_t)fsz) { fprintf(stderr, "upgradeconfig: short read on %s (%zu of %ld bytes)\n", path, nread, fsz); free(bytes); return -1; } uint8_t sha256[SHA256_DIGEST_LENGTH]; SHA256(bytes, (size_t)fsz, sha256); int rc = xcecli_upgradeconfig(active_ctx->ctx, sha256, sizeof(sha256), bytes, (size_t)fsz); free(bytes); if (rc < 0) { fprintf(stderr, "upgradeconfig: failed (rc=%d): %s\n", rc, strerror(-rc)); return rc; } printf("Config upgrade submitted from %s (%ld bytes)\n", path, fsz); return 0; } static const flagspec_t runtasks_flags[] = { { 'm', "host", "host", required_argument, 0, "1 task; repeatable up to 5" }, { 'u', "user", "user", required_argument, 0, NULL }, { 'c', "command-line", "command-line", required_argument, 0, NULL }, { 'x', "exdata", "exdata", required_argument, 0, NULL }, { 'd', "dir", "dir", required_argument, 0, NULL }, { 'j', "jobid", "jobid", required_argument, 0, NULL }, { 't', "launch-type", "launch-type", required_argument, 0, "\"mpi\" if >1 total tasks, else \"sequential\"" }, }; #define RUNTASKS_NFLAGS (sizeof(runtasks_flags) / sizeof(runtasks_flags[0])) int cmd_runtasks(int argc, char* argv[]) { xce_taskhost_t hosts[MAX_TASK_HOSTS]; size_t nhosts = 0; xce_taskrun_t taskrun; int opt; int rc; int count; int totaltasks = 0; char* user = NULL; char* cmdline = NULL; char* dir = NULL; char* exdata = NULL; char* host = NULL; char* launch_type = NULL; int i; uint64_t jobid = 0; struct option long_opts[RUNTASKS_NFLAGS + 1]; char optstring[32]; optind = 1; opterr = 0; if (flagspec_build_opts(runtasks_flags, RUNTASKS_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: taskrun: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'm': if (nhosts >= MAX_TASK_HOSTS) { fprintf(stderr, "too many run hosts, just support %d run hosts\n", MAX_TASK_HOSTS); return -1; } if (strchr(optarg, (int)':') != NULL) { char* p; char buf[256]; p = strchr(optarg, (int)':'); count = atoi(p + 1); memset(buf, 0x0, sizeof(buf)); strncpy(buf, optarg, (int)(p - optarg)); host = strdup(buf); } else { host = strdup(optarg); count = 1; } hosts[nhosts].host = host; hosts[nhosts].ntasks = count; nhosts++; totaltasks += count; break; case 'u': if (user != NULL) { free(user); } user = strdup(optarg); break; case 'c': if (cmdline != NULL) { free(cmdline); } cmdline = strdup(optarg); break; case 'x': if (exdata != NULL) { free(exdata); } exdata = strdup(optarg); break; case 'd': if (dir != NULL) { free(dir); } dir = strdup(optarg); break; case 'j': parse_jobid(&jobid, optarg); break; case 't': launch_type = strdup(optarg); break; case '?': return flagspec_reject( "taskrun", runtasks_flags, RUNTASKS_NFLAGS, argv[optind - 1], "taskrun [-j ] [-m [:]]... [-u ] " "[-c ] [-x ] [-d ] [-t ]"); } } for (i = 0; i < nhosts; i++) { hosts[i].exechome = dir; hosts[i].execcwd = dir; hosts[i].execuser = user; hosts[i].exdata = exdata; hosts[i].cmdline = cmdline; hosts[i].launch_type = launch_type; if (hosts[i].launch_type == NULL) { if (totaltasks > 1) { hosts[i].launch_type = strdup("mpi"); } else { hosts[i].launch_type = strdup("sequential"); } } } taskrun.taskhosts = (xce_taskhost_t**)calloc(nhosts, sizeof(xce_taskhost_t*)); taskrun.clientidx = -1; if (taskrun.taskhosts != NULL) { for (i = 0; i < nhosts; i++) { taskrun.taskhosts[i] = &hosts[i]; } taskrun.nhosts = nhosts; taskrun.jobid = jobid; rc = xcecli_runtask(active_ctx->ctx, &taskrun); } else { rc = -ENOMEM; } if (rc < 0) { errno = -rc; perror("xcecli_runtask"); } else { printf("run task ok\n"); } return rc; } static int add_pendreason(xce_job_preason_t* preason, char* s) { char s1[MAX_LINE_LEN]; char* pcode = NULL; char* phost = NULL; char* pres = NULL; char* p; int code; if ((preason == NULL) || (s == NULL)) { fprintf(stderr, "bad parameter\n"); return -1; } if ((p = strrchr(s, (int)':')) == NULL) { pcode = s; } else { pcode = p + 1; memset(s1, 0x0, sizeof(s1)); strncpy(s1, s, p - s); if ((p = strrchr(s1, (int)':')) == NULL) { phost = s1; } else { *p = 0x0; phost = p + 1; pres = s1; } } code = atoi(pcode); return add_job_preason(preason, code, phost, pres); } static const flagspec_t setpend_flags[] = { { 'j', "jobid", "jobid", required_argument, 1, NULL }, { 'p', "reason", "reason", required_argument, 1, NULL }, }; #define SETPEND_NFLAGS (sizeof(setpend_flags) / sizeof(setpend_flags[0])) int cmd_setpend(int argc, char* argv[]) { int opt; xce_job_preason_t jreasons; xce_pendreason_t preason = { .reasons = &jreasons }; xce_preason_set_t reasons = { .nreasons = 1, .reasons = (xce_pendreason_t*[]) { &preason } }; int rc = 0; struct option long_opts[SETPEND_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; if (flagspec_build_opts(setpend_flags, SETPEND_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: setpend: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { int rc = 0; switch (opt) { case 'j': parse_jobid(&(preason.jobid), optarg); break; case 'p': rc = add_pendreason(&jreasons, optarg); if (rc < 0) { perror("add_pendreason"); return rc; } break; case '?': return flagspec_reject( "setpend", setpend_flags, SETPEND_NFLAGS, argv[optind - 1], "setpend -j -p [-p ...]"); } } if (preason.jobid == 0) { fprintf(stderr, "no or incorrect jobid specified\n"); return -1; } if ((jreasons.main_reason == 0) && (jreasons.nhost_reasons == 0) && (jreasons.nres_reasons == 0)) { fprintf(stderr, "no or incorrect pend reason set\n"); return -1; } rc = xcecli_set_pend_reason(active_ctx->ctx, &reasons); if (rc < 0) { perror("xcecli_set_pend_reason"); erase_job_preason(&jreasons); return rc; } printf("ok\n"); return 0; } int cmd_join_worker(int argc, char* argv[]) { int rc; int role; hostattrs_t attrs; role = HostRole_worker; /* xce-test-cli-ux task 2.6 remediation (bug-1557): the argument-parsing/ * defaulting block (including the maxmem default fix) is extracted into * join_worker_args.c/.h so it can be unit-tested in isolation from this * function's REPL/network state -- see join_worker_args.h for the full * rationale. This is a structural extraction only; the parsing/defaulting * behavior and everything from xcecli_join() onward are unchanged. */ rc = join_worker_parse_attrs(argc, argv, &attrs); if (rc < 0) { return rc; } rc = xcecli_join(active_ctx->ctx, role, &attrs); if (rc < 0) { SET_ERROR(-rc); perror("xce_join"); return rc; } printf("host join ok\n"); return 0; } /* jstart -j jobid [-p pid] — raft-apply-decoupling task 7.2: synthetic * worker-fact-stream primitive. Mirrors cmd_jobfinish's existing pattern * exactly (same getopt/parse_jobid/SET_ERROR shape) but drives the * xcecli_notify_jobstart wire call that cmd_jobfinish's sibling * xcecli_notify_jobfinish already exercises — libxce (client.c) already * implements the full worker-fact API (jobstart/jobfinish/jobfail/...); this * command was the missing REPL-level primitive for jobstart specifically. * Drives the SAME production receipt path (on_job_start -> record_job_start_fact * -> persist_submit_job_fact -> durable append -> commit -> apply pump -> * apply_committed_entry()) that a real worker's report would, without * requiring a real scheduler dispatch (on_job_start only requires the job to * already exist in jobmgr, i.e. have been submitted). */ static const flagspec_t jobstart_flags[] = { { 'j', "jobid", "jobid", required_argument, 1, NULL }, { 'p', "pid", "pid", required_argument, 0, "this xce_test process's own pid" }, }; #define JOBSTART_NFLAGS (sizeof(jobstart_flags) / sizeof(jobstart_flags[0])) int cmd_jobstart(int argc, char* argv[]) { int rc; int opt; uint64_t jobid = 0; unsigned int pid = 0; unsigned int pids[1]; job_pidinfo_t info; struct option long_opts[JOBSTART_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; memset(&info, 0x0, sizeof(info)); if (flagspec_build_opts(jobstart_flags, JOBSTART_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jstart: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'j': parse_jobid(&jobid, optarg); break; case 'p': pid = (unsigned int)atoi(optarg); break; case '?': return flagspec_reject("jstart", jobstart_flags, JOBSTART_NFLAGS, argv[optind - 1], "jstart -j [-p ]"); } } if (jobid == 0) { printf(" you need specify a positive jobid\n"); return -EINVAL; } pids[0] = (pid != 0) ? pid : (unsigned int)getpid(); info.jobid = jobid; info.npids = 1; info.pids = pids; info.npgids = 0; info.pgids = NULL; info.exdata = NULL; rc = xcecli_notify_jobstart(active_ctx->ctx, &info); if (rc < 0) { SET_ERROR(-rc); perror("xcecli_jobstart"); return rc; } printf("ok\n"); return 0; } static const flagspec_t jobfinish_flags[] = { { 'j', "jobid", "jobid", required_argument, 1, NULL }, { 'c', "exit-code", "exit-code", required_argument, 0, "0" }, }; #define JOBFINISH_NFLAGS (sizeof(jobfinish_flags) / sizeof(jobfinish_flags[0])) int cmd_jobfinish(int argc, char* argv[]) { int rc; int opt; uint64_t jobid = 0; int code = 0; xce_jobfinish_t finish; struct option long_opts[JOBFINISH_NFLAGS + 1]; char optstring[16]; /* raft-apply-decoupling task 7.2: reset optind so repeated jfinish * invocations within one xce_test process (the fact-stream soak scenario * calls this many times in a single worker session) each re-parse their * own argv from the start. Without this, getopt's optind carries over * from the PRIOR command's parse position, silently skipping this * command's own "-j jobid" on the second and later calls (jobid stays 0 * and the command errors out) — matches the reset convention most other * cmd_ handlers in this file already follow (e.g. cmd_join_worker just * above, cmd_jobsub, cmd_resupdate). */ optind = 1; opterr = 0; if (flagspec_build_opts(jobfinish_flags, JOBFINISH_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jfinish: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'j': parse_jobid(&jobid, optarg); break; case 'c': code = atoi(optarg); break; case '?': return flagspec_reject("jfinish", jobfinish_flags, JOBFINISH_NFLAGS, argv[optind - 1], "jfinish -j [-c ]"); } } if (jobid == 0) { printf(" you need specify a positive jobid\n"); return -EINVAL; } finish.jobid = jobid; finish.exit_code = code; rc = xcecli_notify_jobfinish(active_ctx->ctx, &finish); if (rc < 0) { SET_ERROR(-rc); perror("xcecli_jobfinish"); return rc; } printf("ok\n"); return 0; } /* jfail -j jobid [-c code] [-s stage] — raft-apply-decoupling task 7.2: * synthetic worker-fact-stream primitive, sibling of jstart/jfinish above. * Drives xcecli_notify_jobfail -> on_job_fail -> record_job_fail_fact -> * persist_submit_job_fact_ex (the _pad-discriminated path, task 6.1 round 2) * -> apply_job_fail_transition() at apply time. Reuses the existing * job_failinfo_t shape already exercised internally by this file's own * notify_job_fail() helper (used by run_job() on a local exec failure). */ static const flagspec_t jobfail_flags[] = { { 'j', "jobid", "jobid", required_argument, 1, NULL }, { 'c', "failure-code", "failure-code", required_argument, 0, "0" }, { 's', "stage", "stage", required_argument, 0, "JOB_EXEC_STAGE_POST_EXEC" }, }; #define JOBFAIL_NFLAGS (sizeof(jobfail_flags) / sizeof(jobfail_flags[0])) int cmd_jobfail(int argc, char* argv[]) { int rc; int opt; uint64_t jobid = 0; int code = 0; int stage = JOB_EXEC_STAGE_POST_EXEC; job_failinfo_t failinfo; struct option long_opts[JOBFAIL_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; memset(&failinfo, 0x0, sizeof(failinfo)); if (flagspec_build_opts(jobfail_flags, JOBFAIL_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: jfail: internal error building flag table\n"); return -EINVAL; } while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'j': parse_jobid(&jobid, optarg); break; case 'c': code = atoi(optarg); break; case 's': stage = atoi(optarg); break; case '?': return flagspec_reject( "jfail", jobfail_flags, JOBFAIL_NFLAGS, argv[optind - 1], "jfail -j [-c ] [-s ]"); } } if (jobid == 0) { printf(" you need specify a positive jobid\n"); return -EINVAL; } failinfo.jobid = jobid; failinfo.stage = stage; failinfo.code = code; failinfo.detail = "synthetic fact-stream failure (xce_test jfail)"; rc = xcecli_notify_jobfail(active_ctx->ctx, &failinfo); if (rc < 0) { SET_ERROR(-rc); perror("xcecli_jobfail"); return rc; } printf("ok\n"); return 0; } /* set_resval() -- thin wrapper (resource-value-verification task 2.1). * * Type resolution (name -> Value_u_which, including the unrecognized-name * numeric-default fallback and `-t` override) now lives in * resval_typeresolve.c/resval_resolve_type(), shared with the standalone * unit tests. This function keeps only what still needs REPL state: the * xce_val_t allocation/parsing per wire type, and `.host` tagging via * get_setting("myname") (unchanged from before this refactor). * * `type_override`: RESVAL_TYPE_UNKNOWN for "no override" (the common case), * or an explicit Value_u_which tag from cmd_resupdate()'s new `-t` flag. * Ignored by resval_resolve_type() for any already-known resource name. */ static int set_resval( xce_resval_t* vals, int idx, char* name, char* val, int type_override) { int valtype; int rc = -1; if (name == NULL) { return 0; } valtype = resval_resolve_type(name, type_override); if (valtype == RESVAL_TYPE_UNKNOWN) { /* Only reachable if resval_resolve_type() itself ever declines to * resolve a non-NULL name -- not possible today (it always falls * back to Value_u_fvalue or the override), kept as a defensive * guard against a future change to that function's contract. */ return rc; } rc = 1; vals[idx].val = (xce_val_t*)calloc(1, sizeof(xce_val_t)); switch (valtype) { case Value_u_fvalue: vals[idx].name = name; vals[idx].val->which = valtype; if (val == NULL) { vals[idx].val->u.fvalue = 0.0; } else { vals[idx].val->u.fvalue = atof(val); } break; case Value_u_ivalue32: vals[idx].name = name; vals[idx].val->which = valtype; if (val == NULL) { vals[idx].val->u.ivalue32 = 0; } else { vals[idx].val->u.ivalue32 = atoi(val); } break; case Value_u_ivalue64: vals[idx].name = name; vals[idx].val->which = valtype; if (val == NULL) { vals[idx].val->u.ivalue64 = 0; } else { vals[idx].val->u.ivalue64 = atol(val); } break; case Value_u_bvalue: vals[idx].name = name; vals[idx].val->which = valtype; if (val == NULL) { vals[idx].val->u.bvalue = 0; } else { if (strcasecmp(val, "t") == 0) { vals[idx].val->u.bvalue = 1; } else { vals[idx].val->u.bvalue = 0; } } break; } /* NOTE (pre-existing, unchanged): Value_u_svalue (type/model, and now * an unrecognized name given `-t string`) has no case here -- the * switch falls through with vals[idx].name/.val->which left unset. * This is set_resval()'s existing behavior for type/model today; * Requirement 1.4 requires preserving it byte-for-byte, so it is * intentionally not fixed by this task. See CONCERNS in this task's * status report. */ vals[idx].host = get_setting("myname"); return rc; } /* * resource-value-verification task 2.2: the controller's own update RPC * reply can never distinguish per-name accept/reject -- update_resval_list() * (src/controller/worker.c:185-198) unconditionally returns success * regardless of any per-item failure (design.md's "set_resval()/ * cmd_resupdate" component, Responsibilities & Constraints). To recover * per-name accept/reject (Requirements 1.2, 1.3), cmd_resupdate() issues one * follow-up xcecli_resval() query per submitted (resource, worker) pair -- * the exact client-library call resval/cmd_resval already use -- and * compares the returned value against what was just sent. */ #define RESVAL_VERIFY_MAX_ATTEMPTS (5) #define RESVAL_VERIFY_RETRY_MS (100L) /* Compare a just-submitted value against a follow-up query's result. Both * sides are xce_val_t (wire value union) -- a match requires the same * `which` tag and an equal payload. */ static int resval_value_matches(xce_val_t* sent, xce_val_t* got) { if ((sent == NULL) || (got == NULL) || (sent->which != got->which)) { return 0; } switch (sent->which) { case Value_u_ivalue32: return sent->u.ivalue32 == got->u.ivalue32; case Value_u_uvalue32: return sent->u.uvalue32 == got->u.uvalue32; case Value_u_ivalue64: return sent->u.ivalue64 == got->u.ivalue64; case Value_u_uvalue64: return sent->u.uvalue64 == got->u.uvalue64; case Value_u_svalue: return EQUAL_STRING(sent->u.svalue, got->u.svalue); case Value_u_fvalue: return float_equal(sent->u.fvalue, got->u.fvalue); case Value_u_bvalue: return (sent->u.bvalue == 0) == (got->u.bvalue == 0); default: return 0; } } /* * Query a just-submitted (name, host) pair back from the controller and * print its per-name accepted/rejected verdict (Requirements 1.2, 1.3). A * short bounded retry guards against a genuinely-registered resource whose * value hasn't propagated to the query path yet (design.md's Implementation * Notes: sub-second in practice for a controller-local query) -- NOT an * arbitrary long poll. */ static void resupdate_report_one(char* name, char* host, xce_val_t* sent) { int attempt; int ever_found = 0; if (name == NULL) { /* set_resval()'s Value_u_svalue fallback path leaves .name unset * (pre-existing gap predating this task -- see set_resval()'s own * comment above); nothing usable to query or report here. */ return; } for (attempt = 0; attempt < RESVAL_VERIFY_MAX_ATTEMPTS; attempt++) { xce_resval_t** got = NULL; int rc; if (attempt > 0) { usleep((useconds_t)(RESVAL_VERIFY_RETRY_MS * US_PER_MS)); } rc = xcecli_resval(active_ctx->ctx, host, name, &got); if (rc > 0) { int matched = resval_value_matches(sent, got[0]->val); int i; ever_found = 1; for (i = 0; i < rc; i++) { free_ResourceVal_ptr(&got[i]); } FREEUP(got); if (matched) { printf("%s: accepted\n", name); return; } /* Found but the value differs -- could still be a stale value * about to be overwritten by the just-sent update's propagation, * so keep retrying within the bounded budget rather than giving * up immediately. */ continue; } } if (ever_found) { printf("%s: rejected: value mismatch\n", name); } else { printf("%s: rejected: unknown resource\n", name); } } static const flagspec_t resupdate_flags[] = { { 'n', "resname", "resname", required_argument, 1, NULL }, { 'v', "value", "value", required_argument, 1, NULL }, { 't', "value-type", "value-type", required_argument, 0, "numeric unless overridden" }, }; #define RESUPDATE_NFLAGS (sizeof(resupdate_flags) / sizeof(resupdate_flags[0])) int cmd_resupdate(int argc, char* argv[]) { #define MAX_N_RES (16) int nres; xce_resval_t vals[MAX_N_RES]; xce_resval_t* valptrs[MAX_N_RES]; char* subnames[MAX_N_RES]; char* resname = NULL; char* resval_str = NULL; int type_override = RESVAL_TYPE_UNKNOWN; int opt; int rc; struct option long_opts[RESUPDATE_NFLAGS + 1]; char optstring[16]; optind = 1; opterr = 0; nres = 0; if (flagspec_build_opts(resupdate_flags, RESUPDATE_NFLAGS, long_opts, optstring, sizeof(optstring)) < 0) { fprintf(stderr, "xce_test: resupdate: internal error building flag table\n"); return -EINVAL; } /* -t is deferred: it applies to the pair most recently opened by -n * (design.md's documented order is -n/-v/[-t]), so a pending pair's * set_resval() call is deferred until the NEXT -n or end-of-args -- * not fired immediately on -v as before -- so a trailing -t still has * a chance to be seen for that same pair. */ while ((opt = getopt_long(argc, argv, optstring, long_opts, NULL)) != -1) { switch (opt) { case 'n': if (resname != NULL) { if (resval_str == NULL) { printf( "res `%s' has no value specified, default will be used\n", resname); } rc = set_resval(vals, nres, resname, resval_str, type_override); if (rc >= 0) { valptrs[nres] = &vals[nres]; /* Tracked independently of vals[nres].name: set_resval() * leaves .name unset on the Value_u_svalue fallback path * (see its own comment), but `resname` here is always the * exact name that was actually submitted for this pair. */ subnames[nres] = resname; nres++; } resname = NULL; resval_str = NULL; type_override = RESVAL_TYPE_UNKNOWN; } resname = strdup(optarg); break; case 'v': if (resname != NULL && resval_str == NULL) { resval_str = optarg; } else { printf("ignore res value `%s' as no res name found\n", optarg); } break; case 't': if (resname != NULL) { if (strcasecmp(optarg, "numeric") == 0) { type_override = Value_u_fvalue; } else if (strcasecmp(optarg, "string") == 0) { type_override = Value_u_svalue; } else { printf( "ignoring unrecognized -t value `%s' (expected " "`numeric' or `string')\n", optarg); } } else { printf("ignore -t `%s' as no res name found\n", optarg); } break; case '?': return flagspec_reject( "resupdate", resupdate_flags, RESUPDATE_NFLAGS, argv[optind - 1], "resupdate -n -v [-t ] " "[-n -v [-t ] ...]"); } } /* Flush the last pending pair. Matches the pre-existing behavior of a * trailing -n with no -v ever seen (silently dropped, same as before * this change) -- only flush here when a value was actually supplied. */ if (resname != NULL && resval_str != NULL) { rc = set_resval(vals, nres, resname, resval_str, type_override); if (rc >= 0) { valptrs[nres] = &vals[nres]; subnames[nres] = resname; nres++; } } rc = xcecli_update_resvals(active_ctx->ctx, nres, &valptrs[0]); if (rc >= 0) { int i; printf("resource update ok!\n"); /* The update RPC's own reply can't carry per-name accept/reject * (see the comment above resupdate_report_one()) -- verify each * submitted pair individually via a follow-up query instead. */ for (i = 0; i < nres; i++) { resupdate_report_one(subnames[i], vals[i].host, vals[i].val); } } else { SET_ERROR(-rc); perror("xcecli_update_resval"); } return 0; } int cmd_join_sched(int argc, char* argv[]) { int role = HostRole_scheduler; int rc = -1; rc = xcecli_join(active_ctx->ctx, role, NULL); if (rc < 0) { SET_ERROR(-rc); perror("xce_join"); return rc; } printf("scheduler join ok\n"); return 0; } static int print_reslist(xce_reslist_t reslist) { int i; for (i = 0; i < reslist.count; i++) { printf("name: %s\nflags: %d\ninterval: %d\ndesc: %s\n\n", reslist.res[i]->name, reslist.res[i]->flags, reslist.res[i]->interval, reslist.res[i]->desc); } return 0; } static int print_queuelist(xce_queuelist_t qlist) { int i; for (i = 0; i < qlist.count; i++) { printf("name: %s\nstatus: %d\npriority: %d\n\n", qlist.queues[i]->queue, qlist.queues[i]->status, qlist.queues[i]->priority); } return 0; } static int print_workerlist(xce_workerlist_t wlist) { int i; for (i = 0; i < wlist.count; i++) { printf("name: %s\nstatus: %d\n\n", wlist.workers[i]->name, wlist.workers[i]->status); } return 0; } /* bug-1711 (Requirement 11.8): print the already-fetched group records * (conf->uglist / conf->hglist) instead of the previous no-op stub, which * always left the "User Groups"/"Host Groups" header followed by silent * nothing -- indistinguishable from a genuinely empty cluster. Mirrors * print_reslist/print_queuelist/print_workerlist's own one-record-per-block * convention just above. */ static int print_grouplist(xce_group_configlist_t glist) { int i, j; for (i = 0; i < glist.count; i++) { printf("name: %s\nmembers:", glist.groups[i]->name); for (j = 0; j < glist.groups[i]->members.count; j++) { printf(" %s", glist.groups[i]->members.data[j]); } printf("\nex_members:"); for (j = 0; j < glist.groups[i]->ex_members.count; j++) { printf(" %s", glist.groups[i]->ex_members.data[j]); } printf("\n\n"); } return 0; } /* bug-1711 (Requirement 11.8): print the already-fetched config records * (conf->cfglist) instead of the previous no-op stub -- see print_grouplist * above for the same fix applied to the "Configs" section. */ static int print_configlist(xce_configlist_t clist) { int i; for (i = 0; i < clist.count; i++) { printf("name: %s\nvalue: %s\n\n", clist.configs[i]->name, clist.configs[i]->value); } return 0; } int cmd_getconf(int argc, char* argv[]) { xce_clusterconf_t* conf; int rc; rc = xcecli_clusterconf(active_ctx->ctx, &conf); if (rc < 0) { SET_ERROR(-rc); perror("xce_clusterconf"); return rc; } printf("Cluster Name: %s\n", conf->name); printf("\nResources\n===========\n"); print_reslist(conf->reslist); printf("\nQueues\n========\n"); print_queuelist(conf->qlist); printf("\nWorkers\n========\n"); print_workerlist(conf->wlist); printf("\nUser Groups\n===========\n"); print_grouplist(conf->uglist); printf("\nHost Groups\n============\n"); print_grouplist(conf->hglist); printf("\nConfigs\n=======\n"); print_configlist(conf->cfglist); free_ClusterConf(conf); return 0; } #define KVAL(v) (((v) == NULL) ? "" : (v)) int cmd_set(int argc, char* argv[]) { int i; if (argc == 1) { for (i = 0; active_ctx->settings[i].key; i++) { printf("%-12s:%s default: %s\n", active_ctx->settings[i].key, KVAL(active_ctx->settings[i].value), KVAL(active_ctx->settings[i].defval)); } return 0; } /* errexit toggle (xce-test-e2e-primitives task 1.1), shell `-e`/`+e` style. * Default is mode-dependent (batch `-e` abort-on-failure, interactive `+e` * continue); these override it for the rest of the session. The session * still exits non-zero if any command failed. Handled here ahead of the * per-context key/value path. `continue-on-error` == `+e` (back-compat). */ if ((argc == 2) && ((strcmp(argv[1], "+e") == 0) || (strcmp(argv[1], "continue-on-error") == 0))) { g_continue_on_error = 1; /* +e: keep the read loop running on failure */ return 0; } if ((argc == 2) && (strcmp(argv[1], "-e") == 0)) { g_continue_on_error = 0; /* -e: abort the read loop on first failure */ return 0; } if (argc != 3) { printf("set \n"); return -EINVAL; } for (i = 0; active_ctx->settings[i].key; i++) { if (strcmp(active_ctx->settings[i].key, argv[1]) == 0) { if (active_ctx->settings[i].value != NULL) { free(active_ctx->settings[i].value); } active_ctx->settings[i].value = strdup(argv[2]); return 0; } } printf("Can not find setting key: %s\n", argv[1]); return -EINVAL; } static int run_command(char* cmdline, pid_t* pid) { pid_t child; child = fork(); *pid = child; if (child < 0) { *pid = -errno; return -errno; } if (child == 0) { execlp("/bin/sh", "/bin/sh", "-c", cmdline, NULL); perror("execlp"); exit(-1); } return 0; } static int run_task(char* cmdline, pid_t* pid) { return run_command(cmdline, pid); } static int get_avail_run_job_slots() { int i; for (i = 0; i < MAX_RUN_JOBS; i++) { if (run_jobs[i].jobid == 0) { return i; } } return -1; } static int get_avail_run_task_slots() { int i; for (i = 0; i < MAX_RUN_TASKS; i++) { if (run_tasks[i].pid <= 0) { return i; } } return -1; } static int sched_on_jobs(int njobs, xce_job_t** jobs) { printf("%s %d jobs gotten\n", __func__, njobs); return 0; } static int sched_on_workers(int nworkers, xce_worker_t** workers) { printf("%s %d workers gotten\n", __func__, nworkers); return 0; } static void notify_job_fail(uint64_t jobid, int code) { /* clang-format off */ job_failinfo_t failinfo = { .jobid = jobid, .stage = JOB_EXEC_STAGE_MAIN, .code = code, .detail = "no detail information" }; /*clang-format on*/ xcecli_notify_jobfail(active_ctx->ctx, &failinfo); return; } static void notify_task_fail(uint64_t jobid, int clientidx, int taskidx, int code) { xce_task_failinfo_t failinfo = { .jobid = jobid, .host = get_setting("myname"), .clientidx = clientidx, .taskidx = taskidx, .code = code, .detail = NULL }; xcecli_notify_taskfail(active_ctx->ctx, &failinfo); } static void notify_jobstart(uint64_t jobid, int pid, int pgid) { /* clang-format off */ job_pidinfo_t pidinfo = { .jobid = jobid, .npids = 1, .pids = (unsigned int []){(unsigned int)pid}, .npgids = 0, .pgids = (unsigned int []){(unsigned int)pgid}, .exdata = NULL }; /* clang-format on */ xcecli_notify_jobstart(active_ctx->ctx, &pidinfo); return; } static void notify_task_start(uint64_t jobid, int clientidx, int taskidx, int pid, int pgid) { xce_task_pidinfo_t pidinfo = { .jobid = jobid, .host = get_setting("myname"), .clientidx = clientidx, .taskidx = taskidx, .npids = 1, .pids = (unsigned int[]) { (unsigned int)pid }, .npgids = 1, .pgids = (unsigned int[]) { (unsigned int)pgid }, .exdata = NULL }; xcecli_notify_taskstart(active_ctx->ctx, &pidinfo); return; } static int run_task_remote(uint64_t jobid, job_startinfo_t* startinfo, xce_launch_t* launchinfo) { xce_taskhost_t hosts[launchinfo->nhosts - 1]; xce_taskrun_t taskrun; int i; int rc; taskrun.taskhosts = (xce_taskhost_t**)calloc(launchinfo->nhosts - 1, sizeof(xce_taskhost_t*)); if (taskrun.taskhosts == NULL) { return -ENOMEM; } taskrun.clientidx = -1; for (i = 1; i < launchinfo->nhosts; i++) { hosts[i - 1].exechome = startinfo->exechome; hosts[i - 1].execcwd = startinfo->execcwd; hosts[i - 1].cmdline = startinfo->cmdline; hosts[i - 1].exdata = startinfo->exdata; hosts[i - 1].execuser = startinfo->execuser; hosts[i - 1].launch_type = startinfo->launch_type; hosts[i - 1].host = launchinfo->hosts[i]->host; hosts[i - 1].ntasks = launchinfo->hosts[i]->nranks; if (hosts[i - 1].launch_type == NULL) { /* use mpi as default */ hosts[i - 1].launch_type = "mpi"; } taskrun.taskhosts[i - 1] = &hosts[i - 1]; } taskrun.nhosts = launchinfo->nhosts - 1; taskrun.jobid = jobid; xcecli_unblock_tasks(active_ctx->ctx); rc = xcecli_runtask(active_ctx->ctx, &taskrun); if (rc < 0) { fprintf(stderr, "run remote task failed\n"); } FREEUP(taskrun.taskhosts); return rc; } static int run_task_local(uint64_t jobid, job_startinfo_t* startinfo, int ntasks) { pid_t pid; char jobidStr[64]; pid = fork(); if (pid < 0) { return pid; } if (pid == 0) { int i; pid_t pids[ntasks]; int rc = 0; int nlive = 0; /* disable detection of address sanitizer as * the request structure will not be freed in * child and we will exit directly */ setenv("ASAN_OPTIONS", "detect_leaks=false", 1); for (i = 0; i < ntasks; i++) { rc = run_command(startinfo->cmdline, &pids[i]); if (rc >= 0) { nlive++; } } if (rc >= 0) { rc = 0; } while (nlive > 0) { sleep(1); for (i = 0; i < ntasks; i++) { int wstatus; if (pids[i] <= 0) { continue; } if (waitpid(pids[i], &wstatus, WNOHANG) <= 0) { continue; } if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) { nlive--; if (WIFSIGNALED(wstatus)) { rc = 128 + WTERMSIG(wstatus); } else { if (WEXITSTATUS(wstatus) != 0) { rc = WEXITSTATUS(wstatus); } } } } } exit(rc); } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, jobid); printf("Job %s started, PID: %d\n", jobidStr, pid); return pid; } int run_job(xce_job_exec_t* startmsg) { int rc = 0; uint64_t jobid = startmsg->jobid; job_startinfo_t* startinfo = startmsg->start_info; xce_launch_t* launchinfo = startmsg->launchinfo; printf("%s: exechome= %s\n", __func__, (startinfo->exechome == NULL) ? "NULL" : startinfo->exechome); printf("%s: execcwd= %s\n", __func__, (startinfo->execcwd == NULL) ? "NULL" : startinfo->execcwd); printf("%s: execuser= %s\n", __func__, (startinfo->execuser == NULL) ? "NULL" : startinfo->execuser); printf("%s: cmdline= %s\n", __func__, (startinfo->cmdline == NULL) ? "NULL" : startinfo->cmdline); printf("%s: exdata= %s\n", __func__, (startinfo->exdata == NULL) ? "NULL" : startinfo->exdata); printf("%s: launch_type= %s\n", __func__, (startinfo->launch_type == NULL) ? "NULL" : startinfo->launch_type); printf("%s: q_pre_exec= %s\n", __func__, (startinfo->q_pre_exec == NULL) ? "NULL" : startinfo->q_pre_exec); printf("%s: q_post_exec= %s\n", __func__, (startinfo->q_post_exec == NULL) ? "NULL" : startinfo->q_post_exec); printf("%s: job_pre_exec= %s\n", __func__, (startinfo->job_pre_exec == NULL) ? "NULL" : startinfo->job_pre_exec); printf("%s: job_post_exec= %s\n", __func__, (startinfo->job_post_exec == NULL) ? "NULL" : startinfo->job_post_exec); if (launchinfo->nhosts > 1) { rc = run_task_remote(jobid, startinfo, launchinfo); if (rc < 0) { fprintf(stderr, "run_task_remote() fail, rc %d\n", rc); return rc; } } rc = run_task_local(jobid, startinfo, launchinfo->hosts[0]->nranks); if (rc < 0) { notify_job_fail(jobid, rc); } else { int idx; int pid; pid = rc; idx = get_avail_run_job_slots(); if (idx < 0) { printf("too many jobs run in this worker\n"); } else { run_jobs[idx].jobid = jobid; run_jobs[idx].pid = pid; run_jobs[idx].ts = 0; } notify_jobstart(jobid, pid, pid); } return rc; } static int handle_startjob(xce_request_t* request) { return run_job(request->body.jobstartmsg); } static int handle_notify(xce_request_t* request) { xce_notify_data_t* data = request->body.notifymsg; char jobidStr[64]; switch (data->event) { case NotifyData_data_taskPidInfo: fmt_jobid(jobidStr, sizeof(jobidStr) - 1, data->data.taskinfo->jobid); printf("job %s host %s taskidx %d pid %d\n", jobidStr, data->data.taskinfo->host, data->data.taskinfo->taskidx, data->data.taskinfo->pids[0]); break; case NotifyData_data_taskFailInfo: fmt_jobid(jobidStr, sizeof(jobidStr) - 1, data->data.taskfailinfo->jobid); printf("job %s host %s task %d error code %d\n", jobidStr, data->data.taskfailinfo->host, data->data.taskfailinfo->taskidx, data->data.taskfailinfo->code); break; case NotifyData_data_taskFinish: fmt_jobid(jobidStr, sizeof(jobidStr) - 1, data->data.taskfinish->jobid); printf("job %s host %s task %d exit code %d\n", jobidStr, data->data.taskfinish->host, data->data.taskfinish->taskidx, data->data.taskfinish->exit_code); break; } return 0; } static int job_kill(uint64_t jobid, int signo) { int i; int killed = 0; for (i = 0; i < MAX_RUN_JOBS; i++) { if ((run_jobs[i].jobid == 0) || (run_jobs[i].jobid != jobid)) { continue; } if (run_jobs[i].pid > 1) { kill(run_jobs[i].pid, signo); killed++; } } return killed; } static int job_chkpnt(uint64_t jobid, char* method, int kill) { char buf[128]; char info[256]; printf("checkpoint job %ld by method %s\n", jobid, method); sprintf(buf, "/tmp/%ld/ckpt_dir", jobid); /* do some actions to checkpoint jobs */ sprintf(info, "{\nhost_select=\"criu\"; \nrestart_command=\"cd %s; criu restore " "-vvv\"}\n", buf); xcecli_notify_jobchkpt(active_ctx->ctx, jobid, info); if (kill) { job_kill(jobid, SIGTERM); } return 0; } static int handle_jobcontrol(xce_request_t* request) { uint64_t jobid; int signo; xce_jobctrl_t* jobctrl = request->body.jobctrlmsg; int rc = 0; switch (jobctrl->action) { case JobControl_u_siginfo: jobid = jobctrl->jobid; signo = jobctrl->u.siginfo->signo; rc = job_kill(jobid, signo); break; case JobControl_u_ckptoption: rc = job_chkpnt(jobctrl->jobid, jobctrl->u.ckptoption->method, jobctrl->u.ckptoption->kill); break; default: rc = -EINVAL; ERROR("unknown job control action %d", jobctrl->action); break; } return rc; } static int handle_runtasks(xce_request_t* request) { xce_taskrun_t* runmsg = request->body.runtaskmsg; int rc = 0; int i = 0; char jobidStr[64]; #if 0 job_startinfo_t startinfo; #endif if (runmsg->nhosts != 1) { /* something wrong */ rc = -EINVAL; notify_task_fail(runmsg->jobid, runmsg->clientidx, -1, rc); return rc; } if (strcmp(runmsg->taskhosts[0]->host, get_setting("myname")) != 0) { /* something wrong */ rc = -EINVAL; notify_task_fail(runmsg->jobid, runmsg->clientidx, -1, rc); return rc; } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, runmsg->jobid); printf("run task: jobid = %s\n", jobidStr); printf("run task: user = %s\n", runmsg->taskhosts[0]->execuser); printf("run task: cmdline = %s\n", runmsg->taskhosts[0]->cmdline); printf("run task: exdata = %s\n", runmsg->taskhosts[0]->exdata); printf("run task: ntasks =%d\n", runmsg->taskhosts[0]->ntasks); #if 0 startinfo.exechome = runmsg->taskhosts[0]->exechome; startinfo.execcwd = runmsg->taskhosts[0]->execcwd; startinfo.execuser = runmsg->taskhosts[0]->execuser; startinfo.cmdline = runmsg->taskhosts[0]->cmdline; startinfo.exdata = runmsg->taskhosts[0]->exdata; startinfo.launch_type = runmsg->taskhosts[0]->launch_type; #endif for (i = 0; i < runmsg->taskhosts[0]->ntasks; i++) { int pid = 0; int pgid = 0; rc = run_task(runmsg->taskhosts[0]->cmdline, &pid); if (rc < 0) { notify_task_fail(runmsg->jobid, runmsg->clientidx, i, rc); } else { int idx; printf("job %s task %d started, PID: %d\n", jobidStr, i, pid); idx = get_avail_run_task_slots(); if (idx < 0) { printf("too many running tasks\n"); } else { run_tasks[idx].jobid = runmsg->jobid; run_tasks[idx].taskid = i; run_tasks[idx].clientidx = runmsg->clientidx; run_tasks[idx].pid = pid; run_tasks[idx].ts = 0; } notify_task_start(runmsg->jobid, runmsg->clientidx, i, pid, pgid); } } return 0; } static int myreq_handler(xce_request_t* request, void* args) { int rc = 0; switch (request->opcode) { case OPCode_push: if (request->body_which == Request_body_datamsg) { dataset_t* datamsg = request->body.datamsg; switch (datamsg->table) { case XceData_data_schedjobs: case XceData_data_jobs: sched_on_jobs(datamsg->nrows, (xce_job_t**)datamsg->rows); break; case XceData_data_schedworkers: case XceData_data_workers: sched_on_workers(datamsg->nrows, (xce_worker_t**)datamsg->rows); break; default: break; } } break; case OPCode_startjob: rc = handle_startjob(request); break; case OPCode_notify: rc = handle_notify(request); break; case OPCode_jobcontrol: rc = handle_jobcontrol(request); break; case OPCode_runtasks: rc = handle_runtasks(request); break; default: break; } return rc; } static void check_run_jobs() { int i; time_t now = time(NULL); for (i = 0; i < MAX_RUN_JOBS; i++) { int wstatus; if (run_jobs[i].jobid == 0) { continue; } if (run_jobs[i].pid <= 0) { continue; } if (waitpid(run_jobs[i].pid, &wstatus, WNOHANG | WUNTRACED | WCONTINUED) <= 0) { /* fake job rusage */ int npids = 1; int npgids = 0; int nusage = 2; /* bug-1735: match the real worker's collect_job_rusages() * resnames (xcecd_cgroups.c), not the old never-matching * "cputime"/"mem" strings. */ xce_rusage_t data[] = { { "cpu_time_sec", 100.0 }, { "memory_rss", 200.0 } }; xce_rusage_t* usages[] = { &(data[0]), &(data[1]) }; if (now - run_jobs[i].ts > RUSAGE_UPDATE_INTERVAL) { xcecli_update_jusage(active_ctx->ctx, run_jobs[i].jobid, npids, &(run_jobs[i].pid), npgids, NULL, nusage, &usages[0]); run_jobs[i].ts = now; } continue; } if (WIFSTOPPED(wstatus)) { xcecli_notify_jobstop(active_ctx->ctx, run_jobs[i].jobid); } else if (WIFCONTINUED(wstatus)) { xcecli_notify_jobresume(active_ctx->ctx, run_jobs[i].jobid); } else if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) { xce_jobfinish_t finish; char jobidStr[64]; finish.jobid = run_jobs[i].jobid; if (WIFEXITED(wstatus)) { finish.exit_code = WEXITSTATUS(wstatus); } else { finish.exit_code = 128 + WTERMSIG(wstatus); } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, finish.jobid); printf("job %s exit code %d\n", jobidStr, finish.exit_code); xcecli_notify_jobfinish(active_ctx->ctx, &finish); run_jobs[i].jobid = 0; run_jobs[i].pid = 0; } } } static void check_run_tasks() { int i; time_t now = time(NULL); for (i = 0; i < MAX_RUN_TASKS; i++) { int wstatus; if (run_tasks[i].jobid == 0) { continue; } if (run_tasks[i].pid <= 0) { continue; } if (waitpid(run_tasks[i].pid, &wstatus, WNOHANG | WUNTRACED | WCONTINUED) <= 0) { /* fake job rusage */ int npids = 1; int npgids = 0; int nusage = 2; /* bug-1735: match the real worker's collect_job_rusages() * resnames (xcecd_cgroups.c), not the old never-matching * "cputime"/"mem" strings. */ xce_rusage_t data[] = { { "cpu_time_sec", 100.0 }, { "memory_rss", 200.0 } }; xce_rusage_t* usages[] = { &(data[0]), &(data[1]) }; if (now - run_tasks[i].ts > RUSAGE_UPDATE_INTERVAL) { xcecli_update_jusage(active_ctx->ctx, run_tasks[i].jobid, npids, &(run_tasks[i].pid), npgids, NULL, nusage, &usages[0]); run_tasks[i].ts = now; } continue; } if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) { xce_task_finish_t finish = { .jobid = run_tasks[i].jobid, .host = get_setting("myname"), .taskidx = run_tasks[i].taskid, .clientidx = run_tasks[i].clientidx, .exit_code = 0 }; char jobidStr[64]; if (WIFEXITED(wstatus)) { finish.exit_code = WEXITSTATUS(wstatus); } else { finish.exit_code = 128 + WTERMSIG(wstatus); } fmt_jobid(jobidStr, sizeof(jobidStr) - 1, finish.jobid); printf("job %s task %d exit code %d\n", jobidStr, finish.taskidx, finish.exit_code); xcecli_notify_taskfinish(active_ctx->ctx, &finish); run_tasks[i].jobid = 0; run_tasks[i].pid = 0; } } } #define HEARTBEAT_INTERVAL (60) static int worker_house_keeping(void* arg) { command_ctx_t* ctx = (command_ctx_t*)arg; int rc = 0; xce_use(ctx); check_run_jobs(); check_run_tasks(); rc = xcecli_heartbeat(ctx->ctx, HEARTBEAT_INTERVAL); if (rc < 0) { perror("xcecli_heartbeat"); } return rc; } int main(int argc, char* argv[]) { struct arguments arguments; char* buf; char* logfile = "/tmp/xce_test.log"; char* loglevel = "info"; int logsize = 64 * 1024 * 1024; int logcount = 10; /* build-time-versioning task 3.2: xce_test's own --version/-V output * (via argp_program_version) -- static so it outlives this point in * main() (contract documented on xce_compose_version() in utils.c). */ static char version_buf[128]; memset(&arguments, 0x0, sizeof(arguments)); /* dirty-content-fingerprint task 3.2: xce_test is not a daemon -- it has * no startup log line at all (see anatomy/build-time-versioning notes), * so there is no second call site to keep divergent here (unlike * controller/main.c and scheduler/src/main.c, which each keep a * separate, untouched xce_compose_version() buffer for their startup * INFO log per Req 3.2). version_buf can safely be composed directly * via xce_compose_version_ex() to pick up the dirty_fingerprint suffix. * Must run before argp_parse(): argp reads argp_program_version by * pointer when --version/-V is processed. */ xce_compose_version_ex(version_buf, sizeof(version_buf), "xce v0.0.1-", STR(VERSION_COMMIT_ID), STR(VERSION_BUILD_TIME), STR(VERSION_DIRTY_FINGERPRINT)); argp_program_version = version_buf; argp_parse(&argp, argc, argv, 0, 0, &arguments); if (arguments.verbose != 0) { loglevel = verbose_to_loglevel(arguments.verbose); } /* Remember the user's level so wait-primary can restore it after temporarily * raising the threshold to silence its poll-loop failover spam. */ g_active_loglevel = loglevel; mylog_init(logfile, loglevel, logsize, logcount); /* xce-test-cli-ux task 1.3, Requirement 9.9: replaces the former bare * `signal(SIGINT, sigbreak);` call. glibc's signal() installs with * SA_RESTART SET (BSD semantics), which silently restarts a blocking * slow syscall (read(), etc.) interrupted by SIGINT instead of * returning EINTR -- see xcetest_interrupt.h's file header for the * full rationale. xcetest_install_sigint() installs the same * sigbreak() handler via sigaction() with SA_RESTART explicitly * cleared, so a later blocking read (e.g. the confirmation prompt * built on top of this task) reliably observes EINTR. */ xcetest_install_sigint(sigbreak); memset(&run_jobs, 0x0, sizeof(run_jobs)); memset(&run_tasks, 0x0, sizeof(run_tasks)); idgen_ctx_init(&idctx); /* bug-1420 follow-up: folding pid into apiserverId alone (4 bits, 16 * buckets) at the jsub call site still collides too often across this * harness's rapid, per-invocation-fresh xce_test processes (confirmed * empirically -- a 10-round chaotic soak still produced same-second * duplicate jobids after that fix landed). idgen_ctx_init() always * resets sequence to 0, and MAKE_JOBID's sequence field alone has 20 * bits (config.h N_SEQUENCE_BITS) of headroom -- two orders of * magnitude more than the namespacing fields. Re-seed it here with a * value that mixes pid (stable per process) with monotonic-clock * nanoseconds (varies even across processes launched back-to-back in * the same wall-clock second, unlike idgen's own second-resolution * timestamp), masked to N_SEQUENCE_BITS so it can never bleed into the * adjacent tag/timestamp bits MAKE_JOBID packs beside it. idgen()'s own * ROUNDUP-to-N_ARRAY_IDX_BITS logic for array jobs still applies * unchanged on top of this seed. */ { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); idctx.sequence = ((uint32_t)getpid() ^ (uint32_t)ts.tv_nsec) & (uint32_t)UMAX_BITS(N_SEQUENCE_BITS); } /* errexit default, modelled on shell `-e`/`+e`, keyed on the stdin mode: * batch (piped/redirected stdin) → `-e`: abort the read loop on the first * failing command and exit non-zero (a script wants fail-fast + verdict). * interactive (TTY) REPL → `+e`: a failed command prints its error and the * loop CONTINUES (never exit on a typo). * Override at any time with `set -e` / `set +e` (or `set continue-on-error`). */ g_interactive = isatty(STDIN_FILENO); g_continue_on_error = g_interactive ? 1 : 0; /* xce-test-cli-ux task 2.3, Requirement 7: register command-name tab * completion, but ONLY for a genuinely interactive session -- design.md's * explicit precondition on xcetest_complete(). A piped/redirected/ * scripted stdin never calls readline()'s completion machinery anyway * (no terminal to press Tab on), but gating the registration itself * (rather than relying on that) keeps the precondition explicit and * matches this file's existing g_interactive-gated conventions. */ if (g_interactive) { rl_attempted_completion_function = xcetest_complete; } /* xce-test-rc-file task 2.2: auto-load $XDG_CONFIG_HOME/xce_test/rc (or * $HOME/.config/xce_test/rc) before the read loop starts, for both * interactive and scripted sessions alike -- unless --norc was given. * A missing/unresolvable rc file is silent (Req 1.3); a failing rc-file * line is handled entirely by preprocess_and_run's existing abort/continue * accounting (Req 3) -- an abort short-circuits straight to the same * exit-code convention used at the bottom of this function, skipping the * read loop entirely (Req 1.4/3.3). */ if (!arguments.norc) { char* rcpath = rcfile_resolve_path(); if (rcpath != NULL) { int rcrc = rcfile_load(rcpath, preprocess_and_run); free(rcpath); if (rcrc) { return g_session_failed ? 1 : 0; } } } while (!done) { char* s; int rc; prompt[0] = 0x0; fill_prompt(active_ctx, prompt); strcat(prompt, ">"); buf = readline(prompt); if (buf == NULL) { execute_line("quit"); continue; } if (strlen(buf) == 0) { free(buf); continue; } s = buf; rc = preprocess_and_run(s); free(buf); if (rc) { break; /* abort-on-failure requested loop stop */ } } return g_session_failed ? 1 : 0; }