Rongsong Shen
878a42fb48
cmd_join_worker's simulated-worker maxmem default was 64 bytes -- a leftover from when the field was MB-scale, unfixed when maxmem became bytes-canonical elsewhere. Now 64 * 1024 * 1024. Extracted the argument-parsing/defaulting logic into join_worker_parse_attrs() (join_worker_args.c), since it runs entirely before the function's network call and is a genuine testable seam -- same precedent as task 2.5's joblist_display.c. Verified live against a real deployed cluster (workerlist reports MaxMem: 64.00 MB) plus a permanent 5-test regression suite. maxswap/maxtmp have the identical stale-default bug (logged as bug-1793) but are deliberately left unfixed here since USAGE_JOIN_WORKER's help text currently documents their stale values accurately -- fixing only the code would create a new docs mismatch; a locked-in sibling-defaults test documents this for the follow-up. 1353/1353 across the full controller suite (+5), 0 regressions. Task 2.6 of .kiro/specs/xce-test-cli-ux/tasks.md. Requirements: 12.1 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019jaMf9QEwpugH2W9KAuc7f
4872 lines
166 KiB
C
4872 lines
166 KiB
C
#include <argp.h>
|
|
#include <errno.h>
|
|
#include <getopt.h>
|
|
#include <inttypes.h>
|
|
#include <limits.h> /* PATH_MAX -- cmd_source's realpath() canonicalization
|
|
* buffer (xce-test-cli-ux task 2.4) */
|
|
#include <signal.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/wait.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
|
|
#include <editline/readline.h> /* libedit (BSD) -- provides readline()/
|
|
* add_history(), same as GNU readline's
|
|
* split readline.h/history.h pair */
|
|
|
|
#include <cjson/cJSON.h>
|
|
#include <cjson/s2j.h>
|
|
#include <openssl/sha.h> /* SHA256() — cmd_upgradeconfig hashes the config blob */
|
|
|
|
#include "client.h"
|
|
#include "config.h" /* XCE_MAXNAME_LEN */
|
|
#include "controller.capnp.h"
|
|
#include "ctrlstatus_fmt.h" /* ctrlstatus_format_time_ms (task 6.3) */
|
|
#include "delay.h"
|
|
#include "idgen.h"
|
|
#include "preason.h"
|
|
#include "walcompare.h" /* cross-member committed-prefix divergence (cmd_walcompare) */
|
|
#include "waldump.h" /* offline WAL inspector (cmd_waldump) */
|
|
#include "snapdiff.h" /* byte-level checkpoint-file comparator (cmd_snapdiff) */
|
|
#include "snapverify.h" /* offline checkpoint-file structural validator (cmd_snapverify) */
|
|
#include "configcompare.h" /* cross-member active-config comparator (cmd_configcompare) */
|
|
#include "confirm.h" /* destructive-command confirmation gate (xce-test-cli-ux task 2.1) */
|
|
#include "joblist_display.h" /* worker_status_str/job_state_str/format_job_time
|
|
* (xce-test-cli-ux task 2.5 remediation) */
|
|
#include "join_worker_args.h" /* join_worker_parse_attrs (xce-test-cli-ux
|
|
* task 2.6 remediation, bug-1557) */
|
|
#include "rcfile.h" /* startup rc-file path resolution + loading (xce-test-rc-file task 2.2) */
|
|
#include "resval_typeresolve.h" /* resval_resolve_type (resource-value-verification task 2.1) */
|
|
#include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */
|
|
#include "xce/types.h"
|
|
#include "xcetest_interrupt.h" /* xcetest_install_sigint, xcetest_interrupted (xce-test-cli-ux task 1.3) */
|
|
|
|
typedef int (*cmd_func_t)(int, char*[]);
|
|
|
|
typedef struct {
|
|
char* name;
|
|
cmd_func_t func;
|
|
char* doc;
|
|
const char* usage;
|
|
} command_t;
|
|
|
|
typedef struct {
|
|
char* key;
|
|
char* value;
|
|
char* defval;
|
|
} setting_t;
|
|
|
|
typedef struct _command_ctx {
|
|
struct _command_ctx* parent;
|
|
char* prompt;
|
|
setting_t* settings;
|
|
command_t* commands;
|
|
xcecli_ctx_t ctx;
|
|
} command_ctx_t;
|
|
|
|
typedef struct {
|
|
uint64_t jobid;
|
|
int pid;
|
|
time_t ts;
|
|
} run_job_t;
|
|
|
|
typedef struct {
|
|
uint64_t jobid;
|
|
uint32_t taskid;
|
|
int clientidx;
|
|
int pid;
|
|
time_t ts;
|
|
} run_task_t;
|
|
|
|
typedef struct {
|
|
char* host;
|
|
int ntasks;
|
|
} host_task_t;
|
|
|
|
typedef struct {
|
|
char* PATH;
|
|
} sub_envs_t;
|
|
|
|
typedef struct {
|
|
sub_envs_t sub_env_vars;
|
|
} exdata_t;
|
|
|
|
#define MAX_RUN_JOBS (20)
|
|
#define MAX_TASK_HOSTS (5)
|
|
#define MAX_RUN_TASKS (50)
|
|
#define RUSAGE_UPDATE_INTERVAL (60)
|
|
#define MAX_LINE_LEN (40960)
|
|
|
|
#define S2J_JSON_SET_var_string_ELEMENT(to_json, from_struct, _element) \
|
|
if (((from_struct) != NULL) && (((from_struct)->_element) != NULL)) { \
|
|
cJSON_AddStringToObject((to_json), #_element, \
|
|
(from_struct)->_element); \
|
|
}
|
|
|
|
static run_job_t run_jobs[MAX_RUN_JOBS];
|
|
static run_task_t run_tasks[MAX_RUN_TASKS];
|
|
|
|
static int myreq_handler(xce_request_t*, void*);
|
|
|
|
extern int cmd_help(int, char*[]);
|
|
extern int cmd_up(int, char*[]);
|
|
extern int cmd_worker(int, char*[]);
|
|
extern int cmd_scheduler(int, char*[]);
|
|
extern int cmd_client(int, char*[]);
|
|
extern int cmd_quit(int, char*[]);
|
|
extern int cmd_hello(int, char*[]);
|
|
extern int cmd_shutdown(int, char*[]);
|
|
extern int cmd_set(int, char*[]);
|
|
extern int cmd_reslist(int, char*[]);
|
|
extern int cmd_join_worker(int, char*[]);
|
|
extern int cmd_jobstart(int, char*[]);
|
|
extern int cmd_jobfail(int, char*[]);
|
|
extern int cmd_join_sched(int, char*[]);
|
|
extern int cmd_workerlist(int, char*[]);
|
|
extern int cmd_joblist(int, char*[]);
|
|
extern int cmd_jobrun(int, char*[]);
|
|
extern int cmd_jobsub(int, char*[]);
|
|
extern int cmd_getconf(int, char*[]);
|
|
extern int cmd_jobfinish(int, char*[]);
|
|
extern int cmd_resupdate(int, char*[]);
|
|
extern int cmd_resval(int, char*[]);
|
|
extern int cmd_jobkill(int, char*[]);
|
|
extern int cmd_jobckpt(int, char*[]);
|
|
extern int cmd_qctrl(int, char*[]);
|
|
extern int cmd_hostctrl(int, char*[]);
|
|
extern int cmd_upgradeconfig(int, char*[]);
|
|
extern int cmd_runtasks(int, char*[]);
|
|
extern int cmd_setpend(int, char*[]);
|
|
extern int cmd_jobinfo(int, char*[]);
|
|
extern int cmd_echo(int, char*[]);
|
|
extern int cmd_expect_jobs(int, char*[]);
|
|
extern int cmd_expect_job(int, char*[]);
|
|
extern int cmd_wait_primary(int, char*[]);
|
|
extern int cmd_ctrlstatus(int, char*[]);
|
|
extern int cmd_waldump(int, char*[]);
|
|
extern int cmd_walcompare(int, char*[]);
|
|
extern int cmd_snapverify(int, char*[]);
|
|
extern int cmd_snapdiff(int, char*[]);
|
|
extern int cmd_configcompare(int, char*[]);
|
|
extern int cmd_source(int, char*[]);
|
|
static int worker_house_keeping(void*);
|
|
|
|
/* Session verdict / loop state (xce-test-e2e-primitives task 1.1).
|
|
*
|
|
* g_session_failed set to 1 the first time any command in the session
|
|
* fails; drives the process exit code in main().
|
|
* g_continue_on_error when set (via `set continue-on-error`), a failing
|
|
* command does not stop the read loop; the session still
|
|
* exits non-zero at the end.
|
|
* g_last_status status of the most recently executed command
|
|
* (0 == success, non-zero == failure). Future tasks
|
|
* expose this as the `$?` auto-variable.
|
|
*/
|
|
static int g_session_failed = 0;
|
|
static int g_continue_on_error = 0;
|
|
static int g_last_status = 0;
|
|
/* Interactive (TTY) sessions are a REPL: a failed command prints its error and
|
|
* the loop CONTINUES — never abort the read loop on a typo. The abort-on-first-
|
|
* failure + non-zero-exit verdict is a SCRIPTED/piped-mode concept only. Set
|
|
* from isatty(stdin) in main. */
|
|
static int g_interactive = 0;
|
|
|
|
/* Auto-variable state for the fixed `$jobid` / `$jobcount` substitution
|
|
* (xce-test-e2e-primitives task 3.1, Req 3.1/3.3/3.5/3.6).
|
|
*
|
|
* g_last_jobid most recently acknowledged JobId as a string; empty ("")
|
|
* means unset. Populated later by cmd_jobsub (task 4.1).
|
|
* g_last_jobcount most recent job count; -1 means unset. Populated by
|
|
* cmd_joblist on a successful `joblist` (task 5.1, Req 3.4);
|
|
* a failed joblist leaves it untouched so no fake count is
|
|
* advertised.
|
|
*
|
|
* Until a command sets one, it stays unset — which is exactly what makes an
|
|
* unset `$jobid`/`$jobcount` reference fail the command (Req 3.6). */
|
|
static char g_last_jobid[XCE_MAXNAME_LEN] = { 0 };
|
|
static int g_last_jobcount = -1;
|
|
|
|
static idgen_ctx_t idctx;
|
|
delaymgr_t delaymgr;
|
|
|
|
static setting_t settings[] = {
|
|
/* HA controller list: comma-separated "name=host:port" or "host:port"
|
|
* entries (same syntax as the controller --peers flag), enabling the
|
|
* test client to fail over across controller candidates. */
|
|
{ (char*)"masters", (char*)NULL, (char*)"localhost:9990" },
|
|
{ (char*)"secret", (char*)NULL, (char*)"keepSecret" },
|
|
{ (char*)"myname", (char*)NULL, (char*)"localhost" },
|
|
{ (char*)NULL, (char*)NULL, (char*)NULL }
|
|
};
|
|
|
|
/* Detailed per-command `help <cmd>` text (design: keep these OUT of the
|
|
* command_t table initializers below so the tables stay readable). Each
|
|
* constant's shape: first line = command name + flag/positional syntax,
|
|
* then one indented line per flag/positional explaining what it does and
|
|
* whether it's required, then (where useful) an `example:` line. Derived
|
|
* from actually reading each cmd_* handler's getopt() string and switch
|
|
* cases (or its positional argv[] handling) -- not guessed from the name.
|
|
*
|
|
* Commands that are identical (same func, same semantics) across more than
|
|
* one of the 4 command tables share ONE constant here, referenced from every
|
|
* table, so the text can't drift out of sync between contexts. */
|
|
|
|
static const char USAGE_HELP[] =
|
|
"help [<cmd>]\n"
|
|
" <cmd> optional command name; prints its detailed usage instead of\n"
|
|
" the one-line summary\n"
|
|
" With no argument, lists every command in the current context with its\n"
|
|
" one-line summary (unaffected by this detailed-usage text).\n"
|
|
"example: help resval\n";
|
|
|
|
static const char USAGE_QUIT[] =
|
|
"quit\n"
|
|
" (no arguments)\n"
|
|
" Leaves the current context (if any) and exits xce_test.\n";
|
|
|
|
static const char USAGE_ECHO[] =
|
|
"echo [<message...>]\n"
|
|
" <message...> zero or more words, printed space-joined followed by a\n"
|
|
" newline (always succeeds)\n"
|
|
" Useful to print the $jobid / $jobcount / $? auto-variables in scripts.\n"
|
|
"example: echo job is $jobid\n";
|
|
|
|
static const char USAGE_UP[] =
|
|
"..\n"
|
|
" (no arguments)\n"
|
|
" Leaves the current context and returns to its parent, closing the\n"
|
|
" underlying client connection for this context.\n";
|
|
|
|
static const char USAGE_WALDUMP[] =
|
|
"waldump <wal_dir|data_dir|seg_file> [more...]\n"
|
|
" <path> one or more paths: a seg-*.wal file, a wal/ dir, or a\n"
|
|
" node/data dir\n"
|
|
" Offline, read-only WAL inspector: lists every entry, decodes\n"
|
|
" JOB_SUBMIT jobids, and flags LSN gaps and a CRC-broken tail.\n"
|
|
" Returns nonzero if any path fails to load or any gap/CRC break is\n"
|
|
" found.\n"
|
|
"example: waldump /var/xce/ctrl-1/wal\n";
|
|
|
|
static const char USAGE_WALCOMPARE[] =
|
|
"walcompare [--report] [--committed <N>] <memberA> <memberB> [more...]\n"
|
|
" --report also emit a full per-member (lsn,term,pcrc,status)\n"
|
|
" machine-checkable table, not just the verdict\n"
|
|
" --committed <N> use LSN N as the shared committed boundary instead\n"
|
|
" of deriving it as the minimum CRC-intact tip across\n"
|
|
" members\n"
|
|
" <memberX> 2-16 member paths (seg-*.wal file, wal/ dir, or\n"
|
|
" node/data dir)\n"
|
|
" Returns 0 if all members' committed WAL prefixes are identical,\n"
|
|
" nonzero on divergence or load error.\n"
|
|
"example: walcompare --report /var/xce/ctrl-1/wal /var/xce/ctrl-2/wal\n";
|
|
|
|
static const char USAGE_SNAPVERIFY[] =
|
|
"snapverify <snap_file> [<snap_file> ...]\n"
|
|
" <snap_file> one or more snap-*.capnp or snap-*.bin checkpoint files\n"
|
|
" Offline structural validator: walks the same layout\n"
|
|
" persist_load_snapshot() parses, without restoring anything. Reports\n"
|
|
" the byte offset of the first structural defect and how many frames\n"
|
|
" parsed cleanly before it.\n"
|
|
" Returns nonzero if any file fails to verify.\n"
|
|
"example: snapverify /var/xce/ctrl-1/snap/snap-000123.capnp\n";
|
|
|
|
static const char USAGE_SNAPDIFF[] =
|
|
"snapdiff <source_file> <received_file>\n"
|
|
" <source_file> the primary's own locally generated checkpoint\n"
|
|
" (snap-*.capnp)\n"
|
|
" <received_file> the copy a standby assembled from the wire\n"
|
|
" (snap-*.bin)\n"
|
|
" Byte-level comparator; reports the first differing offset and\n"
|
|
" whether it lands on an OPCode_snapshotTransfer chunk boundary.\n"
|
|
" Returns 0 if identical, nonzero otherwise.\n"
|
|
"example: snapdiff snap-000123.capnp snap-000123.bin\n";
|
|
|
|
static const char USAGE_CONFIGCOMPARE[] =
|
|
"configcompare <member_dir> <member_dir> [<member_dir> ...]\n"
|
|
" <member_dir> a member's workdir/name conf-root directory (NOT a raw\n"
|
|
" config file path) -- the dir under which conf/current\n"
|
|
" resolves\n"
|
|
" Recomputes each member's active-config sha256 fresh from its own\n"
|
|
" on-disk bytes and reports whether all members match.\n"
|
|
" Returns 0 if all members are present and identical, nonzero if any\n"
|
|
" member is unreadable or divergent.\n"
|
|
"example: configcompare /var/xce/ctrl-1 /var/xce/ctrl-2 /var/xce/ctrl-3\n";
|
|
|
|
static const char USAGE_SOURCE[] =
|
|
"source <file>\n"
|
|
" <file> path to a file containing xce_test commands, one per line\n"
|
|
" (required)\n"
|
|
" Reads and executes each line of <file> in order, exactly as if it\n"
|
|
" had been typed at the current prompt -- including a nested `source`\n"
|
|
" command, so one script file can compose others. A file that does\n"
|
|
" not exist or cannot be read is rejected with an explicit error and\n"
|
|
" no line of it is executed; a file that would source itself,\n"
|
|
" directly or transitively, is rejected as a cycle instead of\n"
|
|
" hanging. A failing line inside the file follows the session's\n"
|
|
" current -e/+e/continue-on-error setting (see `help set`), exactly\n"
|
|
" as it does for lines typed at the top-level prompt.\n"
|
|
"example: source /path/to/script.xt\n";
|
|
|
|
static const char USAGE_WORKER[] =
|
|
"worker\n"
|
|
" (no arguments)\n"
|
|
" Enters the worker context: connects to the controller cluster as a\n"
|
|
" worker client and starts the local delay-manager housekeeping loop.\n";
|
|
|
|
static const char USAGE_SCHEDULER[] =
|
|
"scheduler\n"
|
|
" (no arguments)\n"
|
|
" Enters the scheduler context: connects to the controller cluster as\n"
|
|
" a scheduler client.\n";
|
|
|
|
static const char USAGE_CLIENT[] =
|
|
"client\n"
|
|
" (no arguments)\n"
|
|
" Enters the client context: connects to the controller cluster as a\n"
|
|
" regular API client (job submission, resource/worker queries, etc.).\n";
|
|
|
|
static const char USAGE_SET[] =
|
|
"set\n"
|
|
"set <key> <value>\n"
|
|
"set -e | +e | continue-on-error\n"
|
|
" (no args) list all settings in the current context\n"
|
|
" (current value + default)\n"
|
|
" <key> <value> set setting <key> (e.g. masters, secret,\n"
|
|
" myname) to <value>\n"
|
|
" -e abort the read loop on the first failing\n"
|
|
" command (errexit)\n"
|
|
" +e | continue-on-error keep the read loop running after a\n"
|
|
" failing command\n"
|
|
"example: set masters ctrl1=localhost:9991,ctrl2=localhost:9992\n";
|
|
|
|
static const char USAGE_JOIN_WORKER[] =
|
|
"join [-f <cpufactor>] [-c <maxcpus>] [-m <maxmem>] [-s <maxswap>]\n"
|
|
" [-t <maxtmp>] [-d <ndisks>]\n"
|
|
" -f <cpufactor> relative CPU speed factor (default 1.0)\n"
|
|
" -c <maxcpus> max CPU count (default 4)\n"
|
|
" -m <maxmem> max memory in bytes (default 64)\n"
|
|
" -s <maxswap> max swap in bytes (default 1024)\n"
|
|
" -t <maxtmp> max tmp space in bytes (default 2048)\n"
|
|
" -d <ndisks> number of local disks (default 1)\n"
|
|
" Joins the cluster as a worker (hosttype LINUX64, hostmodel Intel64).\n"
|
|
"example: join -c 8 -m 16384\n";
|
|
|
|
static const char USAGE_JOBSTART[] =
|
|
"jstart -j <jobid> [-p <pid>]\n"
|
|
" -j <jobid> jobid to report a synthetic job-start fact for\n"
|
|
" (required)\n"
|
|
" -p <pid> pid to report (default: this xce_test process's own\n"
|
|
" pid)\n"
|
|
"example: jstart -j 12345 -p 4242\n";
|
|
|
|
static const char USAGE_JOBFINISH[] =
|
|
"jfinish -j <jobid> [-c <exitcode>]\n"
|
|
" -j <jobid> jobid to report a synthetic job-finish fact for\n"
|
|
" (required)\n"
|
|
" -c <exitcode> exit code to report (default 0)\n"
|
|
"example: jfinish -j 12345 -c 0\n";
|
|
|
|
static const char USAGE_JOBFAIL[] =
|
|
"jfail -j <jobid> [-c <code>] [-s <stage>]\n"
|
|
" -j <jobid> jobid to report a synthetic job-fail fact for\n"
|
|
" (required)\n"
|
|
" -c <code> failure code to report (default 0)\n"
|
|
" -s <stage> exec-stage code to report (default\n"
|
|
" JOB_EXEC_STAGE_POST_EXEC)\n"
|
|
"example: jfail -j 12345 -c 1\n";
|
|
|
|
static const char USAGE_RESUPDATE[] =
|
|
"resupdate -n <resname> -v <value> [-t <numeric|string>]\n"
|
|
" [-n <resname> -v <value> [-t <numeric|string>] ...]\n"
|
|
" -n <resname> resource name to update; any name the cluster has\n"
|
|
" registered is accepted (e.g. slots, mem, cpuf,\n"
|
|
" maxswap, maxtmp, maxcpus, ndisks, type, model,\n"
|
|
" dynres1, dynres2, r1m, r15s, r15m, pg, ls, it, swp,\n"
|
|
" io, tmp); may repeat for multiple resources\n"
|
|
" -v <value> value for the immediately preceding -n (numeric for\n"
|
|
" most resources, string for type/model)\n"
|
|
" -t <numeric|string> optional: force the wire type for the\n"
|
|
" immediately preceding -n/-v pair. Only meaningful for\n"
|
|
" a resource name NOT already known to this tool -- a\n"
|
|
" known name (e.g. mem, type) always keeps its own\n"
|
|
" fixed type, and -t is ignored for it. An unrecognized\n"
|
|
" name defaults to numeric when -t is omitted.\n"
|
|
" Order matters: pair each -n with its -v (and optional -t) right\n"
|
|
" after it.\n"
|
|
"example: resupdate -n slots -v 4 -n mem -v 8192\n"
|
|
"example: resupdate -n r15m -v 1.5\n"
|
|
"example: resupdate -n mylabel -v foo -t string\n";
|
|
|
|
static const char USAGE_HELLO[] =
|
|
"hello [-c <count>] [-m <message>]\n"
|
|
" -c <count> number of times to send the hello (default 1)\n"
|
|
" -m <message> greeting text to send (default \"Greeting from xce\n"
|
|
" client\")\n"
|
|
"example: hello -c 3 -m \"ping\"\n";
|
|
|
|
static const char USAGE_RESLIST[] =
|
|
"reslist\n"
|
|
" (no arguments)\n"
|
|
" Prints all cluster-wide resources (name, flags, interval,\n"
|
|
" description).\n";
|
|
|
|
static const char USAGE_WORKERLIST[] =
|
|
"workerlist\n"
|
|
" (no arguments)\n"
|
|
" Prints all joined workers (host, status, host type/model, cpu\n"
|
|
" factor, max cpu/mem/swap/tmp, ndisks).\n";
|
|
|
|
static const char USAGE_QCTRL[] =
|
|
"qctrl -q <queue> [-a] [-i] [-o] [-c]\n"
|
|
" -q <queue> queue name to control (required)\n"
|
|
" -a set queue ACTIVE (accepts dispatch)\n"
|
|
" -i set queue INACTIVE (rejects dispatch)\n"
|
|
" -o set queue OPEN (accepts new submissions)\n"
|
|
" -c set queue CLOSE (rejects new submissions)\n"
|
|
"example: qctrl -q normal -i\n";
|
|
|
|
static const char USAGE_HOSTCTRL[] =
|
|
"hostctrl -h <hostname> (-o|-c)\n"
|
|
" -h <hostname> host to administratively control (required)\n"
|
|
" -o open the host (allow scheduling)\n"
|
|
" -c close the host (admin-close; shows CLOSED in\n"
|
|
" workerlist)\n"
|
|
"example: hostctrl -h node01 -c\n";
|
|
|
|
static const char USAGE_UPGRADECONFIG[] =
|
|
"upgradeconfig <cluster.bin>\n"
|
|
" <cluster.bin> path to a capnp-encoded ClusterConfig blob (same\n"
|
|
" format `xceadmin config build` produces and\n"
|
|
" deploy-ha.sh installs)\n"
|
|
" Computes its SHA-256 and submits it as a configuration upgrade via\n"
|
|
" OP_upgradeConfig.\n"
|
|
"example: upgradeconfig /var/xce/cluster.bin\n";
|
|
|
|
static const char USAGE_JOBLIST[] =
|
|
"joblist\n"
|
|
" (no arguments)\n"
|
|
" Prints all jobs in the cluster (JobId, user, queue, state, times,\n"
|
|
" ntasks, command) and captures the count into $jobcount.\n";
|
|
|
|
static const char USAGE_EXPECT_JOBS[] =
|
|
"expect-jobs <N>\n"
|
|
" <N> non-negative integer; asserts the current job count equals N\n"
|
|
" Fails (nonzero) on a malformed N, a failed joblist query, or a count\n"
|
|
" mismatch.\n"
|
|
"example: expect-jobs 3\n";
|
|
|
|
static const char USAGE_EXPECT_JOB[] =
|
|
"expect-job <jobid>\n"
|
|
" <jobid> a JobId string (as printed by joblist / captured into\n"
|
|
" $jobid); asserts a job with this id is present in the\n"
|
|
" current job list\n"
|
|
" Fails (nonzero) on a missing argument, a failed joblist query, or if\n"
|
|
" the job is not found.\n"
|
|
"example: expect-job $jobid\n";
|
|
|
|
static const char USAGE_WAIT_PRIMARY[] =
|
|
"wait-primary [<timeout_ms>]\n"
|
|
" <timeout_ms> optional max wait in milliseconds (default 30000);\n"
|
|
" must be a non-negative integer\n"
|
|
" Blocks, polling every 200 ms, until the AUTHORITATIVE controller\n"
|
|
" status probe reaches the serving primary (a standby/hint answer does\n"
|
|
" not count), or the timeout elapses.\n"
|
|
" Returns 0 once a primary answers, nonzero (ETIMEDOUT) on timeout.\n"
|
|
"example: wait-primary 10000\n";
|
|
|
|
static const char USAGE_CTRLSTATUS[] =
|
|
"ctrlstatus\n"
|
|
" (no arguments)\n"
|
|
" Prints the responding controller's identity/role, cluster name,\n"
|
|
" peer topology, config version/checksum, and HA health (term,\n"
|
|
" committedLsn, appliedLsn, fenced) from a single any-node status\n"
|
|
" read.\n";
|
|
|
|
static const char USAGE_SHUTDOWN[] =
|
|
"shutdown\n"
|
|
" (no arguments)\n"
|
|
" Shuts down the xce controller and, on success, leaves the client\n"
|
|
" context (same as running `..`).\n";
|
|
|
|
static const char USAGE_JOBRUN[] =
|
|
"jobrun -j <jobid> [-m <host>[:<ntasks>]] [-m ...]\n"
|
|
" -j <jobid> jobid to dispatch (required, must be > 0)\n"
|
|
" -m <host>[:<n>] host to run tasks on, optional :ntasks count\n"
|
|
" (default 1); may be repeated up to 5 times\n"
|
|
"example: jobrun -j 12345 -m node01:2 -m node02\n";
|
|
|
|
static const char USAGE_JOBSUB[] =
|
|
"jobsub -u <user> -q <queue> -c <cmdline> [-R <resreq>]\n"
|
|
" [-n <min>[,<max>]] [-x <exdata>] [-a <arraysize>]\n"
|
|
" -u <user> submitting user (required)\n"
|
|
" -q <queue> target queue (required)\n"
|
|
" -c <cmdline> command line to run (required)\n"
|
|
" -R <resreq> resource requirement expression (default\n"
|
|
" \"type == any\")\n"
|
|
" -n <min>[,<max>] task count, or a min,max range (default 1)\n"
|
|
" -x <exdata> raw exdata JSON string (overrides the built-in\n"
|
|
" PATH env)\n"
|
|
" -a <arraysize> job array size (default 1)\n"
|
|
" On success, captures the submitted JobId into the $jobid\n"
|
|
" auto-variable.\n"
|
|
"example: jobsub -u alice -q normal -c \"sleep 10\" -n 1,4\n";
|
|
|
|
static const char USAGE_RESVAL[] =
|
|
"resval -n <resname> [-m <host>]\n"
|
|
" -n <resname> resource name to query (required)\n"
|
|
" -m <host> host to query (required unless the resource is a\n"
|
|
" shared/cluster-wide resource, not per-host)\n"
|
|
"example: resval -n slots -m myhost\n";
|
|
|
|
static const char USAGE_JOBKILL[] =
|
|
"jobkill -j <jobid> [-s <signo>]\n"
|
|
" -j <jobid> jobid to signal (required)\n"
|
|
" -s <signo> signal number to send (default SIGTERM)\n"
|
|
"example: jobkill -j 12345 -s 9\n";
|
|
|
|
static const char USAGE_JOBCKPT[] =
|
|
"jobckpt -j <jobid> [-k] [-m <method>]\n"
|
|
" -j <jobid> jobid to checkpoint (required)\n"
|
|
" -k kill the job immediately after checkpointing\n"
|
|
" -m <method> checkpoint method name (default \"default\")\n"
|
|
"example: jobckpt -j 12345 -m default\n";
|
|
|
|
static const char USAGE_JOBINFO[] =
|
|
"jobinfo -j <jobid> [-j <jobid> ...]\n"
|
|
" -j <jobid> jobid to show info for; may be repeated to show\n"
|
|
" several jobs in one call\n"
|
|
"example: jobinfo -j 12345\n";
|
|
|
|
static const char USAGE_TASKRUN[] =
|
|
"taskrun [-j <jobid>] [-m <host>[:<n>]]... [-u <user>] [-c <cmdline>]\n"
|
|
" [-x <exdata>] [-d <dir>] [-t <launch_type>]\n"
|
|
" -j <jobid> jobid this task run belongs to\n"
|
|
" -m <host>[:<n>] host to run on, optional :ntasks count\n"
|
|
" (default 1); may be repeated up to 5 times\n"
|
|
" -u <user> exec user, applied to every host\n"
|
|
" -c <cmdline> command line, applied to every host\n"
|
|
" -x <exdata> exdata string, applied to every host\n"
|
|
" -d <dir> exec home/cwd directory, applied to every host\n"
|
|
" -t <launch_type> launch type (default \"mpi\" if the total task\n"
|
|
" count across all -m hosts is > 1, else a\n"
|
|
" single-task default)\n"
|
|
"example: taskrun -j 12345 -m node01:2 -c \"hostname\" -u alice\n";
|
|
|
|
static const char USAGE_JOIN_SCHED[] =
|
|
"join\n"
|
|
" (no arguments)\n"
|
|
" Joins the cluster as a scheduler.\n";
|
|
|
|
static const char USAGE_GETCONF[] =
|
|
"getconf\n"
|
|
" (no arguments)\n"
|
|
" Prints the full cluster configuration (name, resources, queues,\n"
|
|
" workers, user/host groups, configs) known to the scheduler.\n";
|
|
|
|
static const char USAGE_SETPEND[] =
|
|
"setpend -j <jobid> -p <reason> [-p <reason> ...]\n"
|
|
" -j <jobid> jobid to set the pend reason for (required)\n"
|
|
" -p <reason> pend reason spec, may repeat: `<code>` sets the main\n"
|
|
" reason; `<host>:<code>` adds a host-based reason;\n"
|
|
" `<res>:<host>:<code>` adds a resource-based reason\n"
|
|
"example: setpend -j 12345 -p 5\n";
|
|
|
|
static command_t top_commands[] = {
|
|
{ "help", cmd_help, "Display this text", USAGE_HELP },
|
|
{ "worker", cmd_worker, "In worker context", USAGE_WORKER },
|
|
{ "scheduler", cmd_scheduler, "In scheduler context", USAGE_SCHEDULER },
|
|
{ "client", cmd_client, "In client context", USAGE_CLIENT },
|
|
{ "set", cmd_set, "Show/Set settings", USAGE_SET },
|
|
{ "echo", cmd_echo, "Print a message", USAGE_ECHO },
|
|
{ "waldump", cmd_waldump, "Inspect WAL segments", USAGE_WALDUMP },
|
|
{ "walcompare", cmd_walcompare, "Diff member committed WAL prefixes",
|
|
USAGE_WALCOMPARE },
|
|
{ "snapverify", cmd_snapverify, "Verify checkpoint file structure",
|
|
USAGE_SNAPVERIFY },
|
|
{ "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files",
|
|
USAGE_SNAPDIFF },
|
|
{ "configcompare", cmd_configcompare, "Compare active config hash across members",
|
|
USAGE_CONFIGCOMPARE },
|
|
{ "source", cmd_source, "Run commands from a file", USAGE_SOURCE },
|
|
{ "quit", cmd_quit, "Quit xce test", USAGE_QUIT },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t topctx = { .parent = NULL,
|
|
.prompt = "xce",
|
|
.settings = settings,
|
|
.commands = top_commands };
|
|
|
|
static command_t worker_commands[] = {
|
|
{ "..", cmd_up, "Goto upper context", USAGE_UP },
|
|
{ "help", cmd_help, "Display this text", USAGE_HELP },
|
|
{ "quit", cmd_quit, "Quit xce test", USAGE_QUIT },
|
|
{ "join", cmd_join_worker, "Join the cluster", USAGE_JOIN_WORKER },
|
|
{ "jstart", cmd_jobstart, "tell controller jobstart", USAGE_JOBSTART },
|
|
{ "jfinish", cmd_jobfinish, "tell controller jobfinish", USAGE_JOBFINISH },
|
|
{ "jfail", cmd_jobfail, "tell controller jobfail", USAGE_JOBFAIL },
|
|
{ "resupdate", cmd_resupdate, "update res vals", USAGE_RESUPDATE },
|
|
{ "echo", cmd_echo, "Print a message", USAGE_ECHO },
|
|
{ "source", cmd_source, "Run commands from a file", USAGE_SOURCE },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t workerctx = { .parent = &topctx,
|
|
.prompt = "worker",
|
|
.settings = settings,
|
|
.commands = worker_commands };
|
|
|
|
static command_t client_commands[] = {
|
|
{ "..", cmd_up, "Goto upper context", USAGE_UP },
|
|
{ "hello", cmd_hello, "Say hello to xce controller", USAGE_HELLO },
|
|
{ "reslist", cmd_reslist, "Display cluster resources", USAGE_RESLIST },
|
|
{ "workerlist", cmd_workerlist, "list workers", USAGE_WORKERLIST },
|
|
{ "qctrl", cmd_qctrl, "queue control", USAGE_QCTRL },
|
|
{ "hostctrl", cmd_hostctrl, "host control", USAGE_HOSTCTRL },
|
|
{ "upgradeconfig", cmd_upgradeconfig,
|
|
"submit a config upgrade (cluster.bin file)", USAGE_UPGRADECONFIG },
|
|
{ "joblist", cmd_joblist, "list jobs", USAGE_JOBLIST },
|
|
{ "expect-jobs", cmd_expect_jobs, "assert current job count", USAGE_EXPECT_JOBS },
|
|
{ "expect-job", cmd_expect_job, "assert a JobId is present", USAGE_EXPECT_JOB },
|
|
{ "wait-primary", cmd_wait_primary,
|
|
"wait until a primary serves [timeout_ms]", USAGE_WAIT_PRIMARY },
|
|
{ "ctrlstatus", cmd_ctrlstatus, "show controller/cluster status", USAGE_CTRLSTATUS },
|
|
{ "waldump", cmd_waldump, "Inspect WAL segments", USAGE_WALDUMP },
|
|
{ "walcompare", cmd_walcompare, "Diff member committed WAL prefixes",
|
|
USAGE_WALCOMPARE },
|
|
{ "snapverify", cmd_snapverify, "Verify checkpoint file structure",
|
|
USAGE_SNAPVERIFY },
|
|
{ "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files",
|
|
USAGE_SNAPDIFF },
|
|
{ "configcompare", cmd_configcompare, "Compare active config hash across members",
|
|
USAGE_CONFIGCOMPARE },
|
|
{ "shutdown", cmd_shutdown, "shutdown xce controller", USAGE_SHUTDOWN },
|
|
{ "jobrun", cmd_jobrun, "run a job", USAGE_JOBRUN },
|
|
{ "jobsub", cmd_jobsub, "submit a job", USAGE_JOBSUB },
|
|
{ "jobkill", cmd_jobkill, "signal a job", USAGE_JOBKILL },
|
|
{ "jobckpt", cmd_jobckpt, "checkpoint a job", USAGE_JOBCKPT },
|
|
{ "jobinfo", cmd_jobinfo, "job information", USAGE_JOBINFO },
|
|
{ "resval", cmd_resval, "get value of res", USAGE_RESVAL },
|
|
{ "taskrun", cmd_runtasks, "run tasks", USAGE_TASKRUN },
|
|
{ "help", cmd_help, "Display this text", USAGE_HELP },
|
|
{ "echo", cmd_echo, "Print a message", USAGE_ECHO },
|
|
{ "source", cmd_source, "Run commands from a file", USAGE_SOURCE },
|
|
{ "quit", cmd_quit, "Quit xce test", USAGE_QUIT },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t clientctx = { .parent = &topctx,
|
|
.prompt = "client",
|
|
.settings = settings,
|
|
.commands = client_commands };
|
|
|
|
static command_t scheduler_commands[] = {
|
|
{ "..", cmd_up, "Goto upper context", USAGE_UP },
|
|
{ "help", cmd_help, "Display this text", USAGE_HELP },
|
|
{ "quit", cmd_quit, "Quit xce test", USAGE_QUIT },
|
|
{ "join", cmd_join_sched, "Join the cluster", USAGE_JOIN_SCHED },
|
|
{ "getconf", cmd_getconf, "Get conf of cluster", USAGE_GETCONF },
|
|
{ "setpend", cmd_setpend, "set job pend reason", USAGE_SETPEND },
|
|
{ "echo", cmd_echo, "Print a message", USAGE_ECHO },
|
|
{ "source", cmd_source, "Run commands from a file", USAGE_SOURCE },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL, (const char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t schedulerctx = { .parent = &topctx,
|
|
.prompt = "scheduler",
|
|
.settings = settings,
|
|
.commands = scheduler_commands };
|
|
|
|
static command_ctx_t* active_ctx = &topctx;
|
|
|
|
static char prompt[256];
|
|
static int break_out = 0;
|
|
|
|
/* xce-test-cli-ux task 1.6, Requirement 4.3: describes the in-session
|
|
* `help' command as the way to list/learn available commands, in addition
|
|
* to argp's own auto-generated --help/-?/-V flag documentation below. */
|
|
static char doc[] =
|
|
"xce_test -- interactive/scripted test client for the controller, "
|
|
"worker, and scheduler\v"
|
|
"Once a session is running, type `help' at the prompt to list the "
|
|
"commands available in the current context, or `help <command>' for "
|
|
"details on a specific one.";
|
|
static char args_doc[] = "";
|
|
|
|
/* build-time-versioning task 3.2: no longer a compile-time initializer --
|
|
* argp reads this by pointer when --version/-V is processed, and
|
|
* xce_compose_version() (utils.h) is a runtime function, not a constant
|
|
* expression, so it cannot initialize a file-scope global. main() assigns
|
|
* this to a static buffer, computed via xce_compose_version(), before
|
|
* argp_parse() runs. See utils.h's NOTE FOR TASK 3.2 above MK_VERSION. */
|
|
const char* argp_program_version = NULL;
|
|
const char* argp_program_bug_address = BUG_EMAIL;
|
|
|
|
/* Long-option key (above ASCII to avoid collisions with short opts) --
|
|
* mirrors src/controller/main.c's OPT_PEERS/OPT_CONF_KEEP_N idiom. */
|
|
#define OPT_NORC 256
|
|
|
|
struct arguments {
|
|
int verbose;
|
|
int norc;
|
|
};
|
|
|
|
static struct argp_option options[] = {
|
|
{ "verbose", 'v', 0, 0, "Produce verbose logs" },
|
|
{ "norc", OPT_NORC, 0, 0, "Skip loading the rc file for this invocation" },
|
|
/* xce-test-cli-ux task 1.6, Requirement 4.1: argp's OWN built-in help
|
|
* alias is `-?` (via --help), NOT `-h` -- glibc argp does not add `-h`
|
|
* automatically. `-h` is hidden from the rendered help/usage text
|
|
* (OPTION_HIDDEN) so it doesn't duplicate the auto-generated
|
|
* "-?, --help" entry there; it is still a fully functional alias,
|
|
* handled explicitly in parse_opt() below via argp_state_help(). */
|
|
{ 0, 'h', 0, OPTION_HIDDEN, 0 },
|
|
{ 0 }
|
|
};
|
|
|
|
static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
|
{
|
|
struct arguments* arguments = state->input;
|
|
|
|
switch (key) {
|
|
case 'v':
|
|
arguments->verbose++;
|
|
break;
|
|
case OPT_NORC:
|
|
arguments->norc = 1;
|
|
break;
|
|
case 'h':
|
|
/* ARGP_HELP_STD_HELP includes ARGP_HELP_EXIT_OK, so this prints
|
|
* the same help text as --help/-? and exits(0) -- it does not
|
|
* return. */
|
|
argp_state_help(state, stdout, ARGP_HELP_STD_HELP);
|
|
break;
|
|
default:
|
|
return ARGP_ERR_UNKNOWN;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct argp argp = { options, parse_opt, args_doc, doc };
|
|
|
|
void sigbreak(int signo) { break_out = 1; }
|
|
|
|
/* xce-test-cli-ux task 1.3, Requirement 9.9: read-only accessor over
|
|
* `break_out` for code outside this file (interruptcall.c, task 1.4) to
|
|
* poll -- see xcetest_interrupt.h's file-header comment for the full
|
|
* contract (pure read, no reset side effect, break_out's own reset stays
|
|
* test.c-private). */
|
|
int xcetest_interrupted(void) { return break_out; }
|
|
|
|
/* xce-test-cli-ux task 2.1, Requirement 9.4/9.6/13.7: read-only accessor
|
|
* over `g_interactive` (this file's top-level-session TTY flag, set once
|
|
* in main() from isatty(STDIN_FILENO) and never changed after that) for
|
|
* code outside this file (confirm.c) to consult -- mirrors
|
|
* xcetest_interrupted()'s accessor pattern immediately above. Pure read;
|
|
* declared in confirm.h rather than xcetest_interrupt.h since confirm.c
|
|
* is its only consumer and g_interactive is unrelated to that header's
|
|
* SIGINT/EINTR concern. Deliberately the TOP-LEVEL session's stdin, not
|
|
* "is this line from a sourced file" -- that distinction is Requirement
|
|
* 13.7, a later task; this accessor's contract must already be correct
|
|
* for it, per design.md's Confirmation Gate invariant. */
|
|
int xcetest_is_interactive(void) { return g_interactive; }
|
|
|
|
static char* get_setting(char* key)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; active_ctx->settings[i].key; i++) {
|
|
if (strcmp(key, active_ctx->settings[i].key) != 0) {
|
|
continue;
|
|
}
|
|
|
|
if (active_ctx->settings[i].value != NULL) {
|
|
return active_ctx->settings[i].value;
|
|
}
|
|
|
|
return active_ctx->settings[i].defval;
|
|
}
|
|
|
|
printf("Key `%s' is not in settings\n", key);
|
|
return NULL;
|
|
}
|
|
|
|
static xce_client_config_t config;
|
|
|
|
/* Holds the parsed HA controller list. Static so the hostname[] buffers it
|
|
* owns outlive enter_ctx(): xcecli_ctx_init() stores the config pointer (it
|
|
* does not copy masters[]), so config.masters[i] must reference storage that
|
|
* persists for the lifetime of the client context. */
|
|
static xce_cluster_config_t master_cfg;
|
|
|
|
int enter_ctx(command_ctx_t* ctx)
|
|
{
|
|
char* masters = get_setting("masters");
|
|
int i;
|
|
|
|
/* Parse the controller HA peer list with the shared libxce parser — the
|
|
* same "name=host:port,..."/"host:port,..." syntax used by the controller
|
|
* --peers flag — so the test client fails over across candidates exactly
|
|
* like the worker/scheduler/apiserver do. */
|
|
memset(&master_cfg, 0, sizeof(master_cfg));
|
|
if ((masters == NULL) || (xce_parse_peers(masters, &master_cfg) != 0) ||
|
|
(master_cfg.candidates_count < 1)) {
|
|
fprintf(stderr, "enter_ctx: invalid 'masters' setting `%s'\n",
|
|
masters ? masters : "(null)");
|
|
return -EINVAL;
|
|
}
|
|
|
|
config.nmasters = master_cfg.candidates_count;
|
|
if (config.nmasters > N_MAX_MASTERS) {
|
|
fprintf(stderr,
|
|
"enter_ctx: %d masters configured, using first %d "
|
|
"(N_MAX_MASTERS)\n",
|
|
config.nmasters, N_MAX_MASTERS);
|
|
config.nmasters = N_MAX_MASTERS;
|
|
}
|
|
|
|
for (i = 0; i < config.nmasters; i++) {
|
|
config.masters[i] = master_cfg.candidates[i].hostname;
|
|
config.master_ports[i] = master_cfg.candidates[i].port;
|
|
}
|
|
|
|
config.from = get_setting("myname");
|
|
config.key = get_setting("secret");
|
|
|
|
xcecli_ctx_init(&(ctx->ctx), &config);
|
|
xcecli_set_handler(ctx->ctx, &myreq_handler, NULL);
|
|
xcecli_start(ctx->ctx);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int leave_ctx(command_ctx_t* ctx)
|
|
{
|
|
if (ctx == &topctx) {
|
|
return 0;
|
|
}
|
|
|
|
if (ctx == &workerctx) {
|
|
delaymgr_fini(&delaymgr);
|
|
}
|
|
|
|
xcecli_wakeup(active_ctx->ctx, (int)-EINTR);
|
|
|
|
xcecli_stop(ctx->ctx);
|
|
xcecli_ctx_fini(ctx->ctx);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int fill_prompt(command_ctx_t* ctx, char* buf)
|
|
{
|
|
if (ctx->parent != NULL) {
|
|
fill_prompt(ctx->parent, buf);
|
|
strcat(buf, "/");
|
|
}
|
|
|
|
strcat(buf, ctx->prompt);
|
|
|
|
return strlen(buf);
|
|
}
|
|
|
|
command_t* find_command(char* name)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; active_ctx->commands[i].name; i++) {
|
|
if (strcmp(active_ctx->commands[i].name, name) == 0) {
|
|
return &(active_ctx->commands[i]);
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* xce-test-cli-ux task 2.3, Requirement 7: command-name tab completion.
|
|
*
|
|
* libedit API verification (design.md's flagged P0 risk, research.md's
|
|
* "libedit completion API may not perfectly mirror GNU readline" risk):
|
|
* confirmed by reading the ACTUAL linked headers in this build environment
|
|
* (this project's `src/flake.nix`-pinned libedit, pkg-config name
|
|
* "libedit", providing `editline/readline.h` -- the same header already
|
|
* included at the top of this file) plus a standalone pty-driven probe
|
|
* program built and run against this exact library. Findings:
|
|
*
|
|
* - The GNU-readline-style completion API IS present verbatim:
|
|
* `rl_completion_func_t* rl_attempted_completion_function` (signature
|
|
* `char** (*)(const char*, int, int)`, exactly matching this task's
|
|
* `xcetest_complete` shape from design.md) and
|
|
* `char** rl_completion_matches(const char*, rl_compentry_func_t*)`
|
|
* (matches on a `char* (*)(const char*, int)` generator, exactly
|
|
* `xcetest_complete_generator`'s shape). No lower-level `<histedit.h>`
|
|
* fallback was needed.
|
|
* - `rl_attempted_completion_over` exists and suppresses libedit's
|
|
* default filename-completion fallback -- used below so an empty
|
|
* match list in the active context yields NO completion rather than
|
|
* local filenames.
|
|
* - Single/multiple-match UI behavior is entirely the library's: a sole
|
|
* match is completed in full (plus a trailing space) with ZERO extra
|
|
* code (7.2); multiple matches with no further common prefix ring the
|
|
* bell on the first Tab and print the candidate list (input line left
|
|
* untouched -- confirmed via a raw pty capture) on a SECOND,
|
|
* consecutive Tap of Tab -- standard GNU-readline/bash double-Tab
|
|
* convention (7.3), not something this code implements. This
|
|
* implementation's own responsibility is narrowly the context-scoped
|
|
* candidate lookup below.
|
|
* - `rl_variable_bind("show-all-if-ambiguous", "on")` was probed and
|
|
* returns failure (1) against this libedit build -- that variable is
|
|
* not implemented here, so it is not used; the stock double-Tab
|
|
* behavior above is what ships.
|
|
*/
|
|
|
|
/* xcetest_complete_generator - the readline "compentry" generator: called
|
|
* repeatedly with state==0 on the first call (resets the scan) and
|
|
* state!=0 on each subsequent call, returning one more `strdup()`ed match
|
|
* per call (readline/libedit frees each returned string) and NULL once
|
|
* exhausted. Iterates active_ctx->commands[] -- the CURRENTLY active
|
|
* context at the moment Tab is pressed, read fresh on every call rather
|
|
* than captured at registration time -- so completion always tracks
|
|
* `worker`/`scheduler`/`client`/`..` context switches for the life of the
|
|
* session (Requirement 7.1/7.4: only the active context's commands are
|
|
* ever offered). Plain prefix match against `text`, same convention as
|
|
* find_command()'s exact-match lookup just above. */
|
|
static char* xcetest_complete_generator(const char* text, int state)
|
|
{
|
|
static int idx;
|
|
static size_t len;
|
|
|
|
if (state == 0) {
|
|
idx = 0;
|
|
len = strlen(text);
|
|
}
|
|
|
|
while (active_ctx->commands[idx].name != NULL) {
|
|
const char* name = active_ctx->commands[idx].name;
|
|
|
|
idx++;
|
|
if (strncmp(name, text, len) == 0) {
|
|
return strdup(name);
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* xcetest_complete - the readline "attempted completion" entry point,
|
|
* registered as rl_attempted_completion_function only when g_interactive
|
|
* (main(), below). `start`/`end` are byte offsets of the word being
|
|
* completed within rl_line_buffer; `start == 0` means the FIRST word on
|
|
* the line, i.e. the command name itself. Per this task's explicit
|
|
* boundary (command-NAME completion only -- no flag-value completion of
|
|
* jobids/hostnames/resource names), every other word position (start != 0)
|
|
* returns NULL and leaves rl_attempted_completion_over at its default so
|
|
* libedit's normal fallback applies there, unchanged from today's
|
|
* zero-customization behavior. For the command-name position, this always
|
|
* sets rl_attempted_completion_over so a context with zero matching
|
|
* commands offers NOTHING (never falls back to filename completion). */
|
|
static char** xcetest_complete(const char* text, int start, int end)
|
|
{
|
|
(void)end;
|
|
|
|
if (start != 0) {
|
|
return NULL;
|
|
}
|
|
|
|
rl_attempted_completion_over = 1;
|
|
|
|
return rl_completion_matches(text, xcetest_complete_generator);
|
|
}
|
|
|
|
/* Execute a command line. */
|
|
int execute_line(char* line)
|
|
{
|
|
int rc;
|
|
command_t* command;
|
|
int argc;
|
|
char** argv;
|
|
|
|
argc = parse_argv(line, &argv);
|
|
if (argc < 0) {
|
|
SET_ERROR(-argc);
|
|
perror("execute_line");
|
|
return -1;
|
|
}
|
|
|
|
if ((argv == NULL) || (argv[0] == NULL)) {
|
|
return 0;
|
|
}
|
|
|
|
command = find_command(argv[0]);
|
|
|
|
if (!command) {
|
|
fprintf(stderr, "%s: No such command for xce test.\n", argv[0]);
|
|
free_argv(argv, argc);
|
|
return (-1);
|
|
}
|
|
|
|
rc = (command->func)(argc, argv);
|
|
|
|
free_argv(argv, argc);
|
|
return rc;
|
|
}
|
|
|
|
/* Substitute the fixed auto-variables `$?`, `$jobid`, `$jobcount` on a raw line
|
|
* before tokenizing (xce-test-e2e-primitives task 3.1, Req 3.1/3.3/3.5/3.6).
|
|
*
|
|
* Only WHOLE-TOKEN occurrences are recognized: a `$name` must be bounded by the
|
|
* start of the line / whitespace on its left and whitespace / end-of-line on its
|
|
* right. A `$name` glued to other non-whitespace (e.g. `x$jobid`, `$jobidx`) is
|
|
* NOT a recognized auto-variable and is copied through verbatim. No other `$...`
|
|
* syntax is recognized: an unknown whole token such as `$other` is also copied
|
|
* verbatim (it is simply not one of the three names).
|
|
*
|
|
* Replacements:
|
|
* $? -> decimal of g_last_status (always available).
|
|
* $jobid -> g_last_jobid if set (non-empty); unset => command failure.
|
|
* $jobcount -> decimal of g_last_jobcount if set (>=0); unset => failure.
|
|
*
|
|
* Writes the expanded line into `out` (capacity `outsz`, always NUL-terminated
|
|
* on success). Returns 0 on success, -EINVAL on an unset auto-var reference,
|
|
* -ENOSPC if the expansion would overflow `out`. On any error `out` content is
|
|
* unspecified and the caller must fail the command rather than run it. */
|
|
static int substitute_autovars(const char* line, char* out, size_t outsz)
|
|
{
|
|
const char* p = line;
|
|
size_t o = 0;
|
|
|
|
if ((line == NULL) || (out == NULL) || (outsz == 0)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
while (*p != '\0') {
|
|
/* A `$` starts a candidate auto-var only when it begins a token, i.e.
|
|
* the previous char is the line start or whitespace. */
|
|
int at_token_start
|
|
= (p == line) || (*(p - 1) == ' ') || (*(p - 1) == '\t');
|
|
|
|
if ((*p == '$') && at_token_start) {
|
|
const char* name = p + 1;
|
|
const char* end = name;
|
|
const char* repl = NULL;
|
|
char numbuf[32];
|
|
|
|
/* Scan to the token end (whitespace or NUL). */
|
|
while ((*end != '\0') && (*end != ' ') && (*end != '\t')) {
|
|
end++;
|
|
}
|
|
|
|
size_t namelen = (size_t)(end - name);
|
|
|
|
if ((namelen == 1) && (name[0] == '?')) {
|
|
snprintf(numbuf, sizeof(numbuf), "%d", g_last_status);
|
|
repl = numbuf;
|
|
} else if ((namelen == 5) && (strncmp(name, "jobid", 5) == 0)) {
|
|
if (g_last_jobid[0] == '\0') {
|
|
fprintf(stderr,
|
|
"autovar: $jobid referenced before set\n");
|
|
return -EINVAL;
|
|
}
|
|
repl = g_last_jobid;
|
|
} else if ((namelen == 8) && (strncmp(name, "jobcount", 8) == 0)) {
|
|
if (g_last_jobcount < 0) {
|
|
fprintf(stderr,
|
|
"autovar: $jobcount referenced before set\n");
|
|
return -EINVAL;
|
|
}
|
|
snprintf(numbuf, sizeof(numbuf), "%d", g_last_jobcount);
|
|
repl = numbuf;
|
|
}
|
|
|
|
if (repl != NULL) {
|
|
size_t rlen = strlen(repl);
|
|
|
|
if ((o + rlen) >= outsz) {
|
|
return -ENOSPC;
|
|
}
|
|
memcpy(out + o, repl, rlen);
|
|
o += rlen;
|
|
p = end; /* consume the whole `$name` token */
|
|
continue;
|
|
}
|
|
|
|
/* Not a recognized auto-var: copy the whole token verbatim. */
|
|
while (p < end) {
|
|
if ((o + 1) >= outsz) {
|
|
return -ENOSPC;
|
|
}
|
|
out[o++] = *p++;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if ((o + 1) >= outsz) {
|
|
return -ENOSPC;
|
|
}
|
|
out[o++] = *p++;
|
|
}
|
|
|
|
out[o] = '\0';
|
|
return 0;
|
|
}
|
|
|
|
/* Main-loop wrapper around execute_line that implements the session verdict
|
|
* (xce-test-e2e-primitives task 1.1):
|
|
* - blank / whitespace-only lines and `#`-comment lines are ignored;
|
|
* - normal lines run via execute_line; g_last_status / g_session_failed are
|
|
* updated from the result;
|
|
* - in default (abort) mode a failure stops the loop; under
|
|
* g_continue_on_error the loop keeps running but the failure is recorded.
|
|
*
|
|
* Returns 0 to keep the read loop running, non-zero to request that the loop
|
|
* stop (abort-on-failure). The leading `!` shell-escape and `$`-substitution
|
|
* are intentionally handled in later tasks (2.1 / 3.1); this function is the
|
|
* single chokepoint where they will be added.
|
|
*/
|
|
int preprocess_and_run(char* line)
|
|
{
|
|
char* p = line;
|
|
int rc;
|
|
|
|
/* Skip leading whitespace to classify the line. */
|
|
while ((*p == ' ') || (*p == '\t')) {
|
|
p++;
|
|
}
|
|
|
|
/* Ignore blank / whitespace-only lines and `#` comments. */
|
|
if ((*p == '\0') || (*p == '#')) {
|
|
return 0;
|
|
}
|
|
|
|
/* Shell-escape: a line whose first non-whitespace char is `!` runs the
|
|
* remainder via system(). The shell command's exit status becomes
|
|
* g_last_status (the `$?` auto-variable); a non-zero status is a command
|
|
* failure subject to the same abort-vs-continue rules as execute_line. The
|
|
* remainder is passed raw to the shell (no parse_argv / auto-var
|
|
* substitution — that is task 3.1). (xce-test-e2e-primitives task 2.1) */
|
|
if (*p == '!') {
|
|
const char* rest = p + 1;
|
|
int sysrc = system(rest);
|
|
int status;
|
|
|
|
if (sysrc == -1) {
|
|
/* system() itself failed to run the shell — treat as failure. */
|
|
status = 127;
|
|
} else if (WIFEXITED(sysrc)) {
|
|
status = WEXITSTATUS(sysrc);
|
|
} else {
|
|
/* Signaled / abnormal termination — non-zero failure status. */
|
|
status = 1;
|
|
}
|
|
|
|
g_last_status = status;
|
|
|
|
if (status != 0) {
|
|
g_session_failed = 1;
|
|
if (!g_continue_on_error && !g_interactive) {
|
|
return 1; /* abort: stop the read loop (scripted mode only) */
|
|
}
|
|
} else {
|
|
add_history(line);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Substitute the fixed auto-variables (`$?`, `$jobid`, `$jobcount`) before
|
|
* tokenizing (xce-test-e2e-primitives task 3.1). An unset `$jobid` /
|
|
* `$jobcount` reference (Req 3.6) — or an overflow — fails the command:
|
|
* we do NOT run execute_line, and the result is treated exactly like a
|
|
* failing command (g_last_status non-zero, g_session_failed, abort vs
|
|
* continue). The leading `!` shell-escape above is intentionally NOT
|
|
* substituted: that line goes raw to system() and the shell owns `$?`. */
|
|
{
|
|
char expanded[MAX_LINE_LEN];
|
|
int subrc = substitute_autovars(p, expanded, sizeof(expanded));
|
|
|
|
if (subrc < 0) {
|
|
g_last_status = subrc;
|
|
g_session_failed = 1;
|
|
if (!g_continue_on_error && !g_interactive) {
|
|
return 1; /* abort: stop the read loop (scripted mode only) */
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
rc = execute_line(expanded);
|
|
}
|
|
|
|
g_last_status = (rc < 0) ? (rc) : 0;
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
g_session_failed = 1;
|
|
if (!g_continue_on_error) {
|
|
return 1; /* errexit (-e): stop the read loop on first failure */
|
|
}
|
|
} else {
|
|
add_history(line);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_help(int argc, char* argv[])
|
|
{
|
|
int i;
|
|
int rc = 0;
|
|
|
|
if (argc > 1) {
|
|
char* cmd = argv[1];
|
|
|
|
for (i = 0; active_ctx->commands[i].name != NULL; i++) {
|
|
if (strcmp(argv[1], active_ctx->commands[i].name) == 0) {
|
|
/* `help <cmd>` prints the detailed per-command usage (flags,
|
|
* positionals, example) -- distinct from the compact one-line
|
|
* `doc` the bare `help` listing below still uses unchanged.
|
|
* Every table entry is expected to carry a non-NULL `usage`
|
|
* now; the `doc` fallback stays only as a defensive guard
|
|
* against a future entry that forgets to set it. */
|
|
if (active_ctx->commands[i].usage != NULL) {
|
|
fputs(active_ctx->commands[i].usage, stdout);
|
|
} else {
|
|
printf("%s\t\t%s.\n", active_ctx->commands[i].name,
|
|
active_ctx->commands[i].doc);
|
|
}
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
printf("No commands matchs `%s'. Possibilities are:\n", cmd);
|
|
rc = -1;
|
|
}
|
|
|
|
for (i = 0; active_ctx->commands[i].name != NULL; i++) {
|
|
printf("\t%-12s\t%s\n", active_ctx->commands[i].name,
|
|
active_ctx->commands[i].doc);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/* echo <message...> — print the remainder of the line (argv[1..] joined by
|
|
* single spaces) followed by a newline and succeed (xce-test-e2e-primitives
|
|
* task 1.2, Req 6.3). `echo` with no argument prints an empty line and still
|
|
* succeeds. Returns 0 always so it never marks the session failed. */
|
|
int cmd_echo(int argc, char* argv[])
|
|
{
|
|
int i;
|
|
|
|
for (i = 1; i < argc; i++) {
|
|
if (i > 1) {
|
|
putchar(' ');
|
|
}
|
|
fputs(argv[i], stdout);
|
|
}
|
|
putchar('\n');
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* waldump <path> [<path> ...] — offline WAL inspector for failover/durability
|
|
* triage. Each path may be a seg-*.wal file, a wal/ dir, or a node/data dir.
|
|
* Read-only: lists every entry, decodes JOB_SUBMIT jobids, and flags LSN gaps
|
|
* (a missing committed prefix is THE failover job-loss signal) plus the CRC
|
|
* recovered tip. Returns nonzero if any path failed or any gap was found, so
|
|
* it doubles as an assertion in scripted E2E runs. */
|
|
int cmd_waldump(int argc, char* argv[])
|
|
{
|
|
if (argc < 2) {
|
|
fprintf(stderr,
|
|
"usage: waldump <wal_dir|data_dir|seg_file> [more...]\n");
|
|
return -1;
|
|
}
|
|
|
|
int rc = 0;
|
|
for (int i = 1; i < argc; i++) {
|
|
waldump_stats_t st;
|
|
int r = waldump_path(argv[i], stdout, &st);
|
|
if (r != 0) {
|
|
fprintf(stderr, "waldump: %s: %s\n", argv[i], strerror(-r));
|
|
rc = -1;
|
|
continue;
|
|
}
|
|
if (st.n_gaps > 0 || st.crc_broken)
|
|
rc = -1; /* surface the defect as a nonzero command verdict */
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
/* walcompare [--report] [--committed N] <memberA> <memberB> [more...] — diff
|
|
* member WAL committed prefixes by (lsn -> term, pcrc) and FAIL (nonzero) on any
|
|
* committed-prefix divergence (ha-term-safety R6.1/R6.2/R6.3), reusing the
|
|
* waldump decoder. Each member path may be a seg-*.wal file, a wal/ dir, or a
|
|
* node/data dir (same resolution as waldump). With no --committed, the shared
|
|
* committed boundary = the minimum CRC-intact tip across members. The diverging
|
|
* LSN and per-member (term, pcrc) identities are reported. With --report, also
|
|
* emit a machine-checkable per-member `(lsn, term, pcrc, status)` table and a
|
|
* committed-vs-uncommitted classification for each divergence (task 1.1). Exit
|
|
* 0 = identical, nonzero = divergence or load error — so it doubles as a
|
|
* scripted assertion. */
|
|
int cmd_walcompare(int argc, char* argv[])
|
|
{
|
|
uint64_t committed = 0; /* 0 = derive the shared boundary */
|
|
int report = 0; /* --report => full per-member machine table */
|
|
int ai = 1;
|
|
|
|
/* Optional flags (any order): --report, --committed N. */
|
|
for (;;) {
|
|
if (argc > ai && strcmp(argv[ai], "--report") == 0) {
|
|
report = 1;
|
|
ai += 1;
|
|
continue;
|
|
}
|
|
if (argc > ai && strcmp(argv[ai], "--committed") == 0) {
|
|
if (argc <= ai + 1) {
|
|
fprintf(stderr, "walcompare: --committed needs an LSN\n");
|
|
return -1;
|
|
}
|
|
char* end = NULL;
|
|
unsigned long v = strtoul(argv[ai + 1], &end, 10);
|
|
if (end == argv[ai + 1] || *end != '\0') {
|
|
fprintf(stderr, "walcompare: bad --committed LSN '%s'\n",
|
|
argv[ai + 1]);
|
|
return -1;
|
|
}
|
|
committed = (uint64_t)v;
|
|
ai += 2;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
int n = argc - ai;
|
|
if (n < 2) {
|
|
fprintf(stderr,
|
|
"usage: walcompare [--report] [--committed N] <memberA> "
|
|
"<memberB> [more...]\n");
|
|
return -1;
|
|
}
|
|
if (n > 16) {
|
|
fprintf(stderr, "walcompare: at most 16 members\n");
|
|
return -1;
|
|
}
|
|
|
|
walcompare_member_t members[16];
|
|
for (int i = 0; i < n; i++) {
|
|
members[i].path = argv[ai + i];
|
|
members[i].committed_lsn = committed; /* 0 => derive shared boundary */
|
|
}
|
|
|
|
walcompare_divergence_t div;
|
|
int rc = report ? walcompare_report(members, n, stdout, &div)
|
|
: walcompare(members, n, stdout, &div);
|
|
if (rc == WALCMP_OK)
|
|
return 0;
|
|
if (rc == WALCMP_DIVERGENT) {
|
|
fprintf(stderr,
|
|
"walcompare: FAIL — committed-prefix divergence at LSN %" PRIu64
|
|
"\n",
|
|
div.lsn);
|
|
return -1; /* nonzero session verdict */
|
|
}
|
|
fprintf(stderr, "walcompare: error: %s\n", strerror(-rc));
|
|
return -1;
|
|
}
|
|
|
|
/* snapverify <snap_file> [<snap_file> ...] — offline checkpoint-file
|
|
* structural validator for force-snapshot triage (bug-1389: OPCode_
|
|
* snapshotTransfer chunk-content corruption). Each path is a snap-*.capnp
|
|
* (locally generated) or snap-*.bin (received via ha_handle_snapshot_chunk)
|
|
* file. Read-only: walks the same layout persist_load_snapshot() parses
|
|
* without restoring anything, and on the first structural defect reports the
|
|
* exact byte offset and how many job/queue frames parsed cleanly before it —
|
|
* the diagnostic a bare "persist_load_snapshot failed rc=-5" log line
|
|
* doesn't give you. Returns nonzero if any path failed to verify. */
|
|
int cmd_snapverify(int argc, char* argv[])
|
|
{
|
|
if (argc < 2) {
|
|
fprintf(stderr, "usage: snapverify <snap_file> [more...]\n");
|
|
return -1;
|
|
}
|
|
|
|
int rc = 0;
|
|
for (int i = 1; i < argc; i++) {
|
|
snapverify_report_t rep;
|
|
int r = snapverify_path(argv[i], stdout, &rep);
|
|
if (r != 0)
|
|
rc = -1;
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
/* snapdiff <source_file> <received_file> — byte-level checkpoint-file
|
|
* comparator for force-snapshot triage (bug-1389). `source_file` is the
|
|
* primary's own locally generated checkpoint (snap-*.capnp); `received_file`
|
|
* is the copy a standby assembled from the wire (snap-*.bin). Reports the
|
|
* first differing byte offset and whether it lands exactly on an
|
|
* OPCode_snapshotTransfer chunk boundary — the decisive signal for "the
|
|
* transport corrupted one chunk" vs "these were never the same content".
|
|
* Returns 0 if identical, nonzero otherwise. */
|
|
int cmd_snapdiff(int argc, char* argv[])
|
|
{
|
|
if (argc != 3) {
|
|
fprintf(stderr, "usage: snapdiff <source_file> <received_file>\n");
|
|
return -1;
|
|
}
|
|
|
|
snapdiff_report_t rep;
|
|
int rc = snapdiff_paths(argv[1], argv[2], stdout, &rep);
|
|
return (rc == 0) ? 0 : -1;
|
|
}
|
|
|
|
/* configcompare <member_dir> <member_dir> [more...] — cross-member
|
|
* active-configuration comparator (config-upgrade-snapshot task 8.4, Req
|
|
* 4.2). Each path is a member's `workdir/name` conf-root directory (the
|
|
* same directory persist.c's read_active_config_bytes() and config_repl.c's
|
|
* local_config_sha256() resolve `conf/current` under) — NOT a raw config
|
|
* file path. Recomputes each member's active-config sha256 fresh from its
|
|
* own on-disk bytes and reports whether all members match. Returns 0 if
|
|
* all members are present and identical, nonzero if any member is
|
|
* unreadable or divergent. */
|
|
int cmd_configcompare(int argc, char* argv[])
|
|
{
|
|
if (argc < 3) {
|
|
fprintf(stderr, "usage: configcompare <member_dir> <member_dir> [more...]\n");
|
|
return -1;
|
|
}
|
|
|
|
configcompare_report_t rep;
|
|
int rc = configcompare_paths((const char* const*)&argv[1], argc - 1, stdout, &rep);
|
|
return (rc == 0) ? 0 : -1;
|
|
}
|
|
|
|
/* --------------------------------------------------------------------
|
|
* source <file> -- in-session script sourcing (xce-test-cli-ux task 2.4,
|
|
* Requirements 13.1-13.6; 13.7's TTY-condition clarification relating
|
|
* `source` to the Confirmation Gate is a separate, later validation task
|
|
* and is NOT implemented here).
|
|
*
|
|
* Reuses rcfile_load() (rcfile.c/h, xce-test-rc-file spec) UNCHANGED as the
|
|
* actual line-reading/dispatch loop, passing preprocess_and_run as its
|
|
* runner -- the same per-line command dispatcher the interactive REPL loop
|
|
* and the startup rc-file loader already use. rcfile.c itself is not
|
|
* touched: its own missing-file-is-silent-success policy is correct for
|
|
* its one other caller (the startup rc file) but wrong for an explicit
|
|
* user command, so cmd_source layers its own missing-file check (access())
|
|
* and cycle detection (a small in-flight canonical-path stack) on top,
|
|
* entirely in this file -- per research.md's "source built directly on
|
|
* rcfile_load" decision.
|
|
*
|
|
* Cycle detection: source_push_path()/source_pop_path() maintain a static
|
|
* LIFO stack of realpath()-canonicalized in-flight source paths.
|
|
* source_push_path() scans the stack for the same canonical path already
|
|
* in flight (a real cycle -- including through symlinks or a differently-
|
|
* spelled relative path, since realpath() runs before the scan) and
|
|
* returns -ELOOP without pushing if found; otherwise it pushes and returns
|
|
* 0. cmd_source calls source_pop_path() on EVERY exit path once the push
|
|
* succeeded (success, or rcfile_load returning nonzero because a line
|
|
* inside the file failed under -e/abort-on-failure semantics) -- this is
|
|
* the "path stack popped exactly once per successful push, on every exit
|
|
* path" invariant design.md requires. When source_push_path() itself
|
|
* fails (cycle detected), nothing was pushed, so cmd_source returns
|
|
* immediately WITHOUT calling source_pop_path() -- there is nothing to
|
|
* pop.
|
|
*
|
|
* Abort/continue-on-error composition (Req 13.3): a failing line inside
|
|
* the sourced file makes preprocess_and_run (the runner) return 1 once
|
|
* g_continue_on_error is false, which is exactly rcfile_load's own
|
|
* "stop on nonzero runner return" mechanism -- no extra code is needed
|
|
* here for that part; it falls out of reusing rcfile_load() unchanged.
|
|
* cmd_source's own return value then translates rcfile_load's result
|
|
* (0 == every line ran without requesting an abort) into this codebase's
|
|
* negative-errno failure convention (nonzero -> a generic -1 failure,
|
|
* mirroring cmd_configcompare's/cmd_waldump's own "rc == 0 ? 0 : -1"
|
|
* pattern) rather than forwarding rcfile_load's raw positive "stop" signal
|
|
* verbatim: forwarding it unchanged would be silently treated as SUCCESS
|
|
* by an enclosing preprocess_and_run call (only rc<0 counts as failure
|
|
* there), which would incorrectly let an enclosing script's later lines
|
|
* keep running past a `source` that itself aborted under -e, and would
|
|
* break that composition for nested sourcing (source A containing
|
|
* "source B", where B aborts). Translating to a negative value here makes
|
|
* a failed `source` compose correctly at every nesting depth, exactly
|
|
* like any other failing command.
|
|
*
|
|
* Nested sourcing (Req 13.4): "source" is registered in every command
|
|
* table (see the four command_t[] tables below); a line inside a sourced
|
|
* file that itself reads "source <otherfile>" is dispatched by
|
|
* preprocess_and_run/execute_line exactly like any other line, so nested
|
|
* sourcing falls out of dispatch being generic. The path stack is a
|
|
* single process-wide stack (not scoped per active context), so a cycle
|
|
* is caught regardless of which context each nesting level was sourced
|
|
* from. */
|
|
#define SOURCE_MAX_DEPTH 64
|
|
|
|
static char* g_source_stack[SOURCE_MAX_DEPTH];
|
|
static int g_source_depth = 0;
|
|
|
|
/* Returns 0 and pushes `canonical_path` if it is not already in flight, or
|
|
* -ELOOP without pushing if it is (a cycle -- the same canonicalized path
|
|
* sourcing itself, directly or transitively). Returns -ENOMEM if the stack
|
|
* is exhausted (SOURCE_MAX_DEPTH levels of DISTINCT, non-cyclic nested
|
|
* sourcing) or a heap copy of the path could not be made -- either way,
|
|
* nothing is pushed, so the caller must not call source_pop_path(). */
|
|
static int source_push_path(const char* canonical_path)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < g_source_depth; i++) {
|
|
if (strcmp(g_source_stack[i], canonical_path) == 0) {
|
|
return -ELOOP;
|
|
}
|
|
}
|
|
|
|
if (g_source_depth >= SOURCE_MAX_DEPTH) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
g_source_stack[g_source_depth] = strdup(canonical_path);
|
|
if (g_source_stack[g_source_depth] == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
g_source_depth++;
|
|
return 0;
|
|
}
|
|
|
|
/* Pops the most recently pushed path (LIFO). A no-op if the stack is
|
|
* already empty (defensive; every real call site only calls this after a
|
|
* source_push_path() call that itself returned 0). */
|
|
static void source_pop_path(void)
|
|
{
|
|
if (g_source_depth <= 0) {
|
|
return;
|
|
}
|
|
|
|
g_source_depth--;
|
|
free(g_source_stack[g_source_depth]);
|
|
g_source_stack[g_source_depth] = NULL;
|
|
}
|
|
|
|
/* source <file> -- reads and executes each line of <file> as if it had
|
|
* been typed at the current prompt, in order (Req 13.1). A missing or
|
|
* unreadable file is rejected with an explicit stderr error BEFORE
|
|
* rcfile_load() is ever called, bypassing its silent-missing-file
|
|
* behavior entirely (Req 13.2) -- rcfile.c itself is unmodified. */
|
|
int cmd_source(int argc, char* argv[])
|
|
{
|
|
char canonical[PATH_MAX];
|
|
int rc;
|
|
|
|
if (argc != 2) {
|
|
fprintf(stderr, "usage: source <file>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (access(argv[1], R_OK) != 0) {
|
|
fprintf(stderr, "source: %s: %s\n", argv[1], strerror(errno));
|
|
return -errno;
|
|
}
|
|
|
|
if (realpath(argv[1], canonical) == NULL) {
|
|
fprintf(stderr, "source: %s: %s\n", argv[1], strerror(errno));
|
|
return -errno;
|
|
}
|
|
|
|
rc = source_push_path(canonical);
|
|
if (rc == -ELOOP) {
|
|
fprintf(stderr,
|
|
"source: %s: cycle detected (file sources itself, directly "
|
|
"or transitively)\n",
|
|
canonical);
|
|
return rc;
|
|
} else if (rc != 0) {
|
|
fprintf(stderr, "source: %s: %s\n", canonical, strerror(-rc));
|
|
return rc;
|
|
}
|
|
|
|
rc = rcfile_load(argv[1], preprocess_and_run);
|
|
|
|
source_pop_path(); /* popped on EVERY exit path past the push above */
|
|
|
|
return (rc == 0) ? 0 : -1;
|
|
}
|
|
|
|
int cmd_up(int argc, char* argv[])
|
|
{
|
|
if (active_ctx->parent != NULL) {
|
|
int rc;
|
|
|
|
if ((rc = leave_ctx(active_ctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = active_ctx->parent;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_worker(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
|
|
if ((rc = enter_ctx(&workerctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = &workerctx;
|
|
|
|
delaymgr_init(&delaymgr, 2000 /* precision_ms */, NULL);
|
|
delaymgr_start(&delaymgr);
|
|
delay_task(&delaymgr, 5, -1, (delay_task_entry_fn_t)worker_house_keeping,
|
|
&workerctx);
|
|
return 0;
|
|
}
|
|
|
|
int cmd_scheduler(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
|
|
if ((rc = enter_ctx(&schedulerctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = &schedulerctx;
|
|
return 0;
|
|
}
|
|
|
|
int cmd_client(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
|
|
if ((rc = enter_ctx(&clientctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = &clientctx;
|
|
return 0;
|
|
}
|
|
|
|
static int done = 0;
|
|
|
|
int cmd_quit(int argc, char* argv[])
|
|
{
|
|
leave_ctx(active_ctx);
|
|
|
|
done = 1;
|
|
return 0;
|
|
}
|
|
|
|
int cmd_hello(int argc, char* argv[])
|
|
{
|
|
int rc = 0;
|
|
int opt;
|
|
char* msg = "Greeting from xce client";
|
|
int count = 1;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "c:m:")) != -1) {
|
|
switch (opt) {
|
|
case 'c':
|
|
count = atoi(optarg);
|
|
break;
|
|
case 'm':
|
|
msg = optarg;
|
|
break;
|
|
default:
|
|
/* -w was never implemented (getopt string above is "c:m:" only)
|
|
* -- this hint previously advertised a nonexistent -w wait_usec
|
|
* flag; corrected to match the actual accepted options. */
|
|
fprintf(stderr, " hello [-c count] [-m message]\n");
|
|
return -1;
|
|
break;
|
|
}
|
|
}
|
|
|
|
break_out = 0;
|
|
while ((!break_out) && (count > 0)) {
|
|
rc = xcecli_hello(active_ctx->ctx, msg);
|
|
|
|
count--;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int cmd_shutdown(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
rc = xcecli_shutdown(active_ctx->ctx);
|
|
/* leave client context */
|
|
if (rc >= 0) {
|
|
execute_line("..");
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int cmd_reslist(int argc, char* argv[])
|
|
{
|
|
int rc, i;
|
|
xce_resource_t** res;
|
|
|
|
rc = xcecli_reslist(active_ctx->ctx, &res);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_reslist");
|
|
return rc;
|
|
}
|
|
|
|
/* Note the extra literal space before the Description specifier: unlike
|
|
* cmd_workerlist's all-left-justified text columns (which self-separate
|
|
* via their own trailing padding), Interval stays right-justified here,
|
|
* so without an explicit separator a short Interval value would run
|
|
* directly into the now-left-justified Description text. */
|
|
printf("%-10s%10s%10s %-64s\n", "Name", "Flags", "Interval",
|
|
"Description");
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
/* bug-1711 (Requirement 11.4): prefix as hex ("0x%x"), matching this
|
|
* file's own state_raw convention (print_jobinfo), instead of an
|
|
* unlabeled bare %x that is ambiguous vs decimal. */
|
|
char flags_buf[16];
|
|
|
|
snprintf(flags_buf, sizeof(flags_buf), "0x%x", res[i]->flags);
|
|
|
|
printf("%-10s%10s%10d %-64s\n", res[i]->name, flags_buf,
|
|
res[i]->interval, res[i]->desc);
|
|
}
|
|
|
|
free_Resource_list(rc, res);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* worker_status_str() -- extracted to joblist_display.c/.h (xce-test-cli-ux
|
|
* task 2.5 remediation): pure, dependency-light display formatting, no
|
|
* networking/REPL/global-state coupling, so it can be unit-tested outside
|
|
* of this file (see test_joblist_display.c), mirroring set_resval()'s own
|
|
* extraction to resval_typeresolve.c below. Call sites are unchanged --
|
|
* they now resolve through joblist_display.h's declaration. */
|
|
|
|
/* forward decl: defined near cmd_resval, below, which also uses it */
|
|
static void format_bytes_human(double bytes, char* buf, size_t bufsize);
|
|
|
|
int cmd_workerlist(int argc, char* argv[])
|
|
{
|
|
int rc, i;
|
|
xce_worker_t** workers;
|
|
|
|
rc = xcecli_workerlist(active_ctx->ctx, &workers);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_workerlist");
|
|
return rc;
|
|
}
|
|
|
|
printf("%-20s%-8s%-25s%-25s%12s%12s%12s%12s%12s%12s\n", "HostName",
|
|
"Status", "HostType", "HostModel", "CPUfactor", "MaxCPU", "MaxMem",
|
|
"MaxSwap", "MaxTmp", "NDisks");
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
/* MaxMem/MaxSwap/MaxTmp are stored/wired as raw bytes (see
|
|
* is_bytes_valued_resource) -- humanize for display only, every
|
|
* other column stays as-is. */
|
|
char maxmem_buf[64];
|
|
char maxswap_buf[64];
|
|
char maxtmp_buf[64];
|
|
|
|
format_bytes_human(workers[i]->attrs->maxmem, maxmem_buf,
|
|
sizeof(maxmem_buf));
|
|
format_bytes_human(workers[i]->attrs->maxswap, maxswap_buf,
|
|
sizeof(maxswap_buf));
|
|
format_bytes_human(workers[i]->attrs->maxtmp, maxtmp_buf,
|
|
sizeof(maxtmp_buf));
|
|
|
|
printf("%-20s%-8s%-25s%-25s%12f%12d%12s%12s%12s%12d\n",
|
|
workers[i]->name, worker_status_str(workers[i]->status),
|
|
workers[i]->attrs->hosttype, workers[i]->attrs->hostmodel,
|
|
workers[i]->attrs->cpufactor, workers[i]->attrs->maxcpus,
|
|
maxmem_buf, maxswap_buf, maxtmp_buf,
|
|
workers[i]->attrs->ndisks);
|
|
}
|
|
|
|
free_Worker_list(rc, workers);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* job_state_str()/format_job_time() -- extracted to joblist_display.c/.h
|
|
* (xce-test-cli-ux task 2.5 remediation), same rationale as
|
|
* worker_status_str() above: pure, dependency-light display formatting,
|
|
* unit-tested outside of this file in test_joblist_display.c. Call sites
|
|
* (below, and print_jobinfo's "state:" line) are unchanged. */
|
|
|
|
/* Shared, print-free joblist core for `cmd_joblist` and the `expect-jobs` /
|
|
* `expect-job` assertions (xce-test-e2e-primitives task 5.1, design.md:81,
|
|
* 116-117).
|
|
*
|
|
* Performs the single `xcecli_joblist` call (through the failover-aware client
|
|
* context) and returns the current job count (>= 0), or a negative errno-style
|
|
* value on failure. When `find_jobid != NULL`, `*contains` is set to 1 if a job
|
|
* whose `fmt_jobid(...)` string equals `find_jobid` is present, else 0; the
|
|
* `fmt_jobid` representation matches the one `cmd_joblist` prints and the one
|
|
* `cmd_jobsub` captures into `$jobid` (task 4.1), so `expect-job $jobid` lines
|
|
* up. When `print_table` is set, the human-readable table is printed (the
|
|
* `cmd_joblist` path). The job array is always freed internally.
|
|
*
|
|
* On error this returns the negative rc WITHOUT setting any auto-variable, so a
|
|
* failed joblist never advertises a fake `$jobcount` (Req 3.6). */
|
|
static int joblist_query(const char* find_jobid, int* contains, int print_table,
|
|
int quiet)
|
|
{
|
|
int rc, i;
|
|
xce_job_t** jobs;
|
|
|
|
if (contains != NULL) {
|
|
*contains = 0;
|
|
}
|
|
|
|
rc = xcecli_joblist(active_ctx->ctx, &jobs);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
/* `quiet` callers (e.g. the wait-primary poll, which probes repeatedly
|
|
* and EXPECTS failure until a primary serves) suppress the per-attempt
|
|
* error print so a normal failover wait does not spam the console. */
|
|
if (!quiet) {
|
|
perror("xce_joblist");
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (print_table) {
|
|
printf("%-26s%-10s%-10s%-8s%-24s%-24s%-24s%12s%12s%30s\n", "JobId",
|
|
"User", "Queue", "State", "SubmitTime", "StartTime",
|
|
"FinishTime", "MinTasks", "MaxTasks", "Command");
|
|
}
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
char buf[128];
|
|
|
|
fmt_jobid(buf, sizeof(buf) - 1, jobs[i]->jobid);
|
|
|
|
if ((find_jobid != NULL) && (contains != NULL)
|
|
&& (strcmp(buf, find_jobid) == 0)) {
|
|
*contains = 1;
|
|
}
|
|
|
|
if (print_table) {
|
|
char submit_buf[32];
|
|
char start_buf[32];
|
|
char finish_buf[32];
|
|
|
|
printf("%-26s%-10s%-10s%-8s%-24s%-24s%-24s%12d%12d%30s\n", buf,
|
|
jobs[i]->submit->user, jobs[i]->submit->queue,
|
|
job_state_str(jobs[i]->state),
|
|
format_job_time(jobs[i]->queue_time, submit_buf,
|
|
sizeof(submit_buf)),
|
|
format_job_time(jobs[i]->start_time, start_buf,
|
|
sizeof(start_buf)),
|
|
format_job_time(jobs[i]->finish_time, finish_buf,
|
|
sizeof(finish_buf)),
|
|
jobs[i]->submit->mintasks, jobs[i]->submit->maxtasks,
|
|
jobs[i]->submit->start_info->cmdline);
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
free_Job_ptr(&(jobs[i]));
|
|
}
|
|
|
|
free(jobs);
|
|
|
|
return rc;
|
|
}
|
|
|
|
int cmd_joblist(int argc, char* argv[])
|
|
{
|
|
int count;
|
|
|
|
count = joblist_query(NULL, NULL, 1, /*quiet=*/0);
|
|
|
|
if (count < 0) {
|
|
return count;
|
|
}
|
|
|
|
/* Capture $jobcount (Req 3.4): a successful joblist publishes the returned
|
|
* job count so a later `echo $jobcount` / `expect-jobs` substitutes it. On
|
|
* failure we returned above without touching g_last_jobcount, so a failed
|
|
* joblist never advertises a fake count (Req 3.6). */
|
|
g_last_jobcount = count;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* `expect-jobs N` assertion (xce-test-e2e-primitives task 5.2, Req 5.1/5.2,
|
|
* design.md:116-117).
|
|
*
|
|
* Passes (returns 0) when the current job count equals N; otherwise prints the
|
|
* expected vs actual count and returns a negative errno-style value so
|
|
* preprocess_and_run records the session as failed (Req 1.1).
|
|
*
|
|
* A malformed argument (missing or non-numeric N) is itself an assertion
|
|
* failure, not a no-op. A failed joblist query is treated as a failure too: we
|
|
* cannot determine the count, so we never silently equate "query failed" with
|
|
* "zero jobs" (Req 5.2 reports actual; a failed query has no trustworthy
|
|
* actual). */
|
|
int cmd_expect_jobs(int argc, char* argv[])
|
|
{
|
|
long expected;
|
|
char* end;
|
|
int count;
|
|
|
|
if (argc != 2) {
|
|
printf("expect-jobs <N>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
errno = 0;
|
|
expected = strtol(argv[1], &end, 10);
|
|
|
|
if ((errno != 0) || (end == argv[1]) || (*end != '\0') || (expected < 0)) {
|
|
printf("expect-jobs: invalid count '%s' (expected a non-negative "
|
|
"integer)\n",
|
|
argv[1]);
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* Print-free count via the shared helper. A negative rc means the query
|
|
* itself failed (e.g. no reachable master) — that is an assertion failure,
|
|
* not "0 jobs". */
|
|
count = joblist_query(NULL, NULL, 0, /*quiet=*/0);
|
|
|
|
if (count < 0) {
|
|
printf("expect-jobs: could not determine job count (joblist query "
|
|
"failed)\n");
|
|
return count;
|
|
}
|
|
|
|
if (count != (int)expected) {
|
|
printf("expect-jobs: expected %ld, actual %d\n", expected, count);
|
|
return -EINVAL;
|
|
}
|
|
|
|
printf("expect-jobs: ok (%d)\n", count);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* `expect-job <jobid>` assertion (xce-test-e2e-primitives task 5.3, Req 5.3/5.4,
|
|
* design.md cmd_expect_jobs / cmd_expect_job).
|
|
*
|
|
* Passes (returns 0) when a job whose `fmt_jobid` string equals argv[1] is
|
|
* present in the current job list; otherwise prints the missing id and returns a
|
|
* negative errno-style value so preprocess_and_run records the session as failed
|
|
* (Req 1.1).
|
|
*
|
|
* `$jobid` substitution is transparent: autovar substitution (task 3.1) replaces
|
|
* `$jobid` with the captured id string BEFORE this handler runs, so argv[1] is
|
|
* already the literal id. The membership test in joblist_query uses the SAME
|
|
* `fmt_jobid` representation that cmd_jobsub captured into `$jobid` (task 4.1),
|
|
* so `expect-job $jobid` matches the joblist entry.
|
|
*
|
|
* A missing argument is itself an assertion failure, not a no-op. A failed
|
|
* joblist query is also a failure: we cannot determine membership, so we report
|
|
* the query failure distinctly rather than silently equating it with "absent". */
|
|
int cmd_expect_job(int argc, char* argv[])
|
|
{
|
|
int contains;
|
|
int rc;
|
|
|
|
if (argc != 2) {
|
|
printf("expect-job <jobid>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* Print-free membership query via the shared helper. A negative rc means the
|
|
* query itself failed (e.g. no reachable master) — that is an assertion
|
|
* failure, distinct from "job absent". */
|
|
rc = joblist_query(argv[1], &contains, 0, /*quiet=*/0);
|
|
|
|
if (rc < 0) {
|
|
printf("expect-job: could not determine membership of '%s' (joblist "
|
|
"query failed)\n",
|
|
argv[1]);
|
|
return rc;
|
|
}
|
|
|
|
if (!contains) {
|
|
printf("expect-job: job %s not found\n", argv[1]);
|
|
return -ENOENT;
|
|
}
|
|
|
|
printf("expect-job: ok (%s)\n", argv[1]);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* `wait-primary [timeout_ms]` (controller-status task 4.2, Req 9.1-9.4;
|
|
* originally xce-test-e2e-primitives task 6.1; design.md "cmd_wait_primary probe
|
|
* swap — State contract").
|
|
*
|
|
* Blocks until the AUTHORITATIVE controller status probe reaches the serving
|
|
* primary, or the timeout elapses. The probe is xcecli_ctrlstatus_primary: it
|
|
* drives active_ctx + libxce master-rotation/failover and returns 0 ONLY when a
|
|
* reply with responderRole == PRIMARY was obtained (it internally follows the
|
|
* redirect to the primary), or a typed negative when no primary answered within
|
|
* its budget (Req 9.1). A standby/hint answer is NOT success — the probe keeps
|
|
* polling until the primary itself answers (this avoids a false success on a
|
|
* standby's stale primary view after a primary death).
|
|
*
|
|
* On the first authoritative probe we print the served primary's identity and
|
|
* return 0 (Req 9.2). Otherwise we sleep a short poll interval and retry,
|
|
* re-checking a MONOTONIC deadline between attempts so the command returns
|
|
* within a bounded time of the timeout and never hangs indefinitely when there
|
|
* is no primary (Req 9.3). The deadline is derived from CLOCK_MONOTONIC, never
|
|
* wall-clock, so a clock step cannot skew it.
|
|
*
|
|
* The timeout is argv[1] in milliseconds when supplied, else
|
|
* WAIT_PRIMARY_DEFAULT_MS (Req 9.4). A malformed timeout argument is a usage
|
|
* error (return < 0) rather than a silent fallback to the default. */
|
|
#define WAIT_PRIMARY_DEFAULT_MS 30000L /* default max wait when arg omitted */
|
|
#define WAIT_PRIMARY_POLL_MS 200L /* sleep between probe attempts */
|
|
#define MS_PER_SEC 1000L
|
|
#define NS_PER_MS 1000000L
|
|
#define US_PER_MS 1000L
|
|
|
|
/* The client's configured log level (set in main from -v), so wait-primary can
|
|
* temporarily raise the threshold to silence libxce's per-attempt failover INFO
|
|
* spam during its poll and then RESTORE the user's chosen level. */
|
|
static char* g_active_loglevel = (char*)"info";
|
|
|
|
static long monotonic_ms(void)
|
|
{
|
|
struct timespec ts;
|
|
|
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
|
|
return ((long)ts.tv_sec * MS_PER_SEC) + (ts.tv_nsec / NS_PER_MS);
|
|
}
|
|
|
|
int cmd_wait_primary(int argc, char* argv[])
|
|
{
|
|
long timeout_ms = WAIT_PRIMARY_DEFAULT_MS;
|
|
long start_ms, elapsed_ms;
|
|
|
|
if (argc > 2) {
|
|
printf("wait-primary [timeout_ms]\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (argc == 2) {
|
|
char* end;
|
|
|
|
errno = 0;
|
|
timeout_ms = strtol(argv[1], &end, 10);
|
|
|
|
if ((errno != 0) || (end == argv[1]) || (*end != '\0')
|
|
|| (timeout_ms < 0)) {
|
|
printf("wait-primary: invalid timeout '%s' (expected a "
|
|
"non-negative integer in ms)\n",
|
|
argv[1]);
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
start_ms = monotonic_ms();
|
|
|
|
/* The poll probes repeatedly and EXPECTS failure until a primary serves, so
|
|
* it must not spam the console: xcecli_ctrlstatus_primary suppresses its own
|
|
* per-attempt output, and we raise the client log threshold for the duration
|
|
* of the loop to silence libxce's per-attempt "xcecli failover: attempt N"
|
|
* INFO logs — RESTORED to the user's level on every exit path. */
|
|
mylog_setlevel((char*)"warn");
|
|
|
|
/* Probe, then re-check the monotonic deadline between attempts. Each probe
|
|
* may itself take time (libxce failover + redirect-to-primary has its own
|
|
* internal retry budget); we re-read the clock after every probe returns so
|
|
* the overall loop honors the deadline and returns failure in bounded time. */
|
|
for (;;) {
|
|
xce_ctrlstatus_t* status = NULL;
|
|
int rc = xcecli_ctrlstatus_primary(active_ctx->ctx,
|
|
&status);
|
|
|
|
/* Success ONLY on an authoritative reply from the serving primary
|
|
* (responderRole == PRIMARY, guaranteed by xcecli_ctrlstatus_primary).
|
|
* Print the served primary's identity (the responder IS the primary, so
|
|
* primaryName == responderName) and free the reply (Req 9.1, 9.2). */
|
|
if (rc == 0) {
|
|
const char* primary = (status != NULL)
|
|
? (NULLSTR(status->primaryName)
|
|
? status->responderName
|
|
: status->primaryName)
|
|
: NULL;
|
|
|
|
elapsed_ms = monotonic_ms() - start_ms;
|
|
mylog_setlevel(g_active_loglevel);
|
|
printf("wait-primary: primary '%s' serving after %ld ms\n",
|
|
NULLSTR(primary) ? "unknown" : primary, elapsed_ms);
|
|
xcecli_free_ctrlstatus(status);
|
|
return 0;
|
|
}
|
|
|
|
/* Non-authoritative / failed probe: nothing to free (the typed-failure
|
|
* contract leaves *status untouched), keep polling until the deadline. */
|
|
elapsed_ms = monotonic_ms() - start_ms;
|
|
|
|
if (elapsed_ms >= timeout_ms) {
|
|
mylog_setlevel(g_active_loglevel);
|
|
printf("wait-primary: no primary served within %ld ms (waited "
|
|
"%ld ms)\n",
|
|
timeout_ms, elapsed_ms);
|
|
return -ETIMEDOUT;
|
|
}
|
|
|
|
/* Don't oversleep past the deadline: cap the poll to the time left. */
|
|
{
|
|
long remaining_ms = timeout_ms - elapsed_ms;
|
|
long sleep_ms = (remaining_ms < WAIT_PRIMARY_POLL_MS)
|
|
? remaining_ms
|
|
: WAIT_PRIMARY_POLL_MS;
|
|
|
|
if (sleep_ms > 0) {
|
|
usleep((useconds_t)(sleep_ms * US_PER_MS));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Map a ctrl_role_t raw value (CtrlStatus.responderRole, see ha.h:60-63 and
|
|
* design.md "Schema — reply payload") to a display string. PRIMARY==0,
|
|
* STANDBY==1, INIT==2, ELECTING==3; anything else is reported verbatim so an
|
|
* unexpected role is never silently rendered as a known one. */
|
|
static const char* ctrlstatus_role_str(uint8_t role)
|
|
{
|
|
switch (role) {
|
|
case 0:
|
|
return "primary";
|
|
case 1:
|
|
return "standby";
|
|
case 2:
|
|
return "initializing";
|
|
case 3:
|
|
return "electing";
|
|
default:
|
|
return "unknown-role";
|
|
}
|
|
}
|
|
|
|
/* `ctrlstatus` (controller-status task 4.1, Req 1.3/3.3/5.1/5.2/8.1/8.2/8.3,
|
|
* design.md "cmd_ctrlstatus (test client) — State contract").
|
|
*
|
|
* Issues the any-node status read (xcecli_ctrlstatus, is_write=false: whatever
|
|
* controller answers — primary or standby hint view — rides the configured
|
|
* failover/rotation) and prints the identity/health fields in human-readable
|
|
* form.
|
|
*
|
|
* On failure (no reachable controller etc.) it prints a typed error and returns
|
|
* the negative rc so the session records the command as failed and exits
|
|
* non-zero (Req 8.3). On success it prints the fields and returns 0 (Req 8.2). */
|
|
int cmd_ctrlstatus(int argc, char* argv[])
|
|
{
|
|
xce_ctrlstatus_t* status = NULL;
|
|
int rc;
|
|
int i;
|
|
|
|
(void)argv;
|
|
|
|
if (argc != 1) {
|
|
printf("ctrlstatus\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
rc = xcecli_ctrlstatus(active_ctx->ctx, &status);
|
|
|
|
if (rc < 0) {
|
|
char ebuf[128];
|
|
|
|
ebuf[0] = '\0';
|
|
xce_strerror_r(-rc, ebuf, sizeof(ebuf));
|
|
SET_ERROR(-rc);
|
|
|
|
switch (-rc) {
|
|
case E_XCE_NOMASTER:
|
|
fprintf(stderr, "ctrlstatus failed: no-master (%s)\n", ebuf);
|
|
break;
|
|
case E_XCE_NOT_READY:
|
|
fprintf(stderr, "ctrlstatus failed: not-ready (%s)\n", ebuf);
|
|
break;
|
|
case ETIMEDOUT:
|
|
fprintf(stderr, "ctrlstatus failed: timeout (%s)\n", ebuf);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "ctrlstatus failed: %s\n", ebuf);
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/* Current primary (Req 2.x): an empty primaryName means no elected/known
|
|
* primary — render the well-defined "unknown" (Req 1.3/2.2/2.3) rather than
|
|
* an empty or stale value. */
|
|
printf("Primary: %s\n",
|
|
NULLSTR(status->primaryName) ? "unknown" : status->primaryName);
|
|
|
|
/* Answering node + role, framed as a property of the responder and kept
|
|
* DISTINCT from the authoritative Primary line above (Req 3.1/3.2/3.3): a
|
|
* transient responder role (electing/initializing) must not be misread as
|
|
* the cluster's primary state. */
|
|
printf("answered by %s (%s)\n",
|
|
NULLSTR(status->responderName) ? "unknown" : status->responderName,
|
|
ctrlstatus_role_str(status->responderRole));
|
|
|
|
printf("Cluster: %s\n",
|
|
NULLSTR(status->clusterName) ? "unknown" : status->clusterName);
|
|
|
|
/* Configured controller topology + count (Req 4.1/4.2). */
|
|
printf("Controllers (%u):\n", status->peerCount);
|
|
for (i = 0; i < status->peers.count; i++) {
|
|
xce_ctrlpeer_t* peer = status->peers.list[i];
|
|
|
|
if (peer == NULL) {
|
|
continue;
|
|
}
|
|
|
|
printf("\t%s\t%s:%u\n", NULLSTR(peer->name) ? "?" : peer->name,
|
|
NULLSTR(peer->host) ? "?" : peer->host, peer->port);
|
|
}
|
|
|
|
/* Config version + committed config version (Req 5.1) and the content
|
|
* checksum as the hex string the controller already exposes (Req 5.2). */
|
|
printf("configVersion: %llu\n", (unsigned long long)status->configVersion);
|
|
printf("committedConfigVersion: %llu\n",
|
|
(unsigned long long)status->committedConfigVersion);
|
|
printf("configChecksum: %s\n",
|
|
NULLSTR(status->configChecksum) ? "unknown" : status->configChecksum);
|
|
|
|
/* HA health / progress (Req 6.x). */
|
|
printf("term: %llu\n", (unsigned long long)status->term);
|
|
printf("committedLsn: %llu\n", (unsigned long long)status->committedLsn);
|
|
printf("appliedLsn: %llu\n", (unsigned long long)status->appliedLsn);
|
|
printf("fenced: %s\n", status->fenced ? "true" : "false");
|
|
|
|
/* Responder snapshot timestamp (Req 11.2/11.3): render the responder's
|
|
* sample-time wall clock (epoch ms) in the SAME local-time
|
|
* "YYYY-MM-DD HH:MM:SS,mmm" form the controller logger emits, so it can be
|
|
* compared directly against ctrl-N.log. */
|
|
{
|
|
char tbuf[64];
|
|
if (ctrlstatus_format_time_ms(status->responder_time_ms, tbuf,
|
|
sizeof(tbuf)) != NULL) {
|
|
printf("responderTime: %s\n", tbuf);
|
|
} else {
|
|
printf("responderTime: unknown\n");
|
|
}
|
|
}
|
|
|
|
xcecli_free_ctrlstatus(status);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void print_jobinfo(xce_job_t* job)
|
|
{
|
|
char idstring[128];
|
|
|
|
if (job == NULL) {
|
|
return;
|
|
}
|
|
|
|
fmt_jobid(idstring, sizeof(idstring) - 1, job->jobid);
|
|
printf("JobID: %s\n", idstring);
|
|
printf("state: %s\n", job_state_str(job->state));
|
|
/* raft-apply-decoupling task 7.2: job_state_str's if/else-if chain checks
|
|
* JOB_STAT_RUN before JOB_STAT_EXIT, so a job that is simultaneously RUN
|
|
* and EXIT|PERR (e.g. a worker-reported FAIL received without a prior
|
|
* FINISH — apply_job_fail_transition ORs in EXIT|PERR without clearing
|
|
* RUN) prints as "RUN", silently hiding the EXIT/PERR bits from the
|
|
* pretty string. Print the raw bitmask too so a scriptable caller (the
|
|
* xce-test-ha factstream regression scenario's phantom-fact check) can
|
|
* test for a specific bit's presence unambiguously instead of relying on
|
|
* the display string's priority ordering. */
|
|
printf("state_raw: 0x%x\n", job->state);
|
|
printf("Queue: %s\n", job->submit->queue);
|
|
printf("User: %s\n", job->submit->user);
|
|
printf("Slots: %d,%d\n", job->submit->mintasks, job->submit->maxtasks);
|
|
printf("Command: %s\n", job->submit->start_info->cmdline);
|
|
|
|
if (job->pend_reasons != NULL) {
|
|
printf("Pend Reason:\n");
|
|
if (job->pend_reasons->main_reason != 0) {
|
|
printf("\tmain Reason: %d\n", job->pend_reasons->main_reason);
|
|
} else {
|
|
if (job->pend_reasons->nhost_reasons > 0) {
|
|
int i;
|
|
|
|
printf("\tHost Reasons:\n");
|
|
for (i = 0; i < job->pend_reasons->nhost_reasons; i++) {
|
|
printf("\t\t(%s, %d)\n",
|
|
job->pend_reasons->host_based_reasons[i]->host,
|
|
job->pend_reasons->host_based_reasons[i]->reason);
|
|
}
|
|
}
|
|
|
|
if (job->pend_reasons->nres_reasons > 0) {
|
|
int i;
|
|
|
|
printf("\tResource Reasons:\n");
|
|
for (i = 0; i < job->pend_reasons->nres_reasons; i++) {
|
|
printf("\t\t(%s, %s, %d)\n",
|
|
job->pend_reasons->res_based_reasons[i]->resource,
|
|
job->pend_reasons->res_based_reasons[i]->host,
|
|
job->pend_reasons->res_based_reasons[i]->reason);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
int cmd_jobinfo(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "j:")) != -1) {
|
|
uint64_t jobid;
|
|
xce_job_t* job;
|
|
int rc = 0;
|
|
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
|
|
if (jobid > 0) {
|
|
rc = xcecli_jobinfo(active_ctx->ctx, jobid, &job);
|
|
if (rc < 0) {
|
|
perror("xcecli_jobinfo");
|
|
continue;
|
|
}
|
|
|
|
print_jobinfo(job);
|
|
free_Job_ptr(&job);
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int fill_job_alloc(xce_job_alloc_t* jalloc, int nhosts,
|
|
host_task_t* hosts)
|
|
{
|
|
xce_host_alloc_t** halloc = NULL;
|
|
int i;
|
|
|
|
halloc = (xce_host_alloc_t**)calloc(nhosts, sizeof(xce_host_alloc_t*));
|
|
if (halloc == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
for (i = 0; i < nhosts; i++) {
|
|
xce_alloc_t* alloc;
|
|
xce_alloc_t** allocs;
|
|
|
|
alloc = (xce_alloc_t*)calloc(1, sizeof(*alloc));
|
|
allocs = (xce_alloc_t**)calloc(1, sizeof(*allocs));
|
|
halloc[i] = (xce_host_alloc_t*)calloc(1, sizeof(xce_host_alloc_t));
|
|
if ((halloc[i] == NULL) || (alloc == NULL) || (allocs == NULL)) {
|
|
/* FIXME: handle error */
|
|
return -ENOMEM;
|
|
}
|
|
|
|
alloc->resname = "slots";
|
|
alloc->reserve = "";
|
|
alloc->instid = -1;
|
|
alloc->value = (float)hosts[i].ntasks;
|
|
alloc->from = time(NULL);
|
|
alloc->duration = -1;
|
|
|
|
allocs[0] = alloc;
|
|
|
|
halloc[i]->nallocs = 1;
|
|
halloc[i]->allocs = allocs;
|
|
halloc[i]->host = hosts[i].host;
|
|
}
|
|
|
|
jalloc->nhosts = nhosts;
|
|
jalloc->allocs = halloc;
|
|
|
|
return 0;
|
|
}
|
|
int cmd_jobrun(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
host_task_t hosts[MAX_TASK_HOSTS];
|
|
xce_job_alloc_t jalloc;
|
|
xce_job_dispatch_t jdispatch
|
|
= { .njobs = 1, .allocs = (xce_job_alloc_t*[]) { &jalloc } };
|
|
int nhosts = 0;
|
|
char* host;
|
|
int count;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "m:j:")) != -1) {
|
|
switch (opt) {
|
|
case 'm':
|
|
if (nhosts >= MAX_TASK_HOSTS) {
|
|
fprintf(stderr,
|
|
"too many run hosts, we only support max %d hosts\n",
|
|
MAX_TASK_HOSTS);
|
|
return -1;
|
|
}
|
|
|
|
if (strchr(optarg, (int)':') != NULL) {
|
|
char* p;
|
|
char buf[256];
|
|
|
|
p = strchr(optarg, (int)':');
|
|
count = atoi(p + 1);
|
|
memset(buf, 0x0, sizeof(buf));
|
|
|
|
strncpy(buf, optarg, (int)(p - optarg));
|
|
host = strdup(buf);
|
|
} else {
|
|
host = strdup(optarg);
|
|
count = 1;
|
|
}
|
|
|
|
hosts[nhosts].host = host;
|
|
hosts[nhosts].ntasks = count;
|
|
nhosts++;
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
if (jobid == 0) {
|
|
printf("you need to specify a positive jobid with -j");
|
|
return -1;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf("no jobid specified");
|
|
return -1;
|
|
}
|
|
|
|
fill_job_alloc(&jalloc, nhosts, &hosts[0]);
|
|
|
|
jalloc.jobid = jobid;
|
|
|
|
rc = xcecli_jobdispatch(active_ctx->ctx, &jdispatch);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_jobdispatch");
|
|
return rc;
|
|
}
|
|
|
|
#if 0
|
|
printf("job %s will be started on host: %s\n",
|
|
argv[1], halloc.host);
|
|
#endif
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int build_job_envs(xce_job_submit_t* submit)
|
|
{
|
|
exdata_t exdata = { .sub_env_vars = { .PATH = getenv("PATH") } };
|
|
|
|
s2j_create_json_obj(json_exdata);
|
|
s2j_create_json_obj(json_envs);
|
|
s2j_json_set_basic_element(json_envs, &(exdata.sub_env_vars), var_string,
|
|
PATH);
|
|
cJSON_AddItemToObject(json_exdata, "sub_env_vars", json_envs);
|
|
|
|
submit->submit->start_info->exdata = cJSON_Print(json_exdata);
|
|
cJSON_Delete(json_exdata);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int build_job_exdata(xce_job_submit_t* submit)
|
|
{
|
|
build_job_envs(submit);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobsub(int argc, char* argv[])
|
|
{
|
|
int rc = 0;
|
|
job_submit_data_t submit;
|
|
xce_job_submit_t jobsubmit = { .submit = &submit };
|
|
int opt;
|
|
/* Initialize so a "rc >= 0 but *jobid never written" outcome (a non-clean
|
|
* ack such as a redirect/zero-row/not-ready surfaced as a non-negative rc)
|
|
* is detectable as an invalid (zero) JobId rather than printed as garbage
|
|
* (bug-786). */
|
|
uint64_t jobid = 0;
|
|
uint32_t min = 1, max = 1;
|
|
uint32_t arsize = 1;
|
|
char jobidStr[128];
|
|
|
|
optind = 1;
|
|
|
|
memset(&submit, 0x0, sizeof(submit));
|
|
submit.start_info
|
|
= (job_startinfo_t*)calloc(1, sizeof(*(submit.start_info)));
|
|
if (submit.start_info == NULL) {
|
|
perror("calloc");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
while ((opt = getopt(argc, argv, "u:q:R:c:x:n:a:")) != -1) {
|
|
switch (opt) {
|
|
case 'u':
|
|
submit.user = strdup(optarg);
|
|
break;
|
|
case 'q':
|
|
submit.queue = strdup(optarg);
|
|
break;
|
|
case 'R':
|
|
submit.resreq = strdup(optarg);
|
|
break;
|
|
case 'c':
|
|
submit.start_info->cmdline = strdup(optarg);
|
|
break;
|
|
case 'x':
|
|
submit.start_info->exdata = strdup(optarg);
|
|
break;
|
|
case 'n':
|
|
if (strchr(optarg, (int)',') != NULL) {
|
|
char buf[64];
|
|
char* p = strchr(optarg, (int)',');
|
|
|
|
max = atoi(p + 1);
|
|
memset(buf, 0x0, sizeof(buf));
|
|
strncpy(buf, optarg, (int)(p - optarg));
|
|
min = atoi(buf);
|
|
|
|
if (max == 0) {
|
|
printf("Incorrect setting of max tasks count, set to same "
|
|
"as min task value\n");
|
|
max = min;
|
|
}
|
|
} else {
|
|
min = atoi(optarg);
|
|
max = min;
|
|
}
|
|
break;
|
|
case 'a':
|
|
arsize = atoi(optarg);
|
|
if (arsize <= 0) {
|
|
arsize = 1;
|
|
}
|
|
break;
|
|
default:
|
|
printf(
|
|
"jobsub -u <user> -q <queue> -R <resreq> -c <command line>\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
}
|
|
|
|
if (min == 0) {
|
|
printf("Incorrect setting of min task count, set to 1 as default\n");
|
|
min = 1;
|
|
}
|
|
|
|
if (NULLSTR(submit.user)) {
|
|
printf("you need to specify job user with -u\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
|
|
if (NULLSTR(submit.queue)) {
|
|
printf("you need to specify job queue with -q\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
|
|
if (NULLSTR(submit.start_info->cmdline)) {
|
|
printf("you need to specify job command with -c\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
|
|
if (NULLSTR(submit.resreq)) {
|
|
submit.resreq = strdup("type == any");
|
|
}
|
|
|
|
submit.mintasks = min;
|
|
submit.maxtasks = max;
|
|
|
|
/* apiserverId=0 here let two unrelated jobs collide on the same jobid
|
|
* whenever the harness spawns a fresh xce_test process within the same
|
|
* wall-clock second (idgen_ctx_init resets sequence to 0 each time,
|
|
* and ts/tag/seq alone were then identical for both processes' first
|
|
* submission). Production apiserver instances never hit this: each
|
|
* holds one process-lifetime idgen_ctx_t (clib.go's sync.Once) and is
|
|
* assigned its own distinct apiserverId, so the jobid namespace is
|
|
* already partitioned by config.h's bit layout. Folding this process's
|
|
* pid into apiserverId gives each xce_test invocation its own slice of
|
|
* that same partitioning, making a same-second cross-process collision
|
|
* structurally impossible here too. */
|
|
idgen(&idctx, 0, (size_t)((uint64_t)getpid() & MAX_APISERVER_ID), arsize,
|
|
&(jobsubmit.jobid));
|
|
|
|
if (NULLSTR(submit.start_info->execuser)) {
|
|
submit.start_info->execuser = strdup(submit.user);
|
|
}
|
|
|
|
if (NULLSTR(submit.start_info->exechome)) {
|
|
submit.start_info->exechome = strdup("/tmp");
|
|
}
|
|
|
|
if (NULLSTR(submit.start_info->execcwd)) {
|
|
submit.start_info->execcwd = strdup("/tmp");
|
|
}
|
|
|
|
build_job_exdata(&jobsubmit);
|
|
|
|
jobsubmit.array_size = arsize;
|
|
rc = xcecli_jobsubmit(active_ctx->ctx, &jobsubmit, &jobid);
|
|
|
|
cleanup:
|
|
FREEUP(submit.user);
|
|
FREEUP(submit.queue);
|
|
FREEUP(submit.resreq);
|
|
FREEUP(submit.start_info->cmdline);
|
|
FREEUP(submit.start_info->exdata);
|
|
FREEUP(submit.start_info);
|
|
|
|
/* Report success only on a clean ack that echoes back the EXACT JobId we
|
|
* submitted (bug-786, Req 7.1/7.2/7.3). xce_jobsubmit() writes *jobid only
|
|
* when the reply status is >= 0; XCECLI_FAILOVER_CALL surfaces a non-clean
|
|
* outcome as a negative rc (e.g. -E_XCE_NOT_READY on no-primary budget
|
|
* exhaustion, -E_XCE_REDIRECT / -E_XCE_NOMASTER on a standby, -ETIMEDOUT /
|
|
* -ENOTCONN on transport loss).
|
|
*
|
|
* The `jobid != 0` gate alone is NOT sufficient: on a failover, libxce's
|
|
* handle_sub_reply() returns -1 on a non-jobid reply but LEAVES the stale
|
|
* previous value in ctx->reply.u.jobid, and xceapi.c copies it out whenever
|
|
* status >= 0 — so a rejected submit can surface as rc >= 0 with a NON-ZERO
|
|
* stale-garbage jobid (the live soak captured `63-15--1-1015808[32751]`
|
|
* while the controller logged "handle_jobsubmit fail to lock job" and never
|
|
* committed it). We therefore require the returned jobid to EQUAL the JobId
|
|
* we generated locally and submitted: on a genuine clean ack the controller
|
|
* commits and echoes back exactly submit->jobid (handle_jobsubmit replies
|
|
* reply.data.jobid = job_submit() = base_id = submit->jobid; verified
|
|
* round-trip). `jobid != jobsubmit.jobid` covers BOTH the never-written
|
|
* (zero) case AND the stale non-zero garbage case in one uint64 compare. */
|
|
if (rc < 0) {
|
|
char ebuf[128];
|
|
|
|
ebuf[0] = '\0';
|
|
xce_strerror_r(-rc, ebuf, sizeof(ebuf));
|
|
SET_ERROR(-rc);
|
|
|
|
switch (-rc) {
|
|
case E_XCE_NOMASTER:
|
|
fprintf(stderr, "jobsubmit failed: no-master (%s)\n", ebuf);
|
|
break;
|
|
case E_XCE_REDIRECT:
|
|
fprintf(stderr, "jobsubmit failed: redirect (%s)\n", ebuf);
|
|
break;
|
|
case E_XCE_NOT_READY:
|
|
fprintf(stderr, "jobsubmit failed: not-ready (%s)\n", ebuf);
|
|
break;
|
|
case ETIMEDOUT:
|
|
fprintf(stderr, "jobsubmit failed: timeout (%s)\n", ebuf);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "jobsubmit failed: %s\n", ebuf);
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
if (jobid != jobsubmit.jobid) {
|
|
/* Non-negative rc but the returned JobId does not match the one we
|
|
* submitted: not a clean ack. This rejects BOTH the never-written
|
|
* (jobid == 0) case AND the stale-garbage non-zero case left in the
|
|
* libxce reply union on a failover-rejected submit. Never print "ok"
|
|
* with an uninitialized/garbage id (Req 7.3). */
|
|
char wantStr[128];
|
|
char gotStr[128];
|
|
|
|
fmt_jobid(wantStr, sizeof(wantStr) - 1, jobsubmit.jobid);
|
|
fmt_jobid(gotStr, sizeof(gotStr) - 1, jobid);
|
|
fprintf(stderr,
|
|
"jobsubmit failed: no clean ack (controller did not confirm "
|
|
"the submitted JobId: wanted %s, got %s)\n",
|
|
wantStr, gotStr);
|
|
return -E_XCE_INTERNAL;
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, array_jobid(jobid));
|
|
|
|
/* Capture $jobid (Req 3.2) using the SAME representation cmd_joblist prints
|
|
* for a job's JobId column — fmt_jobid(jobs[i]->jobid), the full jobid
|
|
* WITHOUT array_jobid() stripping — so a later `$jobid` substitution and
|
|
* `expect-job $jobid` compare equal to the joblist entry for this job. Only
|
|
* set on this clean-ack + valid-id path; never on a failure path, so an
|
|
* unset $jobid still fails per Req 3.6. */
|
|
fmt_jobid(g_last_jobid, sizeof(g_last_jobid) - 1, jobid);
|
|
|
|
printf("jobsubmit ok: JobId: %s\n", jobidStr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Resource names whose canonical stored/wire value is raw bytes (not a
|
|
* generic schema-driven unit system -- deliberately kept as a small,
|
|
* explicit, client-side list rather than a wire field, since only a
|
|
* handful of resources need this and a wire-level unit hint would have
|
|
* grown every Resource/ResourceVal message regardless of whether it's
|
|
* used). CLI display picks the best-fit human unit adaptively
|
|
* (format_bytes_human); the stored value is never converted. Covers every
|
|
* memory/disk-scale built-in host attribute and load index (maxmem, mem,
|
|
* tmp, maxswap, maxtmp) -- deliberately NOT maxcpus/ndisks (plain counts)
|
|
* or cpuf/r1m/r15s (unitless factors/averages).
|
|
*/
|
|
static int is_bytes_valued_resource(const char* name)
|
|
{
|
|
static const char* const BYTES_VALUED_RESOURCES[]
|
|
= { "maxmem", "mem", "tmp", "maxswap", "maxtmp" };
|
|
const int nres = (int)(sizeof(BYTES_VALUED_RESOURCES)
|
|
/ sizeof(BYTES_VALUED_RESOURCES[0]));
|
|
int i;
|
|
|
|
if (name == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
for (i = 0; i < nres; i++) {
|
|
if (strcmp(name, BYTES_VALUED_RESOURCES[i]) == 0) {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Render a raw byte count as a human-readable string (e.g. "30.90 GB"),
|
|
* adaptively picking the smallest unit the value fits in. Only used for
|
|
* display -- the stored/wire value stays raw bytes always, this never
|
|
* feeds back into anything that's compared or persisted. A pure function
|
|
* so it's directly unit-testable without a live client connection.
|
|
*/
|
|
static void format_bytes_human(double bytes, char* buf, size_t bufsize)
|
|
{
|
|
static const char* const UNITS[] = { "B", "KB", "MB", "GB", "TB", "PB" };
|
|
const int nunits = (int)(sizeof(UNITS) / sizeof(UNITS[0]));
|
|
double val = bytes;
|
|
int i = 0;
|
|
|
|
while (val >= 1024.0 && i < nunits - 1) {
|
|
val /= 1024.0;
|
|
i++;
|
|
}
|
|
|
|
snprintf(buf, bufsize, "%.2f %s", val, UNITS[i]);
|
|
}
|
|
|
|
int cmd_resval(int argc, char* argv[])
|
|
{
|
|
char* host = NULL;
|
|
char* res = NULL;
|
|
xce_resval_t** vals = NULL;
|
|
int opt;
|
|
int rc;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "n:m:")) != -1) {
|
|
switch (opt) {
|
|
case 'n':
|
|
res = strdup(optarg);
|
|
break;
|
|
case 'm':
|
|
host = strdup(optarg);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "resval -n resname [-m host]\n");
|
|
FREEUP(host);
|
|
FREEUP(res);
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
rc = xcecli_resval(active_ctx->ctx, host, res, &vals);
|
|
if (rc > 0) {
|
|
int i;
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
xce_resval_t* val = vals[i];
|
|
|
|
printf("resource %s@%s:\n", val->name,
|
|
(val->host == NULL) ? "cluster" : val->host);
|
|
printf("value: ");
|
|
switch (val->val->which) {
|
|
case Value_u_svalue:
|
|
printf("%s\n", val->val->u.svalue);
|
|
break;
|
|
case Value_u_fvalue:
|
|
if (is_bytes_valued_resource(val->name)) {
|
|
char hbuf[64];
|
|
|
|
format_bytes_human(val->val->u.fvalue, hbuf,
|
|
sizeof(hbuf));
|
|
printf("%f (%s)\n", val->val->u.fvalue, hbuf);
|
|
} else {
|
|
printf("%f\n", val->val->u.fvalue);
|
|
}
|
|
break;
|
|
case Value_u_bvalue:
|
|
printf("%s\n", (val->val->u.bvalue == 0) ? "false" : "true");
|
|
break;
|
|
default:
|
|
printf(" <unknown type>\n");
|
|
break;
|
|
}
|
|
|
|
free_ResourceVal_ptr(&val);
|
|
}
|
|
|
|
FREEUP(vals);
|
|
} else {
|
|
printf("no resource value found in given host\n");
|
|
}
|
|
|
|
FREEUP(host);
|
|
FREEUP(res);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobkill(int argc, char* argv[])
|
|
{
|
|
int signo;
|
|
uint64_t jobid = 0;
|
|
int opt;
|
|
int rc;
|
|
xce_jobctrl_t jobctrl;
|
|
xce_signal_t siginfo;
|
|
char jobidStr[64];
|
|
|
|
optind = 1;
|
|
signo = SIGTERM;
|
|
|
|
while ((opt = getopt(argc, argv, "s:j:")) != -1) {
|
|
switch (opt) {
|
|
case 's':
|
|
signo = atoi(optarg);
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
fprintf(stderr, "you need to specify jobid with -j\n");
|
|
return -1;
|
|
}
|
|
|
|
jobctrl.action = JobControl_u_siginfo;
|
|
jobctrl.jobid = jobid;
|
|
jobctrl.u.siginfo = &siginfo;
|
|
siginfo.signo = signo;
|
|
siginfo.force = 1;
|
|
|
|
rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_jobctrl");
|
|
return rc;
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, jobid);
|
|
printf("Job %s signaled\n", jobidStr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobckpt(int argc, char* argv[])
|
|
{
|
|
uint64_t jobid = 0;
|
|
int opt;
|
|
int rc;
|
|
xce_jobctrl_t jobctrl;
|
|
xce_ckpt_options_t ckptoption;
|
|
char jobidStr[64];
|
|
|
|
optind = 1;
|
|
ckptoption.kill = 0;
|
|
|
|
while ((opt = getopt(argc, argv, "kj:m:")) != -1) {
|
|
switch (opt) {
|
|
case 'k':
|
|
ckptoption.kill = 1;
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'm':
|
|
ckptoption.method = optarg;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
fprintf(stderr, "you need to specify jobid with -j\n");
|
|
return -1;
|
|
}
|
|
|
|
if (ckptoption.method == NULL) {
|
|
ckptoption.method = "default";
|
|
}
|
|
jobctrl.action = JobControl_u_ckptoption;
|
|
jobctrl.jobid = jobid;
|
|
jobctrl.u.ckptoption = &ckptoption;
|
|
|
|
rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_jobctrl");
|
|
return rc;
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, jobid);
|
|
printf("Job %s checkpointed\n", jobidStr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_qctrl(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
int rc;
|
|
xce_queuectrl_t queuectrl;
|
|
int action;
|
|
char* queue = NULL;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "aiociq:")) != -1) {
|
|
switch (opt) {
|
|
case 'a':
|
|
action = QUEUE_ACTIVE;
|
|
break;
|
|
case 'i':
|
|
action = QUEUE_INACTIVE;
|
|
break;
|
|
case 'o':
|
|
action = QUEUE_OPEN;
|
|
break;
|
|
case 'c':
|
|
action = QUEUE_CLOSE;
|
|
break;
|
|
case 'q':
|
|
queue = strdup(optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (queue == NULL) {
|
|
fprintf(stderr, "you need to specify queue name with -q\n");
|
|
return -1;
|
|
}
|
|
|
|
queuectrl.action = action;
|
|
queuectrl.queue = queue;
|
|
|
|
rc = xcecli_queuectrl(active_ctx->ctx, &queuectrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_queuectrl");
|
|
return rc;
|
|
}
|
|
|
|
printf("Operation on queue %s done\n", queue);
|
|
|
|
free(queue);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* hostctrl -o|-c -h <hostname> — administratively open/close a host
|
|
* (xce_hostctrl_t{action, hostname} via xcecli_hostctrl). Mirrors cmd_qctrl's
|
|
* exact getopt shape; only -o/-c are wired (matching xce_hostctrl_req's own
|
|
* accepted action range, request.c — HOST_TAG/HOST_UNTAG exist in the enum
|
|
* but are rejected -EINVAL server-side today, so no flag is offered for them
|
|
* here). Used by the config-upgrade chaotic soak (task 8.6) to set a host's
|
|
* admin status before/around a config upgrade and assert it survives. */
|
|
int cmd_hostctrl(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
int rc;
|
|
xce_hostctrl_t hostctrl;
|
|
int action = HOST_DO_NONE;
|
|
char* hostname = NULL;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "och:")) != -1) {
|
|
switch (opt) {
|
|
case 'o':
|
|
action = HOST_OPEN;
|
|
break;
|
|
case 'c':
|
|
action = HOST_CLOSE;
|
|
break;
|
|
case 'h':
|
|
hostname = strdup(optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (hostname == NULL) {
|
|
fprintf(stderr, "you need to specify host name with -h\n");
|
|
return -1;
|
|
}
|
|
|
|
if (action == HOST_DO_NONE) {
|
|
fprintf(stderr, "you need to specify -o (open) or -c (close)\n");
|
|
free(hostname);
|
|
return -1;
|
|
}
|
|
|
|
hostctrl.action = action;
|
|
hostctrl.hostname = hostname;
|
|
|
|
rc = xcecli_hostctrl(active_ctx->ctx, &hostctrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_hostctrl");
|
|
free(hostname);
|
|
return rc;
|
|
}
|
|
|
|
printf("Operation on host %s done\n", hostname);
|
|
|
|
free(hostname);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* upgradeconfig <cluster.bin> — submit a configuration upgrade
|
|
* (config-upgrade-snapshot task 8.6). Reads the given capnp-encoded
|
|
* ClusterConfig blob (the same format `xceadmin config build` produces and
|
|
* deploy-ha.sh installs as cluster.bin), computes its SHA-256 (OpenSSL
|
|
* SHA256(), matching config_repl.c's do_upgrade_config_with_bytes transit
|
|
* verification), and drives it through xcecli_upgradeconfig ->
|
|
* xce_upgradeconfig_req_raw (OP_upgradeConfig). Mirrors cmd_snapdiff's
|
|
* bounds-checked file-reading style. */
|
|
int cmd_upgradeconfig(int argc, char* argv[])
|
|
{
|
|
if (argc != 2) {
|
|
fprintf(stderr, "usage: upgradeconfig <cluster.bin>\n");
|
|
return -1;
|
|
}
|
|
|
|
const char* path = argv[1];
|
|
FILE* fp = fopen(path, "rb");
|
|
if (fp == NULL) {
|
|
fprintf(stderr, "upgradeconfig: cannot open %s: %s\n", path,
|
|
strerror(errno));
|
|
return -1;
|
|
}
|
|
|
|
if (fseek(fp, 0, SEEK_END) != 0) {
|
|
fprintf(stderr, "upgradeconfig: fseek failed on %s: %s\n", path,
|
|
strerror(errno));
|
|
fclose(fp);
|
|
return -1;
|
|
}
|
|
|
|
long fsz = ftell(fp);
|
|
if (fsz <= 0) {
|
|
fprintf(stderr, "upgradeconfig: %s is empty or ftell failed\n", path);
|
|
fclose(fp);
|
|
return -1;
|
|
}
|
|
rewind(fp);
|
|
|
|
uint8_t* bytes = (uint8_t*)malloc((size_t)fsz);
|
|
if (bytes == NULL) {
|
|
fprintf(stderr, "upgradeconfig: out of memory reading %s\n", path);
|
|
fclose(fp);
|
|
return -1;
|
|
}
|
|
|
|
size_t nread = fread(bytes, 1, (size_t)fsz, fp);
|
|
fclose(fp);
|
|
if (nread != (size_t)fsz) {
|
|
fprintf(stderr, "upgradeconfig: short read on %s (%zu of %ld bytes)\n",
|
|
path, nread, fsz);
|
|
free(bytes);
|
|
return -1;
|
|
}
|
|
|
|
uint8_t sha256[SHA256_DIGEST_LENGTH];
|
|
SHA256(bytes, (size_t)fsz, sha256);
|
|
|
|
int rc = xcecli_upgradeconfig(active_ctx->ctx, sha256,
|
|
sizeof(sha256), bytes, (size_t)fsz);
|
|
free(bytes);
|
|
|
|
if (rc < 0) {
|
|
fprintf(stderr, "upgradeconfig: failed (rc=%d): %s\n", rc,
|
|
strerror(-rc));
|
|
return rc;
|
|
}
|
|
|
|
printf("Config upgrade submitted from %s (%ld bytes)\n", path, fsz);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_runtasks(int argc, char* argv[])
|
|
{
|
|
xce_taskhost_t hosts[MAX_TASK_HOSTS];
|
|
size_t nhosts = 0;
|
|
xce_taskrun_t taskrun;
|
|
int opt;
|
|
int rc;
|
|
int count;
|
|
int totaltasks = 0;
|
|
char* user = NULL;
|
|
char* cmdline = NULL;
|
|
char* dir = NULL;
|
|
char* exdata = NULL;
|
|
char* host = NULL;
|
|
char* launch_type = NULL;
|
|
int i;
|
|
uint64_t jobid = 0;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "m:u:c:x:d:j:t:")) != -1) {
|
|
switch (opt) {
|
|
case 'm':
|
|
if (nhosts >= MAX_TASK_HOSTS) {
|
|
fprintf(stderr,
|
|
"too many run hosts, just support %d run hosts\n",
|
|
MAX_TASK_HOSTS);
|
|
return -1;
|
|
}
|
|
|
|
if (strchr(optarg, (int)':') != NULL) {
|
|
char* p;
|
|
char buf[256];
|
|
|
|
p = strchr(optarg, (int)':');
|
|
count = atoi(p + 1);
|
|
memset(buf, 0x0, sizeof(buf));
|
|
strncpy(buf, optarg, (int)(p - optarg));
|
|
host = strdup(buf);
|
|
} else {
|
|
host = strdup(optarg);
|
|
count = 1;
|
|
}
|
|
|
|
hosts[nhosts].host = host;
|
|
hosts[nhosts].ntasks = count;
|
|
nhosts++;
|
|
totaltasks += count;
|
|
break;
|
|
case 'u':
|
|
if (user != NULL) {
|
|
free(user);
|
|
}
|
|
user = strdup(optarg);
|
|
break;
|
|
case 'c':
|
|
if (cmdline != NULL) {
|
|
free(cmdline);
|
|
}
|
|
cmdline = strdup(optarg);
|
|
break;
|
|
case 'x':
|
|
if (exdata != NULL) {
|
|
free(exdata);
|
|
}
|
|
exdata = strdup(optarg);
|
|
break;
|
|
case 'd':
|
|
if (dir != NULL) {
|
|
free(dir);
|
|
}
|
|
dir = strdup(optarg);
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 't':
|
|
launch_type = strdup(optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < nhosts; i++) {
|
|
hosts[i].exechome = dir;
|
|
hosts[i].execcwd = dir;
|
|
hosts[i].execuser = user;
|
|
hosts[i].exdata = exdata;
|
|
hosts[i].cmdline = cmdline;
|
|
hosts[i].launch_type = launch_type;
|
|
if (hosts[i].launch_type == NULL) {
|
|
if (totaltasks > 1) {
|
|
hosts[i].launch_type = strdup("mpi");
|
|
} else {
|
|
hosts[i].launch_type = strdup("sequential");
|
|
}
|
|
}
|
|
}
|
|
|
|
taskrun.taskhosts
|
|
= (xce_taskhost_t**)calloc(nhosts, sizeof(xce_taskhost_t*));
|
|
taskrun.clientidx = -1;
|
|
|
|
if (taskrun.taskhosts != NULL) {
|
|
for (i = 0; i < nhosts; i++) {
|
|
taskrun.taskhosts[i] = &hosts[i];
|
|
}
|
|
|
|
taskrun.nhosts = nhosts;
|
|
taskrun.jobid = jobid;
|
|
rc = xcecli_runtask(active_ctx->ctx, &taskrun);
|
|
} else {
|
|
rc = -ENOMEM;
|
|
}
|
|
|
|
if (rc < 0) {
|
|
errno = -rc;
|
|
perror("xcecli_runtask");
|
|
} else {
|
|
printf("run task ok\n");
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int add_pendreason(xce_job_preason_t* preason, char* s)
|
|
{
|
|
char s1[MAX_LINE_LEN];
|
|
char* pcode = NULL;
|
|
char* phost = NULL;
|
|
char* pres = NULL;
|
|
char* p;
|
|
int code;
|
|
|
|
if ((preason == NULL) || (s == NULL)) {
|
|
fprintf(stderr, "bad parameter\n");
|
|
return -1;
|
|
}
|
|
|
|
if ((p = strrchr(s, (int)':')) == NULL) {
|
|
pcode = s;
|
|
} else {
|
|
pcode = p + 1;
|
|
memset(s1, 0x0, sizeof(s1));
|
|
strncpy(s1, s, p - s);
|
|
|
|
if ((p = strrchr(s1, (int)':')) == NULL) {
|
|
phost = s1;
|
|
} else {
|
|
*p = 0x0;
|
|
phost = p + 1;
|
|
pres = s1;
|
|
}
|
|
}
|
|
|
|
code = atoi(pcode);
|
|
|
|
return add_job_preason(preason, code, phost, pres);
|
|
}
|
|
|
|
int cmd_setpend(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
xce_job_preason_t jreasons;
|
|
xce_pendreason_t preason = { .reasons = &jreasons };
|
|
xce_preason_set_t reasons
|
|
= { .nreasons = 1, .reasons = (xce_pendreason_t*[]) { &preason } };
|
|
int rc = 0;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "j:p:")) != -1) {
|
|
int rc = 0;
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&(preason.jobid), optarg);
|
|
break;
|
|
case 'p':
|
|
rc = add_pendreason(&jreasons, optarg);
|
|
if (rc < 0) {
|
|
perror("add_pendreason");
|
|
return rc;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (preason.jobid == 0) {
|
|
fprintf(stderr, "no or incorrect jobid specified\n");
|
|
return -1;
|
|
}
|
|
|
|
if ((jreasons.main_reason == 0) && (jreasons.nhost_reasons == 0)
|
|
&& (jreasons.nres_reasons == 0)) {
|
|
fprintf(stderr, "no or incorrect pend reason set\n");
|
|
return -1;
|
|
}
|
|
|
|
rc = xcecli_set_pend_reason(active_ctx->ctx, &reasons);
|
|
if (rc < 0) {
|
|
perror("xcecli_set_pend_reason");
|
|
erase_job_preason(&jreasons);
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_join_worker(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int role;
|
|
hostattrs_t attrs;
|
|
|
|
role = HostRole_worker;
|
|
|
|
/* xce-test-cli-ux task 2.6 remediation (bug-1557): the argument-parsing/
|
|
* defaulting block (including the maxmem default fix) is extracted into
|
|
* join_worker_args.c/.h so it can be unit-tested in isolation from this
|
|
* function's REPL/network state -- see join_worker_args.h for the full
|
|
* rationale. This is a structural extraction only; the parsing/defaulting
|
|
* behavior and everything from xcecli_join() onward are unchanged. */
|
|
rc = join_worker_parse_attrs(argc, argv, &attrs);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = xcecli_join(active_ctx->ctx, role, &attrs);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_join");
|
|
return rc;
|
|
}
|
|
|
|
printf("host join ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* jstart -j jobid [-p pid] — raft-apply-decoupling task 7.2: synthetic
|
|
* worker-fact-stream primitive. Mirrors cmd_jobfinish's existing pattern
|
|
* exactly (same getopt/parse_jobid/SET_ERROR shape) but drives the
|
|
* xcecli_notify_jobstart wire call that cmd_jobfinish's sibling
|
|
* xcecli_notify_jobfinish already exercises — libxce (client.c) already
|
|
* implements the full worker-fact API (jobstart/jobfinish/jobfail/...); this
|
|
* command was the missing REPL-level primitive for jobstart specifically.
|
|
* Drives the SAME production receipt path (on_job_start -> record_job_start_fact
|
|
* -> persist_submit_job_fact -> durable append -> commit -> apply pump ->
|
|
* apply_committed_entry()) that a real worker's report would, without
|
|
* requiring a real scheduler dispatch (on_job_start only requires the job to
|
|
* already exist in jobmgr, i.e. have been submitted). */
|
|
int cmd_jobstart(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
unsigned int pid = 0;
|
|
unsigned int pids[1];
|
|
job_pidinfo_t info;
|
|
|
|
optind = 1;
|
|
memset(&info, 0x0, sizeof(info));
|
|
|
|
while ((opt = getopt(argc, argv, "j:p:")) != -1) {
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'p':
|
|
pid = (unsigned int)atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" jstart -j jobid [-p pid]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf(" you need specify a positive jobid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
pids[0] = (pid != 0) ? pid : (unsigned int)getpid();
|
|
info.jobid = jobid;
|
|
info.npids = 1;
|
|
info.pids = pids;
|
|
info.npgids = 0;
|
|
info.pgids = NULL;
|
|
info.exdata = NULL;
|
|
|
|
rc = xcecli_notify_jobstart(active_ctx->ctx, &info);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_jobstart");
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobfinish(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
int code = 0;
|
|
xce_jobfinish_t finish;
|
|
|
|
/* raft-apply-decoupling task 7.2: reset optind so repeated jfinish
|
|
* invocations within one xce_test process (the fact-stream soak scenario
|
|
* calls this many times in a single worker session) each re-parse their
|
|
* own argv from the start. Without this, getopt's optind carries over
|
|
* from the PRIOR command's parse position, silently skipping this
|
|
* command's own "-j jobid" on the second and later calls (jobid stays 0
|
|
* and the command errors out) — matches the reset convention most other
|
|
* cmd_ handlers in this file already follow (e.g. cmd_join_worker just
|
|
* above, cmd_jobsub, cmd_resupdate). */
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "j:c:")) != -1) {
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'c':
|
|
code = atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" jfinish -j jobid [-c exitcode]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf(" you need specify a positive jobid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
finish.jobid = jobid;
|
|
finish.exit_code = code;
|
|
|
|
rc = xcecli_notify_jobfinish(active_ctx->ctx, &finish);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_jobfinish");
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* jfail -j jobid [-c code] [-s stage] — raft-apply-decoupling task 7.2:
|
|
* synthetic worker-fact-stream primitive, sibling of jstart/jfinish above.
|
|
* Drives xcecli_notify_jobfail -> on_job_fail -> record_job_fail_fact ->
|
|
* persist_submit_job_fact_ex (the _pad-discriminated path, task 6.1 round 2)
|
|
* -> apply_job_fail_transition() at apply time. Reuses the existing
|
|
* job_failinfo_t shape already exercised internally by this file's own
|
|
* notify_job_fail() helper (used by run_job() on a local exec failure). */
|
|
int cmd_jobfail(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
int code = 0;
|
|
int stage = JOB_EXEC_STAGE_POST_EXEC;
|
|
job_failinfo_t failinfo;
|
|
|
|
optind = 1;
|
|
memset(&failinfo, 0x0, sizeof(failinfo));
|
|
|
|
while ((opt = getopt(argc, argv, "j:c:s:")) != -1) {
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'c':
|
|
code = atoi(optarg);
|
|
break;
|
|
case 's':
|
|
stage = atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" jfail -j jobid [-c code] [-s stage]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf(" you need specify a positive jobid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
failinfo.jobid = jobid;
|
|
failinfo.stage = stage;
|
|
failinfo.code = code;
|
|
failinfo.detail = "synthetic fact-stream failure (xce_test jfail)";
|
|
|
|
rc = xcecli_notify_jobfail(active_ctx->ctx, &failinfo);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_jobfail");
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* set_resval() -- thin wrapper (resource-value-verification task 2.1).
|
|
*
|
|
* Type resolution (name -> Value_u_which, including the unrecognized-name
|
|
* numeric-default fallback and `-t` override) now lives in
|
|
* resval_typeresolve.c/resval_resolve_type(), shared with the standalone
|
|
* unit tests. This function keeps only what still needs REPL state: the
|
|
* xce_val_t allocation/parsing per wire type, and `.host` tagging via
|
|
* get_setting("myname") (unchanged from before this refactor).
|
|
*
|
|
* `type_override`: RESVAL_TYPE_UNKNOWN for "no override" (the common case),
|
|
* or an explicit Value_u_which tag from cmd_resupdate()'s new `-t` flag.
|
|
* Ignored by resval_resolve_type() for any already-known resource name.
|
|
*/
|
|
static int set_resval(
|
|
xce_resval_t* vals, int idx, char* name, char* val, int type_override)
|
|
{
|
|
int valtype;
|
|
int rc = -1;
|
|
|
|
if (name == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
valtype = resval_resolve_type(name, type_override);
|
|
if (valtype == RESVAL_TYPE_UNKNOWN) {
|
|
/* Only reachable if resval_resolve_type() itself ever declines to
|
|
* resolve a non-NULL name -- not possible today (it always falls
|
|
* back to Value_u_fvalue or the override), kept as a defensive
|
|
* guard against a future change to that function's contract. */
|
|
return rc;
|
|
}
|
|
|
|
rc = 1;
|
|
vals[idx].val = (xce_val_t*)calloc(1, sizeof(xce_val_t));
|
|
|
|
switch (valtype) {
|
|
case Value_u_fvalue:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.fvalue = 0.0;
|
|
} else {
|
|
vals[idx].val->u.fvalue = atof(val);
|
|
}
|
|
break;
|
|
case Value_u_ivalue32:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.ivalue32 = 0;
|
|
} else {
|
|
vals[idx].val->u.ivalue32 = atoi(val);
|
|
}
|
|
break;
|
|
case Value_u_ivalue64:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.ivalue64 = 0;
|
|
} else {
|
|
vals[idx].val->u.ivalue64 = atol(val);
|
|
}
|
|
break;
|
|
case Value_u_bvalue:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.bvalue = 0;
|
|
} else {
|
|
if (strcasecmp(val, "t") == 0) {
|
|
vals[idx].val->u.bvalue = 1;
|
|
} else {
|
|
vals[idx].val->u.bvalue = 0;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
/* NOTE (pre-existing, unchanged): Value_u_svalue (type/model, and now
|
|
* an unrecognized name given `-t string`) has no case here -- the
|
|
* switch falls through with vals[idx].name/.val->which left unset.
|
|
* This is set_resval()'s existing behavior for type/model today;
|
|
* Requirement 1.4 requires preserving it byte-for-byte, so it is
|
|
* intentionally not fixed by this task. See CONCERNS in this task's
|
|
* status report. */
|
|
|
|
vals[idx].host = get_setting("myname");
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* resource-value-verification task 2.2: the controller's own update RPC
|
|
* reply can never distinguish per-name accept/reject -- update_resval_list()
|
|
* (src/controller/worker.c:185-198) unconditionally returns success
|
|
* regardless of any per-item failure (design.md's "set_resval()/
|
|
* cmd_resupdate" component, Responsibilities & Constraints). To recover
|
|
* per-name accept/reject (Requirements 1.2, 1.3), cmd_resupdate() issues one
|
|
* follow-up xcecli_resval() query per submitted (resource, worker) pair --
|
|
* the exact client-library call resval/cmd_resval already use -- and
|
|
* compares the returned value against what was just sent.
|
|
*/
|
|
#define RESVAL_VERIFY_MAX_ATTEMPTS (5)
|
|
#define RESVAL_VERIFY_RETRY_MS (100L)
|
|
|
|
/* Compare a just-submitted value against a follow-up query's result. Both
|
|
* sides are xce_val_t (wire value union) -- a match requires the same
|
|
* `which` tag and an equal payload. */
|
|
static int resval_value_matches(xce_val_t* sent, xce_val_t* got)
|
|
{
|
|
if ((sent == NULL) || (got == NULL) || (sent->which != got->which)) {
|
|
return 0;
|
|
}
|
|
|
|
switch (sent->which) {
|
|
case Value_u_ivalue32:
|
|
return sent->u.ivalue32 == got->u.ivalue32;
|
|
case Value_u_uvalue32:
|
|
return sent->u.uvalue32 == got->u.uvalue32;
|
|
case Value_u_ivalue64:
|
|
return sent->u.ivalue64 == got->u.ivalue64;
|
|
case Value_u_uvalue64:
|
|
return sent->u.uvalue64 == got->u.uvalue64;
|
|
case Value_u_svalue:
|
|
return EQUAL_STRING(sent->u.svalue, got->u.svalue);
|
|
case Value_u_fvalue:
|
|
return float_equal(sent->u.fvalue, got->u.fvalue);
|
|
case Value_u_bvalue:
|
|
return (sent->u.bvalue == 0) == (got->u.bvalue == 0);
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Query a just-submitted (name, host) pair back from the controller and
|
|
* print its per-name accepted/rejected verdict (Requirements 1.2, 1.3). A
|
|
* short bounded retry guards against a genuinely-registered resource whose
|
|
* value hasn't propagated to the query path yet (design.md's Implementation
|
|
* Notes: sub-second in practice for a controller-local query) -- NOT an
|
|
* arbitrary long poll.
|
|
*/
|
|
static void resupdate_report_one(char* name, char* host, xce_val_t* sent)
|
|
{
|
|
int attempt;
|
|
int ever_found = 0;
|
|
|
|
if (name == NULL) {
|
|
/* set_resval()'s Value_u_svalue fallback path leaves .name unset
|
|
* (pre-existing gap predating this task -- see set_resval()'s own
|
|
* comment above); nothing usable to query or report here. */
|
|
return;
|
|
}
|
|
|
|
for (attempt = 0; attempt < RESVAL_VERIFY_MAX_ATTEMPTS; attempt++) {
|
|
xce_resval_t** got = NULL;
|
|
int rc;
|
|
|
|
if (attempt > 0) {
|
|
usleep((useconds_t)(RESVAL_VERIFY_RETRY_MS * US_PER_MS));
|
|
}
|
|
|
|
rc = xcecli_resval(active_ctx->ctx, host, name, &got);
|
|
if (rc > 0) {
|
|
int matched = resval_value_matches(sent, got[0]->val);
|
|
int i;
|
|
|
|
ever_found = 1;
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
free_ResourceVal_ptr(&got[i]);
|
|
}
|
|
FREEUP(got);
|
|
|
|
if (matched) {
|
|
printf("%s: accepted\n", name);
|
|
return;
|
|
}
|
|
/* Found but the value differs -- could still be a stale value
|
|
* about to be overwritten by the just-sent update's propagation,
|
|
* so keep retrying within the bounded budget rather than giving
|
|
* up immediately. */
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (ever_found) {
|
|
printf("%s: rejected: value mismatch\n", name);
|
|
} else {
|
|
printf("%s: rejected: unknown resource\n", name);
|
|
}
|
|
}
|
|
|
|
int cmd_resupdate(int argc, char* argv[])
|
|
{
|
|
#define MAX_N_RES (16)
|
|
int nres;
|
|
xce_resval_t vals[MAX_N_RES];
|
|
xce_resval_t* valptrs[MAX_N_RES];
|
|
char* subnames[MAX_N_RES];
|
|
char* resname = NULL;
|
|
char* resval_str = NULL;
|
|
int type_override = RESVAL_TYPE_UNKNOWN;
|
|
int opt;
|
|
int rc;
|
|
|
|
optind = 1;
|
|
nres = 0;
|
|
/* -t is deferred: it applies to the pair most recently opened by -n
|
|
* (design.md's documented order is -n/-v/[-t]), so a pending pair's
|
|
* set_resval() call is deferred until the NEXT -n or end-of-args --
|
|
* not fired immediately on -v as before -- so a trailing -t still has
|
|
* a chance to be seen for that same pair. */
|
|
while ((opt = getopt(argc, argv, "n:v:t:")) != -1) {
|
|
switch (opt) {
|
|
case 'n':
|
|
if (resname != NULL) {
|
|
if (resval_str == NULL) {
|
|
printf(
|
|
"res `%s' has no value specified, default will be used\n",
|
|
resname);
|
|
}
|
|
rc = set_resval(vals, nres, resname, resval_str, type_override);
|
|
if (rc >= 0) {
|
|
valptrs[nres] = &vals[nres];
|
|
/* Tracked independently of vals[nres].name: set_resval()
|
|
* leaves .name unset on the Value_u_svalue fallback path
|
|
* (see its own comment), but `resname` here is always the
|
|
* exact name that was actually submitted for this pair. */
|
|
subnames[nres] = resname;
|
|
nres++;
|
|
}
|
|
resname = NULL;
|
|
resval_str = NULL;
|
|
type_override = RESVAL_TYPE_UNKNOWN;
|
|
}
|
|
|
|
resname = strdup(optarg);
|
|
break;
|
|
case 'v':
|
|
if (resname != NULL && resval_str == NULL) {
|
|
resval_str = optarg;
|
|
} else {
|
|
printf("ignore res value `%s' as no res name found\n", optarg);
|
|
}
|
|
break;
|
|
case 't':
|
|
if (resname != NULL) {
|
|
if (strcasecmp(optarg, "numeric") == 0) {
|
|
type_override = Value_u_fvalue;
|
|
} else if (strcasecmp(optarg, "string") == 0) {
|
|
type_override = Value_u_svalue;
|
|
} else {
|
|
printf(
|
|
"ignoring unrecognized -t value `%s' (expected "
|
|
"`numeric' or `string')\n",
|
|
optarg);
|
|
}
|
|
} else {
|
|
printf("ignore -t `%s' as no res name found\n", optarg);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
/* Flush the last pending pair. Matches the pre-existing behavior of a
|
|
* trailing -n with no -v ever seen (silently dropped, same as before
|
|
* this change) -- only flush here when a value was actually supplied. */
|
|
if (resname != NULL && resval_str != NULL) {
|
|
rc = set_resval(vals, nres, resname, resval_str, type_override);
|
|
if (rc >= 0) {
|
|
valptrs[nres] = &vals[nres];
|
|
subnames[nres] = resname;
|
|
nres++;
|
|
}
|
|
}
|
|
|
|
rc = xcecli_update_resvals(active_ctx->ctx, nres, &valptrs[0]);
|
|
if (rc >= 0) {
|
|
int i;
|
|
|
|
printf("resource update ok!\n");
|
|
|
|
/* The update RPC's own reply can't carry per-name accept/reject
|
|
* (see the comment above resupdate_report_one()) -- verify each
|
|
* submitted pair individually via a follow-up query instead. */
|
|
for (i = 0; i < nres; i++) {
|
|
resupdate_report_one(subnames[i], vals[i].host, vals[i].val);
|
|
}
|
|
} else {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_update_resval");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cmd_join_sched(int argc, char* argv[])
|
|
{
|
|
int role = HostRole_scheduler;
|
|
int rc = -1;
|
|
|
|
rc = xcecli_join(active_ctx->ctx, role, NULL);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_join");
|
|
return rc;
|
|
}
|
|
|
|
printf("scheduler join ok\n");
|
|
return 0;
|
|
}
|
|
|
|
static int print_reslist(xce_reslist_t reslist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < reslist.count; i++) {
|
|
printf("name: %s\nflags: %d\ninterval: %d\ndesc: %s\n\n",
|
|
reslist.res[i]->name, reslist.res[i]->flags,
|
|
reslist.res[i]->interval, reslist.res[i]->desc);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int print_queuelist(xce_queuelist_t qlist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < qlist.count; i++) {
|
|
printf("name: %s\nstatus: %d\npriority: %d\n\n", qlist.queues[i]->queue,
|
|
qlist.queues[i]->status, qlist.queues[i]->priority);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int print_workerlist(xce_workerlist_t wlist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < wlist.count; i++) {
|
|
printf("name: %s\nstatus: %d\n\n", wlist.workers[i]->name,
|
|
wlist.workers[i]->status);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* bug-1711 (Requirement 11.8): print the already-fetched group records
|
|
* (conf->uglist / conf->hglist) instead of the previous no-op stub, which
|
|
* always left the "User Groups"/"Host Groups" header followed by silent
|
|
* nothing -- indistinguishable from a genuinely empty cluster. Mirrors
|
|
* print_reslist/print_queuelist/print_workerlist's own one-record-per-block
|
|
* convention just above. */
|
|
static int print_grouplist(xce_group_configlist_t glist)
|
|
{
|
|
int i, j;
|
|
|
|
for (i = 0; i < glist.count; i++) {
|
|
printf("name: %s\nmembers:", glist.groups[i]->name);
|
|
for (j = 0; j < glist.groups[i]->members.count; j++) {
|
|
printf(" %s", glist.groups[i]->members.data[j]);
|
|
}
|
|
printf("\nex_members:");
|
|
for (j = 0; j < glist.groups[i]->ex_members.count; j++) {
|
|
printf(" %s", glist.groups[i]->ex_members.data[j]);
|
|
}
|
|
printf("\n\n");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* bug-1711 (Requirement 11.8): print the already-fetched config records
|
|
* (conf->cfglist) instead of the previous no-op stub -- see print_grouplist
|
|
* above for the same fix applied to the "Configs" section. */
|
|
static int print_configlist(xce_configlist_t clist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < clist.count; i++) {
|
|
printf("name: %s\nvalue: %s\n\n", clist.configs[i]->name,
|
|
clist.configs[i]->value);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cmd_getconf(int argc, char* argv[])
|
|
{
|
|
xce_clusterconf_t* conf;
|
|
int rc;
|
|
|
|
rc = xcecli_clusterconf(active_ctx->ctx, &conf);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_clusterconf");
|
|
return rc;
|
|
}
|
|
|
|
printf("Cluster Name: %s\n", conf->name);
|
|
|
|
printf("\nResources\n===========\n");
|
|
print_reslist(conf->reslist);
|
|
|
|
printf("\nQueues\n========\n");
|
|
print_queuelist(conf->qlist);
|
|
|
|
printf("\nWorkers\n========\n");
|
|
print_workerlist(conf->wlist);
|
|
|
|
printf("\nUser Groups\n===========\n");
|
|
print_grouplist(conf->uglist);
|
|
|
|
printf("\nHost Groups\n============\n");
|
|
print_grouplist(conf->hglist);
|
|
|
|
printf("\nConfigs\n=======\n");
|
|
print_configlist(conf->cfglist);
|
|
|
|
free_ClusterConf(conf);
|
|
|
|
return 0;
|
|
}
|
|
#define KVAL(v) (((v) == NULL) ? "<not set>" : (v))
|
|
|
|
int cmd_set(int argc, char* argv[])
|
|
{
|
|
int i;
|
|
|
|
if (argc == 1) {
|
|
for (i = 0; active_ctx->settings[i].key; i++) {
|
|
printf("%-12s:%s default: %s\n", active_ctx->settings[i].key,
|
|
KVAL(active_ctx->settings[i].value),
|
|
KVAL(active_ctx->settings[i].defval));
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* errexit toggle (xce-test-e2e-primitives task 1.1), shell `-e`/`+e` style.
|
|
* Default is mode-dependent (batch `-e` abort-on-failure, interactive `+e`
|
|
* continue); these override it for the rest of the session. The session
|
|
* still exits non-zero if any command failed. Handled here ahead of the
|
|
* per-context key/value path. `continue-on-error` == `+e` (back-compat). */
|
|
if ((argc == 2) &&
|
|
((strcmp(argv[1], "+e") == 0) ||
|
|
(strcmp(argv[1], "continue-on-error") == 0))) {
|
|
g_continue_on_error = 1; /* +e: keep the read loop running on failure */
|
|
return 0;
|
|
}
|
|
if ((argc == 2) && (strcmp(argv[1], "-e") == 0)) {
|
|
g_continue_on_error = 0; /* -e: abort the read loop on first failure */
|
|
return 0;
|
|
}
|
|
|
|
if (argc != 3) {
|
|
printf("set <key> <value>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
for (i = 0; active_ctx->settings[i].key; i++) {
|
|
if (strcmp(active_ctx->settings[i].key, argv[1]) == 0) {
|
|
if (active_ctx->settings[i].value != NULL) {
|
|
free(active_ctx->settings[i].value);
|
|
}
|
|
|
|
active_ctx->settings[i].value = strdup(argv[2]);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
printf("Can not find setting key: %s\n", argv[1]);
|
|
return -EINVAL;
|
|
}
|
|
|
|
static int run_command(char* cmdline, pid_t* pid)
|
|
{
|
|
pid_t child;
|
|
|
|
child = fork();
|
|
|
|
*pid = child;
|
|
if (child < 0) {
|
|
*pid = -errno;
|
|
|
|
return -errno;
|
|
}
|
|
|
|
if (child == 0) {
|
|
execlp("/bin/sh", "/bin/sh", "-c", cmdline, NULL);
|
|
perror("execlp");
|
|
exit(-1);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int run_task(char* cmdline, pid_t* pid)
|
|
{
|
|
return run_command(cmdline, pid);
|
|
}
|
|
|
|
static int get_avail_run_job_slots()
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < MAX_RUN_JOBS; i++) {
|
|
if (run_jobs[i].jobid == 0) {
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
static int get_avail_run_task_slots()
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < MAX_RUN_TASKS; i++) {
|
|
if (run_tasks[i].pid <= 0) {
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
static int sched_on_jobs(int njobs, xce_job_t** jobs)
|
|
{
|
|
printf("%s %d jobs gotten\n", __func__, njobs);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_on_workers(int nworkers, xce_worker_t** workers)
|
|
{
|
|
printf("%s %d workers gotten\n", __func__, nworkers);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void notify_job_fail(uint64_t jobid, int code)
|
|
{
|
|
/* clang-format off */
|
|
job_failinfo_t failinfo = {
|
|
.jobid = jobid,
|
|
.stage = JOB_EXEC_STAGE_MAIN,
|
|
.code = code,
|
|
.detail = "no detail information"
|
|
};
|
|
/*clang-format on*/
|
|
|
|
xcecli_notify_jobfail(active_ctx->ctx, &failinfo);
|
|
return;
|
|
}
|
|
|
|
static void notify_task_fail(uint64_t jobid,
|
|
int clientidx,
|
|
int taskidx,
|
|
int code) {
|
|
xce_task_failinfo_t failinfo = {
|
|
.jobid = jobid,
|
|
.host = get_setting("myname"),
|
|
.clientidx = clientidx,
|
|
.taskidx = taskidx,
|
|
.code = code,
|
|
.detail = NULL
|
|
};
|
|
|
|
xcecli_notify_taskfail(active_ctx->ctx, &failinfo);
|
|
}
|
|
|
|
static void notify_jobstart(uint64_t jobid,
|
|
int pid,
|
|
int pgid) {
|
|
/* clang-format off */
|
|
job_pidinfo_t pidinfo = {
|
|
.jobid = jobid,
|
|
.npids = 1,
|
|
.pids = (unsigned int []){(unsigned int)pid},
|
|
.npgids = 0,
|
|
.pgids = (unsigned int []){(unsigned int)pgid},
|
|
.exdata = NULL
|
|
};
|
|
/* clang-format on */
|
|
|
|
xcecli_notify_jobstart(active_ctx->ctx, &pidinfo);
|
|
|
|
return;
|
|
}
|
|
|
|
static void notify_task_start(uint64_t jobid, int clientidx, int taskidx,
|
|
int pid, int pgid)
|
|
{
|
|
xce_task_pidinfo_t pidinfo
|
|
= { .jobid = jobid,
|
|
.host = get_setting("myname"),
|
|
.clientidx = clientidx,
|
|
.taskidx = taskidx,
|
|
.npids = 1,
|
|
.pids = (unsigned int[]) { (unsigned int)pid },
|
|
.npgids = 1,
|
|
.pgids = (unsigned int[]) { (unsigned int)pgid },
|
|
.exdata = NULL };
|
|
|
|
xcecli_notify_taskstart(active_ctx->ctx, &pidinfo);
|
|
|
|
return;
|
|
}
|
|
|
|
static int run_task_remote(uint64_t jobid, job_startinfo_t* startinfo,
|
|
xce_launch_t* launchinfo)
|
|
{
|
|
xce_taskhost_t hosts[launchinfo->nhosts - 1];
|
|
xce_taskrun_t taskrun;
|
|
int i;
|
|
int rc;
|
|
|
|
taskrun.taskhosts = (xce_taskhost_t**)calloc(launchinfo->nhosts - 1,
|
|
sizeof(xce_taskhost_t*));
|
|
if (taskrun.taskhosts == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
taskrun.clientidx = -1;
|
|
|
|
for (i = 1; i < launchinfo->nhosts; i++) {
|
|
hosts[i - 1].exechome = startinfo->exechome;
|
|
hosts[i - 1].execcwd = startinfo->execcwd;
|
|
hosts[i - 1].cmdline = startinfo->cmdline;
|
|
hosts[i - 1].exdata = startinfo->exdata;
|
|
hosts[i - 1].execuser = startinfo->execuser;
|
|
hosts[i - 1].launch_type = startinfo->launch_type;
|
|
hosts[i - 1].host = launchinfo->hosts[i]->host;
|
|
hosts[i - 1].ntasks = launchinfo->hosts[i]->nranks;
|
|
|
|
if (hosts[i - 1].launch_type == NULL) {
|
|
/* use mpi as default */
|
|
hosts[i - 1].launch_type = "mpi";
|
|
}
|
|
taskrun.taskhosts[i - 1] = &hosts[i - 1];
|
|
}
|
|
|
|
taskrun.nhosts = launchinfo->nhosts - 1;
|
|
taskrun.jobid = jobid;
|
|
|
|
xcecli_unblock_tasks(active_ctx->ctx);
|
|
|
|
rc = xcecli_runtask(active_ctx->ctx, &taskrun);
|
|
if (rc < 0) {
|
|
fprintf(stderr, "run remote task failed\n");
|
|
}
|
|
|
|
FREEUP(taskrun.taskhosts);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int run_task_local(uint64_t jobid, job_startinfo_t* startinfo,
|
|
int ntasks)
|
|
{
|
|
pid_t pid;
|
|
char jobidStr[64];
|
|
|
|
pid = fork();
|
|
if (pid < 0) {
|
|
return pid;
|
|
}
|
|
|
|
if (pid == 0) {
|
|
int i;
|
|
pid_t pids[ntasks];
|
|
int rc = 0;
|
|
int nlive = 0;
|
|
|
|
/* disable detection of address sanitizer as
|
|
* the request structure will not be freed in
|
|
* child and we will exit directly
|
|
*/
|
|
setenv("ASAN_OPTIONS", "detect_leaks=false", 1);
|
|
|
|
for (i = 0; i < ntasks; i++) {
|
|
rc = run_command(startinfo->cmdline, &pids[i]);
|
|
if (rc >= 0) {
|
|
nlive++;
|
|
}
|
|
}
|
|
|
|
if (rc >= 0) {
|
|
rc = 0;
|
|
}
|
|
|
|
while (nlive > 0) {
|
|
sleep(1);
|
|
for (i = 0; i < ntasks; i++) {
|
|
int wstatus;
|
|
|
|
if (pids[i] <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (waitpid(pids[i], &wstatus, WNOHANG) <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
|
|
nlive--;
|
|
|
|
if (WIFSIGNALED(wstatus)) {
|
|
rc = 128 + WTERMSIG(wstatus);
|
|
} else {
|
|
if (WEXITSTATUS(wstatus) != 0) {
|
|
rc = WEXITSTATUS(wstatus);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
exit(rc);
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, jobid);
|
|
printf("Job %s started, PID: %d\n", jobidStr, pid);
|
|
|
|
return pid;
|
|
}
|
|
|
|
int run_job(xce_job_exec_t* startmsg)
|
|
{
|
|
int rc = 0;
|
|
uint64_t jobid = startmsg->jobid;
|
|
job_startinfo_t* startinfo = startmsg->start_info;
|
|
xce_launch_t* launchinfo = startmsg->launchinfo;
|
|
|
|
printf("%s: exechome= %s\n", __func__,
|
|
(startinfo->exechome == NULL) ? "NULL" : startinfo->exechome);
|
|
printf("%s: execcwd= %s\n", __func__,
|
|
(startinfo->execcwd == NULL) ? "NULL" : startinfo->execcwd);
|
|
printf("%s: execuser= %s\n", __func__,
|
|
(startinfo->execuser == NULL) ? "NULL" : startinfo->execuser);
|
|
printf("%s: cmdline= %s\n", __func__,
|
|
(startinfo->cmdline == NULL) ? "NULL" : startinfo->cmdline);
|
|
printf("%s: exdata= %s\n", __func__,
|
|
(startinfo->exdata == NULL) ? "NULL" : startinfo->exdata);
|
|
printf("%s: launch_type= %s\n", __func__,
|
|
(startinfo->launch_type == NULL) ? "NULL" : startinfo->launch_type);
|
|
printf("%s: q_pre_exec= %s\n", __func__,
|
|
(startinfo->q_pre_exec == NULL) ? "NULL" : startinfo->q_pre_exec);
|
|
printf("%s: q_post_exec= %s\n", __func__,
|
|
(startinfo->q_post_exec == NULL) ? "NULL" : startinfo->q_post_exec);
|
|
printf("%s: job_pre_exec= %s\n", __func__,
|
|
(startinfo->job_pre_exec == NULL) ? "NULL"
|
|
: startinfo->job_pre_exec);
|
|
printf("%s: job_post_exec= %s\n", __func__,
|
|
(startinfo->job_post_exec == NULL) ? "NULL"
|
|
: startinfo->job_post_exec);
|
|
|
|
if (launchinfo->nhosts > 1) {
|
|
rc = run_task_remote(jobid, startinfo, launchinfo);
|
|
if (rc < 0) {
|
|
fprintf(stderr, "run_task_remote() fail, rc %d\n", rc);
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
rc = run_task_local(jobid, startinfo, launchinfo->hosts[0]->nranks);
|
|
if (rc < 0) {
|
|
notify_job_fail(jobid, rc);
|
|
} else {
|
|
int idx;
|
|
int pid;
|
|
|
|
pid = rc;
|
|
idx = get_avail_run_job_slots();
|
|
if (idx < 0) {
|
|
printf("too many jobs run in this worker\n");
|
|
} else {
|
|
run_jobs[idx].jobid = jobid;
|
|
run_jobs[idx].pid = pid;
|
|
run_jobs[idx].ts = 0;
|
|
}
|
|
|
|
notify_jobstart(jobid, pid, pid);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int handle_startjob(xce_request_t* request)
|
|
{
|
|
return run_job(request->body.jobstartmsg);
|
|
}
|
|
|
|
static int handle_notify(xce_request_t* request)
|
|
{
|
|
xce_notify_data_t* data = request->body.notifymsg;
|
|
char jobidStr[64];
|
|
|
|
switch (data->event) {
|
|
case NotifyData_data_taskPidInfo:
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, data->data.taskinfo->jobid);
|
|
printf("job %s host %s taskidx %d pid %d\n",
|
|
jobidStr, data->data.taskinfo->host,
|
|
data->data.taskinfo->taskidx, data->data.taskinfo->pids[0]);
|
|
break;
|
|
case NotifyData_data_taskFailInfo:
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1,
|
|
data->data.taskfailinfo->jobid);
|
|
printf("job %s host %s task %d error code %d\n",
|
|
jobidStr, data->data.taskfailinfo->host,
|
|
data->data.taskfailinfo->taskidx, data->data.taskfailinfo->code);
|
|
break;
|
|
case NotifyData_data_taskFinish:
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1,
|
|
data->data.taskfinish->jobid);
|
|
printf("job %s host %s task %d exit code %d\n",
|
|
jobidStr, data->data.taskfinish->host,
|
|
data->data.taskfinish->taskidx,
|
|
data->data.taskfinish->exit_code);
|
|
break;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int job_kill(uint64_t jobid, int signo)
|
|
{
|
|
int i;
|
|
int killed = 0;
|
|
|
|
for (i = 0; i < MAX_RUN_JOBS; i++) {
|
|
if ((run_jobs[i].jobid == 0) || (run_jobs[i].jobid != jobid)) {
|
|
continue;
|
|
}
|
|
|
|
if (run_jobs[i].pid > 1) {
|
|
kill(run_jobs[i].pid, signo);
|
|
killed++;
|
|
}
|
|
}
|
|
|
|
return killed;
|
|
}
|
|
|
|
static int job_chkpnt(uint64_t jobid, char* method, int kill)
|
|
{
|
|
char buf[128];
|
|
char info[256];
|
|
|
|
printf("checkpoint job %ld by method %s\n", jobid, method);
|
|
sprintf(buf, "/tmp/%ld/ckpt_dir", jobid);
|
|
|
|
/* do some actions to checkpoint jobs */
|
|
|
|
sprintf(info,
|
|
"{\nhost_select=\"criu\"; \nrestart_command=\"cd %s; criu restore "
|
|
"-vvv\"}\n",
|
|
buf);
|
|
xcecli_notify_jobchkpt(active_ctx->ctx, jobid, info);
|
|
|
|
if (kill) {
|
|
job_kill(jobid, SIGTERM);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int handle_jobcontrol(xce_request_t* request)
|
|
{
|
|
uint64_t jobid;
|
|
int signo;
|
|
xce_jobctrl_t* jobctrl = request->body.jobctrlmsg;
|
|
int rc = 0;
|
|
|
|
switch (jobctrl->action) {
|
|
case JobControl_u_siginfo:
|
|
jobid = jobctrl->jobid;
|
|
signo = jobctrl->u.siginfo->signo;
|
|
rc = job_kill(jobid, signo);
|
|
break;
|
|
case JobControl_u_ckptoption:
|
|
rc = job_chkpnt(jobctrl->jobid, jobctrl->u.ckptoption->method,
|
|
jobctrl->u.ckptoption->kill);
|
|
break;
|
|
default:
|
|
rc = -EINVAL;
|
|
ERROR("unknown job control action %d", jobctrl->action);
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int handle_runtasks(xce_request_t* request)
|
|
{
|
|
xce_taskrun_t* runmsg = request->body.runtaskmsg;
|
|
int rc = 0;
|
|
int i = 0;
|
|
char jobidStr[64];
|
|
#if 0
|
|
job_startinfo_t startinfo;
|
|
#endif
|
|
|
|
if (runmsg->nhosts != 1) {
|
|
/* something wrong */
|
|
rc = -EINVAL;
|
|
|
|
notify_task_fail(runmsg->jobid, runmsg->clientidx, -1, rc);
|
|
return rc;
|
|
}
|
|
|
|
if (strcmp(runmsg->taskhosts[0]->host, get_setting("myname")) != 0) {
|
|
/* something wrong */
|
|
rc = -EINVAL;
|
|
|
|
notify_task_fail(runmsg->jobid, runmsg->clientidx, -1, rc);
|
|
return rc;
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, runmsg->jobid);
|
|
printf("run task: jobid = %s\n", jobidStr);
|
|
printf("run task: user = %s\n", runmsg->taskhosts[0]->execuser);
|
|
printf("run task: cmdline = %s\n", runmsg->taskhosts[0]->cmdline);
|
|
printf("run task: exdata = %s\n", runmsg->taskhosts[0]->exdata);
|
|
printf("run task: ntasks =%d\n", runmsg->taskhosts[0]->ntasks);
|
|
|
|
#if 0
|
|
startinfo.exechome = runmsg->taskhosts[0]->exechome;
|
|
startinfo.execcwd = runmsg->taskhosts[0]->execcwd;
|
|
startinfo.execuser = runmsg->taskhosts[0]->execuser;
|
|
startinfo.cmdline = runmsg->taskhosts[0]->cmdline;
|
|
startinfo.exdata = runmsg->taskhosts[0]->exdata;
|
|
startinfo.launch_type = runmsg->taskhosts[0]->launch_type;
|
|
#endif
|
|
|
|
for (i = 0; i < runmsg->taskhosts[0]->ntasks; i++) {
|
|
int pid = 0;
|
|
int pgid = 0;
|
|
|
|
rc = run_task(runmsg->taskhosts[0]->cmdline, &pid);
|
|
if (rc < 0) {
|
|
notify_task_fail(runmsg->jobid, runmsg->clientidx, i, rc);
|
|
} else {
|
|
int idx;
|
|
|
|
printf("job %s task %d started, PID: %d\n", jobidStr, i, pid);
|
|
|
|
idx = get_avail_run_task_slots();
|
|
if (idx < 0) {
|
|
printf("too many running tasks\n");
|
|
} else {
|
|
run_tasks[idx].jobid = runmsg->jobid;
|
|
run_tasks[idx].taskid = i;
|
|
run_tasks[idx].clientidx = runmsg->clientidx;
|
|
run_tasks[idx].pid = pid;
|
|
run_tasks[idx].ts = 0;
|
|
}
|
|
notify_task_start(runmsg->jobid, runmsg->clientidx, i, pid, pgid);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int myreq_handler(xce_request_t* request, void* args)
|
|
{
|
|
int rc = 0;
|
|
|
|
switch (request->opcode) {
|
|
case OPCode_push:
|
|
if (request->body_which == Request_body_datamsg) {
|
|
dataset_t* datamsg = request->body.datamsg;
|
|
|
|
switch (datamsg->table) {
|
|
case XceData_data_schedjobs:
|
|
case XceData_data_jobs:
|
|
sched_on_jobs(datamsg->nrows, (xce_job_t**)datamsg->rows);
|
|
break;
|
|
case XceData_data_schedworkers:
|
|
case XceData_data_workers:
|
|
sched_on_workers(datamsg->nrows, (xce_worker_t**)datamsg->rows);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
case OPCode_startjob:
|
|
rc = handle_startjob(request);
|
|
break;
|
|
case OPCode_notify:
|
|
rc = handle_notify(request);
|
|
break;
|
|
case OPCode_jobcontrol:
|
|
rc = handle_jobcontrol(request);
|
|
break;
|
|
case OPCode_runtasks:
|
|
rc = handle_runtasks(request);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static void check_run_jobs()
|
|
{
|
|
int i;
|
|
time_t now = time(NULL);
|
|
|
|
for (i = 0; i < MAX_RUN_JOBS; i++) {
|
|
int wstatus;
|
|
|
|
if (run_jobs[i].jobid == 0) {
|
|
continue;
|
|
}
|
|
|
|
if (run_jobs[i].pid <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (waitpid(run_jobs[i].pid, &wstatus, WNOHANG | WUNTRACED | WCONTINUED)
|
|
<= 0) {
|
|
/* fake job rusage */
|
|
int npids = 1;
|
|
int npgids = 0;
|
|
int nusage = 2;
|
|
/* bug-1735: match the real worker's collect_job_rusages()
|
|
* resnames (xcecd_cgroups.c), not the old never-matching
|
|
* "cputime"/"mem" strings. */
|
|
xce_rusage_t data[] = {
|
|
{ "cpu_time_sec", 100.0 },
|
|
{ "memory_rss", 200.0 }
|
|
};
|
|
xce_rusage_t* usages[] = { &(data[0]), &(data[1]) };
|
|
|
|
if (now - run_jobs[i].ts > RUSAGE_UPDATE_INTERVAL) {
|
|
xcecli_update_jusage(active_ctx->ctx, run_jobs[i].jobid, npids,
|
|
&(run_jobs[i].pid), npgids, NULL, nusage,
|
|
&usages[0]);
|
|
run_jobs[i].ts = now;
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
if (WIFSTOPPED(wstatus)) {
|
|
xcecli_notify_jobstop(active_ctx->ctx, run_jobs[i].jobid);
|
|
} else if (WIFCONTINUED(wstatus)) {
|
|
xcecli_notify_jobresume(active_ctx->ctx, run_jobs[i].jobid);
|
|
} else if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
|
|
xce_jobfinish_t finish;
|
|
char jobidStr[64];
|
|
|
|
finish.jobid = run_jobs[i].jobid;
|
|
|
|
if (WIFEXITED(wstatus)) {
|
|
finish.exit_code = WEXITSTATUS(wstatus);
|
|
} else {
|
|
finish.exit_code = 128 + WTERMSIG(wstatus);
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, finish.jobid);
|
|
printf("job %s exit code %d\n", jobidStr, finish.exit_code);
|
|
xcecli_notify_jobfinish(active_ctx->ctx, &finish);
|
|
run_jobs[i].jobid = 0;
|
|
run_jobs[i].pid = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void check_run_tasks()
|
|
{
|
|
int i;
|
|
time_t now = time(NULL);
|
|
|
|
for (i = 0; i < MAX_RUN_TASKS; i++) {
|
|
int wstatus;
|
|
|
|
if (run_tasks[i].jobid == 0) {
|
|
continue;
|
|
}
|
|
|
|
if (run_tasks[i].pid <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (waitpid(run_tasks[i].pid, &wstatus,
|
|
WNOHANG | WUNTRACED | WCONTINUED)
|
|
<= 0) {
|
|
/* fake job rusage */
|
|
int npids = 1;
|
|
int npgids = 0;
|
|
int nusage = 2;
|
|
/* bug-1735: match the real worker's collect_job_rusages()
|
|
* resnames (xcecd_cgroups.c), not the old never-matching
|
|
* "cputime"/"mem" strings. */
|
|
xce_rusage_t data[] = {
|
|
{ "cpu_time_sec", 100.0 },
|
|
{ "memory_rss", 200.0 }
|
|
};
|
|
xce_rusage_t* usages[] = { &(data[0]), &(data[1]) };
|
|
|
|
if (now - run_tasks[i].ts > RUSAGE_UPDATE_INTERVAL) {
|
|
xcecli_update_jusage(active_ctx->ctx, run_tasks[i].jobid, npids,
|
|
&(run_tasks[i].pid), npgids, NULL, nusage,
|
|
&usages[0]);
|
|
run_tasks[i].ts = now;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
|
|
xce_task_finish_t finish = { .jobid = run_tasks[i].jobid,
|
|
.host = get_setting("myname"),
|
|
.taskidx = run_tasks[i].taskid,
|
|
.clientidx = run_tasks[i].clientidx,
|
|
.exit_code = 0 };
|
|
char jobidStr[64];
|
|
|
|
if (WIFEXITED(wstatus)) {
|
|
finish.exit_code = WEXITSTATUS(wstatus);
|
|
} else {
|
|
finish.exit_code = 128 + WTERMSIG(wstatus);
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, finish.jobid);
|
|
printf("job %s task %d exit code %d\n", jobidStr,
|
|
finish.taskidx, finish.exit_code);
|
|
xcecli_notify_taskfinish(active_ctx->ctx, &finish);
|
|
run_tasks[i].jobid = 0;
|
|
run_tasks[i].pid = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
#define HEARTBEAT_INTERVAL (60)
|
|
|
|
static int worker_house_keeping(void* arg)
|
|
{
|
|
command_ctx_t* ctx = (command_ctx_t*)arg;
|
|
int rc = 0;
|
|
|
|
xce_use(ctx);
|
|
|
|
check_run_jobs();
|
|
check_run_tasks();
|
|
|
|
rc = xcecli_heartbeat(ctx->ctx, HEARTBEAT_INTERVAL);
|
|
if (rc < 0) {
|
|
perror("xcecli_heartbeat");
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
struct arguments arguments;
|
|
char* buf;
|
|
char* logfile = "/tmp/xce_test.log";
|
|
char* loglevel = "info";
|
|
int logsize = 64 * 1024 * 1024;
|
|
int logcount = 10;
|
|
/* build-time-versioning task 3.2: xce_test's own --version/-V output
|
|
* (via argp_program_version) -- static so it outlives this point in
|
|
* main() (contract documented on xce_compose_version() in utils.c). */
|
|
static char version_buf[128];
|
|
|
|
memset(&arguments, 0x0, sizeof(arguments));
|
|
|
|
/* dirty-content-fingerprint task 3.2: xce_test is not a daemon -- it has
|
|
* no startup log line at all (see anatomy/build-time-versioning notes),
|
|
* so there is no second call site to keep divergent here (unlike
|
|
* controller/main.c and scheduler/src/main.c, which each keep a
|
|
* separate, untouched xce_compose_version() buffer for their startup
|
|
* INFO log per Req 3.2). version_buf can safely be composed directly
|
|
* via xce_compose_version_ex() to pick up the dirty_fingerprint suffix.
|
|
* Must run before argp_parse(): argp reads argp_program_version by
|
|
* pointer when --version/-V is processed. */
|
|
xce_compose_version_ex(version_buf, sizeof(version_buf), "xce v0.0.1-",
|
|
STR(VERSION_COMMIT_ID), STR(VERSION_BUILD_TIME),
|
|
STR(VERSION_DIRTY_FINGERPRINT));
|
|
argp_program_version = version_buf;
|
|
|
|
argp_parse(&argp, argc, argv, 0, 0, &arguments);
|
|
|
|
if (arguments.verbose != 0) {
|
|
loglevel = verbose_to_loglevel(arguments.verbose);
|
|
}
|
|
|
|
/* Remember the user's level so wait-primary can restore it after temporarily
|
|
* raising the threshold to silence its poll-loop failover spam. */
|
|
g_active_loglevel = loglevel;
|
|
|
|
mylog_init(logfile, loglevel, logsize, logcount);
|
|
|
|
/* xce-test-cli-ux task 1.3, Requirement 9.9: replaces the former bare
|
|
* `signal(SIGINT, sigbreak);` call. glibc's signal() installs with
|
|
* SA_RESTART SET (BSD semantics), which silently restarts a blocking
|
|
* slow syscall (read(), etc.) interrupted by SIGINT instead of
|
|
* returning EINTR -- see xcetest_interrupt.h's file header for the
|
|
* full rationale. xcetest_install_sigint() installs the same
|
|
* sigbreak() handler via sigaction() with SA_RESTART explicitly
|
|
* cleared, so a later blocking read (e.g. the confirmation prompt
|
|
* built on top of this task) reliably observes EINTR. */
|
|
xcetest_install_sigint(sigbreak);
|
|
|
|
memset(&run_jobs, 0x0, sizeof(run_jobs));
|
|
memset(&run_tasks, 0x0, sizeof(run_tasks));
|
|
|
|
idgen_ctx_init(&idctx);
|
|
|
|
/* bug-1420 follow-up: folding pid into apiserverId alone (4 bits, 16
|
|
* buckets) at the jsub call site still collides too often across this
|
|
* harness's rapid, per-invocation-fresh xce_test processes (confirmed
|
|
* empirically -- a 10-round chaotic soak still produced same-second
|
|
* duplicate jobids after that fix landed). idgen_ctx_init() always
|
|
* resets sequence to 0, and MAKE_JOBID's sequence field alone has 20
|
|
* bits (config.h N_SEQUENCE_BITS) of headroom -- two orders of
|
|
* magnitude more than the namespacing fields. Re-seed it here with a
|
|
* value that mixes pid (stable per process) with monotonic-clock
|
|
* nanoseconds (varies even across processes launched back-to-back in
|
|
* the same wall-clock second, unlike idgen's own second-resolution
|
|
* timestamp), masked to N_SEQUENCE_BITS so it can never bleed into the
|
|
* adjacent tag/timestamp bits MAKE_JOBID packs beside it. idgen()'s own
|
|
* ROUNDUP-to-N_ARRAY_IDX_BITS logic for array jobs still applies
|
|
* unchanged on top of this seed. */
|
|
{
|
|
struct timespec ts;
|
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
idctx.sequence = ((uint32_t)getpid() ^ (uint32_t)ts.tv_nsec)
|
|
& (uint32_t)UMAX_BITS(N_SEQUENCE_BITS);
|
|
}
|
|
|
|
/* errexit default, modelled on shell `-e`/`+e`, keyed on the stdin mode:
|
|
* batch (piped/redirected stdin) → `-e`: abort the read loop on the first
|
|
* failing command and exit non-zero (a script wants fail-fast + verdict).
|
|
* interactive (TTY) REPL → `+e`: a failed command prints its error and the
|
|
* loop CONTINUES (never exit on a typo).
|
|
* Override at any time with `set -e` / `set +e` (or `set continue-on-error`). */
|
|
g_interactive = isatty(STDIN_FILENO);
|
|
g_continue_on_error = g_interactive ? 1 : 0;
|
|
|
|
/* xce-test-cli-ux task 2.3, Requirement 7: register command-name tab
|
|
* completion, but ONLY for a genuinely interactive session -- design.md's
|
|
* explicit precondition on xcetest_complete(). A piped/redirected/
|
|
* scripted stdin never calls readline()'s completion machinery anyway
|
|
* (no terminal to press Tab on), but gating the registration itself
|
|
* (rather than relying on that) keeps the precondition explicit and
|
|
* matches this file's existing g_interactive-gated conventions. */
|
|
if (g_interactive) {
|
|
rl_attempted_completion_function = xcetest_complete;
|
|
}
|
|
|
|
/* xce-test-rc-file task 2.2: auto-load $XDG_CONFIG_HOME/xce_test/rc (or
|
|
* $HOME/.config/xce_test/rc) before the read loop starts, for both
|
|
* interactive and scripted sessions alike -- unless --norc was given.
|
|
* A missing/unresolvable rc file is silent (Req 1.3); a failing rc-file
|
|
* line is handled entirely by preprocess_and_run's existing abort/continue
|
|
* accounting (Req 3) -- an abort short-circuits straight to the same
|
|
* exit-code convention used at the bottom of this function, skipping the
|
|
* read loop entirely (Req 1.4/3.3). */
|
|
if (!arguments.norc) {
|
|
char* rcpath = rcfile_resolve_path();
|
|
|
|
if (rcpath != NULL) {
|
|
int rcrc = rcfile_load(rcpath, preprocess_and_run);
|
|
|
|
free(rcpath);
|
|
|
|
if (rcrc) {
|
|
return g_session_failed ? 1 : 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
while (!done) {
|
|
char* s;
|
|
int rc;
|
|
|
|
prompt[0] = 0x0;
|
|
fill_prompt(active_ctx, prompt);
|
|
strcat(prompt, ">");
|
|
buf = readline(prompt);
|
|
|
|
if (buf == NULL) {
|
|
execute_line("quit");
|
|
continue;
|
|
}
|
|
|
|
if (strlen(buf) == 0) {
|
|
free(buf);
|
|
continue;
|
|
}
|
|
|
|
s = buf;
|
|
|
|
rc = preprocess_and_run(s);
|
|
free(buf);
|
|
if (rc) {
|
|
break; /* abort-on-failure requested loop stop */
|
|
}
|
|
}
|
|
|
|
return g_session_failed ? 1 : 0;
|
|
}
|