forked from lei.chen/myxce
Rongsong Shen
f28ce57993
jobid_range_exists() (c->jobmgr) and the in-flight registry (job.c,
per-process, only populated by this node's own handle_jobsubmit calls)
both miss a real cross-node case: a client's XCECLI_FAILOVER_CALL retry
resends the same client-generated jobid after a connection drop, and the
resend lands on a newly elected primary that never processed the original
request. If the original was already committed before the old primary
died, the new primary's own local WAL already holds it (Raft Leader
Completeness) but jobmgr hasn't applied it yet — both existing guards see
nothing, so the resend appends a second, genuinely committed WAL_JOB_SUBMIT
entry for the same jobid. A live chaotic soak reproduced this exactly
(waldump: two committed entries, same term, matching apiserverId/timestamp/
sequence, one second apart) — apply's own idempotent dedup keeps jobmgr
correct, but replay's raw WAL_JOB_SUBMIT count no longer matches jobmgr's
growth, firing a false APPLY_DIVERGENT fence and deadlocking promotion.
Fix: apply_committed_entry()'s WAL_JOB_SUBMIT case now claims the jobid in
job.c's in-flight registry (new job_wal_commit_claim/release, wrapping the
existing inflight_range_try_add/remove) for the duration of dispatch,
regardless of whether this node originated the submission or is applying a
replicated/replayed entry. A resend landing mid-dispatch on any node now
sees the same -EEXIST rejection self-originated in-flight submits already
get. Release only fires when the claim call added the entry, so a
self-originated submission's own claim/release lifecycle (unchanged) is
never double-released or stolen.
Also fixes 4 pre-existing tests that broke under the earlier checkpoint_lsn
fix (0eb1054) and were missed in that commit's regression sweep: cluster-
less fixtures where last_applied_lsn never advances now correctly get
checkpoint_lsn=0 unless the test simulates apply catch-up, matching the
new (correct) semantics. One assertion in
test_checkpoint_tail_with_no_snapshot_is_apply_clean compared Phase 1's
checkpoint_lsn against Phase 2's independently-reconstructed physical WAL
tip — that equality only held under the pre-fix current_lsn-tracking
behavior and is removed, not patched around.
Also hardens run-ha-soak.sh's start_node() with setsid+nohup: a forked/
background test-harness invocation whose session tears down mid-run was
silently killing every controller process via SIGHUP forwarding (observed
twice — deploy completes, ~1s of activity, then all 3 nodes die together
with no error logged).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VgT3n4vTv4ZjCuCTmzed1F
3677 lines
111 KiB
C
3677 lines
111 KiB
C
#include <argp.h>
|
|
#include <errno.h>
|
|
#include <getopt.h>
|
|
#include <inttypes.h>
|
|
#include <signal.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/wait.h>
|
|
#include <time.h>
|
|
#include <unistd.h>
|
|
|
|
#include <readline/history.h>
|
|
#include <readline/readline.h>
|
|
|
|
#include <cjson/cJSON.h>
|
|
#include <cjson/s2j.h>
|
|
|
|
#include "client.h"
|
|
#include "config.h" /* XCE_MAXNAME_LEN */
|
|
#include "controller.capnp.h"
|
|
#include "ctrlstatus_fmt.h" /* ctrlstatus_format_time_ms (task 6.3) */
|
|
#include "delay.h"
|
|
#include "idgen.h"
|
|
#include "preason.h"
|
|
#include "walcompare.h" /* cross-member committed-prefix divergence (cmd_walcompare) */
|
|
#include "waldump.h" /* offline WAL inspector (cmd_waldump) */
|
|
#include "snapdiff.h" /* byte-level checkpoint-file comparator (cmd_snapdiff) */
|
|
#include "snapverify.h" /* offline checkpoint-file structural validator (cmd_snapverify) */
|
|
#include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */
|
|
#include "xce/types.h"
|
|
|
|
typedef int (*cmd_func_t)(int, char*[]);
|
|
|
|
typedef struct {
|
|
char* name;
|
|
cmd_func_t func;
|
|
char* doc;
|
|
} command_t;
|
|
|
|
typedef struct {
|
|
char* key;
|
|
char* value;
|
|
char* defval;
|
|
} setting_t;
|
|
|
|
typedef struct _command_ctx {
|
|
struct _command_ctx* parent;
|
|
char* prompt;
|
|
setting_t* settings;
|
|
command_t* commands;
|
|
xcecli_ctx_t ctx;
|
|
} command_ctx_t;
|
|
|
|
typedef struct {
|
|
uint64_t jobid;
|
|
int pid;
|
|
time_t ts;
|
|
} run_job_t;
|
|
|
|
typedef struct {
|
|
uint64_t jobid;
|
|
uint32_t taskid;
|
|
int clientidx;
|
|
int pid;
|
|
time_t ts;
|
|
} run_task_t;
|
|
|
|
typedef struct {
|
|
char* host;
|
|
int ntasks;
|
|
} host_task_t;
|
|
|
|
typedef struct {
|
|
char* PATH;
|
|
} sub_envs_t;
|
|
|
|
typedef struct {
|
|
sub_envs_t sub_env_vars;
|
|
} exdata_t;
|
|
|
|
#define MAX_RUN_JOBS (20)
|
|
#define MAX_TASK_HOSTS (5)
|
|
#define MAX_RUN_TASKS (50)
|
|
#define RUSAGE_UPDATE_INTERVAL (60)
|
|
#define MAX_LINE_LEN (40960)
|
|
|
|
#define S2J_JSON_SET_var_string_ELEMENT(to_json, from_struct, _element) \
|
|
if (((from_struct) != NULL) && (((from_struct)->_element) != NULL)) { \
|
|
cJSON_AddStringToObject((to_json), #_element, \
|
|
(from_struct)->_element); \
|
|
}
|
|
|
|
static run_job_t run_jobs[MAX_RUN_JOBS];
|
|
static run_task_t run_tasks[MAX_RUN_TASKS];
|
|
|
|
static int myreq_handler(xce_request_t*, void*);
|
|
|
|
extern int cmd_help(int, char*[]);
|
|
extern int cmd_up(int, char*[]);
|
|
extern int cmd_worker(int, char*[]);
|
|
extern int cmd_scheduler(int, char*[]);
|
|
extern int cmd_client(int, char*[]);
|
|
extern int cmd_quit(int, char*[]);
|
|
extern int cmd_hello(int, char*[]);
|
|
extern int cmd_shutdown(int, char*[]);
|
|
extern int cmd_set(int, char*[]);
|
|
extern int cmd_reslist(int, char*[]);
|
|
extern int cmd_join_worker(int, char*[]);
|
|
extern int cmd_jobstart(int, char*[]);
|
|
extern int cmd_jobfail(int, char*[]);
|
|
extern int cmd_join_sched(int, char*[]);
|
|
extern int cmd_workerlist(int, char*[]);
|
|
extern int cmd_joblist(int, char*[]);
|
|
extern int cmd_jobrun(int, char*[]);
|
|
extern int cmd_jobsub(int, char*[]);
|
|
extern int cmd_getconf(int, char*[]);
|
|
extern int cmd_jobfinish(int, char*[]);
|
|
extern int cmd_resupdate(int, char*[]);
|
|
extern int cmd_resval(int, char*[]);
|
|
extern int cmd_jobkill(int, char*[]);
|
|
extern int cmd_jobckpt(int, char*[]);
|
|
extern int cmd_qctrl(int, char*[]);
|
|
extern int cmd_runtasks(int, char*[]);
|
|
extern int cmd_setpend(int, char*[]);
|
|
extern int cmd_jobinfo(int, char*[]);
|
|
extern int cmd_echo(int, char*[]);
|
|
extern int cmd_expect_jobs(int, char*[]);
|
|
extern int cmd_expect_job(int, char*[]);
|
|
extern int cmd_wait_primary(int, char*[]);
|
|
extern int cmd_ctrlstatus(int, char*[]);
|
|
extern int cmd_waldump(int, char*[]);
|
|
extern int cmd_walcompare(int, char*[]);
|
|
extern int cmd_snapverify(int, char*[]);
|
|
extern int cmd_snapdiff(int, char*[]);
|
|
static int worker_house_keeping(void*);
|
|
|
|
/* Session verdict / loop state (xce-test-e2e-primitives task 1.1).
|
|
*
|
|
* g_session_failed set to 1 the first time any command in the session
|
|
* fails; drives the process exit code in main().
|
|
* g_continue_on_error when set (via `set continue-on-error`), a failing
|
|
* command does not stop the read loop; the session still
|
|
* exits non-zero at the end.
|
|
* g_last_status status of the most recently executed command
|
|
* (0 == success, non-zero == failure). Future tasks
|
|
* expose this as the `$?` auto-variable.
|
|
*/
|
|
static int g_session_failed = 0;
|
|
static int g_continue_on_error = 0;
|
|
static int g_last_status = 0;
|
|
/* Interactive (TTY) sessions are a REPL: a failed command prints its error and
|
|
* the loop CONTINUES — never abort the read loop on a typo. The abort-on-first-
|
|
* failure + non-zero-exit verdict is a SCRIPTED/piped-mode concept only. Set
|
|
* from isatty(stdin) in main. */
|
|
static int g_interactive = 0;
|
|
|
|
/* Auto-variable state for the fixed `$jobid` / `$jobcount` substitution
|
|
* (xce-test-e2e-primitives task 3.1, Req 3.1/3.3/3.5/3.6).
|
|
*
|
|
* g_last_jobid most recently acknowledged JobId as a string; empty ("")
|
|
* means unset. Populated later by cmd_jobsub (task 4.1).
|
|
* g_last_jobcount most recent job count; -1 means unset. Populated by
|
|
* cmd_joblist on a successful `joblist` (task 5.1, Req 3.4);
|
|
* a failed joblist leaves it untouched so no fake count is
|
|
* advertised.
|
|
*
|
|
* Until a command sets one, it stays unset — which is exactly what makes an
|
|
* unset `$jobid`/`$jobcount` reference fail the command (Req 3.6). */
|
|
static char g_last_jobid[XCE_MAXNAME_LEN] = { 0 };
|
|
static int g_last_jobcount = -1;
|
|
|
|
static idgen_ctx_t idctx;
|
|
delaymgr_t delaymgr;
|
|
|
|
static setting_t settings[] = {
|
|
/* HA controller list: comma-separated "name=host:port" or "host:port"
|
|
* entries (same syntax as the controller --peers flag), enabling the
|
|
* test client to fail over across controller candidates. */
|
|
{ (char*)"masters", (char*)NULL, (char*)"localhost:9990" },
|
|
{ (char*)"secret", (char*)NULL, (char*)"keepSecret" },
|
|
{ (char*)"myname", (char*)NULL, (char*)"localhost" },
|
|
{ (char*)NULL, (char*)NULL, (char*)NULL }
|
|
};
|
|
|
|
static command_t top_commands[] = {
|
|
{ "help", cmd_help, "Display this text" },
|
|
{ "worker", cmd_worker, "In worker context" },
|
|
{ "scheduler", cmd_scheduler, "In scheduler context" },
|
|
{ "client", cmd_client, "In client context" },
|
|
{ "set", cmd_set, "Show/Set settings" },
|
|
{ "echo", cmd_echo, "Print a message" },
|
|
{ "waldump", cmd_waldump, "Inspect WAL segments" },
|
|
{ "walcompare", cmd_walcompare, "Diff member committed WAL prefixes" },
|
|
{ "snapverify", cmd_snapverify, "Verify checkpoint file structure" },
|
|
{ "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files" },
|
|
{ "quit", cmd_quit, "Quit xce test" },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t topctx = { .parent = NULL,
|
|
.prompt = "xce",
|
|
.settings = settings,
|
|
.commands = top_commands };
|
|
|
|
static command_t worker_commands[] = {
|
|
{ "..", cmd_up, "Goto upper context" },
|
|
{ "help", cmd_help, "Display this text" },
|
|
{ "quit", cmd_quit, "Quit xce test" },
|
|
{ "join", cmd_join_worker, "Join the cluster" },
|
|
{ "jstart", cmd_jobstart, "tell controller jobstart" },
|
|
{ "jfinish", cmd_jobfinish, "tell controller jobfinish" },
|
|
{ "jfail", cmd_jobfail, "tell controller jobfail" },
|
|
{ "resupdate", cmd_resupdate, "update res vals" },
|
|
{ "echo", cmd_echo, "Print a message" },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t workerctx = { .parent = &topctx,
|
|
.prompt = "worker",
|
|
.settings = settings,
|
|
.commands = worker_commands };
|
|
|
|
static command_t client_commands[] = {
|
|
{ "..", cmd_up, "Goto upper context" },
|
|
{ "hello", cmd_hello, "Say hello to xce controller" },
|
|
{ "reslist", cmd_reslist, "Display cluster resources" },
|
|
{ "workerlist", cmd_workerlist, "list workers" },
|
|
{ "qctrl", cmd_qctrl, "queue control" },
|
|
{ "joblist", cmd_joblist, "list jobs" },
|
|
{ "expect-jobs", cmd_expect_jobs, "assert current job count" },
|
|
{ "expect-job", cmd_expect_job, "assert a JobId is present" },
|
|
{ "wait-primary", cmd_wait_primary,
|
|
"wait until a primary serves [timeout_ms]" },
|
|
{ "ctrlstatus", cmd_ctrlstatus, "show controller/cluster status" },
|
|
{ "waldump", cmd_waldump, "Inspect WAL segments" },
|
|
{ "walcompare", cmd_walcompare, "Diff member committed WAL prefixes" },
|
|
{ "snapverify", cmd_snapverify, "Verify checkpoint file structure" },
|
|
{ "snapdiff", cmd_snapdiff, "Byte-diff two checkpoint files" },
|
|
{ "shutdown", cmd_shutdown, "shutdown xce controller" },
|
|
{ "jobrun", cmd_jobrun, "run a job" },
|
|
{ "jobsub", cmd_jobsub, "submit a job" },
|
|
{ "jobkill", cmd_jobkill, "signal a job" },
|
|
{ "jobckpt", cmd_jobckpt, "checkpoint a job" },
|
|
{ "jobinfo", cmd_jobinfo, "job information" },
|
|
{ "resval", cmd_resval, "get value of res" },
|
|
{ "taskrun", cmd_runtasks, "run tasks" },
|
|
{ "help", cmd_help, "Display this text" },
|
|
{ "echo", cmd_echo, "Print a message" },
|
|
{ "quit", cmd_quit, "Quit xce test" },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t clientctx = { .parent = &topctx,
|
|
.prompt = "client",
|
|
.settings = settings,
|
|
.commands = client_commands };
|
|
|
|
static command_t scheduler_commands[] = {
|
|
{ "..", cmd_up, "Goto upper context" },
|
|
{ "help", cmd_help, "Display this text" },
|
|
{ "quit", cmd_quit, "Quit xce test" },
|
|
{ "join", cmd_join_sched, "Join the cluster" },
|
|
{ "getconf", cmd_getconf, "Get conf of cluster" },
|
|
{ "setpend", cmd_setpend, "set job pend reason" },
|
|
{ "echo", cmd_echo, "Print a message" },
|
|
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
|
};
|
|
|
|
static command_ctx_t schedulerctx = { .parent = &topctx,
|
|
.prompt = "scheduler",
|
|
.settings = settings,
|
|
.commands = scheduler_commands };
|
|
|
|
static command_ctx_t* active_ctx = &topctx;
|
|
|
|
static char prompt[256];
|
|
static int break_out = 0;
|
|
|
|
static char doc[] = "xce_test";
|
|
static char args_doc[] = "";
|
|
|
|
const char* argp_program_version
|
|
= MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID));
|
|
const char* argp_program_bug_address = BUG_EMAIL;
|
|
|
|
struct arguments {
|
|
int verbose;
|
|
};
|
|
|
|
static struct argp_option options[] = {
|
|
{ "verbose", 'v', 0, 0, "Produce verbose logs" },
|
|
{ 0 }
|
|
};
|
|
|
|
static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
|
{
|
|
struct arguments* arguments = state->input;
|
|
|
|
switch (key) {
|
|
case 'v':
|
|
arguments->verbose++;
|
|
break;
|
|
default:
|
|
return ARGP_ERR_UNKNOWN;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct argp argp = { options, parse_opt, args_doc, doc };
|
|
|
|
void sigbreak(int signo) { break_out = 1; }
|
|
|
|
static char* get_setting(char* key)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; active_ctx->settings[i].key; i++) {
|
|
if (strcmp(key, active_ctx->settings[i].key) != 0) {
|
|
continue;
|
|
}
|
|
|
|
if (active_ctx->settings[i].value != NULL) {
|
|
return active_ctx->settings[i].value;
|
|
}
|
|
|
|
return active_ctx->settings[i].defval;
|
|
}
|
|
|
|
printf("Key `%s' is not in settings\n", key);
|
|
return NULL;
|
|
}
|
|
|
|
static xce_client_config_t config;
|
|
|
|
/* Holds the parsed HA controller list. Static so the hostname[] buffers it
|
|
* owns outlive enter_ctx(): xcecli_ctx_init() stores the config pointer (it
|
|
* does not copy masters[]), so config.masters[i] must reference storage that
|
|
* persists for the lifetime of the client context. */
|
|
static xce_cluster_config_t master_cfg;
|
|
|
|
int enter_ctx(command_ctx_t* ctx)
|
|
{
|
|
char* masters = get_setting("masters");
|
|
int i;
|
|
|
|
/* Parse the controller HA peer list with the shared libxce parser — the
|
|
* same "name=host:port,..."/"host:port,..." syntax used by the controller
|
|
* --peers flag — so the test client fails over across candidates exactly
|
|
* like the worker/scheduler/apiserver do. */
|
|
memset(&master_cfg, 0, sizeof(master_cfg));
|
|
if ((masters == NULL) || (xce_parse_peers(masters, &master_cfg) != 0) ||
|
|
(master_cfg.candidates_count < 1)) {
|
|
fprintf(stderr, "enter_ctx: invalid 'masters' setting `%s'\n",
|
|
masters ? masters : "(null)");
|
|
return -EINVAL;
|
|
}
|
|
|
|
config.nmasters = master_cfg.candidates_count;
|
|
if (config.nmasters > N_MAX_MASTERS) {
|
|
fprintf(stderr,
|
|
"enter_ctx: %d masters configured, using first %d "
|
|
"(N_MAX_MASTERS)\n",
|
|
config.nmasters, N_MAX_MASTERS);
|
|
config.nmasters = N_MAX_MASTERS;
|
|
}
|
|
|
|
for (i = 0; i < config.nmasters; i++) {
|
|
config.masters[i] = master_cfg.candidates[i].hostname;
|
|
config.master_ports[i] = master_cfg.candidates[i].port;
|
|
}
|
|
|
|
config.from = get_setting("myname");
|
|
config.key = get_setting("secret");
|
|
|
|
xcecli_ctx_init(&(ctx->ctx), &config);
|
|
xcecli_set_handler(ctx->ctx, &myreq_handler, NULL);
|
|
xcecli_start(ctx->ctx);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int leave_ctx(command_ctx_t* ctx)
|
|
{
|
|
if (ctx == &topctx) {
|
|
return 0;
|
|
}
|
|
|
|
if (ctx == &workerctx) {
|
|
delaymgr_fini(&delaymgr);
|
|
}
|
|
|
|
xcecli_wakeup(active_ctx->ctx, (int)-EINTR);
|
|
|
|
xcecli_stop(ctx->ctx);
|
|
xcecli_ctx_fini(ctx->ctx);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int fill_prompt(command_ctx_t* ctx, char* buf)
|
|
{
|
|
if (ctx->parent != NULL) {
|
|
fill_prompt(ctx->parent, buf);
|
|
strcat(buf, "/");
|
|
}
|
|
|
|
strcat(buf, ctx->prompt);
|
|
|
|
return strlen(buf);
|
|
}
|
|
|
|
command_t* find_command(char* name)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; active_ctx->commands[i].name; i++) {
|
|
if (strcmp(active_ctx->commands[i].name, name) == 0) {
|
|
return &(active_ctx->commands[i]);
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
/* Execute a command line. */
|
|
int execute_line(char* line)
|
|
{
|
|
int rc;
|
|
command_t* command;
|
|
int argc;
|
|
char** argv;
|
|
|
|
argc = parse_argv(line, &argv);
|
|
if (argc < 0) {
|
|
SET_ERROR(-argc);
|
|
perror("execute_line");
|
|
return -1;
|
|
}
|
|
|
|
if ((argv == NULL) || (argv[0] == NULL)) {
|
|
return 0;
|
|
}
|
|
|
|
command = find_command(argv[0]);
|
|
|
|
if (!command) {
|
|
fprintf(stderr, "%s: No such command for xce test.\n", argv[0]);
|
|
free_argv(argv, argc);
|
|
return (-1);
|
|
}
|
|
|
|
rc = (command->func)(argc, argv);
|
|
|
|
free_argv(argv, argc);
|
|
return rc;
|
|
}
|
|
|
|
/* Substitute the fixed auto-variables `$?`, `$jobid`, `$jobcount` on a raw line
|
|
* before tokenizing (xce-test-e2e-primitives task 3.1, Req 3.1/3.3/3.5/3.6).
|
|
*
|
|
* Only WHOLE-TOKEN occurrences are recognized: a `$name` must be bounded by the
|
|
* start of the line / whitespace on its left and whitespace / end-of-line on its
|
|
* right. A `$name` glued to other non-whitespace (e.g. `x$jobid`, `$jobidx`) is
|
|
* NOT a recognized auto-variable and is copied through verbatim. No other `$...`
|
|
* syntax is recognized: an unknown whole token such as `$other` is also copied
|
|
* verbatim (it is simply not one of the three names).
|
|
*
|
|
* Replacements:
|
|
* $? -> decimal of g_last_status (always available).
|
|
* $jobid -> g_last_jobid if set (non-empty); unset => command failure.
|
|
* $jobcount -> decimal of g_last_jobcount if set (>=0); unset => failure.
|
|
*
|
|
* Writes the expanded line into `out` (capacity `outsz`, always NUL-terminated
|
|
* on success). Returns 0 on success, -EINVAL on an unset auto-var reference,
|
|
* -ENOSPC if the expansion would overflow `out`. On any error `out` content is
|
|
* unspecified and the caller must fail the command rather than run it. */
|
|
static int substitute_autovars(const char* line, char* out, size_t outsz)
|
|
{
|
|
const char* p = line;
|
|
size_t o = 0;
|
|
|
|
if ((line == NULL) || (out == NULL) || (outsz == 0)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
while (*p != '\0') {
|
|
/* A `$` starts a candidate auto-var only when it begins a token, i.e.
|
|
* the previous char is the line start or whitespace. */
|
|
int at_token_start
|
|
= (p == line) || (*(p - 1) == ' ') || (*(p - 1) == '\t');
|
|
|
|
if ((*p == '$') && at_token_start) {
|
|
const char* name = p + 1;
|
|
const char* end = name;
|
|
const char* repl = NULL;
|
|
char numbuf[32];
|
|
|
|
/* Scan to the token end (whitespace or NUL). */
|
|
while ((*end != '\0') && (*end != ' ') && (*end != '\t')) {
|
|
end++;
|
|
}
|
|
|
|
size_t namelen = (size_t)(end - name);
|
|
|
|
if ((namelen == 1) && (name[0] == '?')) {
|
|
snprintf(numbuf, sizeof(numbuf), "%d", g_last_status);
|
|
repl = numbuf;
|
|
} else if ((namelen == 5) && (strncmp(name, "jobid", 5) == 0)) {
|
|
if (g_last_jobid[0] == '\0') {
|
|
fprintf(stderr,
|
|
"autovar: $jobid referenced before set\n");
|
|
return -EINVAL;
|
|
}
|
|
repl = g_last_jobid;
|
|
} else if ((namelen == 8) && (strncmp(name, "jobcount", 8) == 0)) {
|
|
if (g_last_jobcount < 0) {
|
|
fprintf(stderr,
|
|
"autovar: $jobcount referenced before set\n");
|
|
return -EINVAL;
|
|
}
|
|
snprintf(numbuf, sizeof(numbuf), "%d", g_last_jobcount);
|
|
repl = numbuf;
|
|
}
|
|
|
|
if (repl != NULL) {
|
|
size_t rlen = strlen(repl);
|
|
|
|
if ((o + rlen) >= outsz) {
|
|
return -ENOSPC;
|
|
}
|
|
memcpy(out + o, repl, rlen);
|
|
o += rlen;
|
|
p = end; /* consume the whole `$name` token */
|
|
continue;
|
|
}
|
|
|
|
/* Not a recognized auto-var: copy the whole token verbatim. */
|
|
while (p < end) {
|
|
if ((o + 1) >= outsz) {
|
|
return -ENOSPC;
|
|
}
|
|
out[o++] = *p++;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if ((o + 1) >= outsz) {
|
|
return -ENOSPC;
|
|
}
|
|
out[o++] = *p++;
|
|
}
|
|
|
|
out[o] = '\0';
|
|
return 0;
|
|
}
|
|
|
|
/* Main-loop wrapper around execute_line that implements the session verdict
|
|
* (xce-test-e2e-primitives task 1.1):
|
|
* - blank / whitespace-only lines and `#`-comment lines are ignored;
|
|
* - normal lines run via execute_line; g_last_status / g_session_failed are
|
|
* updated from the result;
|
|
* - in default (abort) mode a failure stops the loop; under
|
|
* g_continue_on_error the loop keeps running but the failure is recorded.
|
|
*
|
|
* Returns 0 to keep the read loop running, non-zero to request that the loop
|
|
* stop (abort-on-failure). The leading `!` shell-escape and `$`-substitution
|
|
* are intentionally handled in later tasks (2.1 / 3.1); this function is the
|
|
* single chokepoint where they will be added.
|
|
*/
|
|
int preprocess_and_run(char* line)
|
|
{
|
|
char* p = line;
|
|
int rc;
|
|
|
|
/* Skip leading whitespace to classify the line. */
|
|
while ((*p == ' ') || (*p == '\t')) {
|
|
p++;
|
|
}
|
|
|
|
/* Ignore blank / whitespace-only lines and `#` comments. */
|
|
if ((*p == '\0') || (*p == '#')) {
|
|
return 0;
|
|
}
|
|
|
|
/* Shell-escape: a line whose first non-whitespace char is `!` runs the
|
|
* remainder via system(). The shell command's exit status becomes
|
|
* g_last_status (the `$?` auto-variable); a non-zero status is a command
|
|
* failure subject to the same abort-vs-continue rules as execute_line. The
|
|
* remainder is passed raw to the shell (no parse_argv / auto-var
|
|
* substitution — that is task 3.1). (xce-test-e2e-primitives task 2.1) */
|
|
if (*p == '!') {
|
|
const char* rest = p + 1;
|
|
int sysrc = system(rest);
|
|
int status;
|
|
|
|
if (sysrc == -1) {
|
|
/* system() itself failed to run the shell — treat as failure. */
|
|
status = 127;
|
|
} else if (WIFEXITED(sysrc)) {
|
|
status = WEXITSTATUS(sysrc);
|
|
} else {
|
|
/* Signaled / abnormal termination — non-zero failure status. */
|
|
status = 1;
|
|
}
|
|
|
|
g_last_status = status;
|
|
|
|
if (status != 0) {
|
|
g_session_failed = 1;
|
|
if (!g_continue_on_error && !g_interactive) {
|
|
return 1; /* abort: stop the read loop (scripted mode only) */
|
|
}
|
|
} else {
|
|
add_history(line);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Substitute the fixed auto-variables (`$?`, `$jobid`, `$jobcount`) before
|
|
* tokenizing (xce-test-e2e-primitives task 3.1). An unset `$jobid` /
|
|
* `$jobcount` reference (Req 3.6) — or an overflow — fails the command:
|
|
* we do NOT run execute_line, and the result is treated exactly like a
|
|
* failing command (g_last_status non-zero, g_session_failed, abort vs
|
|
* continue). The leading `!` shell-escape above is intentionally NOT
|
|
* substituted: that line goes raw to system() and the shell owns `$?`. */
|
|
{
|
|
char expanded[MAX_LINE_LEN];
|
|
int subrc = substitute_autovars(p, expanded, sizeof(expanded));
|
|
|
|
if (subrc < 0) {
|
|
g_last_status = subrc;
|
|
g_session_failed = 1;
|
|
if (!g_continue_on_error && !g_interactive) {
|
|
return 1; /* abort: stop the read loop (scripted mode only) */
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
rc = execute_line(expanded);
|
|
}
|
|
|
|
g_last_status = (rc < 0) ? (rc) : 0;
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
g_session_failed = 1;
|
|
if (!g_continue_on_error) {
|
|
return 1; /* errexit (-e): stop the read loop on first failure */
|
|
}
|
|
} else {
|
|
add_history(line);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_help(int argc, char* argv[])
|
|
{
|
|
int i;
|
|
int rc = 0;
|
|
|
|
if (argc > 1) {
|
|
char* cmd = argv[1];
|
|
|
|
for (i = 0; active_ctx->commands[i].name != NULL; i++) {
|
|
if (strcmp(argv[1], active_ctx->commands[i].name) == 0) {
|
|
printf("%s\t\t%s.\n", active_ctx->commands[i].name,
|
|
active_ctx->commands[i].doc);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
printf("No commands matchs `%s'. Possibilities are:\n", cmd);
|
|
rc = -1;
|
|
}
|
|
|
|
for (i = 0; active_ctx->commands[i].name != NULL; i++) {
|
|
printf("\t%-12s\t%s\n", active_ctx->commands[i].name,
|
|
active_ctx->commands[i].doc);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/* echo <message...> — print the remainder of the line (argv[1..] joined by
|
|
* single spaces) followed by a newline and succeed (xce-test-e2e-primitives
|
|
* task 1.2, Req 6.3). `echo` with no argument prints an empty line and still
|
|
* succeeds. Returns 0 always so it never marks the session failed. */
|
|
int cmd_echo(int argc, char* argv[])
|
|
{
|
|
int i;
|
|
|
|
for (i = 1; i < argc; i++) {
|
|
if (i > 1) {
|
|
putchar(' ');
|
|
}
|
|
fputs(argv[i], stdout);
|
|
}
|
|
putchar('\n');
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* waldump <path> [<path> ...] — offline WAL inspector for failover/durability
|
|
* triage. Each path may be a seg-*.wal file, a wal/ dir, or a node/data dir.
|
|
* Read-only: lists every entry, decodes JOB_SUBMIT jobids, and flags LSN gaps
|
|
* (a missing committed prefix is THE failover job-loss signal) plus the CRC
|
|
* recovered tip. Returns nonzero if any path failed or any gap was found, so
|
|
* it doubles as an assertion in scripted E2E runs. */
|
|
int cmd_waldump(int argc, char* argv[])
|
|
{
|
|
if (argc < 2) {
|
|
fprintf(stderr,
|
|
"usage: waldump <wal_dir|data_dir|seg_file> [more...]\n");
|
|
return -1;
|
|
}
|
|
|
|
int rc = 0;
|
|
for (int i = 1; i < argc; i++) {
|
|
waldump_stats_t st;
|
|
int r = waldump_path(argv[i], stdout, &st);
|
|
if (r != 0) {
|
|
fprintf(stderr, "waldump: %s: %s\n", argv[i], strerror(-r));
|
|
rc = -1;
|
|
continue;
|
|
}
|
|
if (st.n_gaps > 0 || st.crc_broken)
|
|
rc = -1; /* surface the defect as a nonzero command verdict */
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
/* walcompare [--report] [--committed N] <memberA> <memberB> [more...] — diff
|
|
* member WAL committed prefixes by (lsn -> term, pcrc) and FAIL (nonzero) on any
|
|
* committed-prefix divergence (ha-term-safety R6.1/R6.2/R6.3), reusing the
|
|
* waldump decoder. Each member path may be a seg-*.wal file, a wal/ dir, or a
|
|
* node/data dir (same resolution as waldump). With no --committed, the shared
|
|
* committed boundary = the minimum CRC-intact tip across members. The diverging
|
|
* LSN and per-member (term, pcrc) identities are reported. With --report, also
|
|
* emit a machine-checkable per-member `(lsn, term, pcrc, status)` table and a
|
|
* committed-vs-uncommitted classification for each divergence (task 1.1). Exit
|
|
* 0 = identical, nonzero = divergence or load error — so it doubles as a
|
|
* scripted assertion. */
|
|
int cmd_walcompare(int argc, char* argv[])
|
|
{
|
|
uint64_t committed = 0; /* 0 = derive the shared boundary */
|
|
int report = 0; /* --report => full per-member machine table */
|
|
int ai = 1;
|
|
|
|
/* Optional flags (any order): --report, --committed N. */
|
|
for (;;) {
|
|
if (argc > ai && strcmp(argv[ai], "--report") == 0) {
|
|
report = 1;
|
|
ai += 1;
|
|
continue;
|
|
}
|
|
if (argc > ai && strcmp(argv[ai], "--committed") == 0) {
|
|
if (argc <= ai + 1) {
|
|
fprintf(stderr, "walcompare: --committed needs an LSN\n");
|
|
return -1;
|
|
}
|
|
char* end = NULL;
|
|
unsigned long v = strtoul(argv[ai + 1], &end, 10);
|
|
if (end == argv[ai + 1] || *end != '\0') {
|
|
fprintf(stderr, "walcompare: bad --committed LSN '%s'\n",
|
|
argv[ai + 1]);
|
|
return -1;
|
|
}
|
|
committed = (uint64_t)v;
|
|
ai += 2;
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
|
|
int n = argc - ai;
|
|
if (n < 2) {
|
|
fprintf(stderr,
|
|
"usage: walcompare [--report] [--committed N] <memberA> "
|
|
"<memberB> [more...]\n");
|
|
return -1;
|
|
}
|
|
if (n > 16) {
|
|
fprintf(stderr, "walcompare: at most 16 members\n");
|
|
return -1;
|
|
}
|
|
|
|
walcompare_member_t members[16];
|
|
for (int i = 0; i < n; i++) {
|
|
members[i].path = argv[ai + i];
|
|
members[i].committed_lsn = committed; /* 0 => derive shared boundary */
|
|
}
|
|
|
|
walcompare_divergence_t div;
|
|
int rc = report ? walcompare_report(members, n, stdout, &div)
|
|
: walcompare(members, n, stdout, &div);
|
|
if (rc == WALCMP_OK)
|
|
return 0;
|
|
if (rc == WALCMP_DIVERGENT) {
|
|
fprintf(stderr,
|
|
"walcompare: FAIL — committed-prefix divergence at LSN %" PRIu64
|
|
"\n",
|
|
div.lsn);
|
|
return -1; /* nonzero session verdict */
|
|
}
|
|
fprintf(stderr, "walcompare: error: %s\n", strerror(-rc));
|
|
return -1;
|
|
}
|
|
|
|
/* snapverify <snap_file> [<snap_file> ...] — offline checkpoint-file
|
|
* structural validator for force-snapshot triage (bug-1389: OPCode_
|
|
* snapshotTransfer chunk-content corruption). Each path is a snap-*.capnp
|
|
* (locally generated) or snap-*.bin (received via ha_handle_snapshot_chunk)
|
|
* file. Read-only: walks the same layout persist_load_snapshot() parses
|
|
* without restoring anything, and on the first structural defect reports the
|
|
* exact byte offset and how many job/queue frames parsed cleanly before it —
|
|
* the diagnostic a bare "persist_load_snapshot failed rc=-5" log line
|
|
* doesn't give you. Returns nonzero if any path failed to verify. */
|
|
int cmd_snapverify(int argc, char* argv[])
|
|
{
|
|
if (argc < 2) {
|
|
fprintf(stderr, "usage: snapverify <snap_file> [more...]\n");
|
|
return -1;
|
|
}
|
|
|
|
int rc = 0;
|
|
for (int i = 1; i < argc; i++) {
|
|
snapverify_report_t rep;
|
|
int r = snapverify_path(argv[i], stdout, &rep);
|
|
if (r != 0)
|
|
rc = -1;
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
/* snapdiff <source_file> <received_file> — byte-level checkpoint-file
|
|
* comparator for force-snapshot triage (bug-1389). `source_file` is the
|
|
* primary's own locally generated checkpoint (snap-*.capnp); `received_file`
|
|
* is the copy a standby assembled from the wire (snap-*.bin). Reports the
|
|
* first differing byte offset and whether it lands exactly on an
|
|
* OPCode_snapshotTransfer chunk boundary — the decisive signal for "the
|
|
* transport corrupted one chunk" vs "these were never the same content".
|
|
* Returns 0 if identical, nonzero otherwise. */
|
|
int cmd_snapdiff(int argc, char* argv[])
|
|
{
|
|
if (argc != 3) {
|
|
fprintf(stderr, "usage: snapdiff <source_file> <received_file>\n");
|
|
return -1;
|
|
}
|
|
|
|
snapdiff_report_t rep;
|
|
int rc = snapdiff_paths(argv[1], argv[2], stdout, &rep);
|
|
return (rc == 0) ? 0 : -1;
|
|
}
|
|
|
|
int cmd_up(int argc, char* argv[])
|
|
{
|
|
if (active_ctx->parent != NULL) {
|
|
int rc;
|
|
|
|
if ((rc = leave_ctx(active_ctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = active_ctx->parent;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_worker(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
|
|
if ((rc = enter_ctx(&workerctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = &workerctx;
|
|
|
|
delaymgr_init(&delaymgr, 2000 /* precision_ms */, NULL);
|
|
delaymgr_start(&delaymgr);
|
|
delay_task(&delaymgr, 5, -1, (delay_task_entry_fn_t)worker_house_keeping,
|
|
&workerctx);
|
|
return 0;
|
|
}
|
|
|
|
int cmd_scheduler(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
|
|
if ((rc = enter_ctx(&schedulerctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = &schedulerctx;
|
|
return 0;
|
|
}
|
|
|
|
int cmd_client(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
|
|
if ((rc = enter_ctx(&clientctx)) < 0) {
|
|
return rc;
|
|
}
|
|
|
|
active_ctx = &clientctx;
|
|
return 0;
|
|
}
|
|
|
|
static int done = 0;
|
|
|
|
int cmd_quit(int argc, char* argv[])
|
|
{
|
|
leave_ctx(active_ctx);
|
|
|
|
done = 1;
|
|
return 0;
|
|
}
|
|
|
|
int cmd_hello(int argc, char* argv[])
|
|
{
|
|
int rc = 0;
|
|
int opt;
|
|
char* msg = "Greeting from xce client";
|
|
int count = 1;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "c:m:")) != -1) {
|
|
switch (opt) {
|
|
case 'c':
|
|
count = atoi(optarg);
|
|
break;
|
|
case 'm':
|
|
msg = optarg;
|
|
break;
|
|
default:
|
|
fprintf(stderr, " hello [-c count] [-w wait_usec] [-m message]\n");
|
|
return -1;
|
|
break;
|
|
}
|
|
}
|
|
|
|
break_out = 0;
|
|
while ((!break_out) && (count > 0)) {
|
|
rc = xcecli_hello(active_ctx->ctx, msg);
|
|
|
|
count--;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int cmd_shutdown(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
rc = xcecli_shutdown(active_ctx->ctx);
|
|
/* leave client context */
|
|
if (rc >= 0) {
|
|
execute_line("..");
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int cmd_reslist(int argc, char* argv[])
|
|
{
|
|
int rc, i;
|
|
xce_resource_t** res;
|
|
|
|
rc = xcecli_reslist(active_ctx->ctx, &res);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_reslist");
|
|
return rc;
|
|
}
|
|
|
|
printf("%10s%10s%10s%64s\n", "Name", "Flags", "Interval", "Description");
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
printf("%10s%10x%10d%64s\n", res[i]->name, res[i]->flags,
|
|
res[i]->interval, res[i]->desc);
|
|
}
|
|
|
|
free_Resource_list(rc, res);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static char* worker_status_str(uint32_t status)
|
|
{
|
|
if ((status & XCE_WORKER_UNAVAIL) != 0) {
|
|
return "UNAVAIL";
|
|
} else if ((status & XCE_WORKER_LOCKEDU) != 0) {
|
|
return "LOCKU";
|
|
} else if ((status & XCE_WORKER_LOCKEDW) != 0) {
|
|
return "LOCKW";
|
|
} else if ((status & XCE_WORKER_BUSY) != 0) {
|
|
return "BUSY";
|
|
} else if (status == 0) {
|
|
return "OK";
|
|
}
|
|
|
|
return "UNKOWN";
|
|
}
|
|
|
|
int cmd_workerlist(int argc, char* argv[])
|
|
{
|
|
int rc, i;
|
|
xce_worker_t** workers;
|
|
|
|
rc = xcecli_workerlist(active_ctx->ctx, &workers);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_workerlist");
|
|
return rc;
|
|
}
|
|
|
|
printf("%-20s%-8s%-25s%-25s%12s%12s%12s%12s%12s%12s\n", "HostName",
|
|
"Status", "HostType", "HostModel", "CPUfactor", "MaxCPU", "MaxMem",
|
|
"MaxSwap", "MaxTmp", "NDisks");
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
printf("%-20s%-8s%-25s%-25s%12f%12d%12f%12f%12f%12d\n",
|
|
workers[i]->name, worker_status_str(workers[i]->status),
|
|
workers[i]->attrs->hosttype, workers[i]->attrs->hostmodel,
|
|
workers[i]->attrs->cpufactor, workers[i]->attrs->maxcpus,
|
|
workers[i]->attrs->maxmem, workers[i]->attrs->maxswap,
|
|
workers[i]->attrs->maxtmp, workers[i]->attrs->ndisks);
|
|
}
|
|
|
|
free_Worker_list(rc, workers);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static char* job_state_str(int state)
|
|
{
|
|
if ((state & JOB_STAT_PEND) != 0) {
|
|
if ((state & JOB_STAT_PSUSP) != 0) {
|
|
return "PSUSP";
|
|
}
|
|
|
|
return "PEND";
|
|
} else if ((state & JOB_STAT_RUN) != 0) {
|
|
if ((state & JOB_STAT_USUSP) != 0) {
|
|
return "USUSP";
|
|
} else if ((state & JOB_STAT_SSUSP) != 0) {
|
|
return "SSUSP";
|
|
}
|
|
return "RUN";
|
|
} else if ((state & JOB_STAT_EXIT) != 0) {
|
|
return "EXIT";
|
|
} else if ((state & JOB_STAT_DONE) != 0) {
|
|
return "DONE";
|
|
} else if ((state & JOB_STAT_ERR) != 0) {
|
|
return "ERROR";
|
|
}
|
|
|
|
return "UNKNOWN";
|
|
}
|
|
|
|
/* Shared, print-free joblist core for `cmd_joblist` and the `expect-jobs` /
|
|
* `expect-job` assertions (xce-test-e2e-primitives task 5.1, design.md:81,
|
|
* 116-117).
|
|
*
|
|
* Performs the single `xcecli_joblist` call (through the failover-aware client
|
|
* context) and returns the current job count (>= 0), or a negative errno-style
|
|
* value on failure. When `find_jobid != NULL`, `*contains` is set to 1 if a job
|
|
* whose `fmt_jobid(...)` string equals `find_jobid` is present, else 0; the
|
|
* `fmt_jobid` representation matches the one `cmd_joblist` prints and the one
|
|
* `cmd_jobsub` captures into `$jobid` (task 4.1), so `expect-job $jobid` lines
|
|
* up. When `print_table` is set, the human-readable table is printed (the
|
|
* `cmd_joblist` path). The job array is always freed internally.
|
|
*
|
|
* On error this returns the negative rc WITHOUT setting any auto-variable, so a
|
|
* failed joblist never advertises a fake `$jobcount` (Req 3.6). */
|
|
static int joblist_query(const char* find_jobid, int* contains, int print_table,
|
|
int quiet)
|
|
{
|
|
int rc, i;
|
|
xce_job_t** jobs;
|
|
|
|
if (contains != NULL) {
|
|
*contains = 0;
|
|
}
|
|
|
|
rc = xcecli_joblist(active_ctx->ctx, &jobs);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
/* `quiet` callers (e.g. the wait-primary poll, which probes repeatedly
|
|
* and EXPECTS failure until a primary serves) suppress the per-attempt
|
|
* error print so a normal failover wait does not spam the console. */
|
|
if (!quiet) {
|
|
perror("xce_joblist");
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (print_table) {
|
|
printf("%-26s%-10s%-10s%-8s%-16s%-16s%-16s%10s%30s\n", "JobId", "User",
|
|
"Queue", "State", "SubmitTime", "StartTime", "FinishTime",
|
|
"NTasks", "Command");
|
|
}
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
char buf[128];
|
|
|
|
fmt_jobid(buf, sizeof(buf) - 1, jobs[i]->jobid);
|
|
|
|
if ((find_jobid != NULL) && (contains != NULL)
|
|
&& (strcmp(buf, find_jobid) == 0)) {
|
|
*contains = 1;
|
|
}
|
|
|
|
if (print_table) {
|
|
printf("%-26s%-10s%-10s%-8s%-16ld%-16ld%-16ld%4d,%-5d%30s\n", buf,
|
|
jobs[i]->submit->user, jobs[i]->submit->queue,
|
|
job_state_str(jobs[i]->state), jobs[i]->queue_time,
|
|
jobs[i]->start_time, jobs[i]->finish_time,
|
|
jobs[i]->submit->mintasks, jobs[i]->submit->maxtasks,
|
|
jobs[i]->submit->start_info->cmdline);
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
free_Job_ptr(&(jobs[i]));
|
|
}
|
|
|
|
free(jobs);
|
|
|
|
return rc;
|
|
}
|
|
|
|
int cmd_joblist(int argc, char* argv[])
|
|
{
|
|
int count;
|
|
|
|
count = joblist_query(NULL, NULL, 1, /*quiet=*/0);
|
|
|
|
if (count < 0) {
|
|
return count;
|
|
}
|
|
|
|
/* Capture $jobcount (Req 3.4): a successful joblist publishes the returned
|
|
* job count so a later `echo $jobcount` / `expect-jobs` substitutes it. On
|
|
* failure we returned above without touching g_last_jobcount, so a failed
|
|
* joblist never advertises a fake count (Req 3.6). */
|
|
g_last_jobcount = count;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* `expect-jobs N` assertion (xce-test-e2e-primitives task 5.2, Req 5.1/5.2,
|
|
* design.md:116-117).
|
|
*
|
|
* Passes (returns 0) when the current job count equals N; otherwise prints the
|
|
* expected vs actual count and returns a negative errno-style value so
|
|
* preprocess_and_run records the session as failed (Req 1.1).
|
|
*
|
|
* A malformed argument (missing or non-numeric N) is itself an assertion
|
|
* failure, not a no-op. A failed joblist query is treated as a failure too: we
|
|
* cannot determine the count, so we never silently equate "query failed" with
|
|
* "zero jobs" (Req 5.2 reports actual; a failed query has no trustworthy
|
|
* actual). */
|
|
int cmd_expect_jobs(int argc, char* argv[])
|
|
{
|
|
long expected;
|
|
char* end;
|
|
int count;
|
|
|
|
if (argc != 2) {
|
|
printf("expect-jobs <N>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
errno = 0;
|
|
expected = strtol(argv[1], &end, 10);
|
|
|
|
if ((errno != 0) || (end == argv[1]) || (*end != '\0') || (expected < 0)) {
|
|
printf("expect-jobs: invalid count '%s' (expected a non-negative "
|
|
"integer)\n",
|
|
argv[1]);
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* Print-free count via the shared helper. A negative rc means the query
|
|
* itself failed (e.g. no reachable master) — that is an assertion failure,
|
|
* not "0 jobs". */
|
|
count = joblist_query(NULL, NULL, 0, /*quiet=*/0);
|
|
|
|
if (count < 0) {
|
|
printf("expect-jobs: could not determine job count (joblist query "
|
|
"failed)\n");
|
|
return count;
|
|
}
|
|
|
|
if (count != (int)expected) {
|
|
printf("expect-jobs: expected %ld, actual %d\n", expected, count);
|
|
return -EINVAL;
|
|
}
|
|
|
|
printf("expect-jobs: ok (%d)\n", count);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* `expect-job <jobid>` assertion (xce-test-e2e-primitives task 5.3, Req 5.3/5.4,
|
|
* design.md cmd_expect_jobs / cmd_expect_job).
|
|
*
|
|
* Passes (returns 0) when a job whose `fmt_jobid` string equals argv[1] is
|
|
* present in the current job list; otherwise prints the missing id and returns a
|
|
* negative errno-style value so preprocess_and_run records the session as failed
|
|
* (Req 1.1).
|
|
*
|
|
* `$jobid` substitution is transparent: autovar substitution (task 3.1) replaces
|
|
* `$jobid` with the captured id string BEFORE this handler runs, so argv[1] is
|
|
* already the literal id. The membership test in joblist_query uses the SAME
|
|
* `fmt_jobid` representation that cmd_jobsub captured into `$jobid` (task 4.1),
|
|
* so `expect-job $jobid` matches the joblist entry.
|
|
*
|
|
* A missing argument is itself an assertion failure, not a no-op. A failed
|
|
* joblist query is also a failure: we cannot determine membership, so we report
|
|
* the query failure distinctly rather than silently equating it with "absent". */
|
|
int cmd_expect_job(int argc, char* argv[])
|
|
{
|
|
int contains;
|
|
int rc;
|
|
|
|
if (argc != 2) {
|
|
printf("expect-job <jobid>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* Print-free membership query via the shared helper. A negative rc means the
|
|
* query itself failed (e.g. no reachable master) — that is an assertion
|
|
* failure, distinct from "job absent". */
|
|
rc = joblist_query(argv[1], &contains, 0, /*quiet=*/0);
|
|
|
|
if (rc < 0) {
|
|
printf("expect-job: could not determine membership of '%s' (joblist "
|
|
"query failed)\n",
|
|
argv[1]);
|
|
return rc;
|
|
}
|
|
|
|
if (!contains) {
|
|
printf("expect-job: job %s not found\n", argv[1]);
|
|
return -ENOENT;
|
|
}
|
|
|
|
printf("expect-job: ok (%s)\n", argv[1]);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* `wait-primary [timeout_ms]` (controller-status task 4.2, Req 9.1-9.4;
|
|
* originally xce-test-e2e-primitives task 6.1; design.md "cmd_wait_primary probe
|
|
* swap — State contract").
|
|
*
|
|
* Blocks until the AUTHORITATIVE controller status probe reaches the serving
|
|
* primary, or the timeout elapses. The probe is xcecli_ctrlstatus_primary: it
|
|
* drives active_ctx + libxce master-rotation/failover and returns 0 ONLY when a
|
|
* reply with responderRole == PRIMARY was obtained (it internally follows the
|
|
* redirect to the primary), or a typed negative when no primary answered within
|
|
* its budget (Req 9.1). A standby/hint answer is NOT success — the probe keeps
|
|
* polling until the primary itself answers (this avoids a false success on a
|
|
* standby's stale primary view after a primary death).
|
|
*
|
|
* On the first authoritative probe we print the served primary's identity and
|
|
* return 0 (Req 9.2). Otherwise we sleep a short poll interval and retry,
|
|
* re-checking a MONOTONIC deadline between attempts so the command returns
|
|
* within a bounded time of the timeout and never hangs indefinitely when there
|
|
* is no primary (Req 9.3). The deadline is derived from CLOCK_MONOTONIC, never
|
|
* wall-clock, so a clock step cannot skew it.
|
|
*
|
|
* The timeout is argv[1] in milliseconds when supplied, else
|
|
* WAIT_PRIMARY_DEFAULT_MS (Req 9.4). A malformed timeout argument is a usage
|
|
* error (return < 0) rather than a silent fallback to the default. */
|
|
#define WAIT_PRIMARY_DEFAULT_MS 30000L /* default max wait when arg omitted */
|
|
#define WAIT_PRIMARY_POLL_MS 200L /* sleep between probe attempts */
|
|
#define MS_PER_SEC 1000L
|
|
#define NS_PER_MS 1000000L
|
|
#define US_PER_MS 1000L
|
|
|
|
/* The client's configured log level (set in main from -v), so wait-primary can
|
|
* temporarily raise the threshold to silence libxce's per-attempt failover INFO
|
|
* spam during its poll and then RESTORE the user's chosen level. */
|
|
static char* g_active_loglevel = (char*)"info";
|
|
|
|
static long monotonic_ms(void)
|
|
{
|
|
struct timespec ts;
|
|
|
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
|
|
return ((long)ts.tv_sec * MS_PER_SEC) + (ts.tv_nsec / NS_PER_MS);
|
|
}
|
|
|
|
int cmd_wait_primary(int argc, char* argv[])
|
|
{
|
|
long timeout_ms = WAIT_PRIMARY_DEFAULT_MS;
|
|
long start_ms, elapsed_ms;
|
|
|
|
if (argc > 2) {
|
|
printf("wait-primary [timeout_ms]\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (argc == 2) {
|
|
char* end;
|
|
|
|
errno = 0;
|
|
timeout_ms = strtol(argv[1], &end, 10);
|
|
|
|
if ((errno != 0) || (end == argv[1]) || (*end != '\0')
|
|
|| (timeout_ms < 0)) {
|
|
printf("wait-primary: invalid timeout '%s' (expected a "
|
|
"non-negative integer in ms)\n",
|
|
argv[1]);
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
start_ms = monotonic_ms();
|
|
|
|
/* The poll probes repeatedly and EXPECTS failure until a primary serves, so
|
|
* it must not spam the console: xcecli_ctrlstatus_primary suppresses its own
|
|
* per-attempt output, and we raise the client log threshold for the duration
|
|
* of the loop to silence libxce's per-attempt "xcecli failover: attempt N"
|
|
* INFO logs — RESTORED to the user's level on every exit path. */
|
|
mylog_setlevel((char*)"warn");
|
|
|
|
/* Probe, then re-check the monotonic deadline between attempts. Each probe
|
|
* may itself take time (libxce failover + redirect-to-primary has its own
|
|
* internal retry budget); we re-read the clock after every probe returns so
|
|
* the overall loop honors the deadline and returns failure in bounded time. */
|
|
for (;;) {
|
|
xce_ctrlstatus_t* status = NULL;
|
|
int rc = xcecli_ctrlstatus_primary(active_ctx->ctx,
|
|
&status);
|
|
|
|
/* Success ONLY on an authoritative reply from the serving primary
|
|
* (responderRole == PRIMARY, guaranteed by xcecli_ctrlstatus_primary).
|
|
* Print the served primary's identity (the responder IS the primary, so
|
|
* primaryName == responderName) and free the reply (Req 9.1, 9.2). */
|
|
if (rc == 0) {
|
|
const char* primary = (status != NULL)
|
|
? (NULLSTR(status->primaryName)
|
|
? status->responderName
|
|
: status->primaryName)
|
|
: NULL;
|
|
|
|
elapsed_ms = monotonic_ms() - start_ms;
|
|
mylog_setlevel(g_active_loglevel);
|
|
printf("wait-primary: primary '%s' serving after %ld ms\n",
|
|
NULLSTR(primary) ? "unknown" : primary, elapsed_ms);
|
|
xcecli_free_ctrlstatus(status);
|
|
return 0;
|
|
}
|
|
|
|
/* Non-authoritative / failed probe: nothing to free (the typed-failure
|
|
* contract leaves *status untouched), keep polling until the deadline. */
|
|
elapsed_ms = monotonic_ms() - start_ms;
|
|
|
|
if (elapsed_ms >= timeout_ms) {
|
|
mylog_setlevel(g_active_loglevel);
|
|
printf("wait-primary: no primary served within %ld ms (waited "
|
|
"%ld ms)\n",
|
|
timeout_ms, elapsed_ms);
|
|
return -ETIMEDOUT;
|
|
}
|
|
|
|
/* Don't oversleep past the deadline: cap the poll to the time left. */
|
|
{
|
|
long remaining_ms = timeout_ms - elapsed_ms;
|
|
long sleep_ms = (remaining_ms < WAIT_PRIMARY_POLL_MS)
|
|
? remaining_ms
|
|
: WAIT_PRIMARY_POLL_MS;
|
|
|
|
if (sleep_ms > 0) {
|
|
usleep((useconds_t)(sleep_ms * US_PER_MS));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Map a ctrl_role_t raw value (CtrlStatus.responderRole, see ha.h:60-63 and
|
|
* design.md "Schema — reply payload") to a display string. PRIMARY==0,
|
|
* STANDBY==1, INIT==2, ELECTING==3; anything else is reported verbatim so an
|
|
* unexpected role is never silently rendered as a known one. */
|
|
static const char* ctrlstatus_role_str(uint8_t role)
|
|
{
|
|
switch (role) {
|
|
case 0:
|
|
return "primary";
|
|
case 1:
|
|
return "standby";
|
|
case 2:
|
|
return "initializing";
|
|
case 3:
|
|
return "electing";
|
|
default:
|
|
return "unknown-role";
|
|
}
|
|
}
|
|
|
|
/* `ctrlstatus` (controller-status task 4.1, Req 1.3/3.3/5.1/5.2/8.1/8.2/8.3,
|
|
* design.md "cmd_ctrlstatus (test client) — State contract").
|
|
*
|
|
* Issues the any-node status read (xcecli_ctrlstatus, is_write=false: whatever
|
|
* controller answers — primary or standby hint view — rides the configured
|
|
* failover/rotation) and prints the identity/health fields in human-readable
|
|
* form.
|
|
*
|
|
* On failure (no reachable controller etc.) it prints a typed error and returns
|
|
* the negative rc so the session records the command as failed and exits
|
|
* non-zero (Req 8.3). On success it prints the fields and returns 0 (Req 8.2). */
|
|
int cmd_ctrlstatus(int argc, char* argv[])
|
|
{
|
|
xce_ctrlstatus_t* status = NULL;
|
|
int rc;
|
|
int i;
|
|
|
|
(void)argv;
|
|
|
|
if (argc != 1) {
|
|
printf("ctrlstatus\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
rc = xcecli_ctrlstatus(active_ctx->ctx, &status);
|
|
|
|
if (rc < 0) {
|
|
char ebuf[128];
|
|
|
|
ebuf[0] = '\0';
|
|
xce_strerror_r(-rc, ebuf, sizeof(ebuf));
|
|
SET_ERROR(-rc);
|
|
|
|
switch (-rc) {
|
|
case E_XCE_NOMASTER:
|
|
fprintf(stderr, "ctrlstatus failed: no-master (%s)\n", ebuf);
|
|
break;
|
|
case E_XCE_NOT_READY:
|
|
fprintf(stderr, "ctrlstatus failed: not-ready (%s)\n", ebuf);
|
|
break;
|
|
case ETIMEDOUT:
|
|
fprintf(stderr, "ctrlstatus failed: timeout (%s)\n", ebuf);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "ctrlstatus failed: %s\n", ebuf);
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/* Current primary (Req 2.x): an empty primaryName means no elected/known
|
|
* primary — render the well-defined "unknown" (Req 1.3/2.2/2.3) rather than
|
|
* an empty or stale value. */
|
|
printf("Primary: %s\n",
|
|
NULLSTR(status->primaryName) ? "unknown" : status->primaryName);
|
|
|
|
/* Answering node + role, framed as a property of the responder and kept
|
|
* DISTINCT from the authoritative Primary line above (Req 3.1/3.2/3.3): a
|
|
* transient responder role (electing/initializing) must not be misread as
|
|
* the cluster's primary state. */
|
|
printf("answered by %s (%s)\n",
|
|
NULLSTR(status->responderName) ? "unknown" : status->responderName,
|
|
ctrlstatus_role_str(status->responderRole));
|
|
|
|
printf("Cluster: %s\n",
|
|
NULLSTR(status->clusterName) ? "unknown" : status->clusterName);
|
|
|
|
/* Configured controller topology + count (Req 4.1/4.2). */
|
|
printf("Controllers (%u):\n", status->peerCount);
|
|
for (i = 0; i < status->peers.count; i++) {
|
|
xce_ctrlpeer_t* peer = status->peers.list[i];
|
|
|
|
if (peer == NULL) {
|
|
continue;
|
|
}
|
|
|
|
printf("\t%s\t%s:%u\n", NULLSTR(peer->name) ? "?" : peer->name,
|
|
NULLSTR(peer->host) ? "?" : peer->host, peer->port);
|
|
}
|
|
|
|
/* Config version + committed config version (Req 5.1) and the content
|
|
* checksum as the hex string the controller already exposes (Req 5.2). */
|
|
printf("configVersion: %llu\n", (unsigned long long)status->configVersion);
|
|
printf("committedConfigVersion: %llu\n",
|
|
(unsigned long long)status->committedConfigVersion);
|
|
printf("configChecksum: %s\n",
|
|
NULLSTR(status->configChecksum) ? "unknown" : status->configChecksum);
|
|
|
|
/* HA health / progress (Req 6.x). */
|
|
printf("term: %llu\n", (unsigned long long)status->term);
|
|
printf("committedLsn: %llu\n", (unsigned long long)status->committedLsn);
|
|
printf("appliedLsn: %llu\n", (unsigned long long)status->appliedLsn);
|
|
printf("fenced: %s\n", status->fenced ? "true" : "false");
|
|
|
|
/* Responder snapshot timestamp (Req 11.2/11.3): render the responder's
|
|
* sample-time wall clock (epoch ms) in the SAME local-time
|
|
* "YYYY-MM-DD HH:MM:SS,mmm" form the controller logger emits, so it can be
|
|
* compared directly against ctrl-N.log. */
|
|
{
|
|
char tbuf[64];
|
|
if (ctrlstatus_format_time_ms(status->responder_time_ms, tbuf,
|
|
sizeof(tbuf)) != NULL) {
|
|
printf("responderTime: %s\n", tbuf);
|
|
} else {
|
|
printf("responderTime: unknown\n");
|
|
}
|
|
}
|
|
|
|
xcecli_free_ctrlstatus(status);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void print_jobinfo(xce_job_t* job)
|
|
{
|
|
char idstring[128];
|
|
|
|
if (job == NULL) {
|
|
return;
|
|
}
|
|
|
|
fmt_jobid(idstring, sizeof(idstring) - 1, job->jobid);
|
|
printf("JobID: %s\n", idstring);
|
|
printf("state: %s\n", job_state_str(job->state));
|
|
/* raft-apply-decoupling task 7.2: job_state_str's if/else-if chain checks
|
|
* JOB_STAT_RUN before JOB_STAT_EXIT, so a job that is simultaneously RUN
|
|
* and EXIT|PERR (e.g. a worker-reported FAIL received without a prior
|
|
* FINISH — apply_job_fail_transition ORs in EXIT|PERR without clearing
|
|
* RUN) prints as "RUN", silently hiding the EXIT/PERR bits from the
|
|
* pretty string. Print the raw bitmask too so a scriptable caller (the
|
|
* xce-test-ha factstream regression scenario's phantom-fact check) can
|
|
* test for a specific bit's presence unambiguously instead of relying on
|
|
* the display string's priority ordering. */
|
|
printf("state_raw: 0x%x\n", job->state);
|
|
printf("Queue: %s\n", job->submit->queue);
|
|
printf("User: %s\n", job->submit->user);
|
|
printf("Slots: %d,%d\n", job->submit->mintasks, job->submit->maxtasks);
|
|
printf("Command: %s\n", job->submit->start_info->cmdline);
|
|
|
|
if (job->pend_reasons != NULL) {
|
|
printf("Pend Reason:\n");
|
|
if (job->pend_reasons->main_reason != 0) {
|
|
printf("\tmain Reason: %d\n", job->pend_reasons->main_reason);
|
|
} else {
|
|
if (job->pend_reasons->nhost_reasons > 0) {
|
|
int i;
|
|
|
|
printf("\tHost Reasons:\n");
|
|
for (i = 0; i < job->pend_reasons->nhost_reasons; i++) {
|
|
printf("\t\t(%s, %d)\n",
|
|
job->pend_reasons->host_based_reasons[i]->host,
|
|
job->pend_reasons->host_based_reasons[i]->reason);
|
|
}
|
|
}
|
|
|
|
if (job->pend_reasons->nres_reasons > 0) {
|
|
int i;
|
|
|
|
printf("\tResource Reasons:\n");
|
|
for (i = 0; i < job->pend_reasons->nres_reasons; i++) {
|
|
printf("\t\t(%s, %s, %d)\n",
|
|
job->pend_reasons->res_based_reasons[i]->resource,
|
|
job->pend_reasons->res_based_reasons[i]->host,
|
|
job->pend_reasons->res_based_reasons[i]->reason);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
int cmd_jobinfo(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "j:")) != -1) {
|
|
uint64_t jobid;
|
|
xce_job_t* job;
|
|
int rc = 0;
|
|
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
|
|
if (jobid > 0) {
|
|
rc = xcecli_jobinfo(active_ctx->ctx, jobid, &job);
|
|
if (rc < 0) {
|
|
perror("xcecli_jobinfo");
|
|
continue;
|
|
}
|
|
|
|
print_jobinfo(job);
|
|
free_Job_ptr(&job);
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int fill_job_alloc(xce_job_alloc_t* jalloc, int nhosts,
|
|
host_task_t* hosts)
|
|
{
|
|
xce_host_alloc_t** halloc = NULL;
|
|
int i;
|
|
|
|
halloc = (xce_host_alloc_t**)calloc(nhosts, sizeof(xce_host_alloc_t*));
|
|
if (halloc == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
for (i = 0; i < nhosts; i++) {
|
|
xce_alloc_t* alloc;
|
|
xce_alloc_t** allocs;
|
|
|
|
alloc = (xce_alloc_t*)calloc(1, sizeof(*alloc));
|
|
allocs = (xce_alloc_t**)calloc(1, sizeof(*allocs));
|
|
halloc[i] = (xce_host_alloc_t*)calloc(1, sizeof(xce_host_alloc_t));
|
|
if ((halloc[i] == NULL) || (alloc == NULL) || (allocs == NULL)) {
|
|
/* FIXME: handle error */
|
|
return -ENOMEM;
|
|
}
|
|
|
|
alloc->resname = "slots";
|
|
alloc->reserve = "";
|
|
alloc->instid = -1;
|
|
alloc->value = (float)hosts[i].ntasks;
|
|
alloc->from = time(NULL);
|
|
alloc->duration = -1;
|
|
|
|
allocs[0] = alloc;
|
|
|
|
halloc[i]->nallocs = 1;
|
|
halloc[i]->allocs = allocs;
|
|
halloc[i]->host = hosts[i].host;
|
|
}
|
|
|
|
jalloc->nhosts = nhosts;
|
|
jalloc->allocs = halloc;
|
|
|
|
return 0;
|
|
}
|
|
int cmd_jobrun(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
host_task_t hosts[MAX_TASK_HOSTS];
|
|
xce_job_alloc_t jalloc;
|
|
xce_job_dispatch_t jdispatch
|
|
= { .njobs = 1, .allocs = (xce_job_alloc_t*[]) { &jalloc } };
|
|
int nhosts = 0;
|
|
char* host;
|
|
int count;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "m:j:")) != -1) {
|
|
switch (opt) {
|
|
case 'm':
|
|
if (nhosts >= MAX_TASK_HOSTS) {
|
|
fprintf(stderr,
|
|
"too many run hosts, we only support max %d hosts\n",
|
|
MAX_TASK_HOSTS);
|
|
return -1;
|
|
}
|
|
|
|
if (strchr(optarg, (int)':') != NULL) {
|
|
char* p;
|
|
char buf[256];
|
|
|
|
p = strchr(optarg, (int)':');
|
|
count = atoi(p + 1);
|
|
memset(buf, 0x0, sizeof(buf));
|
|
|
|
strncpy(buf, optarg, (int)(p - optarg));
|
|
host = strdup(buf);
|
|
} else {
|
|
host = strdup(optarg);
|
|
count = 1;
|
|
}
|
|
|
|
hosts[nhosts].host = host;
|
|
hosts[nhosts].ntasks = count;
|
|
nhosts++;
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
if (jobid == 0) {
|
|
printf("you need to specify a positive jobid with -j");
|
|
return -1;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf("no jobid specified");
|
|
return -1;
|
|
}
|
|
|
|
fill_job_alloc(&jalloc, nhosts, &hosts[0]);
|
|
|
|
jalloc.jobid = jobid;
|
|
|
|
rc = xcecli_jobdispatch(active_ctx->ctx, &jdispatch);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_jobdispatch");
|
|
return rc;
|
|
}
|
|
|
|
#if 0
|
|
printf("job %s will be started on host: %s\n",
|
|
argv[1], halloc.host);
|
|
#endif
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int build_job_envs(xce_job_submit_t* submit)
|
|
{
|
|
exdata_t exdata = { .sub_env_vars = { .PATH = getenv("PATH") } };
|
|
|
|
s2j_create_json_obj(json_exdata);
|
|
s2j_create_json_obj(json_envs);
|
|
s2j_json_set_basic_element(json_envs, &(exdata.sub_env_vars), var_string,
|
|
PATH);
|
|
cJSON_AddItemToObject(json_exdata, "sub_env_vars", json_envs);
|
|
|
|
submit->submit->start_info->exdata = cJSON_Print(json_exdata);
|
|
cJSON_Delete(json_exdata);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int build_job_exdata(xce_job_submit_t* submit)
|
|
{
|
|
build_job_envs(submit);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobsub(int argc, char* argv[])
|
|
{
|
|
int rc = 0;
|
|
job_submit_data_t submit;
|
|
xce_job_submit_t jobsubmit = { .submit = &submit };
|
|
int opt;
|
|
/* Initialize so a "rc >= 0 but *jobid never written" outcome (a non-clean
|
|
* ack such as a redirect/zero-row/not-ready surfaced as a non-negative rc)
|
|
* is detectable as an invalid (zero) JobId rather than printed as garbage
|
|
* (bug-786). */
|
|
uint64_t jobid = 0;
|
|
uint32_t min = 1, max = 1;
|
|
uint32_t arsize = 1;
|
|
char jobidStr[128];
|
|
|
|
optind = 1;
|
|
|
|
memset(&submit, 0x0, sizeof(submit));
|
|
submit.start_info
|
|
= (job_startinfo_t*)calloc(1, sizeof(*(submit.start_info)));
|
|
if (submit.start_info == NULL) {
|
|
perror("calloc");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
while ((opt = getopt(argc, argv, "u:q:R:c:x:n:a:")) != -1) {
|
|
switch (opt) {
|
|
case 'u':
|
|
submit.user = strdup(optarg);
|
|
break;
|
|
case 'q':
|
|
submit.queue = strdup(optarg);
|
|
break;
|
|
case 'R':
|
|
submit.resreq = strdup(optarg);
|
|
break;
|
|
case 'c':
|
|
submit.start_info->cmdline = strdup(optarg);
|
|
break;
|
|
case 'x':
|
|
submit.start_info->exdata = strdup(optarg);
|
|
break;
|
|
case 'n':
|
|
if (strchr(optarg, (int)',') != NULL) {
|
|
char buf[64];
|
|
char* p = strchr(optarg, (int)',');
|
|
|
|
max = atoi(p + 1);
|
|
memset(buf, 0x0, sizeof(buf));
|
|
strncpy(buf, optarg, (int)(p - optarg));
|
|
min = atoi(buf);
|
|
|
|
if (max == 0) {
|
|
printf("Incorrect setting of max tasks count, set to same "
|
|
"as min task value\n");
|
|
max = min;
|
|
}
|
|
} else {
|
|
min = atoi(optarg);
|
|
max = min;
|
|
}
|
|
break;
|
|
case 'a':
|
|
arsize = atoi(optarg);
|
|
if (arsize <= 0) {
|
|
arsize = 1;
|
|
}
|
|
break;
|
|
default:
|
|
printf(
|
|
"jobsub -u <user> -q <queue> -R <resreq> -c <command line>\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
}
|
|
|
|
if (min == 0) {
|
|
printf("Incorrect setting of min task count, set to 1 as default\n");
|
|
min = 1;
|
|
}
|
|
|
|
if (NULLSTR(submit.user)) {
|
|
printf("you need to specify job user with -u\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
|
|
if (NULLSTR(submit.queue)) {
|
|
printf("you need to specify job queue with -q\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
|
|
if (NULLSTR(submit.start_info->cmdline)) {
|
|
printf("you need to specify job command with -c\n");
|
|
rc = -EINVAL;
|
|
goto cleanup;
|
|
}
|
|
|
|
if (NULLSTR(submit.resreq)) {
|
|
submit.resreq = strdup("type == any");
|
|
}
|
|
|
|
submit.mintasks = min;
|
|
submit.maxtasks = max;
|
|
|
|
/* apiserverId=0 here let two unrelated jobs collide on the same jobid
|
|
* whenever the harness spawns a fresh xce_test process within the same
|
|
* wall-clock second (idgen_ctx_init resets sequence to 0 each time,
|
|
* and ts/tag/seq alone were then identical for both processes' first
|
|
* submission). Production apiserver instances never hit this: each
|
|
* holds one process-lifetime idgen_ctx_t (clib.go's sync.Once) and is
|
|
* assigned its own distinct apiserverId, so the jobid namespace is
|
|
* already partitioned by config.h's bit layout. Folding this process's
|
|
* pid into apiserverId gives each xce_test invocation its own slice of
|
|
* that same partitioning, making a same-second cross-process collision
|
|
* structurally impossible here too. */
|
|
idgen(&idctx, 0, (size_t)((uint64_t)getpid() & MAX_APISERVER_ID), arsize,
|
|
&(jobsubmit.jobid));
|
|
|
|
if (NULLSTR(submit.start_info->execuser)) {
|
|
submit.start_info->execuser = strdup(submit.user);
|
|
}
|
|
|
|
if (NULLSTR(submit.start_info->exechome)) {
|
|
submit.start_info->exechome = strdup("/tmp");
|
|
}
|
|
|
|
if (NULLSTR(submit.start_info->execcwd)) {
|
|
submit.start_info->execcwd = strdup("/tmp");
|
|
}
|
|
|
|
build_job_exdata(&jobsubmit);
|
|
|
|
jobsubmit.array_size = arsize;
|
|
rc = xcecli_jobsubmit(active_ctx->ctx, &jobsubmit, &jobid);
|
|
|
|
cleanup:
|
|
FREEUP(submit.user);
|
|
FREEUP(submit.queue);
|
|
FREEUP(submit.resreq);
|
|
FREEUP(submit.start_info->cmdline);
|
|
FREEUP(submit.start_info->exdata);
|
|
FREEUP(submit.start_info);
|
|
|
|
/* Report success only on a clean ack that echoes back the EXACT JobId we
|
|
* submitted (bug-786, Req 7.1/7.2/7.3). xce_jobsubmit() writes *jobid only
|
|
* when the reply status is >= 0; XCECLI_FAILOVER_CALL surfaces a non-clean
|
|
* outcome as a negative rc (e.g. -E_XCE_NOT_READY on no-primary budget
|
|
* exhaustion, -E_XCE_REDIRECT / -E_XCE_NOMASTER on a standby, -ETIMEDOUT /
|
|
* -ENOTCONN on transport loss).
|
|
*
|
|
* The `jobid != 0` gate alone is NOT sufficient: on a failover, libxce's
|
|
* handle_sub_reply() returns -1 on a non-jobid reply but LEAVES the stale
|
|
* previous value in ctx->reply.u.jobid, and xceapi.c copies it out whenever
|
|
* status >= 0 — so a rejected submit can surface as rc >= 0 with a NON-ZERO
|
|
* stale-garbage jobid (the live soak captured `63-15--1-1015808[32751]`
|
|
* while the controller logged "handle_jobsubmit fail to lock job" and never
|
|
* committed it). We therefore require the returned jobid to EQUAL the JobId
|
|
* we generated locally and submitted: on a genuine clean ack the controller
|
|
* commits and echoes back exactly submit->jobid (handle_jobsubmit replies
|
|
* reply.data.jobid = job_submit() = base_id = submit->jobid; verified
|
|
* round-trip). `jobid != jobsubmit.jobid` covers BOTH the never-written
|
|
* (zero) case AND the stale non-zero garbage case in one uint64 compare. */
|
|
if (rc < 0) {
|
|
char ebuf[128];
|
|
|
|
ebuf[0] = '\0';
|
|
xce_strerror_r(-rc, ebuf, sizeof(ebuf));
|
|
SET_ERROR(-rc);
|
|
|
|
switch (-rc) {
|
|
case E_XCE_NOMASTER:
|
|
fprintf(stderr, "jobsubmit failed: no-master (%s)\n", ebuf);
|
|
break;
|
|
case E_XCE_REDIRECT:
|
|
fprintf(stderr, "jobsubmit failed: redirect (%s)\n", ebuf);
|
|
break;
|
|
case E_XCE_NOT_READY:
|
|
fprintf(stderr, "jobsubmit failed: not-ready (%s)\n", ebuf);
|
|
break;
|
|
case ETIMEDOUT:
|
|
fprintf(stderr, "jobsubmit failed: timeout (%s)\n", ebuf);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "jobsubmit failed: %s\n", ebuf);
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
if (jobid != jobsubmit.jobid) {
|
|
/* Non-negative rc but the returned JobId does not match the one we
|
|
* submitted: not a clean ack. This rejects BOTH the never-written
|
|
* (jobid == 0) case AND the stale-garbage non-zero case left in the
|
|
* libxce reply union on a failover-rejected submit. Never print "ok"
|
|
* with an uninitialized/garbage id (Req 7.3). */
|
|
char wantStr[128];
|
|
char gotStr[128];
|
|
|
|
fmt_jobid(wantStr, sizeof(wantStr) - 1, jobsubmit.jobid);
|
|
fmt_jobid(gotStr, sizeof(gotStr) - 1, jobid);
|
|
fprintf(stderr,
|
|
"jobsubmit failed: no clean ack (controller did not confirm "
|
|
"the submitted JobId: wanted %s, got %s)\n",
|
|
wantStr, gotStr);
|
|
return -E_XCE_INTERNAL;
|
|
}
|
|
|
|
fmt_jobid(jobidStr, sizeof(jobidStr) - 1, array_jobid(jobid));
|
|
|
|
/* Capture $jobid (Req 3.2) using the SAME representation cmd_joblist prints
|
|
* for a job's JobId column — fmt_jobid(jobs[i]->jobid), the full jobid
|
|
* WITHOUT array_jobid() stripping — so a later `$jobid` substitution and
|
|
* `expect-job $jobid` compare equal to the joblist entry for this job. Only
|
|
* set on this clean-ack + valid-id path; never on a failure path, so an
|
|
* unset $jobid still fails per Req 3.6. */
|
|
fmt_jobid(g_last_jobid, sizeof(g_last_jobid) - 1, jobid);
|
|
|
|
printf("jobsubmit ok: JobId: %s\n", jobidStr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_resval(int argc, char* argv[])
|
|
{
|
|
char* host = NULL;
|
|
char* res = NULL;
|
|
xce_resval_t** vals = NULL;
|
|
int opt;
|
|
int rc;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "n:m:")) != -1) {
|
|
switch (opt) {
|
|
case 'n':
|
|
res = strdup(optarg);
|
|
break;
|
|
case 'm':
|
|
host = strdup(optarg);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "resval -n resname [-m host]\n");
|
|
FREEUP(host);
|
|
FREEUP(res);
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
rc = xcecli_resval(active_ctx->ctx, host, res, &vals);
|
|
if (rc > 0) {
|
|
int i;
|
|
|
|
for (i = 0; i < rc; i++) {
|
|
xce_resval_t* val = vals[i];
|
|
|
|
printf("resource %s@%s:\n", val->name,
|
|
(val->host == NULL) ? "cluster" : val->host);
|
|
printf("value: ");
|
|
switch (val->val->which) {
|
|
case Value_u_svalue:
|
|
printf("%s\n", val->val->u.svalue);
|
|
break;
|
|
case Value_u_fvalue:
|
|
printf("%f\n", val->val->u.fvalue);
|
|
break;
|
|
case Value_u_bvalue:
|
|
printf("%s\n", (val->val->u.bvalue == 0) ? "false" : "true");
|
|
break;
|
|
default:
|
|
printf(" <unknown type>\n");
|
|
break;
|
|
}
|
|
|
|
free_ResourceVal_ptr(&val);
|
|
}
|
|
|
|
FREEUP(vals);
|
|
} else {
|
|
printf("no resource value found in given host\n");
|
|
}
|
|
|
|
FREEUP(host);
|
|
FREEUP(res);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobkill(int argc, char* argv[])
|
|
{
|
|
int signo;
|
|
uint64_t jobid = 0;
|
|
int opt;
|
|
int rc;
|
|
xce_jobctrl_t jobctrl;
|
|
xce_signal_t siginfo;
|
|
|
|
optind = 1;
|
|
signo = SIGTERM;
|
|
|
|
while ((opt = getopt(argc, argv, "s:j:")) != -1) {
|
|
switch (opt) {
|
|
case 's':
|
|
signo = atoi(optarg);
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
fprintf(stderr, "you need to specify jobid with -j\n");
|
|
return -1;
|
|
}
|
|
|
|
jobctrl.action = JobControl_u_siginfo;
|
|
jobctrl.jobid = jobid;
|
|
jobctrl.u.siginfo = &siginfo;
|
|
siginfo.signo = signo;
|
|
siginfo.force = 1;
|
|
|
|
rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_jobctrl");
|
|
return rc;
|
|
}
|
|
|
|
printf("Job %lu signaled\n", jobid);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobckpt(int argc, char* argv[])
|
|
{
|
|
uint64_t jobid = 0;
|
|
int opt;
|
|
int rc;
|
|
xce_jobctrl_t jobctrl;
|
|
xce_ckpt_options_t ckptoption;
|
|
|
|
optind = 1;
|
|
ckptoption.kill = 0;
|
|
|
|
while ((opt = getopt(argc, argv, "kj:m:")) != -1) {
|
|
switch (opt) {
|
|
case 'k':
|
|
ckptoption.kill = 1;
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'm':
|
|
ckptoption.method = optarg;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
fprintf(stderr, "you need to specify jobid with -j\n");
|
|
return -1;
|
|
}
|
|
|
|
if (ckptoption.method == NULL) {
|
|
ckptoption.method = "default";
|
|
}
|
|
jobctrl.action = JobControl_u_ckptoption;
|
|
jobctrl.jobid = jobid;
|
|
jobctrl.u.ckptoption = &ckptoption;
|
|
|
|
rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_jobctrl");
|
|
return rc;
|
|
}
|
|
|
|
printf("Job %lu checkpointed\n", jobid);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_qctrl(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
int rc;
|
|
xce_queuectrl_t queuectrl;
|
|
int action;
|
|
char* queue = NULL;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "aiociq:")) != -1) {
|
|
switch (opt) {
|
|
case 'a':
|
|
action = QUEUE_ACTIVE;
|
|
break;
|
|
case 'i':
|
|
action = QUEUE_INACTIVE;
|
|
break;
|
|
case 'o':
|
|
action = QUEUE_OPEN;
|
|
break;
|
|
case 'c':
|
|
action = QUEUE_CLOSE;
|
|
break;
|
|
case 'q':
|
|
queue = strdup(optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (queue == NULL) {
|
|
fprintf(stderr, "you need to specify queue name with -q\n");
|
|
return -1;
|
|
}
|
|
|
|
queuectrl.action = action;
|
|
queuectrl.queue = queue;
|
|
|
|
rc = xcecli_queuectrl(active_ctx->ctx, &queuectrl);
|
|
if (rc < 0) {
|
|
perror("xcecli_queuectrl");
|
|
return rc;
|
|
}
|
|
|
|
printf("Operation on queue %s done\n", queue);
|
|
|
|
free(queue);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_runtasks(int argc, char* argv[])
|
|
{
|
|
xce_taskhost_t hosts[MAX_TASK_HOSTS];
|
|
size_t nhosts = 0;
|
|
xce_taskrun_t taskrun;
|
|
int opt;
|
|
int rc;
|
|
int count;
|
|
int totaltasks = 0;
|
|
char* user = NULL;
|
|
char* cmdline = NULL;
|
|
char* dir = NULL;
|
|
char* exdata = NULL;
|
|
char* host = NULL;
|
|
char* launch_type = NULL;
|
|
int i;
|
|
uint64_t jobid = 0;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "m:u:c:x:d:j:t:")) != -1) {
|
|
switch (opt) {
|
|
case 'm':
|
|
if (nhosts >= MAX_TASK_HOSTS) {
|
|
fprintf(stderr,
|
|
"too many run hosts, just support %d run hosts\n",
|
|
MAX_TASK_HOSTS);
|
|
return -1;
|
|
}
|
|
|
|
if (strchr(optarg, (int)':') != NULL) {
|
|
char* p;
|
|
char buf[256];
|
|
|
|
p = strchr(optarg, (int)':');
|
|
count = atoi(p + 1);
|
|
memset(buf, 0x0, sizeof(buf));
|
|
strncpy(buf, optarg, (int)(p - optarg));
|
|
host = strdup(buf);
|
|
} else {
|
|
host = strdup(optarg);
|
|
count = 1;
|
|
}
|
|
|
|
hosts[nhosts].host = host;
|
|
hosts[nhosts].ntasks = count;
|
|
nhosts++;
|
|
totaltasks += count;
|
|
break;
|
|
case 'u':
|
|
if (user != NULL) {
|
|
free(user);
|
|
}
|
|
user = strdup(optarg);
|
|
break;
|
|
case 'c':
|
|
if (cmdline != NULL) {
|
|
free(cmdline);
|
|
}
|
|
cmdline = strdup(optarg);
|
|
break;
|
|
case 'x':
|
|
if (exdata != NULL) {
|
|
free(exdata);
|
|
}
|
|
exdata = strdup(optarg);
|
|
break;
|
|
case 'd':
|
|
if (dir != NULL) {
|
|
free(dir);
|
|
}
|
|
dir = strdup(optarg);
|
|
break;
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 't':
|
|
launch_type = strdup(optarg);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < nhosts; i++) {
|
|
hosts[i].exechome = dir;
|
|
hosts[i].execcwd = dir;
|
|
hosts[i].execuser = user;
|
|
hosts[i].exdata = exdata;
|
|
hosts[i].cmdline = cmdline;
|
|
hosts[i].launch_type = launch_type;
|
|
if (hosts[i].launch_type == NULL) {
|
|
if (totaltasks > 1) {
|
|
hosts[i].launch_type = strdup("mpi");
|
|
} else {
|
|
hosts[i].launch_type = strdup("sequentail");
|
|
}
|
|
}
|
|
}
|
|
|
|
taskrun.taskhosts
|
|
= (xce_taskhost_t**)calloc(nhosts, sizeof(xce_taskhost_t*));
|
|
taskrun.clientidx = -1;
|
|
|
|
if (taskrun.taskhosts != NULL) {
|
|
for (i = 0; i < nhosts; i++) {
|
|
taskrun.taskhosts[i] = &hosts[i];
|
|
}
|
|
|
|
taskrun.nhosts = nhosts;
|
|
taskrun.jobid = jobid;
|
|
rc = xcecli_runtask(active_ctx->ctx, &taskrun);
|
|
} else {
|
|
rc = -ENOMEM;
|
|
}
|
|
|
|
if (rc < 0) {
|
|
errno = -rc;
|
|
perror("xcecli_runtask");
|
|
} else {
|
|
printf("run task ok\n");
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int add_pendreason(xce_job_preason_t* preason, char* s)
|
|
{
|
|
char s1[MAX_LINE_LEN];
|
|
char* pcode = NULL;
|
|
char* phost = NULL;
|
|
char* pres = NULL;
|
|
char* p;
|
|
int code;
|
|
|
|
if ((preason == NULL) || (s == NULL)) {
|
|
fprintf(stderr, "bad parameter\n");
|
|
return -1;
|
|
}
|
|
|
|
if ((p = strrchr(s, (int)':')) == NULL) {
|
|
pcode = s;
|
|
} else {
|
|
pcode = p + 1;
|
|
memset(s1, 0x0, sizeof(s1));
|
|
strncpy(s1, s, p - s);
|
|
|
|
if ((p = strrchr(s1, (int)':')) == NULL) {
|
|
phost = s1;
|
|
} else {
|
|
*p = 0x0;
|
|
phost = p + 1;
|
|
pres = s1;
|
|
}
|
|
}
|
|
|
|
code = atoi(pcode);
|
|
|
|
return add_job_preason(preason, code, phost, pres);
|
|
}
|
|
|
|
int cmd_setpend(int argc, char* argv[])
|
|
{
|
|
int opt;
|
|
xce_job_preason_t jreasons;
|
|
xce_pendreason_t preason = { .reasons = &jreasons };
|
|
xce_preason_set_t reasons
|
|
= { .nreasons = 1, .reasons = (xce_pendreason_t*[]) { &preason } };
|
|
int rc = 0;
|
|
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "j:p:")) != -1) {
|
|
int rc = 0;
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&(preason.jobid), optarg);
|
|
break;
|
|
case 'p':
|
|
rc = add_pendreason(&jreasons, optarg);
|
|
if (rc < 0) {
|
|
perror("add_pendreason");
|
|
return rc;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (preason.jobid == 0) {
|
|
fprintf(stderr, "no or incorrect jobid specified\n");
|
|
return -1;
|
|
}
|
|
|
|
if ((jreasons.main_reason == 0) && (jreasons.nhost_reasons == 0)
|
|
&& (jreasons.nres_reasons == 0)) {
|
|
fprintf(stderr, "no or incorrect pend reason set\n");
|
|
return -1;
|
|
}
|
|
|
|
rc = xcecli_set_pend_reason(active_ctx->ctx, &reasons);
|
|
if (rc < 0) {
|
|
perror("xcecli_set_pend_reason");
|
|
erase_job_preason(&jreasons);
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_join_worker(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int role;
|
|
int opt;
|
|
hostattrs_t attrs = { .cpufactor = 1.0,
|
|
.maxcpus = 4,
|
|
.maxmem = 64,
|
|
.maxswap = 1024,
|
|
.maxtmp = 2048,
|
|
.ndisks = 1 };
|
|
|
|
role = HostRole_worker;
|
|
|
|
optind = 1;
|
|
|
|
attrs.hostmodel = "Intel64";
|
|
attrs.hosttype = "LINUX64";
|
|
|
|
while ((opt = getopt(argc, argv, "f:c:m:s:t:d:")) != -1) {
|
|
switch (opt) {
|
|
case 'f':
|
|
attrs.cpufactor = atof(optarg);
|
|
break;
|
|
case 'c':
|
|
attrs.maxcpus = atoi(optarg);
|
|
break;
|
|
case 'm':
|
|
attrs.maxmem = atoi(optarg);
|
|
break;
|
|
case 's':
|
|
attrs.maxswap = atoi(optarg);
|
|
break;
|
|
case 't':
|
|
attrs.maxtmp = atoi(optarg);
|
|
break;
|
|
case 'd':
|
|
attrs.ndisks = atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" join [-f cpufactor] [-c maxcpus] [-m maxmem] [-s maxswap] "
|
|
"[-t maxtmp] [-d ndisks]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
rc = xcecli_join(active_ctx->ctx, role, &attrs);
|
|
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_join");
|
|
return rc;
|
|
}
|
|
|
|
printf("host join ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* jstart -j jobid [-p pid] — raft-apply-decoupling task 7.2: synthetic
|
|
* worker-fact-stream primitive. Mirrors cmd_jobfinish's existing pattern
|
|
* exactly (same getopt/parse_jobid/SET_ERROR shape) but drives the
|
|
* xcecli_notify_jobstart wire call that cmd_jobfinish's sibling
|
|
* xcecli_notify_jobfinish already exercises — libxce (client.c) already
|
|
* implements the full worker-fact API (jobstart/jobfinish/jobfail/...); this
|
|
* command was the missing REPL-level primitive for jobstart specifically.
|
|
* Drives the SAME production receipt path (on_job_start -> record_job_start_fact
|
|
* -> persist_submit_job_fact -> durable append -> commit -> apply pump ->
|
|
* apply_committed_entry()) that a real worker's report would, without
|
|
* requiring a real scheduler dispatch (on_job_start only requires the job to
|
|
* already exist in jobmgr, i.e. have been submitted). */
|
|
int cmd_jobstart(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
unsigned int pid = 0;
|
|
unsigned int pids[1];
|
|
job_pidinfo_t info;
|
|
|
|
optind = 1;
|
|
memset(&info, 0x0, sizeof(info));
|
|
|
|
while ((opt = getopt(argc, argv, "j:p:")) != -1) {
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'p':
|
|
pid = (unsigned int)atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" jstart -j jobid [-p pid]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf(" you need specify a positive jobid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
pids[0] = (pid != 0) ? pid : (unsigned int)getpid();
|
|
info.jobid = jobid;
|
|
info.npids = 1;
|
|
info.pids = pids;
|
|
info.npgids = 0;
|
|
info.pgids = NULL;
|
|
info.exdata = NULL;
|
|
|
|
rc = xcecli_notify_jobstart(active_ctx->ctx, &info);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_jobstart");
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
int cmd_jobfinish(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
int code = 0;
|
|
xce_jobfinish_t finish;
|
|
|
|
/* raft-apply-decoupling task 7.2: reset optind so repeated jfinish
|
|
* invocations within one xce_test process (the fact-stream soak scenario
|
|
* calls this many times in a single worker session) each re-parse their
|
|
* own argv from the start. Without this, getopt's optind carries over
|
|
* from the PRIOR command's parse position, silently skipping this
|
|
* command's own "-j jobid" on the second and later calls (jobid stays 0
|
|
* and the command errors out) — matches the reset convention most other
|
|
* cmd_ handlers in this file already follow (e.g. cmd_join_worker just
|
|
* above, cmd_jobsub, cmd_resupdate). */
|
|
optind = 1;
|
|
|
|
while ((opt = getopt(argc, argv, "j:c:")) != -1) {
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'c':
|
|
code = atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" jfinish -j jobid [-c exitcode]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf(" you need specify a positive jobid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
finish.jobid = jobid;
|
|
finish.exit_code = code;
|
|
|
|
rc = xcecli_notify_jobfinish(active_ctx->ctx, &finish);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_jobfinish");
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* jfail -j jobid [-c code] [-s stage] — raft-apply-decoupling task 7.2:
|
|
* synthetic worker-fact-stream primitive, sibling of jstart/jfinish above.
|
|
* Drives xcecli_notify_jobfail -> on_job_fail -> record_job_fail_fact ->
|
|
* persist_submit_job_fact_ex (the _pad-discriminated path, task 6.1 round 2)
|
|
* -> apply_job_fail_transition() at apply time. Reuses the existing
|
|
* job_failinfo_t shape already exercised internally by this file's own
|
|
* notify_job_fail() helper (used by run_job() on a local exec failure). */
|
|
int cmd_jobfail(int argc, char* argv[])
|
|
{
|
|
int rc;
|
|
int opt;
|
|
uint64_t jobid = 0;
|
|
int code = 0;
|
|
int stage = JOB_EXEC_STAGE_POST_EXEC;
|
|
job_failinfo_t failinfo;
|
|
|
|
optind = 1;
|
|
memset(&failinfo, 0x0, sizeof(failinfo));
|
|
|
|
while ((opt = getopt(argc, argv, "j:c:s:")) != -1) {
|
|
switch (opt) {
|
|
case 'j':
|
|
parse_jobid(&jobid, optarg);
|
|
break;
|
|
case 'c':
|
|
code = atoi(optarg);
|
|
break;
|
|
case 's':
|
|
stage = atoi(optarg);
|
|
break;
|
|
default:
|
|
printf(" jfail -j jobid [-c code] [-s stage]\n");
|
|
return -EINVAL;
|
|
}
|
|
}
|
|
|
|
if (jobid == 0) {
|
|
printf(" you need specify a positive jobid\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
failinfo.jobid = jobid;
|
|
failinfo.stage = stage;
|
|
failinfo.code = code;
|
|
failinfo.detail = "synthetic fact-stream failure (xce_test jfail)";
|
|
|
|
rc = xcecli_notify_jobfail(active_ctx->ctx, &failinfo);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_jobfail");
|
|
return rc;
|
|
}
|
|
|
|
printf("ok\n");
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int set_resval(xce_resval_t* vals, int idx, char* name, char* val)
|
|
{
|
|
static struct {
|
|
char* name;
|
|
int valtype;
|
|
} resinfo[] = {
|
|
{ "r15s", Value_u_fvalue },
|
|
{ "r1m", Value_u_fvalue },
|
|
{ "slots", Value_u_fvalue },
|
|
{ "ut", Value_u_fvalue },
|
|
{ "mem", Value_u_fvalue },
|
|
{ "maxcpus", Value_u_fvalue },
|
|
{ "ndisks", Value_u_fvalue },
|
|
{ "cpuf", Value_u_fvalue },
|
|
{ "maxswap", Value_u_fvalue },
|
|
{ "maxtmp", Value_u_fvalue },
|
|
{ "type", Value_u_svalue },
|
|
{ "model", Value_u_svalue },
|
|
{ "verilog", Value_u_fvalue },
|
|
{ "dyna", Value_u_fvalue },
|
|
{ NULL, 0 }
|
|
};
|
|
int i;
|
|
int rc = -1;
|
|
|
|
if (name == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
for (i = 0; resinfo[i].name != NULL; i++) {
|
|
if (strcmp(resinfo[i].name, name) == 0) {
|
|
rc = 1;
|
|
vals[idx].val = (xce_val_t*)calloc(1, sizeof(xce_val_t));
|
|
|
|
switch (resinfo[i].valtype) {
|
|
case Value_u_fvalue:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = resinfo[i].valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.fvalue = 0.0;
|
|
} else {
|
|
vals[idx].val->u.fvalue = atof(val);
|
|
}
|
|
break;
|
|
case Value_u_ivalue32:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = resinfo[i].valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.ivalue32 = 0;
|
|
} else {
|
|
vals[idx].val->u.ivalue32 = atoi(val);
|
|
}
|
|
break;
|
|
case Value_u_ivalue64:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = resinfo[i].valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.ivalue64 = 0;
|
|
} else {
|
|
vals[idx].val->u.ivalue64 = atol(val);
|
|
}
|
|
break;
|
|
case Value_u_bvalue:
|
|
vals[idx].name = name;
|
|
vals[idx].val->which = resinfo[i].valtype;
|
|
if (val == NULL) {
|
|
vals[idx].val->u.bvalue = 0;
|
|
} else {
|
|
if (strcasecmp(val, "t") == 0) {
|
|
vals[idx].val->u.bvalue = 1;
|
|
} else {
|
|
vals[idx].val->u.bvalue = 0;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
vals[idx].host = get_setting("myname");
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
int cmd_resupdate(int argc, char* argv[])
|
|
{
|
|
#define MAX_N_RES (16)
|
|
int nres;
|
|
xce_resval_t vals[MAX_N_RES];
|
|
xce_resval_t* valptrs[MAX_N_RES];
|
|
char* resname = NULL;
|
|
int opt;
|
|
int rc;
|
|
|
|
optind = 1;
|
|
nres = 0;
|
|
while ((opt = getopt(argc, argv, "n:v:")) != -1) {
|
|
switch (opt) {
|
|
case 'n':
|
|
if (resname != NULL) {
|
|
printf(
|
|
"res `%s' has no value specified, default will be used\n",
|
|
resname);
|
|
rc = set_resval(vals, nres, resname, NULL);
|
|
if (rc >= 0) {
|
|
valptrs[nres] = &vals[nres];
|
|
nres++;
|
|
}
|
|
resname = NULL;
|
|
}
|
|
|
|
resname = strdup(optarg);
|
|
break;
|
|
case 'v':
|
|
if (resname != NULL) {
|
|
rc = set_resval(vals, nres, resname, optarg);
|
|
if (rc >= 0) {
|
|
valptrs[nres] = &vals[nres];
|
|
nres++;
|
|
}
|
|
resname = NULL;
|
|
} else {
|
|
printf("ignore res value `%s' as no res name found\n", optarg);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
rc = xcecli_update_resvals(active_ctx->ctx, nres, &valptrs[0]);
|
|
if (rc >= 0) {
|
|
printf("resource update ok!\n");
|
|
} else {
|
|
SET_ERROR(-rc);
|
|
perror("xcecli_update_resval");
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int cmd_join_sched(int argc, char* argv[])
|
|
{
|
|
int role = HostRole_scheduler;
|
|
int rc = -1;
|
|
|
|
rc = xcecli_join(active_ctx->ctx, role, NULL);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_join");
|
|
return rc;
|
|
}
|
|
|
|
printf("scheduler join ok\n");
|
|
return 0;
|
|
}
|
|
|
|
static int print_reslist(xce_reslist_t reslist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < reslist.count; i++) {
|
|
printf("name: %s\nflags: %d\ninterval: %d\ndesc: %s\n\n",
|
|
reslist.res[i]->name, reslist.res[i]->flags,
|
|
reslist.res[i]->interval, reslist.res[i]->desc);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int print_queuelist(xce_queuelist_t qlist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < qlist.count; i++) {
|
|
printf("name: %s\nstatus: %d\npriority: %d\n\n", qlist.queues[i]->queue,
|
|
qlist.queues[i]->status, qlist.queues[i]->priority);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int print_workerlist(xce_workerlist_t wlist)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < wlist.count; i++) {
|
|
printf("name: %s\nstatus: %d\n\n", wlist.workers[i]->name,
|
|
wlist.workers[i]->status);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int print_grouplist(xce_group_configlist_t glist) { return 0; }
|
|
|
|
static int print_configlist(xce_configlist_t clist) { return 0; }
|
|
|
|
int cmd_getconf(int argc, char* argv[])
|
|
{
|
|
xce_clusterconf_t* conf;
|
|
int rc;
|
|
|
|
rc = xcecli_clusterconf(active_ctx->ctx, &conf);
|
|
if (rc < 0) {
|
|
SET_ERROR(-rc);
|
|
perror("xce_clusterconf");
|
|
return rc;
|
|
}
|
|
|
|
printf("Cluster Name: %s\n", conf->name);
|
|
|
|
printf("\nResources\n===========\n");
|
|
print_reslist(conf->reslist);
|
|
|
|
printf("\nQueues\n========\n");
|
|
print_queuelist(conf->qlist);
|
|
|
|
printf("\nWorkers\n========\n");
|
|
print_workerlist(conf->wlist);
|
|
|
|
printf("\nUser Groups\n===========\n");
|
|
print_grouplist(conf->uglist);
|
|
|
|
printf("\nHost Groups\n============\n");
|
|
print_grouplist(conf->hglist);
|
|
|
|
printf("\nConfigs\n=======\n");
|
|
print_configlist(conf->cfglist);
|
|
|
|
free_ClusterConf(conf);
|
|
|
|
return 0;
|
|
}
|
|
#define KVAL(v) (((v) == NULL) ? "<not set>" : (v))
|
|
|
|
int cmd_set(int argc, char* argv[])
|
|
{
|
|
int i;
|
|
|
|
if (argc == 1) {
|
|
for (i = 0; active_ctx->settings[i].key; i++) {
|
|
printf("%-12s:%s default: %s\n", active_ctx->settings[i].key,
|
|
KVAL(active_ctx->settings[i].value),
|
|
KVAL(active_ctx->settings[i].defval));
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* errexit toggle (xce-test-e2e-primitives task 1.1), shell `-e`/`+e` style.
|
|
* Default is mode-dependent (batch `-e` abort-on-failure, interactive `+e`
|
|
* continue); these override it for the rest of the session. The session
|
|
* still exits non-zero if any command failed. Handled here ahead of the
|
|
* per-context key/value path. `continue-on-error` == `+e` (back-compat). */
|
|
if ((argc == 2) &&
|
|
((strcmp(argv[1], "+e") == 0) ||
|
|
(strcmp(argv[1], "continue-on-error") == 0))) {
|
|
g_continue_on_error = 1; /* +e: keep the read loop running on failure */
|
|
return 0;
|
|
}
|
|
if ((argc == 2) && (strcmp(argv[1], "-e") == 0)) {
|
|
g_continue_on_error = 0; /* -e: abort the read loop on first failure */
|
|
return 0;
|
|
}
|
|
|
|
if (argc != 3) {
|
|
printf("set <key> <value>\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
for (i = 0; active_ctx->settings[i].key; i++) {
|
|
if (strcmp(active_ctx->settings[i].key, argv[1]) == 0) {
|
|
if (active_ctx->settings[i].value != NULL) {
|
|
free(active_ctx->settings[i].value);
|
|
}
|
|
|
|
active_ctx->settings[i].value = strdup(argv[2]);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
printf("Can not find setting key: %s\n", argv[1]);
|
|
return -EINVAL;
|
|
}
|
|
|
|
static int run_command(char* cmdline, pid_t* pid)
|
|
{
|
|
pid_t child;
|
|
|
|
child = fork();
|
|
|
|
*pid = child;
|
|
if (child < 0) {
|
|
*pid = -errno;
|
|
|
|
return -errno;
|
|
}
|
|
|
|
if (child == 0) {
|
|
execlp("/bin/sh", "/bin/sh", "-c", cmdline, NULL);
|
|
perror("execlp");
|
|
exit(-1);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int run_task(char* cmdline, pid_t* pid)
|
|
{
|
|
return run_command(cmdline, pid);
|
|
}
|
|
|
|
static int get_avail_run_job_slots()
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < MAX_RUN_JOBS; i++) {
|
|
if (run_jobs[i].jobid == 0) {
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
static int get_avail_run_task_slots()
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < MAX_RUN_TASKS; i++) {
|
|
if (run_tasks[i].pid <= 0) {
|
|
return i;
|
|
}
|
|
}
|
|
|
|
return -1;
|
|
}
|
|
|
|
static int sched_on_jobs(int njobs, xce_job_t** jobs)
|
|
{
|
|
printf("%s %d jobs gotten\n", __func__, njobs);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_on_workers(int nworkers, xce_worker_t** workers)
|
|
{
|
|
printf("%s %d workers gotten\n", __func__, nworkers);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void notify_job_fail(uint64_t jobid, int code)
|
|
{
|
|
/* clang-format off */
|
|
job_failinfo_t failinfo = {
|
|
.jobid = jobid,
|
|
.stage = JOB_EXEC_STAGE_MAIN,
|
|
.code = code,
|
|
.detail = "no detail information"
|
|
};
|
|
/*clang-format on*/
|
|
|
|
xcecli_notify_jobfail(active_ctx->ctx, &failinfo);
|
|
return;
|
|
}
|
|
|
|
static void notify_task_fail(uint64_t jobid,
|
|
int clientidx,
|
|
int taskidx,
|
|
int code) {
|
|
xce_task_failinfo_t failinfo = {
|
|
.jobid = jobid,
|
|
.host = get_setting("myname"),
|
|
.clientidx = clientidx,
|
|
.taskidx = taskidx,
|
|
.code = code,
|
|
.detail = NULL
|
|
};
|
|
|
|
xcecli_notify_taskfail(active_ctx->ctx, &failinfo);
|
|
}
|
|
|
|
static void notify_jobstart(uint64_t jobid,
|
|
int pid,
|
|
int pgid) {
|
|
/* clang-format off */
|
|
job_pidinfo_t pidinfo = {
|
|
.jobid = jobid,
|
|
.npids = 1,
|
|
.pids = (unsigned int []){(unsigned int)pid},
|
|
.npgids = 0,
|
|
.pgids = (unsigned int []){(unsigned int)pgid},
|
|
.exdata = NULL
|
|
};
|
|
/* clang-format on */
|
|
|
|
xcecli_notify_jobstart(active_ctx->ctx, &pidinfo);
|
|
|
|
return;
|
|
}
|
|
|
|
static void notify_task_start(uint64_t jobid, int clientidx, int taskidx,
|
|
int pid, int pgid)
|
|
{
|
|
xce_task_pidinfo_t pidinfo
|
|
= { .jobid = jobid,
|
|
.host = get_setting("myname"),
|
|
.clientidx = clientidx,
|
|
.taskidx = taskidx,
|
|
.npids = 1,
|
|
.pids = (unsigned int[]) { (unsigned int)pid },
|
|
.npgids = 1,
|
|
.pgids = (unsigned int[]) { (unsigned int)pgid },
|
|
.exdata = NULL };
|
|
|
|
xcecli_notify_taskstart(active_ctx->ctx, &pidinfo);
|
|
|
|
return;
|
|
}
|
|
|
|
static int run_task_remote(uint64_t jobid, job_startinfo_t* startinfo,
|
|
xce_launch_t* launchinfo)
|
|
{
|
|
xce_taskhost_t hosts[launchinfo->nhosts - 1];
|
|
xce_taskrun_t taskrun;
|
|
int i;
|
|
int rc;
|
|
|
|
taskrun.taskhosts = (xce_taskhost_t**)calloc(launchinfo->nhosts - 1,
|
|
sizeof(xce_taskhost_t*));
|
|
if (taskrun.taskhosts == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
taskrun.clientidx = -1;
|
|
|
|
for (i = 1; i < launchinfo->nhosts; i++) {
|
|
hosts[i - 1].exechome = startinfo->exechome;
|
|
hosts[i - 1].execcwd = startinfo->execcwd;
|
|
hosts[i - 1].cmdline = startinfo->cmdline;
|
|
hosts[i - 1].exdata = startinfo->exdata;
|
|
hosts[i - 1].execuser = startinfo->execuser;
|
|
hosts[i - 1].launch_type = startinfo->launch_type;
|
|
hosts[i - 1].host = launchinfo->hosts[i]->host;
|
|
hosts[i - 1].ntasks = launchinfo->hosts[i]->nranks;
|
|
|
|
if (hosts[i - 1].launch_type == NULL) {
|
|
/* use mpi as default */
|
|
hosts[i - 1].launch_type = "mpi";
|
|
}
|
|
taskrun.taskhosts[i - 1] = &hosts[i - 1];
|
|
}
|
|
|
|
taskrun.nhosts = launchinfo->nhosts - 1;
|
|
taskrun.jobid = jobid;
|
|
|
|
xcecli_unblock_tasks(active_ctx->ctx);
|
|
|
|
rc = xcecli_runtask(active_ctx->ctx, &taskrun);
|
|
if (rc < 0) {
|
|
fprintf(stderr, "run remote task failed\n");
|
|
}
|
|
|
|
FREEUP(taskrun.taskhosts);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int run_task_local(uint64_t jobid, job_startinfo_t* startinfo,
|
|
int ntasks)
|
|
{
|
|
pid_t pid;
|
|
|
|
pid = fork();
|
|
if (pid < 0) {
|
|
return pid;
|
|
}
|
|
|
|
if (pid == 0) {
|
|
int i;
|
|
pid_t pids[ntasks];
|
|
int rc = 0;
|
|
int nlive = 0;
|
|
|
|
/* disable detection of address sanitizer as
|
|
* the request structure will not be freed in
|
|
* child and we will exit directly
|
|
*/
|
|
setenv("ASAN_OPTIONS", "detect_leaks=false", 1);
|
|
|
|
for (i = 0; i < ntasks; i++) {
|
|
rc = run_command(startinfo->cmdline, &pids[i]);
|
|
if (rc >= 0) {
|
|
nlive++;
|
|
}
|
|
}
|
|
|
|
if (rc >= 0) {
|
|
rc = 0;
|
|
}
|
|
|
|
while (nlive > 0) {
|
|
sleep(1);
|
|
for (i = 0; i < ntasks; i++) {
|
|
int wstatus;
|
|
|
|
if (pids[i] <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (waitpid(pids[i], &wstatus, WNOHANG) <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
|
|
nlive--;
|
|
|
|
if (WIFSIGNALED(wstatus)) {
|
|
rc = 128 + WTERMSIG(wstatus);
|
|
} else {
|
|
if (WEXITSTATUS(wstatus) != 0) {
|
|
rc = WEXITSTATUS(wstatus);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
exit(rc);
|
|
}
|
|
|
|
printf("Job %lu started, PID: %d\n", jobid, pid);
|
|
|
|
return pid;
|
|
}
|
|
|
|
int run_job(xce_job_exec_t* startmsg)
|
|
{
|
|
int rc = 0;
|
|
uint64_t jobid = startmsg->jobid;
|
|
job_startinfo_t* startinfo = startmsg->start_info;
|
|
xce_launch_t* launchinfo = startmsg->launchinfo;
|
|
|
|
printf("%s: exechome= %s\n", __func__,
|
|
(startinfo->exechome == NULL) ? "NULL" : startinfo->exechome);
|
|
printf("%s: execcwd= %s\n", __func__,
|
|
(startinfo->execcwd == NULL) ? "NULL" : startinfo->execcwd);
|
|
printf("%s: execuser= %s\n", __func__,
|
|
(startinfo->execuser == NULL) ? "NULL" : startinfo->execuser);
|
|
printf("%s: cmdline= %s\n", __func__,
|
|
(startinfo->cmdline == NULL) ? "NULL" : startinfo->cmdline);
|
|
printf("%s: exdata= %s\n", __func__,
|
|
(startinfo->exdata == NULL) ? "NULL" : startinfo->exdata);
|
|
printf("%s: launch_type= %s\n", __func__,
|
|
(startinfo->launch_type == NULL) ? "NULL" : startinfo->launch_type);
|
|
printf("%s: q_pre_exec= %s\n", __func__,
|
|
(startinfo->q_pre_exec == NULL) ? "NULL" : startinfo->q_pre_exec);
|
|
printf("%s: q_post_exec= %s\n", __func__,
|
|
(startinfo->q_post_exec == NULL) ? "NULL" : startinfo->q_post_exec);
|
|
printf("%s: job_pre_exec= %s\n", __func__,
|
|
(startinfo->job_pre_exec == NULL) ? "NULL"
|
|
: startinfo->job_pre_exec);
|
|
printf("%s: job_post_exec= %s\n", __func__,
|
|
(startinfo->job_post_exec == NULL) ? "NULL"
|
|
: startinfo->job_post_exec);
|
|
|
|
if (launchinfo->nhosts > 1) {
|
|
rc = run_task_remote(jobid, startinfo, launchinfo);
|
|
if (rc < 0) {
|
|
fprintf(stderr, "run_task_remote() fail, rc %d\n", rc);
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
rc = run_task_local(jobid, startinfo, launchinfo->hosts[0]->nranks);
|
|
if (rc < 0) {
|
|
notify_job_fail(jobid, rc);
|
|
} else {
|
|
int idx;
|
|
int pid;
|
|
|
|
pid = rc;
|
|
idx = get_avail_run_job_slots();
|
|
if (idx < 0) {
|
|
printf("too many jobs run in this worker\n");
|
|
} else {
|
|
run_jobs[idx].jobid = jobid;
|
|
run_jobs[idx].pid = pid;
|
|
run_jobs[idx].ts = 0;
|
|
}
|
|
|
|
notify_jobstart(jobid, pid, pid);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int handle_startjob(xce_request_t* request)
|
|
{
|
|
return run_job(request->body.jobstartmsg);
|
|
}
|
|
|
|
static int handle_notify(xce_request_t* request)
|
|
{
|
|
xce_notify_data_t* data = request->body.notifymsg;
|
|
|
|
switch (data->event) {
|
|
case NotifyData_data_taskPidInfo:
|
|
printf("job %lu host %s taskidx %d pid %d\n",
|
|
data->data.taskinfo->jobid, data->data.taskinfo->host,
|
|
data->data.taskinfo->taskidx, data->data.taskinfo->pids[0]);
|
|
break;
|
|
case NotifyData_data_taskFailInfo:
|
|
printf("job %lu host %s task %d error code %d\n",
|
|
data->data.taskfailinfo->jobid, data->data.taskfailinfo->host,
|
|
data->data.taskfailinfo->taskidx, data->data.taskfailinfo->code);
|
|
break;
|
|
case NotifyData_data_taskFinish:
|
|
printf("job %lu host %s task %d exit code %d\n",
|
|
data->data.taskfinish->jobid, data->data.taskfinish->host,
|
|
data->data.taskfinish->taskidx,
|
|
data->data.taskfinish->exit_code);
|
|
break;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int job_kill(uint64_t jobid, int signo)
|
|
{
|
|
int i;
|
|
int killed = 0;
|
|
|
|
for (i = 0; i < MAX_RUN_JOBS; i++) {
|
|
if ((run_jobs[i].jobid == 0) || (run_jobs[i].jobid != jobid)) {
|
|
continue;
|
|
}
|
|
|
|
if (run_jobs[i].pid > 1) {
|
|
kill(run_jobs[i].pid, signo);
|
|
killed++;
|
|
}
|
|
}
|
|
|
|
return killed;
|
|
}
|
|
|
|
static int job_chkpnt(uint64_t jobid, char* method, int kill)
|
|
{
|
|
char buf[128];
|
|
char info[256];
|
|
|
|
printf("checkpoint job %ld by method %s\n", jobid, method);
|
|
sprintf(buf, "/tmp/%ld/ckpt_dir", jobid);
|
|
|
|
/* do some actions to checkpoint jobs */
|
|
|
|
sprintf(info,
|
|
"{\nhost_select=\"criu\"; \nrestart_command=\"cd %s; criu restore "
|
|
"-vvv\"}\n",
|
|
buf);
|
|
xcecli_notify_jobchkpt(active_ctx->ctx, jobid, info);
|
|
|
|
if (kill) {
|
|
job_kill(jobid, SIGTERM);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int handle_jobcontrol(xce_request_t* request)
|
|
{
|
|
uint64_t jobid;
|
|
int signo;
|
|
xce_jobctrl_t* jobctrl = request->body.jobctrlmsg;
|
|
int rc = 0;
|
|
|
|
switch (jobctrl->action) {
|
|
case JobControl_u_siginfo:
|
|
jobid = jobctrl->jobid;
|
|
signo = jobctrl->u.siginfo->signo;
|
|
rc = job_kill(jobid, signo);
|
|
break;
|
|
case JobControl_u_ckptoption:
|
|
rc = job_chkpnt(jobctrl->jobid, jobctrl->u.ckptoption->method,
|
|
jobctrl->u.ckptoption->kill);
|
|
break;
|
|
default:
|
|
rc = -EINVAL;
|
|
ERROR("unknown job control action %d", jobctrl->action);
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int handle_runtasks(xce_request_t* request)
|
|
{
|
|
xce_taskrun_t* runmsg = request->body.runtaskmsg;
|
|
int rc = 0;
|
|
int i = 0;
|
|
#if 0
|
|
job_startinfo_t startinfo;
|
|
#endif
|
|
|
|
if (runmsg->nhosts != 1) {
|
|
/* something wrong */
|
|
rc = -EINVAL;
|
|
|
|
notify_task_fail(runmsg->jobid, runmsg->clientidx, -1, rc);
|
|
return rc;
|
|
}
|
|
|
|
if (strcmp(runmsg->taskhosts[0]->host, get_setting("myname")) != 0) {
|
|
/* something wrong */
|
|
rc = -EINVAL;
|
|
|
|
notify_task_fail(runmsg->jobid, runmsg->clientidx, -1, rc);
|
|
return rc;
|
|
}
|
|
|
|
printf("run task: jobid = %lu\n", runmsg->jobid);
|
|
printf("run task: user = %s\n", runmsg->taskhosts[0]->execuser);
|
|
printf("run task: cmdline = %s\n", runmsg->taskhosts[0]->cmdline);
|
|
printf("run task: exdata = %s\n", runmsg->taskhosts[0]->exdata);
|
|
printf("run task: ntasks =%d\n", runmsg->taskhosts[0]->ntasks);
|
|
|
|
#if 0
|
|
startinfo.exechome = runmsg->taskhosts[0]->exechome;
|
|
startinfo.execcwd = runmsg->taskhosts[0]->execcwd;
|
|
startinfo.execuser = runmsg->taskhosts[0]->execuser;
|
|
startinfo.cmdline = runmsg->taskhosts[0]->cmdline;
|
|
startinfo.exdata = runmsg->taskhosts[0]->exdata;
|
|
startinfo.launch_type = runmsg->taskhosts[0]->launch_type;
|
|
#endif
|
|
|
|
for (i = 0; i < runmsg->taskhosts[0]->ntasks; i++) {
|
|
int pid = 0;
|
|
int pgid = 0;
|
|
|
|
rc = run_task(runmsg->taskhosts[0]->cmdline, &pid);
|
|
if (rc < 0) {
|
|
notify_task_fail(runmsg->jobid, runmsg->clientidx, i, rc);
|
|
} else {
|
|
int idx;
|
|
|
|
printf("job %lu task %d started, PID: %d\n", runmsg->jobid, i, pid);
|
|
|
|
idx = get_avail_run_task_slots();
|
|
if (idx < 0) {
|
|
printf("too many running tasks\n");
|
|
} else {
|
|
run_tasks[idx].jobid = runmsg->jobid;
|
|
run_tasks[idx].taskid = i;
|
|
run_tasks[idx].clientidx = runmsg->clientidx;
|
|
run_tasks[idx].pid = pid;
|
|
run_tasks[idx].ts = 0;
|
|
}
|
|
notify_task_start(runmsg->jobid, runmsg->clientidx, i, pid, pgid);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int myreq_handler(xce_request_t* request, void* args)
|
|
{
|
|
int rc = 0;
|
|
|
|
switch (request->opcode) {
|
|
case OPCode_push:
|
|
if (request->body_which == Request_body_datamsg) {
|
|
dataset_t* datamsg = request->body.datamsg;
|
|
|
|
switch (datamsg->table) {
|
|
case Data_data_schedjobs:
|
|
case Data_data_jobs:
|
|
sched_on_jobs(datamsg->nrows, (xce_job_t**)datamsg->rows);
|
|
break;
|
|
case Data_data_schedworkers:
|
|
case Data_data_workers:
|
|
sched_on_workers(datamsg->nrows, (xce_worker_t**)datamsg->rows);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
break;
|
|
case OPCode_startjob:
|
|
rc = handle_startjob(request);
|
|
break;
|
|
case OPCode_notify:
|
|
rc = handle_notify(request);
|
|
break;
|
|
case OPCode_jobcontrol:
|
|
rc = handle_jobcontrol(request);
|
|
break;
|
|
case OPCode_runtasks:
|
|
rc = handle_runtasks(request);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static void check_run_jobs()
|
|
{
|
|
int i;
|
|
time_t now = time(NULL);
|
|
|
|
for (i = 0; i < MAX_RUN_JOBS; i++) {
|
|
int wstatus;
|
|
|
|
if (run_jobs[i].jobid == 0) {
|
|
continue;
|
|
}
|
|
|
|
if (run_jobs[i].pid <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (waitpid(run_jobs[i].pid, &wstatus, WNOHANG | WUNTRACED | WCONTINUED)
|
|
<= 0) {
|
|
/* fake job rusage */
|
|
int npids = 1;
|
|
int npgids = 0;
|
|
int nusage = 2;
|
|
xce_rusage_t data[] = {
|
|
{ "cputime", 100.0 },
|
|
{ "mem", 200.0 }
|
|
};
|
|
xce_rusage_t* usages[] = { &(data[0]), &(data[1]) };
|
|
|
|
if (now - run_jobs[i].ts > RUSAGE_UPDATE_INTERVAL) {
|
|
xcecli_update_jusage(active_ctx->ctx, run_jobs[i].jobid, npids,
|
|
&(run_jobs[i].pid), npgids, NULL, nusage,
|
|
&usages[0]);
|
|
run_jobs[i].ts = now;
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
if (WIFSTOPPED(wstatus)) {
|
|
xcecli_notify_jobstop(active_ctx->ctx, run_jobs[i].jobid);
|
|
} else if (WIFCONTINUED(wstatus)) {
|
|
xcecli_notify_jobresume(active_ctx->ctx, run_jobs[i].jobid);
|
|
} else if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
|
|
xce_jobfinish_t finish;
|
|
|
|
finish.jobid = run_jobs[i].jobid;
|
|
|
|
if (WIFEXITED(wstatus)) {
|
|
finish.exit_code = WEXITSTATUS(wstatus);
|
|
} else {
|
|
finish.exit_code = 128 + WTERMSIG(wstatus);
|
|
}
|
|
|
|
printf("job %lu exit code %d\n", finish.jobid, finish.exit_code);
|
|
xcecli_notify_jobfinish(active_ctx->ctx, &finish);
|
|
run_jobs[i].jobid = 0;
|
|
run_jobs[i].pid = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void check_run_tasks()
|
|
{
|
|
int i;
|
|
time_t now = time(NULL);
|
|
|
|
for (i = 0; i < MAX_RUN_TASKS; i++) {
|
|
int wstatus;
|
|
|
|
if (run_tasks[i].jobid == 0) {
|
|
continue;
|
|
}
|
|
|
|
if (run_tasks[i].pid <= 0) {
|
|
continue;
|
|
}
|
|
|
|
if (waitpid(run_tasks[i].pid, &wstatus,
|
|
WNOHANG | WUNTRACED | WCONTINUED)
|
|
<= 0) {
|
|
/* fake job rusage */
|
|
int npids = 1;
|
|
int npgids = 0;
|
|
int nusage = 2;
|
|
xce_rusage_t data[] = {
|
|
{ "cputime", 100.0 },
|
|
{ "mem", 200.0 }
|
|
};
|
|
xce_rusage_t* usages[] = { &(data[0]), &(data[1]) };
|
|
|
|
if (now - run_tasks[i].ts > RUSAGE_UPDATE_INTERVAL) {
|
|
xcecli_update_jusage(active_ctx->ctx, run_tasks[i].jobid, npids,
|
|
&(run_tasks[i].pid), npgids, NULL, nusage,
|
|
&usages[0]);
|
|
run_tasks[i].ts = now;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) {
|
|
xce_task_finish_t finish = { .jobid = run_tasks[i].jobid,
|
|
.host = get_setting("myname"),
|
|
.taskidx = run_tasks[i].taskid,
|
|
.clientidx = run_tasks[i].clientidx,
|
|
.exit_code = 0 };
|
|
|
|
if (WIFEXITED(wstatus)) {
|
|
finish.exit_code = WEXITSTATUS(wstatus);
|
|
} else {
|
|
finish.exit_code = 128 + WTERMSIG(wstatus);
|
|
}
|
|
|
|
printf("job %lu task %d exit code %d\n", finish.jobid,
|
|
finish.taskidx, finish.exit_code);
|
|
xcecli_notify_taskfinish(active_ctx->ctx, &finish);
|
|
run_tasks[i].jobid = 0;
|
|
run_tasks[i].pid = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
#define HEARTBEAT_INTERVAL (60)
|
|
|
|
static int worker_house_keeping(void* arg)
|
|
{
|
|
command_ctx_t* ctx = (command_ctx_t*)arg;
|
|
int rc = 0;
|
|
|
|
xce_use(ctx);
|
|
|
|
check_run_jobs();
|
|
check_run_tasks();
|
|
|
|
rc = xcecli_heartbeat(ctx->ctx, HEARTBEAT_INTERVAL);
|
|
if (rc < 0) {
|
|
perror("xcecli_heartbeat");
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
struct arguments arguments;
|
|
char* buf;
|
|
char* logfile = "/tmp/xce_test.log";
|
|
char* loglevel = "info";
|
|
int logsize = 64 * 1024 * 1024;
|
|
int logcount = 10;
|
|
|
|
memset(&arguments, 0x0, sizeof(arguments));
|
|
argp_parse(&argp, argc, argv, 0, 0, &arguments);
|
|
|
|
if (arguments.verbose != 0) {
|
|
loglevel = verbose_to_loglevel(arguments.verbose);
|
|
}
|
|
|
|
/* Remember the user's level so wait-primary can restore it after temporarily
|
|
* raising the threshold to silence its poll-loop failover spam. */
|
|
g_active_loglevel = loglevel;
|
|
|
|
mylog_init(logfile, loglevel, logsize, logcount);
|
|
|
|
signal(SIGINT, sigbreak);
|
|
|
|
memset(&run_jobs, 0x0, sizeof(run_jobs));
|
|
memset(&run_tasks, 0x0, sizeof(run_tasks));
|
|
|
|
idgen_ctx_init(&idctx);
|
|
|
|
/* bug-1420 follow-up: folding pid into apiserverId alone (4 bits, 16
|
|
* buckets) at the jsub call site still collides too often across this
|
|
* harness's rapid, per-invocation-fresh xce_test processes (confirmed
|
|
* empirically -- a 10-round chaotic soak still produced same-second
|
|
* duplicate jobids after that fix landed). idgen_ctx_init() always
|
|
* resets sequence to 0, and MAKE_JOBID's sequence field alone has 20
|
|
* bits (config.h N_SEQUENCE_BITS) of headroom -- two orders of
|
|
* magnitude more than the namespacing fields. Re-seed it here with a
|
|
* value that mixes pid (stable per process) with monotonic-clock
|
|
* nanoseconds (varies even across processes launched back-to-back in
|
|
* the same wall-clock second, unlike idgen's own second-resolution
|
|
* timestamp), masked to N_SEQUENCE_BITS so it can never bleed into the
|
|
* adjacent tag/timestamp bits MAKE_JOBID packs beside it. idgen()'s own
|
|
* ROUNDUP-to-N_ARRAY_IDX_BITS logic for array jobs still applies
|
|
* unchanged on top of this seed. */
|
|
{
|
|
struct timespec ts;
|
|
clock_gettime(CLOCK_MONOTONIC, &ts);
|
|
idctx.sequence = ((uint32_t)getpid() ^ (uint32_t)ts.tv_nsec)
|
|
& (uint32_t)UMAX_BITS(N_SEQUENCE_BITS);
|
|
}
|
|
|
|
/* errexit default, modelled on shell `-e`/`+e`, keyed on the stdin mode:
|
|
* batch (piped/redirected stdin) → `-e`: abort the read loop on the first
|
|
* failing command and exit non-zero (a script wants fail-fast + verdict).
|
|
* interactive (TTY) REPL → `+e`: a failed command prints its error and the
|
|
* loop CONTINUES (never exit on a typo).
|
|
* Override at any time with `set -e` / `set +e` (or `set continue-on-error`). */
|
|
g_interactive = isatty(STDIN_FILENO);
|
|
g_continue_on_error = g_interactive ? 1 : 0;
|
|
|
|
while (!done) {
|
|
char* s;
|
|
int rc;
|
|
|
|
prompt[0] = 0x0;
|
|
fill_prompt(active_ctx, prompt);
|
|
strcat(prompt, ">");
|
|
buf = readline(prompt);
|
|
|
|
if (buf == NULL) {
|
|
execute_line("quit");
|
|
continue;
|
|
}
|
|
|
|
if (strlen(buf) == 0) {
|
|
free(buf);
|
|
continue;
|
|
}
|
|
|
|
s = buf;
|
|
|
|
rc = preprocess_and_run(s);
|
|
free(buf);
|
|
if (rc) {
|
|
break; /* abort-on-failure requested loop stop */
|
|
}
|
|
}
|
|
|
|
return g_session_failed ? 1 : 0;
|
|
}
|