lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/worker/src/controller_connector.c
Rongsong Shen a3d12d37d9
feat(worker-rejoin-resync): task 6.1 - trigger reconciliation on rejoin 
Arm a reconciliation round in apply_join_state() the moment a worker
host becomes connected (first join, reconnect, or failover-reconnect -
all routed through the same shared call site, Req 1.1-1.3), then send
the query from on_worker_join() strictly after the join reply and
existing config push (Req 2.1-2.2), verified via a structural test
that reads real wire order off the endpoint queue.

Also closes Req 3.7's config-delivery seam end-to-end: worker_resync_
window_ms now rides the existing Conf4Worker push (clustername
precedent, no new opcode) and the worker applies it to its retention
window on receipt.

Fixes bug-1752: myreq_handler had no OPCode_push dispatch case at all,
so every config push - including the pre-existing clustername field -
was silently discarded. Confirmed harmless until now (clustername was
never read anywhere in src/worker/); closed narrowly for
worker_resync_window_ms only.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku
2026-09-01 19:43:18 +00:00

2080 lines
74 KiB
C
Executable file
Raw Blame History

#include "client.h"
#include "controller_connector.h"
#include "netio.h"
#include "uv_ipc.h"
#include "xcecd_cgroups.h"
#include "xcecd_exdata.h"
#include "xcecd_resync.h"
#include "xcecd_worker_utils.h"
#include "xcelog.h"
#include <errno.h>
#include <fcntl.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#ifdef _WIN32
#include <windows.h>
#else
#include <pthread.h>
#include <unistd.h>
#endif
#ifdef XRUN
hostattrs_t host_attrs = { 0 };
#else
extern hostattrs_t host_attrs;
#endif
xcecli_ctx_t xcecli_ctx = { 0 };
xce_client_config_t xce_client_cfg = { 0 };
controller_connector_ctx_t* cc_ctx = NULL;
/* Persistent storage for the single candidate list the current connect/join
* attempt is driving (component-failover-adoption, task 2.1, Req 1 AC5).
* Must outlive the xcecli_ctx_t connection: xce_cluster_config_to_client_
* config() points xce_client_cfg.masters[] directly at this struct's
* hostname buffers (non-owning aliasing -- the library keeps a reference to
* xce_client_cfg for the life of the connection, so the source data behind
* it cannot be a stack-local temporary). Populated by build_cluster_config()
* below from either the operator-supplied --peers list or a synthesized
* single candidate -- never both, never a stack copy. */
static xce_cluster_config_t g_cluster_cfg = { 0 };
/* Set by the rejoin callback registered with the client library
* (xcecd_on_reconnect() below) when the library has transparently
* reconnected and this connector needs to re-issue xcecli_join() as a
* separate, later call. Consumed by checking_connection() (task 2.2). */
static bool needs_rejoin = false;
static int myreq_handler(xce_request_t*, void*);
/* make compiler happy */
typedef struct {
task_func_t function;
void* arg;
} thread_wrapper_t;
static const cc_callbacks_t cc_callbacks
= { .job_finished = xcecd_job_finished,
.job_stopped = xcecd_job_stopped,
.job_resumed = xcecd_job_resumed,
.job_start_error = xcecd_job_start_error,
.job_status_error = xcecd_job_status_error,
.job_ckpted = xcecd_job_ckpted,
.job_remote_start = xcecd_job_remote_start,
.reply_startjob = xcecd_reply_startjob };
static void free_startinfo(job_startinfo_t* info)
{
if (!info) {
return;
}
FREEUP(info->cmdline);
FREEUP(info->execuser);
FREEUP(info->exdata);
FREEUP(info->exechome);
FREEUP(info->execcwd);
FREEUP(info->q_pre_exec);
FREEUP(info->q_post_exec);
FREEUP(info->job_pre_exec);
FREEUP(info->job_post_exec);
FREEUP(info);
}
static void free_limits(res_limits_t* limits)
{
if (!limits) {
return;
}
FREEUP(limits->cpu_affinity);
FREEUP(limits->io_device_path);
FREEUP(limits->user_name);
FREEUP(limits);
}
/**
* @brief Initializes the host attributes with default values.
*/
const cc_callbacks_t* get_cc_callbacks(void) { return &cc_callbacks; }
static void* thread_start_wrapper(void* arg)
{
thread_wrapper_t* wrapper = (thread_wrapper_t*)arg;
wrapper->function(wrapper->arg);
FREEUP(wrapper);
return NULL;
}
/* my old implementation and currently is disabled */
int start_controller_connector_thread(task_func_t function, void* arg)
{
XCECD_LOG(XCE_LOG_DEBUG, "Starting controller connector thread...");
#ifdef _WIN32
HANDLE status_thread
= CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)function, arg, 0, NULL);
if (status_thread == NULL) {
return -1;
}
CloseHandle(status_thread);
#else
pthread_t status_thread;
thread_wrapper_t* wrapper = calloc(1, sizeof(thread_wrapper_t));
if (!wrapper) {
return -1;
}
wrapper->function = function;
wrapper->arg = arg;
if (pthread_create(&status_thread, NULL, thread_start_wrapper, wrapper)
!= 0) {
FREEUP(wrapper);
return -1;
}
pthread_detach(status_thread);
#endif
XCECD_LOG(XCE_LOG_DEBUG, "Controller connector thread started.");
return 0;
}
/**
* @brief Sets the caller-owned identity/auth fields (from, key) on the
* client config. Deliberately does NOT touch nmasters/masters[]/
* master_ports[] -- those are owned by the shared
* xce_cluster_config_to_client_config() helper (build_cluster_config()
* + the conversion call in xcecd_join_cluster() below), matching
* that helper's documented contract ("Does NOT set out->from /
* out->key -- caller sets those separately").
*
* @param config Pointer to the xce_client_config_t structure to update.
*/
static void set_client_identity(xce_client_config_t* config)
{
int ret;
char local_host[MAXHOSTNAMELEN] = { 0 };
if (!config) {
return;
}
ret = gethostname(local_host, sizeof(local_host));
if (ret < 0) {
XCECD_LOG(XCE_LOG_ERROR, "gethostname failed: %s", strerror(errno));
strncpy(local_host, "localhost", sizeof(local_host) - 1);
}
#ifdef VIRTUAL_CLUSTER
config->from = strdup(cc_ctx->virtual_name);
#else
config->from = strdup(local_host);
#endif
config->key = strdup("keepSecret");
XCECD_LOG(XCE_LOG_DEBUG,
"Controller connector identity set: from=%s", config->from);
}
/**
* @brief Build the single validated candidate list this connect/reconnect
* attempt will use (component-failover-adoption, task 2.1, Req 1
* AC5).
*
* A single controller is not a separately-maintained code path: when
* --peers configured a real candidate list (cc_ctx->cluster_cfg), it is
* used as-is; otherwise a one-candidate list is synthesized from the legacy
* remote_host/remote_port default. Either source is written into the same
* persistent g_cluster_cfg and then run through the identical
* xce_cluster_config_validate() call -- no branch downstream of this
* function distinguishes the two sources.
*
* @return 0 on a validated config, the validator's negative error code
* (or -EINVAL if cc_ctx is unset) otherwise.
*/
static int build_cluster_config(void)
{
if (!cc_ctx) {
return -EINVAL;
}
if (cc_ctx->cluster_cfg.candidates_count > 0) {
g_cluster_cfg = cc_ctx->cluster_cfg;
} else {
memset(&g_cluster_cfg, 0, sizeof(g_cluster_cfg));
strncpy(g_cluster_cfg.candidates[0].hostname,
cc_ctx->remote_host ? cc_ctx->remote_host : "",
sizeof(g_cluster_cfg.candidates[0].hostname) - 1);
g_cluster_cfg.candidates[0].port = cc_ctx->remote_port;
g_cluster_cfg.candidates_count = 1;
strncpy(g_cluster_cfg.identity,
cc_ctx->virtual_name ? cc_ctx->virtual_name : "xcecd",
sizeof(g_cluster_cfg.identity) - 1);
}
return xce_cluster_config_validate(&g_cluster_cfg);
}
/* Sized for the worst case: MAX_HA_CANDIDATES entries, each up to
* (XCE_MAXHOSTNAME_LEN - 1) hostname chars + ':' + a 5-digit port + a 2-char
* ", " separator (263 bytes/candidate at the max) -- the extra +16 pads that
* comfortably rather than sizing exactly to the boundary. Named rather than
* inlined at each call site so the callers below and the tests that call
* format_candidate_list() directly can't drift out of sync with each other. */
#define CANDIDATE_LIST_BUF_SIZE ((XCE_MAXHOSTNAME_LEN + 16) * MAX_HA_CANDIDATES)
/**
* @brief Renders every candidate in g_cluster_cfg as "host:port, host:port,
* ..." into buf (component-failover-adoption, task 2.3, Req 1 AC6,
* Req 5 AC2).
*
* xcecd_join_cluster()'s all-candidates-exhausted failure paths only had a
* candidate COUNT to log ("all %d candidate(s) unreachable") -- neither
* xcecli_start() nor endpoint_connect() (src/lib/netio.c) log anything
* per-candidate themselves (confirmed by source read: no XCECD_LOG/INFO/WARN
* call anywhere in xcecli_start()'s per-master connect loop), so without
* this the worker's own failure message never actually named which
* candidates it tried, only how many -- a gap against Req 1 AC6 ("identifying
* which candidates were attempted") and Req 5 AC2 ("identifying every
* candidate attempted"). Found during task 2.3's all-candidates-unreachable
* coverage.
*
* Callers must size buf as CANDIDATE_LIST_BUF_SIZE (or larger) -- with an
* undersized buffer this truncates at the last candidate that fits rather
* than overflowing (bufsz - 1 bound below), so a too-small buf loses
* information, not memory safety.
*/
static void format_candidate_list(char* buf, size_t bufsz)
{
size_t used = 0;
buf[0] = '\0';
for (int i = 0; i < g_cluster_cfg.candidates_count && used < bufsz - 1;
i++) {
int written = snprintf(buf + used, bufsz - used, "%s%s:%d",
i > 0 ? ", " : "",
g_cluster_cfg.candidates[i].hostname,
g_cluster_cfg.candidates[i].port);
if (written < 0) {
break;
}
used += (size_t)written;
}
}
/**
* @brief Rejoin callback registered with the client library
* (component-failover-adoption, task 2.1, Req 2 AC1, Req 5 AC1-2).
*
* The library invokes this while still holding its own per-ctx lock
* (ac->failover.lock, a non-recursive mutex held for the whole
* XCECLI_FAILOVER_CALL retry loop) -- per design.md's "Rejoin callback
* constraint", this callback MUST NOT perform I/O or call any xcecli_*
* function: doing so would try to re-acquire the same already-held lock and
* self-deadlock the calling thread forever. It only sets flags/statics and
* returns 0 immediately so the interrupted operation resumes normally.
* Consuming needs_rejoin to issue the actual xcecli_join() as a separate,
* later call (once the triggering call has returned and the lock is free)
* is task 2.2's job.
*
* worker-rejoin-resync task 4.3 remediation (round 2, Requirements 3.8,
* 6.1): also resets xcecd_resync.c's resync_pending gate here. This is the
* moment the worker itself learns its connection to the controller was
* re-established -- any job-state query it was still assembling (or report
* it had not finished sending) belongs to the connection that just died and
* can never complete; the controller's own Requirement 6.1 Acceptance
* Criterion 1 already discards that incomplete round and re-initiates a
* fresh query on this worker's next rejoin. Without this reset,
* resync_pending can only be cleared by xcecd_resync_handle_query_page()
* seeing a query's LAST page -- which never arrives on a dead connection --
* leaving it stuck true forever (round 1's confirmed defect). Calling
* xcecd_resync_reset() here is safe under the constraint above: it only
* touches xcecd_resync.c's own statics (a bool and a small heap buffer), no
* I/O and no xcecli_* call.
*/
static int xcecd_on_reconnect(xcecli_ctx_t ctx, void* arg)
{
(void)ctx;
(void)arg;
needs_rejoin = true;
xcecd_resync_reset();
return 0;
}
/**
* @brief Joins the cluster using the provided host attributes
* (component-failover-adoption, task 2.1, Req 1 AC1-3/AC5, Req 5
* AC1-2).
*
* Builds exactly one validated candidate list (build_cluster_config()),
* converts it via the shared libxce helper, and performs ONE
* init+start+join sequence. The client library's own failover mechanism
* (client_failover.c) rotates across candidates and follows standby
* redirects internally, within the single xcecli_join() call below --
* there is no per-candidate loop here anymore. A single configured
* candidate is simply the candidates_count=1 case of this exact sequence,
* not a separate fallback.
*
* @param attrs Pointer to the host attributes structure.
* @return int 0 on success, negative error code (or -1) on failure.
*/
int xcecd_join_cluster(hostattrs_t* attrs)
{
int rc;
if (!attrs) {
return -1;
}
rc = build_cluster_config();
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: invalid cluster configuration (rc=%d)",
rc);
return rc;
}
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: attempting %d candidate(s), first=%s:%d",
g_cluster_cfg.candidates_count,
g_cluster_cfg.candidates[0].hostname,
g_cluster_cfg.candidates[0].port);
rc = xce_cluster_config_to_client_config(&g_cluster_cfg, &xce_client_cfg);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: candidate list conversion failed "
"(rc=%d)",
rc);
return rc;
}
set_client_identity(&xce_client_cfg);
rc = xcecli_ctx_init(&xcecli_ctx, &xce_client_cfg);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: xcecli_ctx_init failed (rc=%d)", rc);
return rc;
}
xcecli_set_handler(xcecli_ctx, &myreq_handler, NULL);
rc = xcecli_start(xcecli_ctx);
if (rc < 0) {
char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 };
format_candidate_list(attempted, sizeof(attempted));
XCECD_LOG(XCE_LOG_WARN,
"xcecd_join_cluster: all %d candidate(s) unreachable "
"(rc=%d): %s",
g_cluster_cfg.candidates_count, rc, attempted);
xcecli_ctx_fini(xcecli_ctx);
xcecli_ctx = NULL;
return rc;
}
/* Registered AFTER xcecli_start() succeeds, not before (task 2.3 fix --
* found during this task's E2E reconnect validation): xcecli_start()
* unconditionally resets ctx->failover.rejoin_cb/rejoin_arg to NULL as
* part of initializing per-context failover state (client.c, "no rejoin
* hook is installed for pure clients" -- it assumes callers register
* AFTER start, matching every xcecli_* client contract). Registering
* here BEFORE xcecli_start() (the code's shape prior to this fix) meant
* the registration was silently wiped out immediately afterward, so
* xcecd_on_reconnect() could never fire and needs_rejoin was never set
* by a real transparent reconnect -- task 2.1/2.2's whole rejoin-retry
* mechanism was dead in production despite passing unit tests (those
* tests set needs_rejoin directly or call xcecli_set_reconnect_handler()
* standalone, never through this exact xcecli_start()-then-register
* production sequence, so none of them could catch this ordering bug). */
xcecli_set_reconnect_handler(xcecli_ctx, xcecd_on_reconnect, NULL);
rc = xcecli_join(xcecli_ctx, HostRole_worker, attrs);
if (rc < 0) {
char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 };
format_candidate_list(attempted, sizeof(attempted));
errno = -rc;
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: xcecli_join() failed across %d "
"candidate(s) (%s): %s",
g_cluster_cfg.candidates_count, attempted, strerror(errno));
xcecli_stop(xcecli_ctx);
xcecli_ctx_fini(xcecli_ctx);
xcecli_ctx = NULL;
return rc;
}
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: joined cluster (from=%s, %d "
"candidate(s) configured)",
xce_client_cfg.from, g_cluster_cfg.candidates_count);
return 0;
}
/**
* @brief Issues the actual rejoin call: the same join payload used at
* startup, as a separate call once the triggering call (xcecli_hello
* or whatever else invoked the reconnect callback) has returned and
* the client library's internal failover lock is free.
*
* Split out from checking_connection() behind the rejoin_call function
* pointer below purely for testability: xcecli_join() against a real
* candidate goes through XCECLI_FAILOVER_CALL's full retry/backoff budget
* (up to ~100s) on failure, and a genuine success requires a live
* capnp-speaking controller -- neither is practical to drive from a fast
* unit test. Production always runs through this exact function via the
* default rejoin_call pointer; only tests override the pointer to a
* controllable double.
*/
static int perform_pending_rejoin(void)
{
return xcecli_join(xcecli_ctx, HostRole_worker, &host_attrs);
}
/* Defaults to the real production call (see perform_pending_rejoin() above).
* Overridden only by tests. */
static int (*rejoin_call)(void) = perform_pending_rejoin;
/**
* @brief Periodic liveness probe on the connector thread
* (component-failover-adoption, task 2.1, Req 2 AC1-3, Req 3 AC1/AC3;
* task 2.2, Req 2 AC5, rejoin retry-on-failure).
*
* A single xcecli_hello() call. xcecli_hello() is XCECLI_FAILOVER_CALL-
* wrapped: on a transport failure it already retries indefinitely across
* the full configured candidate list (rotate + backoff) and invokes the
* registered rejoin-flag callback on its own -- the client library now
* owns connection recovery. This probe therefore no longer runs its own
* bounded retry-then-give-up loop: an outage that outlasts the old
* MAX_RETRY_COUNT*RETRY_INTERVAL bound no longer tears the worker's
* connection down for good (cc_ctx->running is never forced false here);
* the connector thread's normal periodic cadence simply gives the library
* another chance on the next call.
*
* Immediately after xcecli_hello() returns (its internal failover lock is
* now free), checks needs_rejoin -- set by xcecd_on_reconnect() when the
* library transparently reconnected during some earlier call. If set, issues
* xcecli_join() as a separate call (the same join payload used at startup).
* The flag clears ONLY on that join call's success (Req 2 AC5): if
* xcecli_join() itself returns -E_XCE_NOT_READY (its own retry budget
* exhausted) or any other failure, needs_rejoin stays set, so the worker is
* never left transport-connected but silently un-joined -- the next
* checking_connection() tick (this function's own periodic cadence, no new
* timer needed) retries the rejoin.
*
* Success is rc >= 0, NOT rc == 0 (task 2.3 fix -- found during this task's
* live E2E reconnect validation): xcecli_join()/xce_join() returns the
* controller's reply.status verbatim, and handle_worker_join()'s SUCCESS
* reply.status is the worker's assigned slot index (join.c, `reply.status =
* idx`), which is any non-negative integer, not always 0 -- only a genuine
* failure reply.status is negative. xcecd_join_cluster() above already
* treats its own xcecli_join() call this way (`if (rc < 0)` is its only
* failure test); this rejoin path used `rc == 0` instead, so a real,
* successful rejoin against a controller that happened to assign a non-zero
* worker index was misreported as a failure and retried forever (harmlessly
* idempotent, but never actually clearing needs_rejoin or logging success).
* Confirmed via a real 3-node cluster: after a primary kill and reconnect,
* xcecli_join() against the new primary returned rc=1 (a valid worker
* index) but the old `rc == 0` check logged "rejoin ... failed" on every
* subsequent liveness probe.
*
* worker-rejoin-resync task 4.3 (Requirement 3.8), narrowed per round-2
* remediation: xcecd_resync_is_pending() gates ONLY the needs_rejoin-
* triggered rejoin_call() below -- the "resume normal post-join operation"
* step Requirement 3.8 actually names -- not this function's entry.
* xcecli_hello() and the needs_rejoin check itself both still run every
* tick unconditionally, so checking_connection() keeps the exact
* self-healing property needs_rejoin already had before this task: a fresh
* chance every cycle to detect and recover the worker's own connection
* (src/lib/xcectx.c's handle_disconnect() -- "so that checking_connection
* can detect the failure and attempt to rejoin"). A resync_pending stuck
* true (round 1's confirmed defect: gating this function's entry made the
* worker's sole connection-recovery mechanism permanently inert, breaking
* Requirement 6.1's "disconnect during an incomplete reconciliation round"
* case) can therefore never disable this probe. In practice resync_pending
* is also cleared the instant a reconnect is detected
* (xcecd_on_reconnect() -> xcecd_resync_reset(), above), so this check is a
* defensive backstop against the rejoin racing an in-flight report, not the
* mechanism that unsticks a stalled one.
*
* @return int Always returns 0.
*/
int checking_connection()
{
int ret = xcecli_hello(xcecli_ctx, "hello");
if (ret != 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Connection to controller not currently healthy (rc=%d); "
"the client library will keep retrying/rotating "
"candidates in the background",
ret);
}
if (needs_rejoin) {
if (xcecd_resync_is_pending()) {
XCECD_LOG(XCE_LOG_DEBUG,
"checking_connection: deferring rejoin -- job-state "
"resync report still being sent");
} else {
int rc = rejoin_call();
if (rc >= 0) {
needs_rejoin = false;
XCECD_LOG(XCE_LOG_INFO,
"checking_connection: rejoin after transparent "
"reconnect succeeded");
} else {
XCECD_LOG(XCE_LOG_WARN,
"checking_connection: rejoin after transparent "
"reconnect failed (rc=%d); will retry on the next "
"liveness probe",
rc);
}
}
}
return 0;
}
#ifndef XRUN
static void dump_startjob_request(xce_job_exec_t* d)
{
char jobidStr[64];
if (!d) {
return;
}
fmt_jobid(jobidStr, sizeof(jobidStr), d->jobid);
XCECD_LOG(XCE_LOG_DEBUG,
"Job ID: %s, Command: %s, Exec User: %s, Exdata: %s", jobidStr,
d->start_info->cmdline ? d->start_info->cmdline : "NULL",
d->start_info->execuser ? d->start_info->execuser : "NULL",
d->start_info->exdata ? d->start_info->exdata : "NULL");
}
#endif // XRUN
static char* create_mpi_cmdline(char* cmd, char* type, xce_launch_t* launchinfo)
{
if (!cmd || !type || !launchinfo || launchinfo->nhosts <= 0) {
XCECD_LOG(XCE_LOG_ERROR, "Invalid parameters for create_mpi_cmdline");
return NULL;
}
char* mpi_executable = NULL;
char* app_command = NULL;
char* token = NULL;
char* cmd_copy = strdup(cmd);
if (!cmd_copy) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for cmd_copy");
return NULL;
}
// Parse the original command to extract mpi executable path and application
// command
token = strtok(cmd_copy, " ");
if (!token) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to parse command: %s", cmd);
FREEUP(cmd_copy);
return NULL;
}
// Check if first token contains mpirun or mpiexec
if (strstr(token, "mpirun") || strstr(token, "mpiexec")) {
mpi_executable = strdup(token);
// Get the rest of the command as application
char* rest = cmd + strlen(token);
while (*rest == ' ')
rest++; // skip spaces
app_command = strdup(rest);
} else {
XCECD_LOG(XCE_LOG_ERROR,
"Command does not contain mpirun or mpiexec: %s", cmd);
FREEUP(cmd_copy);
return NULL;
}
FREEUP(cmd_copy);
if (!mpi_executable || !app_command) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to extract mpi executable or application command");
FREEUP(mpi_executable);
FREEUP(app_command);
return NULL;
}
// Calculate the size needed for host list
size_t hostlist_size = 0;
for (int i = 0; i < launchinfo->nhosts; i++) {
if (launchinfo->hosts[i] && launchinfo->hosts[i]->host) {
// host:nrank + comma + null terminator
hostlist_size += strlen(launchinfo->hosts[i]->host)
+ 20; // 20 should be enough for :nrank
}
}
if (hostlist_size == 0) {
XCECD_LOG(XCE_LOG_ERROR, "No valid hosts found in launch info");
FREEUP(mpi_executable);
FREEUP(app_command);
return NULL;
}
// Build host list string
char* hostlist = calloc(hostlist_size, sizeof(char));
if (!hostlist) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for hostlist");
FREEUP(mpi_executable);
FREEUP(app_command);
return NULL;
}
for (int i = 0; i < launchinfo->nhosts; i++) {
if (launchinfo->hosts[i] && launchinfo->hosts[i]->host) {
if (strlen(hostlist) > 0) {
strcat(hostlist, ",");
}
strcat(hostlist, launchinfo->hosts[i]->host);
strcat(hostlist, ":");
char nrank_str[16];
snprintf(nrank_str, sizeof(nrank_str), "%d",
launchinfo->hosts[i]->nranks);
strcat(hostlist, nrank_str);
}
}
char launcher[MAXFILENAMELEN] = { 0 };
char* server_dir = getenv("XCE_SERVER_DIR");
if (server_dir) {
snprintf(launcher, sizeof(launcher), "%s/xrun", server_dir);
} else {
snprintf(launcher, sizeof(launcher), "/opt/xce/bin/xrun");
}
if (access(launcher, X_OK) != 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xrun launcher not found or not executable at %s", launcher);
FREEUP(mpi_executable);
FREEUP(app_command);
FREEUP(hostlist);
return NULL;
}
// Determine the host parameter based on MPI type
const char* host_param = NULL;
const char* launcher_param = NULL;
if (strcmp(type, "openmpi") == 0) {
host_param = "-host";
launcher_param = "--mca plm_rsh_agent";
} else if (strcmp(type, "mpich") == 0) {
host_param = "-hosts";
launcher_param = "-launcher-exec";
} else {
XCECD_LOG(XCE_LOG_ERROR, "Unsupported MPI type: %s", type);
FREEUP(mpi_executable);
FREEUP(app_command);
FREEUP(hostlist);
return NULL;
}
// Calculate final command size
size_t final_cmd_size
= strlen(mpi_executable) + 1 + // executable + space
strlen(host_param) + 1 + // host parameter + space
strlen(hostlist) + 1 + // hostlist + space
strlen(launcher_param) + 1 + // launcher parameter + space
strlen(launcher) + 1 + // launcher + space
strlen(app_command) + 1; // application + null terminator
char* final_cmd = calloc(final_cmd_size, sizeof(char));
if (!final_cmd) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for final command");
FREEUP(mpi_executable);
FREEUP(app_command);
FREEUP(hostlist);
return NULL;
}
// Build the final command
snprintf(final_cmd, final_cmd_size, "%s %s %s %s %s %s", mpi_executable,
host_param, hostlist, launcher_param, launcher, app_command);
XCECD_LOG(XCE_LOG_DEBUG3, "Generated MPI command: %s", final_cmd);
// Cleanup
FREEUP(mpi_executable);
FREEUP(app_command);
FREEUP(hostlist);
return final_cmd;
}
/**
* @brief Runs a job task on remote hosts.
*
* @param jobid The ID of the job to run.
* @param startinfo Pointer to the job start information.
* @param launchinfo Pointer to the job launch information.
* @return int 0 on success, -1 on failure.
*/
static int job_start_remote_task(uint64_t jobid, job_startinfo_t* startinfo,
xce_launch_t* launchinfo)
{
char jobidStr[64];
xce_taskhost_t hosts[launchinfo->nhosts - 1];
xce_taskrun_t taskrun;
int i;
int rc;
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
if (!startinfo || !launchinfo) {
XCECD_LOG(XCE_LOG_ERROR, "Invalid parameters for job_run_task_remote");
return -1;
}
if ((strcmp(startinfo->launch_type, "openmpi") == 0)
|| (strcmp(startinfo->launch_type, "mpich") == 0)) {
XCECD_LOG(XCE_LOG_DEBUG3,
"MPI launch detected, using MPI task run method.");
return 0;
}
for (i = 1; i < launchinfo->nhosts; i++) {
hosts[i - 1].exechome = startinfo->exechome;
hosts[i - 1].execcwd = startinfo->execcwd;
hosts[i - 1].cmdline = startinfo->cmdline;
hosts[i - 1].exdata = startinfo->exdata;
hosts[i - 1].execuser = startinfo->execuser;
hosts[i - 1].launch_type = startinfo->launch_type;
hosts[i - 1].host = launchinfo->hosts[i]->host;
hosts[i - 1].ntasks = launchinfo->hosts[i]->nranks;
XCECD_LOG(XCE_LOG_DEBUG3, "Remote task setup for host %s with %d ranks",
hosts[i - 1].host, hosts[i - 1].ntasks);
}
taskrun.taskhosts = (xce_taskhost_t**)calloc(launchinfo->nhosts - 1,
sizeof(xce_taskhost_t*));
taskrun.clientidx = -1;
if (taskrun.taskhosts != NULL) {
for (i = 1; i < launchinfo->nhosts; i++) {
taskrun.taskhosts[i - 1] = &hosts[i - 1];
}
taskrun.nhosts = launchinfo->nhosts - 1;
taskrun.jobid = jobid;
XCECD_LOG(XCE_LOG_DEBUG, "Launching %d remote tasks for job %s",
taskrun.nhosts, jobidStr);
xcecli_unblock_tasks(xcecli_ctx);
rc = xcecli_runtask(xcecli_ctx, &taskrun);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR, "Job %s run remote task failed.", jobidStr);
}
} else {
rc = -ENOMEM;
}
FREEUP(taskrun.taskhosts);
return rc;
}
/**
* @brief Starts a job task on the local host.
*
* @param request Pointer to the xce_request_t structure containing the request
* data.
* @return int 0 on success, -1 on failure.
*/
static int job_start_local_task(xce_request_t* request)
{
#ifndef XRUN
xce_job_exec_t* d = request->body.jobstartmsg;
char jobidStr[64];
dump_startjob_request(d);
fork_task_t* job_ctx = calloc(1, sizeof(fork_task_t));
if (job_ctx) {
job_ctx->startinfo = calloc(1, sizeof(job_startinfo_t));
if (!job_ctx->startinfo) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for startinfo");
xcecd_job_start_error(d->jobid);
FREEUP(job_ctx);
return -1;
}
if (d->start_info->execuser && d->start_info->execuser[0] != '\0') {
job_ctx->startinfo->execuser
= strdup((char*)d->start_info->execuser);
} else {
job_ctx->startinfo->execuser = strdup("nobody");
XCECD_LOG(XCE_LOG_WARN, "No execuser provided in startjob request, "
"defaulting to 'nobody'");
}
if (!job_ctx->startinfo->execuser) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for execuser");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
#if 0
char data[1024] = {0};
snprintf(data, sizeof(data),
"{\"sub_env_vars\": {\"XCE_TEST1\": \"abab\", \"XCE_TEST2\": \"cdcd\"}, \"transfer_file\": [{\"op\": \">\", \"local_file\": \"/tmp/file1.txt\", \"remote_file\": \"\"}, {\"op\": \">\", \"local_file\": \"/tmp/file2.txt\", \"remote_file\": \"\"}], \"interactive\": 1}");
job_ctx->exdata = strdup(data);
#else
if (!d->start_info->exdata || d->start_info->exdata[0] == '\0') {
job_ctx->startinfo->exdata = NULL;
XCECD_LOG(XCE_LOG_DEBUG3, "No exdata provided in startjob request");
} else {
job_ctx->startinfo->exdata = strdup((char*)d->start_info->exdata);
XCECD_LOG(XCE_LOG_DEBUG3, "Received exdata: %s",
job_ctx->startinfo->exdata);
if (!job_ctx->startinfo->exdata) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for exdata");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
#endif
if (!d->start_info->exechome || d->start_info->exechome[0] == '\0') {
job_ctx->startinfo->exechome = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No exechome provided in startjob request");
} else {
job_ctx->startinfo->exechome
= strdup((char*)d->start_info->exechome);
if (!job_ctx->startinfo->exechome) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for exechome");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
if (!d->start_info->execcwd || d->start_info->execcwd[0] == '\0') {
job_ctx->startinfo->execcwd = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No execcwd provided in startjob request");
} else {
job_ctx->startinfo->execcwd = strdup((char*)d->start_info->execcwd);
if (!job_ctx->startinfo->execcwd) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for execcwd");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
if (!d->start_info->q_pre_exec
|| d->start_info->q_pre_exec[0] == '\0') {
job_ctx->startinfo->q_pre_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No queue level pre-exec provided in startjob request");
} else {
job_ctx->startinfo->q_pre_exec
= strdup((char*)d->start_info->q_pre_exec);
if (!job_ctx->startinfo->q_pre_exec) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for queue "
"level pre-exec command.");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
if (!d->start_info->q_post_exec
|| d->start_info->q_post_exec[0] == '\0') {
job_ctx->startinfo->q_post_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No queue level post-exec provided in startjob request");
} else {
job_ctx->startinfo->q_post_exec
= strdup((char*)d->start_info->q_post_exec);
if (!job_ctx->startinfo->q_post_exec) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for queue "
"level post-exec command.");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
if (!d->start_info->job_pre_exec
|| d->start_info->job_pre_exec[0] == '\0') {
job_ctx->startinfo->job_pre_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No job level pre-exec provided in startjob request");
} else {
job_ctx->startinfo->job_pre_exec
= strdup((char*)d->start_info->job_pre_exec);
if (!job_ctx->startinfo->job_pre_exec) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for job "
"level pre-exec command.");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
if (!d->start_info->job_post_exec
|| d->start_info->job_post_exec[0] == '\0') {
job_ctx->startinfo->job_post_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No job level post-exec provided in startjob request");
} else {
job_ctx->startinfo->job_post_exec
= strdup((char*)d->start_info->job_post_exec);
if (!job_ctx->startinfo->job_post_exec) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for job "
"level post-exec command.");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
#ifndef DBUS_MANAGER
job_ctx->manager = (void*)cc_ctx->process_manager;
#endif
if (d->start_info->launch_type) {
job_ctx->startinfo->launch_type
= strdup((char*)d->start_info->launch_type);
} else {
job_ctx->startinfo->launch_type = NULL;
}
job_ctx->func = xcecd_process_command;
job_ctx->xce_jobid = d->jobid;
job_ctx->ntasks = d->launchinfo->hosts[0]->nranks;
job_ctx->clientidx = -1;
/* Create MPI command line */
if (strcmp(job_ctx->startinfo->launch_type, "openmpi") == 0
|| strcmp(job_ctx->startinfo->launch_type, "mpich") == 0) {
job_ctx->startinfo->cmdline
= create_mpi_cmdline(d->start_info->cmdline,
d->start_info->launch_type, d->launchinfo);
if (job_ctx->startinfo->cmdline == NULL) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to create MPI command line");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
/* In MPI case, we set ntasks to 1 on all nodes as we have passed
* the host options command line to all of them.
*/
job_ctx->ntasks = 1;
} else {
job_ctx->startinfo->cmdline = strdup((char*)d->start_info->cmdline);
if (!job_ctx->startinfo->cmdline) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for command");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
}
fmt_jobid(jobidStr, sizeof(jobidStr), d->jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Job %s ntasks is: %d", jobidStr,
job_ctx->ntasks);
/* Job limits */
job_ctx->limits = (res_limits_t*)calloc(1, sizeof(res_limits_t));
if (!job_ctx->limits) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for job limits");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
FREEUP(job_ctx);
return -1;
}
/* Even exdata is empty, we still need to initialize limits as for
* runtime user */
if (!parse_limits(job_ctx->startinfo->exdata, job_ctx->limits)) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to parse job limits from exdata");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
free_limits(job_ctx->limits);
FREEUP(job_ctx);
return -1;
}
if (job_ctx->startinfo->execuser != NULL) {
job_ctx->limits->user_name = strdup(job_ctx->startinfo->execuser);
} else {
job_ctx->limits->user_name = strdup("nobody");
XCECD_LOG(XCE_LOG_WARN,
"No execuser provided, defaulting to 'nobody'");
}
job_ctx->limits->cpu_affinity = NULL;
job_ctx->limits->io_device_path = NULL;
if (submit_fork_task(cc_ctx->thread_pool, job_ctx) != 0) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to add task to thread pool");
xcecd_job_start_error(d->jobid);
free_startinfo(job_ctx->startinfo);
free_limits(job_ctx->limits);
FREEUP(job_ctx);
return -1;
} else {
XCECD_LOG(XCE_LOG_DEBUG, "Task added to thread pool queue");
}
} else {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for task context");
xcecd_job_start_error(d->jobid);
return -1;
}
#endif
return 0;
}
static int handle_startjob(xce_request_t* request)
{
int rc = 0;
char jobidStr[64];
XCECD_LOG(XCE_LOG_DEBUG, "Received startjob request from controller");
if (request->body.jobstartmsg == NULL
|| request->body.jobstartmsg->start_info == NULL
|| request->body.jobstartmsg->launchinfo == NULL) {
XCECD_LOG(XCE_LOG_ERROR,
"Received startjob request with invalid parameters");
return -1;
}
if (request->body.jobstartmsg->jobid <= 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Received startjob request with invalid job ID");
return -1;
}
uint64_t jobid = request->body.jobstartmsg->jobid;
if (request->body.jobstartmsg->start_info->cmdline == NULL) {
XCECD_LOG(XCE_LOG_ERROR,
"Received startjob request with empty command");
xcecd_job_start_error(jobid);
return -1;
}
if (request->body.jobstartmsg->start_info->execuser == NULL) {
XCECD_LOG(XCE_LOG_ERROR,
"Received startjob request with empty execuser");
xcecd_job_start_error(jobid);
return -1;
}
job_startinfo_t* startinfo = request->body.jobstartmsg->start_info;
xce_launch_t* launchinfo = request->body.jobstartmsg->launchinfo;
if (launchinfo->nhosts > 1) {
rc = job_start_remote_task(jobid, startinfo, launchinfo);
if (rc < 0) {
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
XCECD_LOG(XCE_LOG_ERROR, "Job %s run remote task failed, rc %d.",
jobidStr, rc);
xcecd_task_fail(jobid, request->body.runtaskmsg->clientidx, -1, rc);
return rc;
}
}
rc = job_start_local_task(request);
return rc;
}
/**
* @brief Handles the runtasks request from the controller.
*
* @param request Pointer to the xce_request_t structure containing the request
* data.
* @return int 0 on success, -1 on failure.
*/
static int handle_runtasks(xce_request_t* request)
{
#ifndef XRUN
XCECD_LOG(XCE_LOG_DEBUG, "Received runtasks request from controller");
xce_taskrun_t* runmsg = request->body.runtaskmsg;
char jobidStr[64];
if (!runmsg) {
XCECD_LOG(XCE_LOG_ERROR,
"Received runtasks request with invalid message");
return -1;
}
if (runmsg->jobid <= 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Received runtasks request with invalid job ID");
return -1;
}
if (runmsg->nhosts != 1) {
XCECD_LOG(XCE_LOG_ERROR,
"Received runtasks request with invalid number of hosts: %d",
runmsg->nhosts);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -EINVAL);
return -1;
}
if (runmsg->taskhosts[0]->host == NULL) {
XCECD_LOG(XCE_LOG_ERROR,
"Received runtasks request with empty host name");
return -1;
}
if (strcmp(xce_client_cfg.from, runmsg->taskhosts[0]->host) != 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Received runtasks request for a different host: %s",
runmsg->taskhosts[0]->host);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -EINVAL);
return -1;
}
fmt_jobid(jobidStr, sizeof(jobidStr), runmsg->jobid);
XCECD_LOG(XCE_LOG_DEBUG3, "run task: jobid = %s", jobidStr);
XCECD_LOG(XCE_LOG_DEBUG3, "run task: user = %s",
runmsg->taskhosts[0]->execuser ? runmsg->taskhosts[0]->execuser
: "NULL");
XCECD_LOG(XCE_LOG_DEBUG3, "run task: cmdline = %s",
runmsg->taskhosts[0]->cmdline ? runmsg->taskhosts[0]->cmdline
: "NULL");
XCECD_LOG(XCE_LOG_DEBUG3, "run task: exdata = %s",
runmsg->taskhosts[0]->exdata ? runmsg->taskhosts[0]->exdata
: "NULL");
XCECD_LOG(XCE_LOG_DEBUG3, "run task: ntasks = %d",
runmsg->taskhosts[0]->ntasks);
XCECD_LOG(XCE_LOG_DEBUG3, "run task: exechome = %s",
runmsg->taskhosts[0]->exechome ? runmsg->taskhosts[0]->exechome
: "NULL");
XCECD_LOG(XCE_LOG_DEBUG3, "run task: execcwd = %s",
runmsg->taskhosts[0]->execcwd ? runmsg->taskhosts[0]->execcwd
: "NULL");
fork_task_t* task_ctx = calloc(1, sizeof(fork_task_t));
if (task_ctx) {
task_ctx->startinfo = calloc(1, sizeof(job_startinfo_t));
if (!task_ctx->startinfo) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for startinfo");
FREEUP(task_ctx);
return -1;
}
task_ctx->startinfo->cmdline
= strdup((char*)runmsg->taskhosts[0]->cmdline);
if (!task_ctx->startinfo->cmdline) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for command");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
}
task_ctx->startinfo->execuser
= strdup((char*)runmsg->taskhosts[0]->execuser);
if (!task_ctx->startinfo->execuser) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for execuser");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
}
task_ctx->ntasks = runmsg->taskhosts[0]->ntasks;
if (task_ctx->ntasks <= 0) {
XCECD_LOG(
XCE_LOG_ERROR,
"Received runtasks request with invalid number of tasks: %d",
task_ctx->ntasks);
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -EINVAL);
return -1;
}
if (!runmsg->taskhosts[0]->exdata
|| runmsg->taskhosts[0]->exdata[0] == '\0') {
task_ctx->startinfo->exdata = NULL;
XCECD_LOG(XCE_LOG_DEBUG3, "No exdata provided in runtasks request");
} else {
task_ctx->startinfo->exdata
= strdup((char*)runmsg->taskhosts[0]->exdata);
XCECD_LOG(XCE_LOG_DEBUG3, "Received exdata: %s",
task_ctx->startinfo->exdata);
if (!task_ctx->startinfo->exdata) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for exdata");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
}
}
if (!runmsg->taskhosts[0]->exechome
|| runmsg->taskhosts[0]->exechome[0] == '\0') {
task_ctx->startinfo->exechome = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No exechome provided in runtasks request");
} else {
task_ctx->startinfo->exechome
= strdup((char*)runmsg->taskhosts[0]->exechome);
if (!task_ctx->startinfo->exechome) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for exechome");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
}
}
if (!runmsg->taskhosts[0]->execcwd
|| runmsg->taskhosts[0]->execcwd[0] == '\0') {
task_ctx->startinfo->execcwd = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No execcwd provided in runtasks request");
} else {
task_ctx->startinfo->execcwd
= strdup((char*)runmsg->taskhosts[0]->execcwd);
if (!task_ctx->startinfo->execcwd) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for execcwd");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
}
}
/* currently we do not consider pre/post exec for tasks */
#if 0
if (!runmsg->taskhosts[0]->q_pre_exec ||
runmsg->taskhosts[0]->q_pre_exec[0] == '\0') {
task_ctx->startinfo->q_pre_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No queue level pre-exec provided in runtasks request");
} else {
task_ctx->startinfo->q_pre_exec = strdup((char*)runmsg->taskhosts[0]->q_pre_exec);
if (!task_ctx->startinfo->q_pre_exec) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for queue level pre-exec command.");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid,
runmsg->clientidx,
-1, -ENOMEM);
return -1;
}
}
if (!runmsg->taskhosts[0]->q_post_exec ||
runmsg->taskhosts[0]->q_post_exec[0] == '\0') {
task_ctx->startinfo->q_post_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No queue level post-exec provided in runtasks request");
} else {
task_ctx->startinfo->q_post_exec = strdup((char*)runmsg->taskhosts[0]->q_post_exec);
if (!task_ctx->startinfo->q_post_exec) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for queue level post-exec command.");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid,
runmsg->clientidx,
-1, -ENOMEM);
return -1;
}
}
if (!runmsg->taskhosts[0]->job_pre_exec ||
runmsg->taskhosts[0]->job_pre_exec[0] == '\0') {
task_ctx->startinfo->job_pre_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No job level pre-exec provided in runtasks request");
} else {
task_ctx->startinfo->job_pre_exec = strdup((char*)runmsg->taskhosts[0]->job_pre_exec);
if (!task_ctx->startinfo->job_pre_exec) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for job level pre-exec command.");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid,
runmsg->clientidx,
-1, -ENOMEM);
return -1;
}
}
if (!runmsg->taskhosts[0]->job_post_exec ||
runmsg->taskhosts[0]->job_post_exec[0] == '\0') {
task_ctx->startinfo->job_post_exec = NULL;
XCECD_LOG(XCE_LOG_DEBUG3,
"No job level post-exec provided in runtasks request");
} else {
task_ctx->startinfo->job_post_exec = strdup((char*)runmsg->taskhosts[0]->job_post_exec);
if (!task_ctx->startinfo->job_post_exec) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for job level post-exec command.");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid,
runmsg->clientidx,
-1, -ENOMEM);
return -1;
}
}
#endif
#ifndef DBUS_MANAGER
task_ctx->manager = (void*)cc_ctx->process_manager;
#endif
task_ctx->func = xcecd_process_command;
task_ctx->xce_jobid = runmsg->jobid;
task_ctx->clientidx = runmsg->clientidx;
if (submit_fork_task(cc_ctx->thread_pool, task_ctx) != 0) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to add task to thread pool");
free_startinfo(task_ctx->startinfo);
FREEUP(task_ctx);
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
} else {
XCECD_LOG(XCE_LOG_DEBUG, "Task added to thread pool queue");
}
} else {
XCECD_LOG(XCE_LOG_ERROR, "Failed to allocate memory for task context");
xcecd_task_fail(runmsg->jobid, runmsg->clientidx, -1, -ENOMEM);
return -1;
}
#endif // XRUN
return 0;
}
/**
* @brief Handles the fetchfile request from the controller.
*
* Reads up to msg->max_bytes bytes from msg->filepath starting at msg->offset
* and sends the data back to the controller as an OPCode_reply.
*
* @param request Pointer to the xce_request_t containing the fetchfile msg.
* @return int 0 on success, -1 on failure.
*/
static int handle_fetchfile(xce_request_t* request)
{
xce_fetchfile_t* msg = request->body.fetchfilemsg;
xce_fetchfile_reply_t freply;
struct stat st;
uint8_t* buf = NULL;
int fd;
ssize_t nread;
uint32_t maxbytes;
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->jobid);
XCECD_LOG(XCE_LOG_DEBUG,
"Fetchfile request: jobid=%s path=%s offset=%lu max=%u",
jobidStr, msg->filepath, msg->offset, msg->max_bytes);
memset(&freply, 0, sizeof(freply));
freply.jobid = msg->jobid;
if (msg->filepath == NULL || msg->filepath[0] == '\0') {
XCECD_LOG(XCE_LOG_ERROR, "Fetchfile: empty filepath");
freply.error = -EINVAL;
goto send;
}
/* Bound max_bytes to avoid huge allocations */
maxbytes = (msg->max_bytes == 0 || msg->max_bytes > 4 * 1024 * 1024)
? 4 * 1024 * 1024
: msg->max_bytes;
fd = open(msg->filepath, O_RDONLY);
if (fd < 0) {
XCECD_LOG(XCE_LOG_ERROR, "Fetchfile: open(%s) failed: %s",
msg->filepath, strerror(errno));
freply.error = -errno;
goto send;
}
if (fstat(fd, &st) < 0) {
freply.error = -errno;
close(fd);
goto send;
}
freply.file_size = (uint64_t)st.st_size;
buf = malloc(maxbytes);
if (buf == NULL) {
freply.error = -ENOMEM;
close(fd);
goto send;
}
nread = pread(fd, buf, maxbytes, (off_t)msg->offset);
close(fd);
if (nread < 0) {
XCECD_LOG(XCE_LOG_ERROR, "Fetchfile: pread failed: %s",
strerror(errno));
freply.error = -errno;
freply.bytes_read = 0;
free(buf);
buf = NULL;
} else {
freply.error = 0;
freply.bytes_read = (uint32_t)nread;
freply.content = buf;
buf = NULL; /* ownership passed to freply */
}
send:
xcecli_reply_fetchfile(xcecli_ctx, request, &freply);
free(freply.content);
free(buf);
return 0;
}
/**
* @brief Handles the runtasks request from the controller.
*
* @param request Pointer to the xce_request_t structure containing the request
* data.
* @return int 0 on success, -1 on failure.
*/
static int handle_jobcontrol(xce_request_t* request)
{
#ifndef XRUN
XCECD_LOG(XCE_LOG_DEBUG, "Received job control request from controller");
xce_jobctrl_t* d = request->body.jobctrlmsg;
char jobidStr[64];
if (d->jobid <= 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Received job control request with invalid job ID");
return -1;
}
fmt_jobid(jobidStr, sizeof(jobidStr), d->jobid);
if (d->action == JobControl_u_siginfo) {
XCECD_LOG(
XCE_LOG_DEBUG,
"Received job control request to send SIGINFO for job ID: %s",
jobidStr);
job_control_task_t* task_ctx = calloc(1, sizeof(job_control_task_t));
if (!task_ctx) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to allocate memory for job control task context");
return -1;
}
task_ctx->jobid = d->jobid;
task_ctx->signal = d->u.siginfo->signo;
task_ctx->force = d->u.siginfo->force;
xcecd_worker_t* worker = find_worker_by_jobid(
(xcecd_worker_t*)*cc_ctx->worker_list, d->jobid);
if (worker) {
if (worker->job_status == JOB_PENDING
|| worker->job_status == JOB_STARTING) {
XCECD_LOG(XCE_LOG_ERROR,
"Job is pending or starting, controller should not "
"pass the signal to %s.",
jobidStr);
// really should not happen, but if it does, we handle it
// gracefully
xcecd_job_status_error(d->jobid);
FREEUP(task_ctx);
return 0;
}
} else {
XCECD_LOG(XCE_LOG_ERROR, "Failed to find worker for job ID: %s",
jobidStr);
FREEUP(task_ctx);
return -1;
}
#if !defined(VIRTUAL_CLUSTER)
if (!ipc_master_send_job_signal(task_ctx->jobid, task_ctx->signal,
task_ctx->force)) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to submit job control task for job ID: %s",
jobidStr);
FREEUP(task_ctx);
return -1;
} else {
XCECD_LOG(XCE_LOG_DEBUG,
"Job control task submitted for job ID: %s", jobidStr);
}
FREEUP(task_ctx);
#else
// For virtual cluster, we handle job control tasks differently
XCECD_LOG(XCE_LOG_DEBUG,
"Virtual cluster mode, handling job control task directly");
FREEUP(task_ctx);
XCECD_LOG(XCE_LOG_DEBUG,
"Job control task handled directly for job ID: %s",
jobidStr);
worker->killed_signal = d->u.siginfo->signo; // Mark job as killed
#endif
} else if (d->action == JobControl_u_ckptoption) {
#if !defined(VIRTUAL_CLUSTER)
XCECD_LOG(XCE_LOG_DEBUG,
"Received job control request to set checkpoint options for "
"job ID: %s",
jobidStr);
if (!ipc_master_send_job_ckpt(d->jobid, d->u.ckptoption->kill,
d->u.ckptoption->method)) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to send checkpoint options for job ID: %s",
jobidStr);
return -1;
} else {
XCECD_LOG(XCE_LOG_DEBUG, "Checkpoint options sent for job ID: %s",
jobidStr);
}
#else
XCECD_LOG(XCE_LOG_WARN, "Received job control request in virtual "
"cluster or xrun mode, which is not supported");
#endif
} else {
XCECD_LOG(XCE_LOG_ERROR,
"Received unknown job control action: %d for job ID: %s",
d->action, jobidStr);
return -1;
}
#endif // XRUN
return 0;
}
/* worker-rejoin-resync task 4.3 (Req 8.2/8.3 precedent): page-size cap for
* this worker's own OPCode_jobStateReport sends, matching the pktlimit
* value apiserver.c's own xce_push_data_req() call sites already use for
* the same {pos,count,n_total} pagination idiom. */
#define XCECD_RESYNC_REPORT_PKTLIMIT 100
/* Wall-clock (epoch) milliseconds -- same rationale and implementation as
* xcecd_worker_utils.c's private finished_job_wallclock_now_ms() (not
* reusable here: internal linkage, separate translation unit). Needed here
* because xcecd_resync.c's functions all take now_ms as a caller-captured
* parameter rather than sampling the clock themselves (task 4.1's own
* convention -- deterministic values in tests, real wall-clock in
* production), and this dispatch case is that production caller. */
static uint64_t resync_wallclock_now_ms(void)
{
struct timespec ts;
if (clock_gettime(CLOCK_REALTIME, &ts) != 0) {
return (uint64_t)time(NULL) * 1000ULL;
}
return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL;
}
/**
* @brief Handles one incoming OPCode_jobStateQuery page from the controller
* (worker-rejoin-resync task 4.3, Requirements 3.8, 8.1, 8.3).
*
* Delegates to xcecd_resync_handle_query_page() (xcecd_resync.c), which
* reassembles a query that itself spans multiple incoming pages and, once
* the last page has arrived, answers it with a paginated
* OPCode_jobStateReport -- see that function's own header comment
* (xcecd_resync.h) for the resync_pending gate it also owns.
*
* @param request Pointer to the xce_request_t structure containing the
* request data.
* @return int 0 on success (whether or not this page completed the query),
* negative on error.
*/
static int handle_jobstatequery(xce_request_t* request)
{
endpoint_t* ep = xcecli_get_my_endpoint(xcecli_ctx);
xce_peerinfo_t* from = xcecli_get_myinfo(xcecli_ctx);
xcecd_worker_t* worker_list
= (cc_ctx != NULL) ? (xcecd_worker_t*)*cc_ctx->worker_list : NULL;
int rc;
if (ep == NULL || from == NULL) {
XCECD_LOG(XCE_LOG_ERROR,
"handle_jobstatequery: no active controller connection");
return -EINVAL;
}
rc = xcecd_resync_handle_query_page(
worker_list, ep, from, request->body.jobstatequerymsg,
resync_wallclock_now_ms(), XCECD_RESYNC_REPORT_PKTLIMIT);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"handle_jobstatequery: failed to process query page "
"(rc=%d)",
rc);
return rc;
}
return 0;
}
/**
* @brief Handles an incoming OPCode_push config-push message from the
* controller (worker-rejoin-resync task 6.1, Requirement 3.7).
*
* Confirmed by inspection: before this task, myreq_handler's dispatch
* switch below had NO case for OPCode_push at all -- a config push
* (worker_push_config()/on_worker_join(), controller/worker.c) fell through
* to the `default: break;` case and was silently discarded in its
* entirety, even for the pre-existing clustername field.
* xcecd_finished_job_retention_set_window_ms()'s own doc comment
* (xcecd_worker_utils.h, task 4.1) independently confirms this same gap:
* "the worker daemon has no runtime delivery path for this ClusterConfig
* field today... this setter is the seam a later task wires the real
* operator-configured value into once that delivery path exists." This
* function is that delivery path, scoped narrowly to the ONE field this
* task needs (workerResyncWindowMs) -- clustername/confvars remain
* unconsumed by the worker, unchanged and out of this task's boundary; a
* general config-receive rework is not attempted here.
*
* Fire-and-forget, matching worker_push_config()'s own controller-side
* send (no reply is expected or sent back).
*
* @param request Pointer to the xce_request_t structure containing the
* request data.
* @return int 0 always -- a malformed/unexpected payload shape (wrong
* body_which, wrong ConfigData union arm, NULL conf4worker) is
* simply ignored, not an error to report back.
*/
static int handle_pushconfig(xce_request_t* request)
{
xce_configdata_t* configdata;
if (request->body_which != Request_body_configpushmsg)
return 0;
configdata = request->body.configpushmsg;
if (configdata == NULL
|| configdata->data_which != ConfigData_data_workerconfig
|| configdata->data.conf4worker == NULL) {
return 0;
}
xcecd_finished_job_retention_set_window_ms(
(uint64_t)configdata->data.conf4worker->worker_resync_window_ms);
return 0;
}
/**
* @brief Handles incoming requests from the controller.
*
* @param ctx Pointer to the I/O context structure.
* @return int 0 on success, -1 on failure.
*/
static int myreq_handler(xce_request_t* request, void* args)
{
int ret = 0;
XCECD_LOG(XCE_LOG_DEBUG, "Calling request handler...");
switch (request->opcode) {
case OPCode_startjob:
ret = handle_startjob(request);
break;
case OPCode_runtasks:
ret = handle_runtasks(request);
break;
case OPCode_jobcontrol:
ret = handle_jobcontrol(request);
break;
case OPCode_fetchfile:
ret = handle_fetchfile(request);
break;
case OPCode_jobStateQuery:
ret = handle_jobstatequery(request);
break;
case OPCode_push:
ret = handle_pushconfig(request);
break;
default:
break;
}
return ret;
}
/**
* @brief Sends a start job reply to the controller.
*
* @param job Pointer to the job structure.
* @return bool true on success, false on failure.
*/
bool xcecd_reply_startjob(xcecd_job_t* job)
{
/* clang-format off */
//xce_ctx_t* ctx = cc_ctx->xce_ctx;
unsigned int pids[] = {job->job_pid};
job_pidinfo_t pidinfo = {
.jobid = job->job_id,
.npids = 1,
.pids = &pids[0],
.npgids = 0,
.pgids = NULL,
.exdata = NULL,
};
if (job->exdata[0] != '\0') {
pidinfo.exdata = strdup(job->exdata);
}
xcecli_notify_jobstart(xcecli_ctx, &pidinfo);
FREEUP(pidinfo.exdata);
return true;
}
/**
* @brief Starts the controller connector by joining the cluster and setting up
* the endpoint handler.
*
* @return bool true on success, false on failure.
*/
bool controller_connector_start(void)
{
int rc;
int retry_limit = 0;
static int error_count = 0;
/* No standalone client-config construction here anymore -- the single
* synthesize-validate-convert-connect pipeline inside
* xcecd_join_cluster() below builds and sets xce_client_cfg itself on
* every attempt (component-failover-adoption, task 2.1, Req 1 AC5). */
do {
rc = xcecd_join_cluster(&host_attrs);
if (rc < 0) {
XCECD_LOG(XCE_LOG_DEBUG,
"Failed to join cluster, retrying in %d seconds",
cc_ctx->join_interval);
error_count++;
if ((retry_limit > 0) && (error_count >= retry_limit)) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to join cluster after %d retries",
retry_limit);
return false;
}
}
sleep(cc_ctx->join_interval);
} while (rc < 0 && cc_ctx->running);
// api_ctx_t* api_ctx = (api_ctx_t*)xcecli_ctx;
// cc_ctx->xce_ctx = &(api_ctx->ctx);
return true;
}
int xcecd_notify_job_fail(uint64_t jobid,
int stage,
int code)
{
job_failinfo_t failinfo = {
.jobid = jobid,
.stage = stage,
.code = code,
.detail = "no detail information"
};
XCECD_LOG(XCE_LOG_DEBUG, "Job failed and calling xcecli_notify_jobfail...");
return xcecli_notify_jobfail(xcecli_ctx, &failinfo);
}
/**
* @brief Marks a job as finished and sends the exit code to the controller.
*
* @param job Pointer to the job structure.
* @return int 0 on success, -1 on failure.
*/
int xcecd_job_finished(xcecd_job_t* job)
{
if (job->exit_code != 0) {
return xcecd_notify_job_fail(job->job_id,
job->stage,
job->exit_code);
}
xce_jobfinish_t myjob = {
.jobid = job->job_id,
.exit_code = 0
};
XCECD_LOG(XCE_LOG_DEBUG, "calling xcecli_jobfinish...");
return xcecli_notify_jobfinish(xcecli_ctx, &myjob);
}
/**
* @brief Marks a job as stopped and sends a notification to the controller.
*
* @param jobid The ID of the job to be marked as stopped.
* @return int 0 on success, -1 on failure.
*/
int xcecd_job_stopped(uint64_t jobid)
{
XCECD_LOG(XCE_LOG_DEBUG,
"Job stopped and calling xcecli_notify_jobstop...");
return xcecli_notify_jobstop(xcecli_ctx, jobid);
}
/**
* @brief Marks a job as resumed and sends a notification to the controller.
*
* @param jobid The ID of the job to be marked as resumed.
* @return int 0 on success, -1 on failure.
*/
int xcecd_job_resumed(uint64_t jobid)
{
XCECD_LOG(XCE_LOG_DEBUG,
"Job resumed and calling xcecli_notify_jobresume...");
return xcecli_notify_jobresume(xcecli_ctx, jobid);
}
/**
* @brief Reply job start error information to controller
* for some cases, we need to requeue the job
* and let controller to retry
* @return int 0 on success, -1 on failure.
*/
int xcecd_job_start_error(uint64_t jobid)
{
return xcecd_notify_job_fail(jobid, JOB_EXEC_STAGE_MAIN, -1);
}
/**
* @brief Reply job status error information to controller
* for some cases, we need to requeue the job
* and let controller to retry
* @return int 0 on success, -1 on failure.
*/
int xcecd_job_status_error(uint64_t jobid)
{
xce_jobfinish_t myjob = {
.jobid = jobid,
.exit_code = -1
};
XCECD_LOG(XCE_LOG_DEBUG, "Job status error and calling xcecli_jobfinish...");
return xcecli_jobfinish(xcecli_ctx, &myjob);
}
/**
* @brief Reply job checkpoint result to controller
* @return int 0 on success, -1 on failure
*/
int xcecd_job_ckpted(uint64_t jobid, int status, const char* info)
{
if (0 == status) {
XCECD_LOG(XCE_LOG_DEBUG, "Job checkpoint success and calling xcecli_notify_jobchkpt...");
} else {
XCECD_LOG(XCE_LOG_ERROR, "Job checkpoint failed and calling xcecli_notify_jobchkpt...");
}
return xcecli_notify_jobchkpt(xcecli_ctx, jobid, (char*)info);
}
/**
* @brief Starts a job on a remote host.
*
* @param hostname The name of the remote host.
* @param command The command to be executed on the remote host.
* @return int 0 on success, -1 on failure.
*/
int xcecd_job_remote_start(uint64_t jobid,
const char* hostname,
const char* command)
{
int rc = 0;
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
if (!cc_ctx || !cc_ctx->worker_list) {
XCECD_LOG(XCE_LOG_ERROR,
"Controller connector context is not initialized");
return -1;
}
if (!hostname || !command) {
XCECD_LOG(XCE_LOG_ERROR,
"Invalid parameters for remote job start");
return -1;
}
if (jobid <= 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Invalid job ID for remote job start");
return -1;
}
xcecd_worker_t* worker = find_worker_by_jobid((xcecd_worker_t *)*cc_ctx->worker_list,
jobid);
if (!worker) {
XCECD_LOG(XCE_LOG_ERROR,
"Worker not found for job ID %s", jobidStr);
return -1;
}
xce_taskhost_t host[] = {
{.exechome = worker->exechome,
.execcwd = worker->execcwd,
.cmdline = strdup(command),
.exdata = worker->job_exdata,
.execuser = worker->execuser,
.launch_type = worker->launch_type,
.host = strdup(hostname),
.ntasks = 1},
{0}
};
xce_taskrun_t taskrun = {
.jobid = jobid,
.clientidx = -1,
.taskhosts = (xce_taskhost_t**)calloc(1, sizeof(xce_taskhost_t*)),
.nhosts = 1
};
taskrun.taskhosts[0] = &host[0];
rc = xcecli_runtask(xcecli_ctx, &taskrun);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to start job %s on remote host %s, rc: %d",
jobidStr, hostname, rc);
xcecd_task_fail(jobid, -1, -1, rc);
return rc;
}
XCECD_LOG(XCE_LOG_DEBUG,
"Remote task of job %s will run on %s",
jobidStr, hostname);
return 0;
}
int xcecd_task_fail(uint64_t jobid, int clientidx,
int taskidx, int code)
{
xce_task_failinfo_t failinfo = {
.jobid = jobid,
.host = xce_client_cfg.from,
.clientidx = clientidx,
.taskidx = taskidx,
.code = code,
.detail = NULL
};
return xcecli_notify_taskfail(xcecli_ctx, &failinfo);
}
int xcecd_jobpost_done(uint64_t jobid)
{
XCECD_LOG(XCE_LOG_DEBUG,
"Job post done and calling xcecli_notify_jobpostdone...");
return xcecli_notify_jobpostdone(xcecli_ctx, jobid);
}
int xcecd_jobrusage_update(xce_job_rusages_t* rusages)
{
char jobidStr[64];
if (!rusages) {
XCECD_LOG(XCE_LOG_ERROR,
"Invalid rusage data.");
return -1;
}
fmt_jobid(jobidStr, sizeof(jobidStr), rusages->jobid);
XCECD_LOG(XCE_LOG_DEBUG,
"Updating job rusage to controller for job %s", jobidStr);
return xcecli_update_jusage(xcecli_ctx,
rusages->jobid,
rusages->npids,
rusages->pids,
rusages->npgids,
rusages->pgids,
rusages->count,
rusages->usages);
}
bool xcecd_send_load_indices(xce_resval_t** vals, int count) {
if (vals == NULL || count <= 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Invalid parameters for sending load indices");
return false;
}
if (xcecli_ctx == NULL) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecli_ctx is not initialized");
return false;
}
int result = xcecli_update_resvals(xcecli_ctx, count, &vals[0]);
if (result != 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to send load indices to controller, error code: %d",
result);
return false;
}
XCECD_LOG(XCE_LOG_DEBUG,
"Successfully sent %d load indices to controller", count);
return true;
}
Reference in a new issue View git blame Copy permalink