lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merge: sync Rongsong's scheduler-framework into chatbot

Browse source 
Full merge of origin/scheduler-framework (scheduler-core restructure,
failover-adoption, WAL_JOB_DISPATCH, apiserver backend health/heartbeat,
build-time versioning, bug-1673/1676/1678/1679/1680).

Conflict resolution:
- scheduler: adopt Rongsong's scheduler-core architecture (driver loop,
  queue project/picker fields, resource bug-1569 fix)
- worker/controller_connector.c: keep Rongsong's reconnect/rejoin
  structure, PORT our bug-1509 resend_job_status onto the rejoin success
  path (his reconnect didn't re-report job state)
- apiserver backend: take Rongsong's health/heartbeat (supersedes our
  e505066 circuit breaker)
- group.c: keep our bug-1513 comment
- misc: take theirs for additive fixes (go_build versioning,
  test_group.c, persist.c RUSAGE_UPDATE, FilterRows bug-1680)

NOTE: our scheduler fixes (bug-1494 family dispatch-fail requeue,
PEND-reason classification) are NOT in Rongsong's branch — needs
verification/porting onto scheduler-core (separate follow-up).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-08-25 12:33:16 +00:00
commit 4e894d2184
395 changed files with 88517 additions and 5742 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/worker/CLAUDE.md
View file

@ -5,15 +5,13 @@ C daemon running on each compute node. Manages job execution, hardware monitorin
## Build
```bash
# From workspace root
nix develop
meson build && ninja -C build
/xce-build worker # lib + worker daemon (see build-deployment.md)
```
## Architecture
- `src/xcecd.c` — Main daemon entry point
- `src/hardware_monitor.c` — CPU/memory/GPU resource monitoring
- `src/process_manager.c` — Job process lifecycle (spawn, track, reap)
- `src/main.c` — Main daemon entry point (`int main`); links against `xcecd_srcs` from `src/xcecd.c` and friends
- `src/hardware_monitor.c` — CPU/memory resource monitoring (no GPU)
- `src/process_worker_manager.c` + `src/uv_proc.c` — Job process lifecycle (fork `xcecd_worker`, spawn/exec the job command); `src/process_manager.c` is unused dead code for job spawn (its `process_manager_start_process` is never called) but is still wired up for generic create/update/destroy/reap in `xcecd.c`
Links against `libxce` from `src/lib/`.

6
src/worker/conf/worker.yaml
View file

@ -1,6 +0,0 @@
---
workerconf:
management_list: ["hostA", "hostB"]
controller_port: 9990
log_level: "info"
log_dir: "/tmp"

13
src/worker/include/xcecd.h
View file

@ -11,7 +11,6 @@
#include "system_monitor.h"
#include "thread_pool.h"
#include "uv_ipc.h"
#include "xcecd_conf.h"
#include "xcecd_db.h"
#include "xcecd_dbus_manager.h"
#include "xcecd_worker_utils.h"
@ -57,6 +56,13 @@ typedef struct {
int update_interval;
int domain_sock;
int debug_level;
/* HA candidate list parsed from --peers (component-failover-adoption,
* task 2.2, Req 1.4). Threaded through xcecd_init() -> xcecd_context_t
* -> controller_connector_init() so cc_ctx->cluster_cfg is populated
* before the first join, instead of main.c assigning it after
* xcecd_start() already ran. candidates_count == 0 means no --peers was
* given (single-host default). */
xce_cluster_config_t cluster_cfg;
} xcecd_config_t;
typedef struct {
@ -82,6 +88,11 @@ typedef struct {
char* log_dir;
char* db_path;
char* virtual_name;
/* Copied from xcecd_config_t by xcecd_init(); consumed by
* controller_connector_init() (xcecd_start()) to populate
* cc_ctx->cluster_cfg BEFORE controller_connector_start() fires the
* first join (component-failover-adoption, task 2.2, Req 1.4). */
xce_cluster_config_t cluster_cfg;
} xcecd_context_t;
extern xcecd_context_t xcecd_ctx;

47
src/worker/include/xcecd_conf.h
View file

@ -1,47 +0,0 @@
#ifndef XCECD_CONF_H
#define XCECD_CONF_H
#include <cyaml/cyaml.h>
#include <stdbool.h>
#include <stdint.h>
#define DEFAULT_CONFIG_PATH "/opt/xce/conf/worker.yaml"
#define MAX_HOSTNAME_LEN 256
#define MAX_PATH_LEN 512
#define MAX_LOG_LEVEL_LEN 16
typedef enum {
CONFIG_SUCCESS = 0,
CONFIG_ERROR_FILE_NOT_FOUND,
CONFIG_ERROR_PARSE_FAILED,
CONFIG_ERROR_INVALID_DATA,
CONFIG_ERROR_MEMORY,
CONFIG_ERROR_UNKNOWN
} config_error_t;
typedef struct {
char* hostname;
} management_host_t;
typedef struct {
management_host_t* management_list;
uint32_t management_list_count;
uint16_t controller_port;
char* log_level;
char* log_dir;
} worker_config_t;
typedef struct {
config_error_t error_code;
char error_message[512];
worker_config_t* config;
} config_result_t;
config_result_t* config_load(const char* config_path);
config_result_t* config_load_default(void);
void config_free(config_result_t* result);
const char* config_error_string(config_error_t error);
void config_print(const worker_config_t* config);
int level_to_int(const char* level);
#endif // XCECD_CONF_H

3
src/worker/meson.build
View file

@ -67,7 +67,6 @@ xrun = executable('xrun',
xcecd_srcs = [
'src/static_array.c',
'src/xcecd_conf.c',
'src/inter_job_utils.c',
'src/xcecd_sig.c',
'src/uv_ipc.c',
@ -104,3 +103,5 @@ xcecd_virtual = executable('xcecd_virtual',
include_directories: [incdir],
dependencies:[xcecd_deps],
install:true)
subdir('testsuite')

623
src/worker/src/controller_connector.c
View file

@ -8,7 +8,6 @@
#include "xcelog.h"
#include <errno.h>
#include <fcntl.h>
#include <pwd.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
@ -32,6 +31,23 @@ xcecli_ctx_t xcecli_ctx = { 0 };
xce_client_config_t xce_client_cfg = { 0 };
controller_connector_ctx_t* cc_ctx = NULL;
/* Persistent storage for the single candidate list the current connect/join
* attempt is driving (component-failover-adoption, task 2.1, Req 1 AC5).
* Must outlive the xcecli_ctx_t connection: xce_cluster_config_to_client_
* config() points xce_client_cfg.masters[] directly at this struct's
* hostname buffers (non-owning aliasing -- the library keeps a reference to
* xce_client_cfg for the life of the connection, so the source data behind
* it cannot be a stack-local temporary). Populated by build_cluster_config()
* below from either the operator-supplied --peers list or a synthesized
* single candidate -- never both, never a stack copy. */
static xce_cluster_config_t g_cluster_cfg = { 0 };
/* Set by the rejoin callback registered with the client library
* (xcecd_on_reconnect() below) when the library has transparently
* reconnected and this connector needs to re-issue xcecli_join() as a
* separate, later call. Consumed by checking_connection() (task 2.2). */
static bool needs_rejoin = false;
static int myreq_handler(xce_request_t*, void*);
/* make compiler happy */
@ -123,11 +139,17 @@ int start_controller_connector_thread(task_func_t function, void* arg)
}
/**
* @brief Initializes the client configuration for the controller connector.
* @brief Sets the caller-owned identity/auth fields (from, key) on the
* client config. Deliberately does NOT touch nmasters/masters[]/
* master_ports[] -- those are owned by the shared
* xce_cluster_config_to_client_config() helper (build_cluster_config()
* + the conversion call in xcecd_join_cluster() below), matching
* that helper's documented contract ("Does NOT set out->from /
* out->key -- caller sets those separately").
*
* @param config Pointer to the xce_client_config_t structure to be initialized.
* @param config Pointer to the xce_client_config_t structure to update.
*/
static void init_client_config(xce_client_config_t* config)
static void set_client_identity(xce_client_config_t* config)
{
int ret;
char local_host[MAXHOSTNAMELEN] = { 0 };
@ -142,9 +164,6 @@ static void init_client_config(xce_client_config_t* config)
strncpy(local_host, "localhost", sizeof(local_host) - 1);
}
config->nmasters = 1;
config->masters[0] = cc_ctx->remote_host;
config->master_ports[0] = cc_ctx->remote_port;
#ifdef VIRTUAL_CLUSTER
config->from = strdup(cc_ctx->virtual_name);
#else
@ -152,205 +171,335 @@ static void init_client_config(xce_client_config_t* config)
#endif
config->key = strdup("keepSecret");
XCECD_LOG(XCE_LOG_DEBUG, "Controller connector configuration initialized.");
XCECD_LOG(XCE_LOG_DEBUG,
"Join cluster configuration - Local host: %s, Remote host: %s, "
"Remote port: %d, Key: %s",
config->from, config->masters[0], config->master_ports[0],
config->key);
"Controller connector identity set: from=%s", config->from);
}
/**
* @brief Reconnect callback — invoked by libxce's transparent failover after
* a connection is re-established (and the ctx recovered), BEFORE the
* interrupted op resumes.
* @brief Build the single validated candidate list this connect/reconnect
* attempt will use (component-failover-adoption, task 2.1, Req 1
* AC5).
*
* This covers the case where the worker process stays alive and only the
* TCP link to the controller dropped (or the controller failed over): the
* failover retry loop in xcecli_hello() reconnects transparently and returns
* success, so checking_connection() sees ret==0 and never takes its
* xcecd_join_cluster()/resend_job_status() path. Without this hook, job
* state changes that occurred while disconnected are never re-reported and
* the controller is left with stale RUN/UNKWN jobs.
* A single controller is not a separately-maintained code path: when
* --peers configured a real candidate list (cc_ctx->cluster_cfg), it is
* used as-is; otherwise a one-candidate list is synthesized from the legacy
* remote_host/remote_port default. Either source is written into the same
* persistent g_cluster_cfg and then run through the identical
* xce_cluster_config_validate() call -- no branch downstream of this
* function distinguishes the two sources.
*
* @return 0 to let the interrupted op resume; <0 would be treated by the
* failover loop as "new primary rejected rejoin" and it would rotate.
* @return 0 on a validated config, the validator's negative error code
* (or -EINVAL if cc_ctx is unset) otherwise.
*/
static int xcecd_reconnect_cb(xcecli_ctx_t c, void* arg)
static int build_cluster_config(void)
{
(void)c;
(void)arg;
if (cc_ctx == NULL || cc_ctx->worker_list == NULL
|| *cc_ctx->worker_list == NULL) {
return 0;
if (!cc_ctx) {
return -EINVAL;
}
XCECD_LOG(XCE_LOG_INFO,
"Controller connection restored; resending job status");
if (cc_ctx->cluster_cfg.candidates_count > 0) {
g_cluster_cfg = cc_ctx->cluster_cfg;
} else {
memset(&g_cluster_cfg, 0, sizeof(g_cluster_cfg));
strncpy(g_cluster_cfg.candidates[0].hostname,
cc_ctx->remote_host ? cc_ctx->remote_host : "",
sizeof(g_cluster_cfg.candidates[0].hostname) - 1);
g_cluster_cfg.candidates[0].port = cc_ctx->remote_port;
g_cluster_cfg.candidates_count = 1;
strncpy(g_cluster_cfg.identity,
cc_ctx->virtual_name ? cc_ctx->virtual_name : "xcecd",
sizeof(g_cluster_cfg.identity) - 1);
}
/* Same semantics as checking_connection()'s rejoin resend (bug-1509
* B2/B3): report the ACTUAL job state so the controller reconciles its
* view (clears UNKWN on a live claim; converges to a terminal state for
* processes that are gone). */
(void)resend_job_status(*cc_ctx->worker_list);
return xce_cluster_config_validate(&g_cluster_cfg);
}
/* Sized for the worst case: MAX_HA_CANDIDATES entries, each up to
* (XCE_MAXHOSTNAME_LEN - 1) hostname chars + ':' + a 5-digit port + a 2-char
* ", " separator (263 bytes/candidate at the max) -- the extra +16 pads that
* comfortably rather than sizing exactly to the boundary. Named rather than
* inlined at each call site so the callers below and the tests that call
* format_candidate_list() directly can't drift out of sync with each other. */
#define CANDIDATE_LIST_BUF_SIZE ((XCE_MAXHOSTNAME_LEN + 16) * MAX_HA_CANDIDATES)
/**
* @brief Renders every candidate in g_cluster_cfg as "host:port, host:port,
* ..." into buf (component-failover-adoption, task 2.3, Req 1 AC6,
* Req 5 AC2).
*
* xcecd_join_cluster()'s all-candidates-exhausted failure paths only had a
* candidate COUNT to log ("all %d candidate(s) unreachable") -- neither
* xcecli_start() nor endpoint_connect() (src/lib/netio.c) log anything
* per-candidate themselves (confirmed by source read: no XCECD_LOG/INFO/WARN
* call anywhere in xcecli_start()'s per-master connect loop), so without
* this the worker's own failure message never actually named which
* candidates it tried, only how many -- a gap against Req 1 AC6 ("identifying
* which candidates were attempted") and Req 5 AC2 ("identifying every
* candidate attempted"). Found during task 2.3's all-candidates-unreachable
* coverage.
*
* Callers must size buf as CANDIDATE_LIST_BUF_SIZE (or larger) -- with an
* undersized buffer this truncates at the last candidate that fits rather
* than overflowing (bufsz - 1 bound below), so a too-small buf loses
* information, not memory safety.
*/
static void format_candidate_list(char* buf, size_t bufsz)
{
size_t used = 0;
buf[0] = '\0';
for (int i = 0; i < g_cluster_cfg.candidates_count && used < bufsz - 1;
i++) {
int written = snprintf(buf + used, bufsz - used, "%s%s:%d",
i > 0 ? ", " : "",
g_cluster_cfg.candidates[i].hostname,
g_cluster_cfg.candidates[i].port);
if (written < 0) {
break;
}
used += (size_t)written;
}
}
/**
* @brief Rejoin callback registered with the client library
* (component-failover-adoption, task 2.1, Req 2 AC1, Req 5 AC1-2).
*
* The library invokes this while still holding its own per-ctx lock
* (ac->failover.lock, a non-recursive mutex held for the whole
* XCECLI_FAILOVER_CALL retry loop) -- per design.md's "Rejoin callback
* constraint", this callback MUST NOT perform I/O or call any xcecli_*
* function: doing so would try to re-acquire the same already-held lock and
* self-deadlock the calling thread forever. It only sets a flag and returns
* 0 immediately so the interrupted operation resumes normally. Consuming
* this flag to issue the actual xcecli_join() as a separate, later call
* (once the triggering call has returned and the lock is free) is task
* 2.2's job.
*/
static int xcecd_on_reconnect(xcecli_ctx_t ctx, void* arg)
{
(void)ctx;
(void)arg;
needs_rejoin = true;
return 0;
}
/**
* @brief Handles I/O events for the endpoint, focusing on error conditions
* and connection liveness checks using EPOLLIN/EPOLLOUT.
* @brief Joins the cluster using the provided host attributes
* (component-failover-adoption, task 2.1, Req 1 AC1-3/AC5, Req 5
* AC1-2).
*
* Builds exactly one validated candidate list (build_cluster_config()),
* converts it via the shared libxce helper, and performs ONE
* init+start+join sequence. The client library's own failover mechanism
* (client_failover.c) rotates across candidates and follows standby
* redirects internally, within the single xcecli_join() call below --
* there is no per-candidate loop here anymore. A single configured
* candidate is simply the candidates_count=1 case of this exact sequence,
* not a separate fallback.
*
* @param attrs Pointer to the host attributes structure.
* @return int 0 on success, negative error code (or -1) on failure.
*/
int xcecd_join_cluster(hostattrs_t* attrs)
{
int rc;
if (!attrs) {
return -1;
}
rc = build_cluster_config();
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: invalid cluster configuration (rc=%d)",
rc);
return rc;
}
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: attempting %d candidate(s), first=%s:%d",
g_cluster_cfg.candidates_count,
g_cluster_cfg.candidates[0].hostname,
g_cluster_cfg.candidates[0].port);
rc = xce_cluster_config_to_client_config(&g_cluster_cfg, &xce_client_cfg);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: candidate list conversion failed "
"(rc=%d)",
rc);
return rc;
}
set_client_identity(&xce_client_cfg);
rc = xcecli_ctx_init(&xcecli_ctx, &xce_client_cfg);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: xcecli_ctx_init failed (rc=%d)", rc);
return rc;
}
xcecli_set_handler(xcecli_ctx, &myreq_handler, NULL);
rc = xcecli_start(xcecli_ctx);
if (rc < 0) {
char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 };
format_candidate_list(attempted, sizeof(attempted));
XCECD_LOG(XCE_LOG_WARN,
"xcecd_join_cluster: all %d candidate(s) unreachable "
"(rc=%d): %s",
g_cluster_cfg.candidates_count, rc, attempted);
xcecli_ctx_fini(xcecli_ctx);
xcecli_ctx = NULL;
return rc;
}
/* Registered AFTER xcecli_start() succeeds, not before (task 2.3 fix --
* found during this task's E2E reconnect validation): xcecli_start()
* unconditionally resets ctx->failover.rejoin_cb/rejoin_arg to NULL as
* part of initializing per-context failover state (client.c, "no rejoin
* hook is installed for pure clients" -- it assumes callers register
* AFTER start, matching every xcecli_* client contract). Registering
* here BEFORE xcecli_start() (the code's shape prior to this fix) meant
* the registration was silently wiped out immediately afterward, so
* xcecd_on_reconnect() could never fire and needs_rejoin was never set
* by a real transparent reconnect -- task 2.1/2.2's whole rejoin-retry
* mechanism was dead in production despite passing unit tests (those
* tests set needs_rejoin directly or call xcecli_set_reconnect_handler()
* standalone, never through this exact xcecli_start()-then-register
* production sequence, so none of them could catch this ordering bug). */
xcecli_set_reconnect_handler(xcecli_ctx, xcecd_on_reconnect, NULL);
rc = xcecli_join(xcecli_ctx, HostRole_worker, attrs);
if (rc < 0) {
char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 };
format_candidate_list(attempted, sizeof(attempted));
errno = -rc;
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: xcecli_join() failed across %d "
"candidate(s) (%s): %s",
g_cluster_cfg.candidates_count, attempted, strerror(errno));
xcecli_stop(xcecli_ctx);
xcecli_ctx_fini(xcecli_ctx);
xcecli_ctx = NULL;
return rc;
}
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: joined cluster (from=%s, %d "
"candidate(s) configured)",
xce_client_cfg.from, g_cluster_cfg.candidates_count);
return 0;
}
/**
* @brief Issues the actual rejoin call: the same join payload used at
* startup, as a separate call once the triggering call (xcecli_hello
* or whatever else invoked the reconnect callback) has returned and
* the client library's internal failover lock is free.
*
* Split out from checking_connection() behind the rejoin_call function
* pointer below purely for testability: xcecli_join() against a real
* candidate goes through XCECLI_FAILOVER_CALL's full retry/backoff budget
* (up to ~100s) on failure, and a genuine success requires a live
* capnp-speaking controller -- neither is practical to drive from a fast
* unit test. Production always runs through this exact function via the
* default rejoin_call pointer; only tests override the pointer to a
* controllable double.
*/
static int perform_pending_rejoin(void)
{
return xcecli_join(xcecli_ctx, HostRole_worker, &host_attrs);
}
/* Defaults to the real production call (see perform_pending_rejoin() above).
* Overridden only by tests. */
static int (*rejoin_call)(void) = perform_pending_rejoin;
/**
* @brief Periodic liveness probe on the connector thread
* (component-failover-adoption, task 2.1, Req 2 AC1-3, Req 3 AC1/AC3;
* task 2.2, Req 2 AC5, rejoin retry-on-failure).
*
* A single xcecli_hello() call. xcecli_hello() is XCECLI_FAILOVER_CALL-
* wrapped: on a transport failure it already retries indefinitely across
* the full configured candidate list (rotate + backoff) and invokes the
* registered rejoin-flag callback on its own -- the client library now
* owns connection recovery. This probe therefore no longer runs its own
* bounded retry-then-give-up loop: an outage that outlasts the old
* MAX_RETRY_COUNT*RETRY_INTERVAL bound no longer tears the worker's
* connection down for good (cc_ctx->running is never forced false here);
* the connector thread's normal periodic cadence simply gives the library
* another chance on the next call.
*
* Immediately after xcecli_hello() returns (its internal failover lock is
* now free), checks needs_rejoin -- set by xcecd_on_reconnect() when the
* library transparently reconnected during some earlier call. If set, issues
* xcecli_join() as a separate call (the same join payload used at startup).
* The flag clears ONLY on that join call's success (Req 2 AC5): if
* xcecli_join() itself returns -E_XCE_NOT_READY (its own retry budget
* exhausted) or any other failure, needs_rejoin stays set, so the worker is
* never left transport-connected but silently un-joined -- the next
* checking_connection() tick (this function's own periodic cadence, no new
* timer needed) retries the rejoin.
*
* Success is rc >= 0, NOT rc == 0 (task 2.3 fix -- found during this task's
* live E2E reconnect validation): xcecli_join()/xce_join() returns the
* controller's reply.status verbatim, and handle_worker_join()'s SUCCESS
* reply.status is the worker's assigned slot index (join.c, `reply.status =
* idx`), which is any non-negative integer, not always 0 -- only a genuine
* failure reply.status is negative. xcecd_join_cluster() above already
* treats its own xcecli_join() call this way (`if (rc < 0)` is its only
* failure test); this rejoin path used `rc == 0` instead, so a real,
* successful rejoin against a controller that happened to assign a non-zero
* worker index was misreported as a failure and retried forever (harmlessly
* idempotent, but never actually clearing needs_rejoin or logging success).
* Confirmed via a real 3-node cluster: after a primary kill and reconnect,
* xcecli_join() against the new primary returned rc=1 (a valid worker
* index) but the old `rc == 0` check logged "rejoin ... failed" on every
* subsequent liveness probe.
*
* @return int Always returns 0.
*/
int checking_connection()
{
const int RETRY_INTERVAL = 5; // seconds
int retry_count = 0;
int ret;
int ret = xcecli_hello(xcecli_ctx, "hello");
/* component-failover-adoption Req 2/Req 3: keep retrying indefinitely so
* the worker self-heals when its controller (or a failover primary)
* becomes reachable again — never give up after a bounded retry count and
* never call handle_disconnect() on a transient outage (that sets
* cc_ctx->running=false and exits the connector thread, requiring an
* operator restart). The connector thread is dedicated to this loop, so
* the unbounded retry/sleep cannot block the worker's other liveness
* duties (hardware/process monitoring run on separate threads). */
while (1) {
ret = xcecli_hello(xcecli_ctx, "hello");
if (ret == 0) {
// Connection successful
if (retry_count > 0) {
XCECD_LOG(XCE_LOG_DEBUG,
"Connection to controller restored after %d retries",
retry_count);
}
return 0;
}
retry_count++;
if (ret != 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Connection to controller is broken. Retry %d failed",
retry_count);
"Connection to controller not currently healthy (rc=%d); "
"the client library will keep retrying/rotating "
"candidates in the background",
ret);
}
if (0 == xcecd_join_cluster(&host_attrs)) {
if (needs_rejoin) {
int rc = rejoin_call();
if (rc >= 0) {
needs_rejoin = false;
XCECD_LOG(XCE_LOG_INFO,
"Rejoined cluster successfully after retry %d",
retry_count);
"checking_connection: rejoin after transparent "
"reconnect succeeded");
/* bug-1509 Phase 3 (B2/B3): re-report running jobs on every
* rejoin, not just at startup, so the controller reconciles
* their state (clears UNKWN on a live claim; converges to a
* terminal state for processes that are gone). */
/* bug-1509 Phase 3 (B2/B3): re-report running/finished job state
* on every rejoin, not just at startup, so the controller
* reconciles its view after the disconnect window (clears UNKWN
* on a live claim; converges to a terminal state for processes
* that are gone). */
(void)resend_job_status(*cc_ctx->worker_list);
return 0;
} else {
XCECD_LOG(XCE_LOG_WARN,
"checking_connection: rejoin after transparent "
"reconnect failed (rc=%d); will retry on the next "
"liveness probe",
rc);
}
XCECD_LOG(XCE_LOG_INFO, "Retrying connection in %d seconds...",
RETRY_INTERVAL);
sleep(RETRY_INTERVAL);
}
}
/**
* @brief Joins the cluster using the provided host attributes.
*
* In HA mode (cc_ctx->cluster_cfg.candidates_count > 0), iterates through
* all candidates until one accepts the join as primary. In single-host mode
* the original path is taken unchanged.
*
* @param attrs Pointer to the host attributes structure.
* @return int 0 on success, -1 on failure.
*/
int xcecd_join_cluster(hostattrs_t* attrs)
{
if (!attrs) {
return -1;
}
int role = HostRole_worker;
/* HA mode: try each candidate until we find the primary */
if (cc_ctx != NULL && cc_ctx->cluster_cfg.candidates_count > 0) {
for (int i = 0; i < cc_ctx->cluster_cfg.candidates_count; i++) {
const char* host = cc_ctx->cluster_cfg.candidates[i].hostname;
int port = cc_ctx->cluster_cfg.candidates[i].port;
/* Point cc_ctx at this candidate so init_client_config() picks it up */
cc_ctx->remote_host = (char*)host;
cc_ctx->remote_port = port;
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: trying candidate %d: %s:%d",
i, host, port);
init_client_config(&xce_client_cfg);
xcecli_ctx_init(&xcecli_ctx, &xce_client_cfg);
xcecli_set_handler(xcecli_ctx, &myreq_handler, NULL);
int rc = xcecli_start(xcecli_ctx);
if (rc < 0) {
XCECD_LOG(XCE_LOG_WARN,
"xcecd_join_cluster: connect to %s:%d failed (rc=%d)",
host, port, rc);
xcecli_ctx_fini(xcecli_ctx); /* free context before next attempt */
continue;
}
rc = xcecli_join(xcecli_ctx, role, attrs);
if (rc == 0) {
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: joined primary at %s:%d",
host, port);
/* Re-report running job state when libxce transparently
* reconnects (failover) — checking_connection() only covers
* the explicit xcecd_join_cluster path, not the auto-reconnect
* path. Registered AFTER a successful join: xcecli_join also
* runs the failover loop, and a candidate-iteration rotate
* (standby redirect) must NOT fire the rejoin hook before the
* session exists. */
xcecli_set_reconnect_handler(xcecli_ctx, xcecd_reconnect_cb,
NULL);
return 0;
}
/* Standby replied — try next candidate */
XCECD_LOG(XCE_LOG_INFO,
"xcecd_join_cluster: candidate %s:%d returned %d "
"(likely standby redirect), trying next",
host, port, rc);
xcecli_stop(xcecli_ctx);
xcecli_ctx_fini(xcecli_ctx); /* free context before next attempt */
}
XCECD_LOG(XCE_LOG_ERROR,
"xcecd_join_cluster: all %d candidates failed",
cc_ctx->cluster_cfg.candidates_count);
return -1;
}
/* Single-host fallback (original path) */
init_client_config(&xce_client_cfg);
xcecli_ctx_init(&xcecli_ctx, &xce_client_cfg);
xcecli_set_handler(xcecli_ctx, &myreq_handler, NULL);
xcecli_start(xcecli_ctx);
int rc = xcecli_join(xcecli_ctx, role, attrs);
if (rc < 0) {
errno = -rc;
XCECD_LOG(XCE_LOG_ERROR, "Call xcecli_join() failure: %s",
strerror(errno));
return rc;
}
/* Same reconnect hook as the HA path above — after a successful join so a
* candidate-iteration rotate never fires it pre-session. */
xcecli_set_reconnect_handler(xcecli_ctx, xcecd_reconnect_cb, NULL);
return 0;
}
@ -358,12 +507,15 @@ int xcecd_join_cluster(hostattrs_t* attrs)
#ifndef XRUN
static void dump_startjob_request(xce_job_exec_t* d)
{
char jobidStr[64];
if (!d) {
return;
}
fmt_jobid(jobidStr, sizeof(jobidStr), d->jobid);
XCECD_LOG(XCE_LOG_DEBUG,
"Job ID: %lu, Command: %s, Exec User: %s, Exdata: %s", d->jobid,
"Job ID: %s, Command: %s, Exec User: %s, Exdata: %s", jobidStr,
d->start_info->cmdline ? d->start_info->cmdline : "NULL",
d->start_info->execuser ? d->start_info->execuser : "NULL",
d->start_info->exdata ? d->start_info->exdata : "NULL");
@ -538,11 +690,14 @@ static char* create_mpi_cmdline(char* cmd, char* type, xce_launch_t* launchinfo)
static int job_start_remote_task(uint64_t jobid, job_startinfo_t* startinfo,
xce_launch_t* launchinfo)
{
char jobidStr[64];
xce_taskhost_t hosts[launchinfo->nhosts - 1];
xce_taskrun_t taskrun;
int i;
int rc;
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
if (!startinfo || !launchinfo) {
XCECD_LOG(XCE_LOG_ERROR, "Invalid parameters for job_run_task_remote");
return -1;
@ -582,13 +737,13 @@ static int job_start_remote_task(uint64_t jobid, job_startinfo_t* startinfo,
taskrun.nhosts = launchinfo->nhosts - 1;
taskrun.jobid = jobid;
XCECD_LOG(XCE_LOG_DEBUG, "Launching %d remote tasks for job %lu",
taskrun.nhosts, jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Launching %d remote tasks for job %s",
taskrun.nhosts, jobidStr);
xcecli_unblock_tasks(xcecli_ctx);
rc = xcecli_runtask(xcecli_ctx, &taskrun);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR, "Job %lu run remote task failed.", jobid);
XCECD_LOG(XCE_LOG_ERROR, "Job %s run remote task failed.", jobidStr);
}
} else {
rc = -ENOMEM;
@ -610,6 +765,7 @@ static int job_start_local_task(xce_request_t* request)
{
#ifndef XRUN
xce_job_exec_t* d = request->body.jobstartmsg;
char jobidStr[64];
dump_startjob_request(d);
@ -817,7 +973,8 @@ static int job_start_local_task(xce_request_t* request)
}
}
XCECD_LOG(XCE_LOG_DEBUG, "Job %lu ntasks is: %d", d->jobid,
fmt_jobid(jobidStr, sizeof(jobidStr), d->jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Job %s ntasks is: %d", jobidStr,
job_ctx->ntasks);
/* Job limits */
@ -843,22 +1000,7 @@ static int job_start_local_task(xce_request_t* request)
}
if (job_ctx->startinfo->execuser != NULL) {
/* bug-1507: only use the exec user if it actually exists on this
* host. The API-layer submit user (e.g. "admin") is not
* necessarily a real /etc/passwd account, and systemd then fails
* the worker unit with EXIT_USER (217), killing every job. Fall
* back to nobody so the job still runs. */
struct passwd* pw = getpwnam(job_ctx->startinfo->execuser);
if (pw != NULL) {
job_ctx->limits->user_name
= strdup(job_ctx->startinfo->execuser);
} else {
job_ctx->limits->user_name = strdup("nobody");
XCECD_LOG(XCE_LOG_WARN,
"exec user '%s' does not exist on this host, "
"running job as nobody",
job_ctx->startinfo->execuser);
}
job_ctx->limits->user_name = strdup(job_ctx->startinfo->execuser);
} else {
job_ctx->limits->user_name = strdup("nobody");
XCECD_LOG(XCE_LOG_WARN,
@ -890,7 +1032,8 @@ static int job_start_local_task(xce_request_t* request)
static int handle_startjob(xce_request_t* request)
{
int rc = 0;
int rc = 0;
char jobidStr[64];
XCECD_LOG(XCE_LOG_DEBUG, "Received startjob request from controller");
@ -930,8 +1073,9 @@ static int handle_startjob(xce_request_t* request)
if (launchinfo->nhosts > 1) {
rc = job_start_remote_task(jobid, startinfo, launchinfo);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR, "Job %lu run remote task failed, rc %d.",
jobid, rc);
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
XCECD_LOG(XCE_LOG_ERROR, "Job %s run remote task failed, rc %d.",
jobidStr, rc);
xcecd_task_fail(jobid, request->body.runtaskmsg->clientidx, -1, rc);
return rc;
}
@ -955,6 +1099,7 @@ static int handle_runtasks(xce_request_t* request)
XCECD_LOG(XCE_LOG_DEBUG, "Received runtasks request from controller");
xce_taskrun_t* runmsg = request->body.runtaskmsg;
char jobidStr[64];
if (!runmsg) {
XCECD_LOG(XCE_LOG_ERROR,
@ -990,7 +1135,8 @@ static int handle_runtasks(xce_request_t* request)
return -1;
}
XCECD_LOG(XCE_LOG_DEBUG3, "run task: jobid = %lu", runmsg->jobid);
fmt_jobid(jobidStr, sizeof(jobidStr), runmsg->jobid);
XCECD_LOG(XCE_LOG_DEBUG3, "run task: jobid = %s", jobidStr);
XCECD_LOG(XCE_LOG_DEBUG3, "run task: user = %s",
runmsg->taskhosts[0]->execuser ? runmsg->taskhosts[0]->execuser
: "NULL");
@ -1230,10 +1376,12 @@ static int handle_fetchfile(xce_request_t* request)
int fd;
ssize_t nread;
uint32_t maxbytes;
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->jobid);
XCECD_LOG(XCE_LOG_DEBUG,
"Fetchfile request: jobid=%lu path=%s offset=%lu max=%u",
msg->jobid, msg->filepath, msg->offset, msg->max_bytes);
"Fetchfile request: jobid=%s path=%s offset=%lu max=%u",
jobidStr, msg->filepath, msg->offset, msg->max_bytes);
memset(&freply, 0, sizeof(freply));
freply.jobid = msg->jobid;
@ -1307,6 +1455,7 @@ static int handle_jobcontrol(xce_request_t* request)
#ifndef XRUN
XCECD_LOG(XCE_LOG_DEBUG, "Received job control request from controller");
xce_jobctrl_t* d = request->body.jobctrlmsg;
char jobidStr[64];
if (d->jobid <= 0) {
XCECD_LOG(XCE_LOG_ERROR,
@ -1314,11 +1463,13 @@ static int handle_jobcontrol(xce_request_t* request)
return -1;
}
fmt_jobid(jobidStr, sizeof(jobidStr), d->jobid);
if (d->action == JobControl_u_siginfo) {
XCECD_LOG(
XCE_LOG_DEBUG,
"Received job control request to send SIGINFO for job ID: %lu",
d->jobid);
"Received job control request to send SIGINFO for job ID: %s",
jobidStr);
job_control_task_t* task_ctx = calloc(1, sizeof(job_control_task_t));
if (!task_ctx) {
@ -1339,8 +1490,8 @@ static int handle_jobcontrol(xce_request_t* request)
|| worker->job_status == JOB_STARTING) {
XCECD_LOG(XCE_LOG_ERROR,
"Job is pending or starting, controller should not "
"pass the signal to %lu.",
d->jobid);
"pass the signal to %s.",
jobidStr);
// really should not happen, but if it does, we handle it
// gracefully
@ -1349,8 +1500,8 @@ static int handle_jobcontrol(xce_request_t* request)
return 0;
}
} else {
XCECD_LOG(XCE_LOG_ERROR, "Failed to find worker for job ID: %lu",
d->jobid);
XCECD_LOG(XCE_LOG_ERROR, "Failed to find worker for job ID: %s",
jobidStr);
FREEUP(task_ctx);
return -1;
}
@ -1359,13 +1510,13 @@ static int handle_jobcontrol(xce_request_t* request)
if (!ipc_master_send_job_signal(task_ctx->jobid, task_ctx->signal,
task_ctx->force)) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to submit job control task for job ID: %lu",
d->jobid);
"Failed to submit job control task for job ID: %s",
jobidStr);
FREEUP(task_ctx);
return -1;
} else {
XCECD_LOG(XCE_LOG_DEBUG,
"Job control task submitted for job ID: %lu", d->jobid);
"Job control task submitted for job ID: %s", jobidStr);
}
FREEUP(task_ctx);
#else
@ -1374,8 +1525,8 @@ static int handle_jobcontrol(xce_request_t* request)
"Virtual cluster mode, handling job control task directly");
FREEUP(task_ctx);
XCECD_LOG(XCE_LOG_DEBUG,
"Job control task handled directly for job ID: %lu",
d->jobid);
"Job control task handled directly for job ID: %s",
jobidStr);
worker->killed_signal = d->u.siginfo->signo; // Mark job as killed
#endif
@ -1383,18 +1534,18 @@ static int handle_jobcontrol(xce_request_t* request)
#if !defined(VIRTUAL_CLUSTER)
XCECD_LOG(XCE_LOG_DEBUG,
"Received job control request to set checkpoint options for "
"job ID: %lu",
d->jobid);
"job ID: %s",
jobidStr);
if (!ipc_master_send_job_ckpt(d->jobid, d->u.ckptoption->kill,
d->u.ckptoption->method)) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to send checkpoint options for job ID: %lu",
d->jobid);
"Failed to send checkpoint options for job ID: %s",
jobidStr);
return -1;
} else {
XCECD_LOG(XCE_LOG_DEBUG, "Checkpoint options sent for job ID: %lu",
d->jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Checkpoint options sent for job ID: %s",
jobidStr);
}
#else
XCECD_LOG(XCE_LOG_WARN, "Received job control request in virtual "
@ -1402,8 +1553,8 @@ static int handle_jobcontrol(xce_request_t* request)
#endif
} else {
XCECD_LOG(XCE_LOG_ERROR,
"Received unknown job control action: %d for job ID: %lu",
d->action, d->jobid);
"Received unknown job control action: %d for job ID: %s",
d->action, jobidStr);
return -1;
}
#endif // XRUN
@ -1485,8 +1636,10 @@ bool controller_connector_start(void)
int retry_limit = 0;
static int error_count = 0;
init_client_config(&xce_client_cfg);
/* No standalone client-config construction here anymore -- the single
* synthesize-validate-convert-connect pipeline inside
* xcecd_join_cluster() below builds and sets xce_client_cfg itself on
* every attempt (component-failover-adoption, task 2.1, Req 1 AC5). */
do {
rc = xcecd_join_cluster(&host_attrs);
if (rc < 0) {
@ -1630,7 +1783,10 @@ int xcecd_job_remote_start(uint64_t jobid,
const char* hostname,
const char* command)
{
int rc = 0;
int rc = 0;
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
if (!cc_ctx || !cc_ctx->worker_list) {
XCECD_LOG(XCE_LOG_ERROR,
@ -1654,7 +1810,7 @@ int xcecd_job_remote_start(uint64_t jobid,
jobid);
if (!worker) {
XCECD_LOG(XCE_LOG_ERROR,
"Worker not found for job ID %lu", jobid);
"Worker not found for job ID %s", jobidStr);
return -1;
}
@ -1682,15 +1838,15 @@ int xcecd_job_remote_start(uint64_t jobid,
rc = xcecli_runtask(xcecli_ctx, &taskrun);
if (rc < 0) {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to start job %lu on remote host %s, rc: %d",
jobid, hostname, rc);
"Failed to start job %s on remote host %s, rc: %d",
jobidStr, hostname, rc);
xcecd_task_fail(jobid, -1, -1, rc);
return rc;
}
XCECD_LOG(XCE_LOG_DEBUG,
"Remote task of job %lu will run on %s",
jobid, hostname);
"Remote task of job %s will run on %s",
jobidStr, hostname);
return 0;
}
@ -1718,14 +1874,17 @@ int xcecd_jobpost_done(uint64_t jobid)
int xcecd_jobrusage_update(xce_job_rusages_t* rusages)
{
char jobidStr[64];
if (!rusages) {
XCECD_LOG(XCE_LOG_ERROR,
"Invalid rusage data.");
return -1;
}
fmt_jobid(jobidStr, sizeof(jobidStr), rusages->jobid);
XCECD_LOG(XCE_LOG_DEBUG,
"Updating job rusage to controller for job %lu", rusages->jobid);
"Updating job rusage to controller for job %s", jobidStr);
return xcecli_update_jusage(xcecli_ctx,
rusages->jobid,

242
src/worker/src/main.c
View file

@ -157,15 +157,20 @@ static void handle_message_from_queue(void)
static void master_disconnect_callback(uint64_t job_id)
{
XCECD_LOG(XCE_LOG_DEBUG, "Worker with client ID %lu disconnected", job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Worker with client ID %s disconnected", jobidStr);
}
static void master_job_status_callback(job_status_t* job_status)
{
int ret = XCECD_OK;
int ret = XCECD_OK;
char jobidStr[64];
XCECD_LOG(XCE_LOG_INFO, "Job %lu status: %d, exdata: %s, %s",
job_status->job_id, job_status->status_code,
fmt_jobid(jobidStr, sizeof(jobidStr), job_status->job_id);
XCECD_LOG(XCE_LOG_INFO, "Job %s status: %d, exdata: %s, %s",
jobidStr, job_status->status_code,
job_status->start_exdata[0] ? job_status->start_exdata
: "no exdata",
job_status->status_message);
@ -222,10 +227,13 @@ static void master_heartbeat_callback(uv_stream_t* client)
static void master_ckpt_result_callback(uint64_t job_id, int status,
const char* restore_cmd)
{
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_LOG(
XCE_LOG_DEBUG,
"Received checkpoint result for job %lu: status %d, restore_cmd: %s",
job_id, status, restore_cmd ? restore_cmd : "NULL");
"Received checkpoint result for job %s: status %d, restore_cmd: %s",
jobidStr, status, restore_cmd ? restore_cmd : "NULL");
if (job_id <= 0) {
XCECD_LOG(XCE_LOG_ERROR, "Job ID is not correct.");
@ -238,12 +246,15 @@ static void master_ckpt_result_callback(uint64_t job_id, int status,
static void master_rusages_callback(xce_job_rusages_t* rusages)
{
char jobidStr[64];
if (!rusages) {
return;
}
XCECD_LOG(XCE_LOG_DEBUG, "Received rusage for job %lu with %d metrics.",
rusages->jobid, rusages->count);
fmt_jobid(jobidStr, sizeof(jobidStr), rusages->jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Received rusage for job %s with %d metrics.",
jobidStr, rusages->count);
xcecd_jobrusage_update(rusages);
@ -280,31 +291,62 @@ static int check_xce_server_dir(char* myfile)
static int check_user(void)
{
#ifndef VIRTUAL_CLUSTER
/* Root is required for the real production daemon (job execution as
* arbitrary users, cgroup management, etc.). This check is fully
* unconditional here -- zero runtime bypass surface -- for the real
* `xcecd` binary (worker/meson.build, no VIRTUAL_CLUSTER). */
if (geteuid() != 0) {
fprintf(stderr, "FATAL: xcecd should be run as root");
exit(-1);
}
#endif
/* VIRTUAL_CLUSTER (xcecd_virtual, worker/meson.build) already skips the
* sqlite/db-recovery block and other root-dependent subsystems (see
* xcecd_start()), so it does not need real root -- this lets it run as
* an ordinary user in a sandboxed dev/CI environment with no root/sudo
* and no permission to create a user namespace (unshare --map-root-user
* fails with EPERM there), which component-failover-adoption task 2.3
* relies on to drive a REAL worker process against a REAL local HA
* controller cluster for its reconnect E2E test. Decided at build time,
* not by an environment variable reachable in the production binary. */
return 1;
}
/* Long-option identifiers (beyond ASCII range to avoid collisions) */
#define OPT_PEERS 256
#define OPT_PEERS 256
#define OPT_VERSION 257
int main(int argc, char* argv[])
{
int opt;
config_result_t* config_result = NULL;
int ret = XCECD_OK;
int config_flag = 0;
char* custom_config_path = NULL;
int verbose = 0;
__attribute__((unused)) char worker_log[1024] = { 0 };
const char* peers_str = NULL;
/* build-time-versioning task 3.3: worker uses plain getopt_long(), not
* argp, so there is no free argp_program_version auto-handling here
* (unlike controller/main.c, scheduler/src/main.c). version_buf is
* computed once, before argument parsing even starts, via the same
* xce_compose_version() runtime helper those two use -- both the
* --version case below and the startup INFO log lines further down
* read this SAME buffer, never recomputing it independently, so the
* two can never disagree (Requirement 2.2). */
static char version_buf[128];
/* dirty-content-fingerprint task 3.3: SEPARATE buffer/composition, used
* ONLY for the --version case below. version_buf above is left
* completely untouched -- its own xce_compose_version() call and the
* two mutually-exclusive startup INFO log lines reading it (further
* down) are unaffected by this feature, per Requirement 3.2. This
* divergence (the two call sites now showing different strings) is
* deliberate, not an oversight -- mirrors controller/main.c's and
* scheduler/src/main.c's identical version_display_buf pattern
* (dirty-content-fingerprint task 3.2). */
static char version_display_buf[128];
xce_cluster_config_t cluster_cfg = {0};
int use_ha = 0; /* 1 when --peers given */
check_user();
signal(SIGPIPE, SIG_IGN);
struct sigaction sa;
@ -335,7 +377,7 @@ int main(int argc, char* argv[])
.domain_sock_file = "/var/run/.xcecd.sock",
.virutal_name = "localhost01",
.remote_host = "localhost",
.remote_port = 6789,
.remote_port = 9990,
.join_interval = 5,
.update_interval = 5,
.resource_update_interval = 1,
@ -344,89 +386,36 @@ int main(int argc, char* argv[])
char logfile[1024] = { 0 };
char* logdir = NULL;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-c") == 0 && i + 1 < argc) {
custom_config_path = argv[i + 1];
config_flag = 1;
break;
}
}
if (config_flag == 0) {
custom_config_path = strdup(DEFAULT_CONFIG_PATH);
}
if (access(custom_config_path, F_OK) == 0) {
/* load config from custom file */
config_result = config_load(custom_config_path);
if (!config_result) {
fprintf(stderr, "Failed to load config from %s\n",
custom_config_path);
return -1;
}
if (config_result->error_code == CONFIG_SUCCESS) {
if (config_result->config->controller_port > 0) {
config.remote_port = config_result->config->controller_port;
}
if (config_result->config->log_dir) {
logdir = strdup(config_result->config->log_dir);
}
if (config_result->config->log_level) {
config.debug_level
= level_to_int(config_result->config->log_level);
}
if (config_result->config->management_list_count > 0) {
config.remote_host
= config_result->config->management_list[0].hostname;
}
}
} else {
fprintf(stderr, "Configuration file %s not found.\n",
custom_config_path);
return -1;
}
static struct option long_opts[] = {
{ "peers", required_argument, NULL, OPT_PEERS },
{ "help", no_argument, NULL, 'h' },
{ NULL, 0, NULL, 0 }
{ "peers", required_argument, NULL, OPT_PEERS },
{ "version", no_argument, NULL, OPT_VERSION },
{ "help", no_argument, NULL, 'h' },
{ NULL, 0, NULL, 0 }
};
while ((opt = getopt_long(argc, argv, "c:p:m:n:l:h", long_opts, NULL))
/* Computed before argument parsing begins (mirrors controller/main.c's
* and scheduler/src/main.c's "compute before argp_parse()" ordering) so
* the OPT_VERSION case immediately below can print it, and so it is
* ready by the time the startup INFO log lines run later in main(). */
xce_compose_version(version_buf, sizeof(version_buf), "xce v0.0.1-",
STR(VERSION_COMMIT_ID), STR(VERSION_BUILD_TIME));
/* dirty-content-fingerprint task 3.3: --version-only composition --
* appends the dirty_fingerprint suffix (if any) on top of the same
* commit/build_time inputs. Do NOT fold this into version_buf above; the
* startup INFO log lines must keep showing the original, fingerprint-
* less format (Req 3.2). */
xce_compose_version_ex(version_display_buf, sizeof(version_display_buf),
"xce v0.0.1-", STR(VERSION_COMMIT_ID),
STR(VERSION_BUILD_TIME),
STR(VERSION_DIRTY_FINGERPRINT));
while ((opt = getopt_long(argc, argv, "n:l:vh", long_opts, NULL))
!= -1) {
switch (opt) {
case 'c':
break; /* Custom config file already handled above */
case 'p':
if (config_flag) {
fprintf(stderr, "Custom config file already specified, "
"ignoring -p option.\n");
break;
}
config.remote_port = atoi(optarg);
break;
case 'm':
if (config_flag) {
fprintf(stderr, "Custom config file already specified, "
"ignoring -m option.\n");
break;
}
config.remote_host = optarg;
break;
case 'n':
config.virutal_name = optarg;
break;
case 'l':
if (config_flag) {
fprintf(stderr, "Custom config file already specified, "
"ignoring -l option.\n");
break;
}
if (optarg) {
if (logdir) {
FREEUP(logdir);
@ -434,26 +423,49 @@ int main(int argc, char* argv[])
logdir = strdup(optarg);
}
break;
case 'v':
verbose++;
break;
case OPT_PEERS:
peers_str = optarg;
break;
case OPT_VERSION:
/* Prints and exits before any other startup work (peer
* parsing, check_user(), daemon init, etc.) -- same contract
* as controller/scheduler's --version. dirty-content-fingerprint
* task 3.3: prints version_display_buf (fingerprint-aware), not
* version_buf -- the startup INFO log lines further down keep
* reading the original version_buf, untouched. */
printf("%s\n", version_display_buf);
return 0;
case 'h':
default:
#ifdef VIRTUAL_CLUSTER
printf("Usage: %s -n virtual_name [-p remote_port] [-m "
"remote_host] [-l log_dir] [--peers list] "
"[-h help]\n",
argv[0]);
#else
printf("Usage: %s [-p remote_port] [-m remote_host] [-l log_dir] "
printf("Usage: %s [-n name] [-l log_dir] [-v] "
"[--peers ctrl-1=host:port,...] "
"[-h help]\n",
argv[0]);
#endif
return -1;
return 0;
}
}
/* Map -v count to log level (same pattern as controller/scheduler).
* 0=INFO (default), 1=DEBUG, 2=DEBUG2, 3+=DEBUG3. */
if (verbose >= 3) {
config.debug_level = XCE_LOG_DEBUG3;
} else if (verbose == 2) {
config.debug_level = XCE_LOG_DEBUG2;
} else if (verbose == 1) {
config.debug_level = XCE_LOG_DEBUG;
}
/* Performed after argument parsing so -h/--help exits immediately
* without requiring root privileges. */
check_user();
if (logdir == NULL) {
logdir = strdup("/tmp");
}
/* P2.5-5: populate and validate xce_cluster_config_t when --peers given */
if (peers_str) {
if (xce_parse_peers(peers_str, &cluster_cfg) != 0) {
@ -471,23 +483,35 @@ int main(int argc, char* argv[])
"Use -n.\n");
return 1;
}
if (cluster_cfg.candidates_count > 3) {
if (cluster_cfg.candidates_count > N_MAX_MASTERS) {
fprintf(stderr,
"xcecd: WARN: --peers has %d candidates; "
"only <=3 validated for HA\n",
cluster_cfg.candidates_count);
"only <=%d validated for HA\n",
cluster_cfg.candidates_count, N_MAX_MASTERS);
}
XCECD_LOG(XCE_LOG_INFO,
"HA cluster config: %d candidate(s), identity=%s",
cluster_cfg.candidates_count, cluster_cfg.identity);
"HA cluster config: %d candidate(s), identity=%s, "
"version %s",
cluster_cfg.candidates_count, cluster_cfg.identity,
version_buf);
use_ha = 1;
} else {
XCECD_LOG(XCE_LOG_INFO,
"No --peers flag; using single-controller path "
"(host=%s port=%d)",
config.remote_host, config.remote_port);
"(host=%s port=%d), version %s",
config.remote_host, config.remote_port, version_buf);
}
/* use_ha and cluster_cfg are consumed after xcecd_start() sets cc_ctx */
/* P2.2 fix (component-failover-adoption, task 2.2, Req 1.4): assign
* cluster_cfg onto config BEFORE xcecd_init()/xcecd_start() run, so it
* is threaded through to cc_ctx->cluster_cfg (via xcecd_ctx) before
* controller_connector_start() fires the very first join. Previously
* this was assigned directly onto cc_ctx AFTER xcecd_start() returned,
* which was already too late -- the first join had already run against
* an empty (zero-initialized) cluster_cfg, silently ignoring --peers on
* startup. When no --peers was given, cluster_cfg stays {0}
* (candidates_count == 0), which is exactly the "no HA list configured"
* case the synthesize-single-candidate path already handles. */
config.cluster_cfg = cluster_cfg;
struct stat st = { 0 };
@ -594,8 +618,10 @@ int main(int argc, char* argv[])
return 1;
}
if (use_ha && cc_ctx != NULL) {
cc_ctx->cluster_cfg = cluster_cfg;
if (use_ha) {
/* cluster_cfg was already threaded into cc_ctx->cluster_cfg before
* xcecd_start() ran (see config.cluster_cfg assignment above) --
* this is confirmation logging only, not the point of assignment. */
XCECD_LOG(XCE_LOG_INFO,
"HA mode active: %d candidates, primary will be auto-discovered",
cluster_cfg.candidates_count);

29
src/worker/src/node_information.c
View file

@ -799,8 +799,12 @@ static int read_memory_info(double* mem_total, double* mem_free,
}
/**
* @brief The mem index gives the currently available physical memory in MB.
* @return A string representing the available physical memory in megabytes.
* @brief The mem index gives the currently available physical memory in
* bytes (canonical internal/wire unit -- same convention as the
* maxmem host attribute; CLI display converts to a human-readable
* unit, config-time thresholds may be written with a unit suffix
* like "500MB" and are converted to bytes when parsed).
* @return A string representing the available physical memory in bytes.
*/
char* node_info_get_mem(void)
{
@ -813,14 +817,14 @@ char* node_info_get_mem(void)
return NULL;
}
// Read memory information
// Read memory information (read_memory_info reports MB)
if (read_memory_info(&mem_total, &mem_free, &mem_used) != 0) {
free(result);
return NULL;
}
// Format the available memory as a string (in MB)
int ret = snprintf(result, 64, "%.2f", mem_free);
// Format the available memory as a string (in bytes)
int ret = snprintf(result, 64, "%.2f", mem_free * 1024.0 * 1024.0);
if (ret < 0 || ret >= 64) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to format memory information string");
free(result);
@ -831,9 +835,11 @@ char* node_info_get_mem(void)
}
/**
* @brief The tmp index is the space available in MB or in units set
* on the file system that contains the temporary directory.
* @return A string representing the available space in temporary directory.
* @brief The tmp index is the space available in bytes (canonical internal/
* wire unit -- see node_info_get_mem) on the file system that
* contains the temporary directory.
* @return A string representing the available space in the temporary
* directory, in bytes.
*/
char* node_info_get_tmp(void)
{
@ -859,11 +865,8 @@ char* node_info_get_tmp(void)
unsigned long long free_bytes
= (unsigned long long)fs_stat.f_bavail * fs_stat.f_frsize;
// Convert to MB
double tmp_free_mb = (double)free_bytes / (1024.0 * 1024.0);
// Format the available tmp space as a string (in MB)
int ret = snprintf(result, 64, "%.2f", tmp_free_mb);
// Format the available tmp space as a string (in bytes)
int ret = snprintf(result, 64, "%llu", free_bytes);
if (ret < 0 || ret >= 64) {
XCECD_LOG(XCE_LOG_ERROR, "Failed to format tmp space string");
free(result);

26
src/worker/src/resource_sync.c
View file

@ -203,9 +203,29 @@ static xce_resval_t* collect_load_indices(int* count)
return NULL;
}
char hostname[256];
if (gethostname(hostname, sizeof(hostname)) != 0) {
strcpy(hostname, "unknown");
/* bug-1569: this worker's JOIN identity (the -n/--name value, e.g.
* "pc13v" -- xcecd_ctx.virtual_name, set from config->virutal_name in
* xcecd_init(), xcecd.c) is what the controller/scheduler key every
* OTHER per-worker resource value by (mark_host_res()/update_val_res(),
* worker.c; sched_get_worker(), core_events.c). Using gethostname()
* (the raw OS/container hostname, which is NOT the same value and can
* be identical across every worker in a multi-worker-per-host test
* setup, or simply a generic sandbox/container name unrelated to any
* configured worker identity) here instead made every periodic load-
* index report's .host wrong -- the controller forwards it to the
* scheduler unmodified (do_update_resvals(), opupd.c), which then
* fails sched_get_worker(core, rv->host) for the REAL worker and
* silently drops the value ("RESVALS_DELTA: unknown worker <host>,
* skipping", core_events.c) -- live-confirmed: gethostname() here
* returned "xce" in this sandbox, not "pc13v". */
const char* hostname = xcecd_ctx.virtual_name;
char hostname_buf[256];
if (hostname == NULL || hostname[0] == '\0') {
if (gethostname(hostname_buf, sizeof(hostname_buf)) != 0) {
strcpy(hostname_buf, "unknown");
}
hostname = hostname_buf;
}
for (int i = 0; i < rescount; i++) {

103
src/worker/src/uv_ipc.c
View file

@ -62,6 +62,7 @@ static job_client_map_t* job_client_map = NULL;
__attribute__((unused)) static void add_job_client_mapping(uint64_t job_id,
client_t* client)
{
char jobidStr[64];
job_client_map_t* mapping
= (job_client_map_t*)calloc(1, sizeof(job_client_map_t));
if (!mapping)
@ -74,7 +75,8 @@ __attribute__((unused)) static void add_job_client_mapping(uint64_t job_id,
client->current_job_id = job_id;
XCECD_LOG(XCE_LOG_DEBUG, "Added job %lu to client %lu mapping", job_id,
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Added job %s to client %lu mapping", jobidStr,
client->id);
}
@ -87,6 +89,7 @@ __attribute__((unused)) static void remove_job_client_mapping(uint64_t job_id)
{
job_client_map_t* current = job_client_map;
job_client_map_t* prev = NULL;
char jobidStr[64];
while (current) {
if (current->job_id == job_id) {
@ -100,8 +103,9 @@ __attribute__((unused)) static void remove_job_client_mapping(uint64_t job_id)
current->client->current_job_id = 0;
}
XCECD_LOG(XCE_LOG_DEBUG, "Removed job %lu from client mapping",
job_id);
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Removed job %s from client mapping",
jobidStr);
free(current);
return;
@ -488,28 +492,36 @@ static void on_master_read(uv_stream_t* client, ssize_t nread,
master_status_callback(&msg->data.status);
}
} else if (msg->type == MSG_JOB_SIGNAL_ACK) {
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->data.signal.job_id);
XCECD_LOG(XCE_LOG_DEBUG,
"Received signal acknowledgment for job %lu, signal %d, "
"Received signal acknowledgment for job %s, signal %d, "
"force %d",
msg->data.signal.job_id, msg->data.signal.signal,
jobidStr, msg->data.signal.signal,
msg->data.signal.force);
} else if (msg->type == MSG_JOB_CKPT_ACK) {
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->data.ckpt.job_id);
XCECD_LOG(XCE_LOG_DEBUG,
"Received checkpoint acknowledgment for job %lu, signal "
"Received checkpoint acknowledgment for job %s, signal "
"%d, method %s",
msg->data.ckpt.job_id, msg->data.ckpt.kill,
jobidStr, msg->data.ckpt.kill,
msg->data.ckpt.method);
} else if (msg->type == MSG_JOB_CKPT_RESULT) {
XCECD_LOG(XCE_LOG_DEBUG, "Received checkpoint result for job %lu",
msg->data.ckpt.job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->data.ckpt.job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Received checkpoint result for job %s",
jobidStr);
if (master_ckpt_result_callback) {
master_ckpt_result_callback(msg->data.ckpt_result.job_id,
msg->data.ckpt_result.status,
msg->data.ckpt_result.restore_cmd);
}
} else if (msg->type == MSG_JOB_RUSAGES) {
XCECD_LOG(XCE_LOG_DEBUG, "Received rusage info for job %lu",
msg->data.rusages.job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->data.rusages.job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Received rusage info for job %s",
jobidStr);
if (master_rusages_callback) {
// deserialize payload
xce_job_rusages_t* rusages
@ -638,8 +650,10 @@ static void on_worker_read(uv_stream_t* server, ssize_t nread,
worker_id = msg->data.assignment.job_id;
worker_job_callback(&msg->data.assignment);
} else if (msg->type == MSG_JOB_SIGNAL && worker_signal_callback) {
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Received signal %d for job %lu",
msg->data.signal.signal, msg->data.signal.job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->data.signal.job_id);
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Received signal %d for job %s",
msg->data.signal.signal, jobidStr);
worker_signal_callback(msg->data.signal.job_id,
msg->data.signal.signal,
@ -649,9 +663,11 @@ static void on_worker_read(uv_stream_t* server, ssize_t nread,
msg->data.signal.signal,
msg->data.signal.force);
} else if (msg->type == MSG_JOB_CKPT && worker_ckpt_callback) {
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), msg->data.ckpt.job_id);
XCECD_WORKER_LOG(XCE_LOG_DEBUG,
"Received checkpoint request for job %lu",
msg->data.ckpt.job_id);
"Received checkpoint request for job %s",
jobidStr);
worker_ckpt_callback(msg->data.ckpt.job_id, msg->data.ckpt.kill,
msg->data.ckpt.method);
@ -977,6 +993,7 @@ void ipc_worker_send_status(job_runtime_info_t* job_info,
return;
ipc_message_t msg;
char jobidStr[64];
memset(&msg, 0, sizeof(msg));
msg.type = MSG_JOB_STATUS;
msg.data.status.job_id = job_info->job_id;
@ -1020,15 +1037,16 @@ void ipc_worker_send_status(job_runtime_info_t* job_info,
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sending job status to master: %s",
status_message);
fmt_jobid(jobidStr, sizeof(jobidStr), job_info->job_id);
if (job_info->job_status == JOB_DONE) {
XCECD_WORKER_LOG(XCE_LOG_INFO, "Job %lu completed successfully.",
job_info->job_id);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Job %s completed successfully.",
jobidStr);
} else if (job_info->job_status == JOB_FAILED) {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Job %lu failed: %s", job_info->job_id,
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Job %s failed: %s", jobidStr,
status_message);
} else if (job_info->job_status == JOB_EXITED) {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Job %lu exited with code %d: %s",
job_info->job_id, job_info->exit_code, status_message);
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Job %s exited with code %d: %s",
jobidStr, job_info->exit_code, status_message);
}
}
@ -1232,9 +1250,12 @@ void ipc_worker_set_reconnection_callback(worker_reconnect_callback_t callback)
bool ipc_master_send_job_signal(uint64_t job_id, int signal, bool force)
{
client_t* target_client = find_client_by_job_id(job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
if (!target_client) {
XCECD_LOG(XCE_LOG_ERROR, "No worker found for job %lu", job_id);
XCECD_LOG(XCE_LOG_ERROR, "No worker found for job %s", jobidStr);
return false;
}
@ -1251,12 +1272,12 @@ bool ipc_master_send_job_signal(uint64_t job_id, int signal, bool force)
int r = uv_write(req, (uv_stream_t*)target_client->pipe, &buf, 1,
on_write_complete);
if (r == 0) {
XCECD_LOG(XCE_LOG_DEBUG, "Sent signal %d for job %lu to worker %d",
signal, job_id, target_client->worker_pid);
XCECD_LOG(XCE_LOG_DEBUG, "Sent signal %d for job %s to worker %d",
signal, jobidStr, target_client->worker_pid);
return true;
} else {
XCECD_LOG(XCE_LOG_ERROR, "Failed to send signal %d for job %lu: %s",
signal, job_id, uv_strerror(r));
XCECD_LOG(XCE_LOG_ERROR, "Failed to send signal %d for job %s: %s",
signal, jobidStr, uv_strerror(r));
free(req);
return false;
}
@ -1278,6 +1299,7 @@ void ipc_worker_send_signal_ack(uint64_t job_id, int signal, bool force)
return;
ipc_message_t msg;
char jobidStr[64];
memset(&msg, 0, sizeof(msg));
msg.type = MSG_JOB_SIGNAL_ACK;
msg.data.signal.job_id = job_id;
@ -1298,9 +1320,10 @@ void ipc_worker_send_signal_ack(uint64_t job_id, int signal, bool force)
}
}
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_WORKER_LOG(
XCE_LOG_DEBUG,
"Sent signal acknowledgment for job %lu, signal %d, force %d", job_id,
"Sent signal acknowledgment for job %s, signal %d, force %d", jobidStr,
signal, force);
}
@ -1334,9 +1357,12 @@ void ipc_worker_set_ckpt_callback(job_ckpt_callback_t callback)
bool ipc_master_send_job_ckpt(uint64_t job_id, int kill, const char* method)
{
client_t* target_client = find_client_by_job_id(job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
if (!target_client) {
XCECD_LOG(XCE_LOG_ERROR, "No worker found for job %lu", job_id);
XCECD_LOG(XCE_LOG_ERROR, "No worker found for job %s", jobidStr);
return false;
}
@ -1354,13 +1380,13 @@ bool ipc_master_send_job_ckpt(uint64_t job_id, int kill, const char* method)
on_write_complete);
if (r == 0) {
XCECD_LOG(XCE_LOG_DEBUG,
"Sent checkpoint request for job %lu to worker %d",
job_id, target_client->worker_pid);
"Sent checkpoint request for job %s to worker %d",
jobidStr, target_client->worker_pid);
return true;
} else {
XCECD_LOG(XCE_LOG_ERROR,
"Failed to send checkpoint request for job %lu: %s",
job_id, uv_strerror(r));
"Failed to send checkpoint request for job %s: %s",
jobidStr, uv_strerror(r));
free(req);
return false;
}
@ -1382,6 +1408,7 @@ bool ipc_worker_send_ckpt_ack(uint64_t job_id, int kill, const char* method)
return false;
ipc_message_t msg;
char jobidStr[64];
memset(&msg, 0, sizeof(msg));
msg.type = MSG_JOB_CKPT;
msg.data.ckpt.job_id = job_id;
@ -1401,7 +1428,8 @@ bool ipc_worker_send_ckpt_ack(uint64_t job_id, int kill, const char* method)
}
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent job checkpoint for job %lu", job_id);
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent job checkpoint for job %s", jobidStr);
return true;
}
@ -1440,6 +1468,7 @@ bool ipc_worker_send_ckpt_result(uint64_t job_id, int status,
return false;
ipc_message_t msg;
char jobidStr[64];
memset(&msg, 0, sizeof(msg));
msg.type = MSG_JOB_CKPT_RESULT;
msg.data.ckpt_result.job_id = job_id;
@ -1461,8 +1490,9 @@ bool ipc_worker_send_ckpt_result(uint64_t job_id, int status,
}
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent job checkpoint result for job %lu",
job_id);
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent job checkpoint result for job %s",
jobidStr);
return true;
}
@ -1478,6 +1508,7 @@ bool ipc_worker_send_rusages(xce_job_rusages_t* rusages)
return false;
ipc_message_t msg;
char jobidStr[64];
memset(&msg, 0, sizeof(msg));
msg.type = MSG_JOB_RUSAGES;
msg.data.rusages.job_id = rusages->jobid;
@ -1538,7 +1569,7 @@ bool ipc_worker_send_rusages(xce_job_rusages_t* rusages)
}
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent rusage info for job %lu",
rusages->jobid);
fmt_jobid(jobidStr, sizeof(jobidStr), rusages->jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Sent rusage info for job %s", jobidStr);
return true;
}

24
src/worker/src/xcecd.c
View file

@ -34,6 +34,7 @@ void xcecd_process_command(void* arg)
{
pid_t pid = 0;
char unit_name[256];
char jobidStr[64];
fork_task_t* ctx = (fork_task_t*)arg;
if (!ctx || !ctx->startinfo || !ctx->startinfo->cmdline) {
@ -57,8 +58,9 @@ void xcecd_process_command(void* arg)
if (!ctx->startinfo) {
xcecd_job_start_error(ctx->xce_jobid);
XCECD_LOG(XCE_LOG_FATAL, "No startinfo provided, job %lu start failed.",
ctx->xce_jobid);
fmt_jobid(jobidStr, sizeof(jobidStr), ctx->xce_jobid);
XCECD_LOG(XCE_LOG_FATAL, "No startinfo provided, job %s start failed.",
jobidStr);
return;
}
@ -328,6 +330,11 @@ controller_connector_init(xcecd_context_t* ctx)
cc_ctx->thread_pool = ctx->thread_pool;
cc_ctx->process_manager = ctx->process_manager;
cc_ctx->worker_list = (void**)&ctx->worker_list;
/* component-failover-adoption task 2.2 (Req 1.4): populate the full
* candidate list here, before controller_connector_start() (called
* right after this returns, still inside xcecd_start()) fires the very
* first join -- so the first connect attempt already sees it. */
cc_ctx->cluster_cfg = ctx->cluster_cfg;
return cc_ctx;
}
@ -343,8 +350,12 @@ static void init_host_attrs(void)
= hardware_monitor_get_total_memory(xcecd_ctx.hardware_monitor);
}
host_attrs.maxswap = 1024;
host_attrs.maxtmp = 2048;
/* Placeholder stub values (bug-1554, real swap/tmp-disk detection not
* yet implemented) -- kept as bytes to match every other memory-scale
* host attribute's canonical unit (maxmem, mem, tmp). Magnitude chosen
* to match the pre-existing MB-scale placeholders (1024MB/2048MB). */
host_attrs.maxswap = 1024.0 * 1024.0 * 1024.0;
host_attrs.maxtmp = 2048.0 * 1024.0 * 1024.0;
host_attrs.ndisks = 1;
host_attrs.hostmodel = strdup("Intel64");
@ -384,6 +395,11 @@ int xcecd_init(const xcecd_config_t* config)
xcecd_ctx.update_interval = config->update_interval;
xcecd_ctx.join_interval = config->join_interval;
xcecd_ctx.debug_level = config->debug_level;
/* component-failover-adoption task 2.2 (Req 1.4): thread the parsed
* --peers candidate list through so it reaches cc_ctx before the first
* join, instead of main.c assigning cc_ctx->cluster_cfg after
* xcecd_start() already ran. */
xcecd_ctx.cluster_cfg = config->cluster_cfg;
/* thread pool initializing */
xcecd_ctx.thread_pool = thread_pool_create(config->thread_pool_size);

6
src/worker/src/xcecd_cgroups.c
View file

@ -1384,6 +1384,7 @@ xce_job_rusages_t* collect_job_rusages(uint64_t job_id)
int r;
char unit_name[256];
char unit_path[512];
char jobidStr[64];
xce_job_rusages_t* rusages = NULL;
rusages = (xce_job_rusages_t*)calloc(1, sizeof(xce_job_rusages_t));
@ -1699,10 +1700,11 @@ xce_job_rusages_t* collect_job_rusages(uint64_t job_id)
strcpy(pgids_str, "none");
}
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_LOG(XCE_LOG_DEBUG3,
"Job %lu rusage: CPU: %.2fs, Mem: %.2fMB, Swap: %.2fMB, Tasks: "
"Job %s rusage: CPU: %.2fs, Mem: %.2fMB, Swap: %.2fMB, Tasks: "
"%d, PIDs: %d [%s], PGIDs: %d [%s]",
job_id, rusages->usages[0]->value, rusages->usages[1]->value,
jobidStr, rusages->usages[0]->value, rusages->usages[1]->value,
rusages->usages[2]->value, (int)rusages->usages[3]->value,
rusages->npids, pids_str, rusages->npgids, pgids_str);
#endif

239
src/worker/src/xcecd_conf.c
View file

@ -1,239 +0,0 @@
#include "xcecd_conf.h"
#include "xcelog.h"
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
static const cyaml_schema_value_t management_host_schema = {
.type = CYAML_STRING,
.flags = CYAML_FLAG_POINTER,
.data_size = sizeof(char*),
.string = {
.min = 0,
.max = MAX_HOSTNAME_LEN,
},
};
static const cyaml_schema_field_t worker_config_fields_schema[] = {
{
.key = "management_list",
.data_offset = offsetof(worker_config_t, management_list),
.value = {
.type = CYAML_SEQUENCE,
.flags = CYAML_FLAG_POINTER,
.data_size = sizeof(char*),
.sequence = {
.entry = &management_host_schema,
.min = 0,
.max = CYAML_UNLIMITED,
},
},
.count_offset = offsetof(worker_config_t, management_list_count),
.count_size = sizeof(uint32_t),
},
{
.key = "controller_port",
.data_offset = offsetof(worker_config_t, controller_port),
.value = {
.type = CYAML_UINT,
.flags = CYAML_FLAG_DEFAULT,
.data_size = sizeof(uint16_t),
},
},
{
.key = "log_level",
.data_offset = offsetof(worker_config_t, log_level),
.value = {
.type = CYAML_STRING,
.flags = CYAML_FLAG_POINTER,
.data_size = sizeof(char*),
.string = {
.min = 0,
.max = MAX_LOG_LEVEL_LEN,
},
},
},
{
.key = "log_dir",
.data_offset = offsetof(worker_config_t, log_dir),
.value = {
.type = CYAML_STRING,
.flags = CYAML_FLAG_POINTER,
.data_size = sizeof(char*),
.string = {
.min = 0,
.max = MAX_PATH_LEN,
},
},
},
{ .key = NULL }
};
static const cyaml_schema_field_t root_fields_schema[] = {
{
.key = "workerconf",
.value = {
.type = CYAML_MAPPING,
.flags = CYAML_FLAG_DEFAULT,
.data_size = sizeof(worker_config_t),
.mapping = {worker_config_fields_schema},
},
},
CYAML_FIELD_END
};
static const cyaml_schema_value_t root_schema = {
.type = CYAML_MAPPING,
.flags = CYAML_FLAG_POINTER,
.data_size = sizeof(worker_config_t),
.mapping = { root_fields_schema },
};
static const cyaml_config_t cyaml_config = {
.log_fn = cyaml_log,
.mem_fn = cyaml_mem,
.log_level = CYAML_LOG_WARNING,
};
config_result_t* config_load(const char* config_path)
{
config_result_t* result = calloc(1, sizeof(config_result_t));
if (!result) {
return NULL;
}
result->error_code = CONFIG_SUCCESS;
result->config = NULL;
FILE* file = fopen(config_path, "r");
if (!file) {
result->error_code = CONFIG_ERROR_FILE_NOT_FOUND;
snprintf(result->error_message, sizeof(result->error_message),
"Cannot open config file '%s': %s", config_path,
strerror(errno));
return result;
}
fclose(file);
cyaml_err_t err = cyaml_load_file(config_path, &cyaml_config, &root_schema,
(void**)&result->config, NULL);
if (err != CYAML_OK) {
result->error_code = CONFIG_ERROR_PARSE_FAILED;
snprintf(result->error_message, sizeof(result->error_message),
"YAML parse error: %s", cyaml_strerror(err));
return result;
}
if (!result->config) {
result->error_code = CONFIG_ERROR_INVALID_DATA;
snprintf(result->error_message, sizeof(result->error_message),
"Configuration data is null");
return result;
}
if (result->config->controller_port == 0) {
result->error_code = CONFIG_ERROR_INVALID_DATA;
snprintf(result->error_message, sizeof(result->error_message),
"controller_port must be greater than 0");
return result;
}
if (!result->config->log_level || strlen(result->config->log_level) == 0) {
result->error_code = CONFIG_ERROR_INVALID_DATA;
snprintf(result->error_message, sizeof(result->error_message),
"log_level cannot be empty");
return result;
}
if (!result->config->log_dir || strlen(result->config->log_dir) == 0) {
result->error_code = CONFIG_ERROR_INVALID_DATA;
snprintf(result->error_message, sizeof(result->error_message),
"log_dir cannot be empty");
return result;
}
return result;
}
config_result_t* config_load_default(void)
{
return config_load(DEFAULT_CONFIG_PATH);
}
void config_free(config_result_t* result)
{
if (!result)
return;
if (result->config) {
cyaml_free(&cyaml_config, &root_schema, result->config, 0);
}
free(result);
}
const char* config_error_string(config_error_t error)
{
switch (error) {
case CONFIG_SUCCESS:
return "Success";
case CONFIG_ERROR_FILE_NOT_FOUND:
return "Configuration file not found";
case CONFIG_ERROR_PARSE_FAILED:
return "YAML parsing failed";
case CONFIG_ERROR_INVALID_DATA:
return "Invalid configuration data";
case CONFIG_ERROR_MEMORY:
return "Memory allocation error";
case CONFIG_ERROR_UNKNOWN:
default:
return "Unknown error";
}
}
int level_to_int(const char* level)
{
if (!level)
return XCE_LOG_INFO; // Default to INFO if NULL
if (strcmp(level, "debug3") == 0) {
return XCE_LOG_DEBUG3;
} else if (strcmp(level, "debug2") == 0) {
return XCE_LOG_DEBUG2;
} else if (strcmp(level, "debug") == 0) {
return XCE_LOG_DEBUG;
} else if (strcmp(level, "notice") == 0) {
return XCE_LOG_NOTICE;
} else if (strcmp(level, "info") == 0) {
return XCE_LOG_INFO;
} else if (strcmp(level, "warn") == 0) {
return XCE_LOG_WARN;
} else if (strcmp(level, "error") == 0) {
return XCE_LOG_ERROR;
} else if (strcmp(level, "fatal") == 0) {
return XCE_LOG_FATAL;
} else if (strcmp(level, "off") == 0) {
return XCE_LOG_OFF;
} else {
return XCE_LOG_INFO; // Default to INFO for unknown levels
}
}
void config_print(const worker_config_t* config)
{
if (!config) {
printf("Configuration is NULL\n");
return;
}
printf("Worker Configuration:\n");
printf(" Controller Port: %u\n", config->controller_port);
printf(" Log Level: %s\n", config->log_level);
printf(" Log Directory: %s\n", config->log_dir);
printf(" Management List (%u hosts):\n", config->management_list_count);
for (uint32_t i = 0; i < config->management_list_count; i++) {
printf(" - %s\n", config->management_list[i].hostname);
}
}

34
src/worker/src/xcecd_dbus_manager.c
View file

@ -188,6 +188,25 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
xcecd_worker_t* worker = *worker_list;
while (worker) {
/* bug-1589: capture the NEXT node before doing any work on this one.
* Every early "nothing more to do for this worker on this pass"
* exit below must advance to `next` and CONTINUE scanning the rest
* of the list -- not silently stay stuck on this same node (several
* branches below used a bare `continue`, which re-tests the SAME
* `worker` pointer forever) and not RETURN out of the whole function
* (the old `worker->worker_status == WORKER_DONE` branch did
* `return true` -- true == 1 == XCECD_ERROR, and worse, it aborted
* the scan of every worker after the first DONE/EXITED one it saw).
* Net effect on a virtual/real worker with N concurrently-dispatched
* jobs in worker_list: only the head of the list ever got processed
* -- job [0] of a job array got its on_job_start report, jobs
* [1..N-1] were starved forever (until job [0] finished AND got
* reaped by cleanup_worker_list(), at which point the NEXT worker
* became the new head and hit the identical bug). Capturing `next`
* up front and always advancing to it is the general fix for every
* branch in this loop, real-mode and VIRTUAL_CLUSTER alike. */
xcecd_worker_t* next = worker->next;
#ifdef VIRTUAL_CLUSTER
if (worker->worker_status == WORKER_RUNNING
&& worker->job_status == JOB_STARTING) {
@ -198,14 +217,15 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
ret = update_worker_status(*worker_list, worker->worker_pid,
WORKER_RUNNING, job, db);
worker = next;
continue;
}
#endif
if (worker->worker_status == WORKER_DONE
|| worker->worker_status == WORKER_EXITED) {
worker = worker->next;
return true;
worker = next;
continue;
}
char unit_name[256];
@ -222,6 +242,7 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
if (worker->job_status == JOB_STARTING) {
worker->job_status = JOB_RUNNING;
worker->worker_status = WORKER_RUNNING;
worker = next;
continue;
}
@ -238,6 +259,7 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
ret = update_worker_status(*worker_list, worker->worker_pid,
WORKER_EXITED, job, db);
}
worker = next;
continue;
#endif
XCECD_LOG(XCE_LOG_DEBUG, "Worker %d is done",
@ -265,6 +287,7 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
ret = update_worker_status(*worker_list, worker->worker_pid,
WORKER_DONE, job, db);
}
worker = next;
continue;
#endif
XCECD_LOG(XCE_LOG_WARN, "Failed to get worker exit code");
@ -288,6 +311,7 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
if (worker->job_status == JOB_STARTING) {
worker->job_status = JOB_RUNNING;
worker->worker_status = WORKER_RUNNING;
worker = next;
continue;
}
@ -329,7 +353,7 @@ int dbus_workers_update(xcecd_worker_t** worker_list, xcecd_job_t* job,
// worker->worker_pid);
}
}
worker = worker->next;
worker = next;
}
return ret;
@ -384,6 +408,7 @@ void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db)
xcecd_worker_t* worker = *worker_list;
while (worker) {
char unit_name[256];
char jobidStr[64];
make_job_sd_unit_name(unit_name, sizeof(unit_name) - 1, "xcecd-worker",
worker->job_info.jobid);
@ -396,7 +421,8 @@ void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db)
update_worker_status(*worker_list, worker->worker_pid, WORKER_EXITED,
&job, db);
XCECD_LOG(XCE_LOG_WARN, "Job %lu killed.", worker->job_info.jobid);
fmt_jobid(jobidStr, sizeof(jobidStr), worker->job_info.jobid);
XCECD_LOG(XCE_LOG_WARN, "Job %s killed.", jobidStr);
worker = worker->next;
}

116
src/worker/src/xcecd_worker.c
View file

@ -80,7 +80,10 @@ static void free_xce_job_rusages(xce_job_rusages_t* rusages)
static void on_rusage_timer(uv_timer_t* timer)
{
if (jobid > 0) {
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Collecting rusage for job %lu", jobid);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG, "Collecting rusage for job %s",
jobidStr);
xce_job_rusages_t* rusages = collect_job_rusages(jobid);
@ -589,8 +592,10 @@ static void check_and_reconnect()
static void xjob_start_vnc()
{
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG3,
"Starting VNC server for interactive job %lu", jobid);
"Starting VNC server for interactive job %s", jobidStr);
char vnc_start_cmd[MAXLINELEN] = { 0 };
if (access(XJOB_PRE_EXEC, F_OK) != 0) {
@ -652,13 +657,16 @@ static void xjob_start_vnc()
elapsed_time++;
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "VNC server started for job %lu", jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "VNC server started for job %s",
jobidStr);
}
static void xjob_stop_vnc()
{
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG3,
"Stopping VNC server for interactive job %lu", jobid);
"Stopping VNC server for interactive job %s", jobidStr);
char vnc_stop_cmd[MAXLINELEN] = { 0 };
if (access(XJOB_POST_EXEC, F_OK) != 0) {
@ -686,7 +694,8 @@ static void xjob_stop_vnc()
exit(1);
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "VNC server stopped for job %lu", jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "VNC server stopped for job %s",
jobidStr);
}
static void worker_job_callback(job_assignment_t* info)
@ -695,6 +704,7 @@ static void worker_job_callback(job_assignment_t* info)
char* url = NULL;
char* start_exdata_info = NULL;
int rc = 0;
char jobidStr[64];
if (!info) {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Received null job assignment");
@ -715,6 +725,8 @@ static void worker_job_callback(job_assignment_t* info)
uv_proc_manager.job_id = jobid;
job_runtime_info.job_id = jobid;
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
if (info->q_pre_exec[0]) {
strncpy(job_runtime_info.q_pre_exec, info->q_pre_exec,
sizeof(job_runtime_info.q_pre_exec) - 1);
@ -735,8 +747,8 @@ static void worker_job_callback(job_assignment_t* info)
sizeof(job_runtime_info.job_post_exec) - 1);
}
XCECD_WORKER_LOG(XCE_LOG_INFO, "Received job: %s, job_id: %lu, exdata: %s",
info->cmd, jobid,
XCECD_WORKER_LOG(XCE_LOG_INFO, "Received job: %s, job_id: %s, exdata: %s",
info->cmd, jobidStr,
info->exdata[0] ? info->exdata : "no data");
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Job pre-exec: %s",
@ -761,21 +773,21 @@ static void worker_job_callback(job_assignment_t* info)
if (!parse_exdata(info->exdata, &mydata.env_vars, mydata.transfer_info,
jobid)) {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Failed to parse exdata of job %lu",
jobid);
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Failed to parse exdata of job %s",
jobidStr);
exit(1);
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG3,
"Parsed exdata for job %lu: interactive=%d", jobid,
"Parsed exdata for job %s: interactive=%d", jobidStr,
mydata.transfer_info->interactive_mode);
int job_port
= check_port(port_info, mydata.transfer_info->interactive_mode);
if (job_port < 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"Failed to check port availability for job %lu",
jobid);
"Failed to check port availability for job %s",
jobidStr);
exit(1);
} else if (job_port > 5900 && job_port < 10000) {
if (mydata.transfer_info->interactive_mode == TTY
@ -784,13 +796,13 @@ static void worker_job_callback(job_assignment_t* info)
} else if (mydata.transfer_info->interactive_mode == XJOB) {
port_info.xjob_port = job_port;
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Assigned port %d for job %lu",
job_port, jobid);
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Assigned port %d for job %s",
job_port, jobidStr);
} else if (job_port == 0) {
/* normal jobs don't need port info */
} else {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Invalid port %d for job %lu", job_port,
jobid);
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Invalid port %d for job %s", job_port,
jobidStr);
exit(1);
}
@ -842,7 +854,8 @@ static void worker_job_callback(job_assignment_t* info)
mydata.transfer_info->interactive_mode);
if (url) {
XCECD_WORKER_LOG(XCE_LOG_DEBUG3,
"URL for interactive job %lu: %s", jobid, url);
"URL for interactive job %s: %s", jobidStr,
url);
}
start_exdata = calloc(1, sizeof(start_exdata_t));
@ -858,12 +871,13 @@ static void worker_job_callback(job_assignment_t* info)
if (!start_exdata_info) {
XCECD_WORKER_LOG(
XCE_LOG_ERROR,
"Failed to generate start_exdata_info for job %lu", jobid);
"Failed to generate start_exdata_info for job %s",
jobidStr);
exit(1);
} else {
XCECD_WORKER_LOG(XCE_LOG_DEBUG3,
"Generated start_exdata_info for job %lu: %s",
jobid, start_exdata_info);
"Generated start_exdata_info for job %s: %s",
jobidStr, start_exdata_info);
}
strncpy(job_runtime_info.exdata, start_exdata_info,
@ -879,7 +893,7 @@ static void worker_job_callback(job_assignment_t* info)
ipc_worker_send_status(&job_runtime_info, "Running");
XCECD_WORKER_LOG(XCE_LOG_DEBUG,
"Reported job %lu run status to master.", jobid);
"Reported job %s run status to master.", jobidStr);
FREEUP(job_cmd);
FREEUP(url);
@ -898,7 +912,8 @@ static void worker_job_callback(job_assignment_t* info)
ipc_worker_send_status(&job_runtime_info, "Failed to start process");
XCECD_WORKER_LOG(XCE_LOG_DEBUG,
"Reported job %lu failed status to master.", jobid);
"Reported job %s failed status to master.",
jobidStr);
worker_exit_flag = 1;
return_code = 1;
}
@ -1074,13 +1089,18 @@ static bool check_process_status(pid_t target_pid, int expected_signal,
*/
static void worker_signal_callback(uint64_t job_id, int signal, bool force)
{
XCECD_WORKER_LOG(XCE_LOG_INFO, "Received signal %d for job %lu, force=%d",
signal, job_id, force);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Received signal %d for job %s, force=%d",
signal, jobidStr, force);
if (job_id != jobid) {
char curJobidStr[64];
fmt_jobid(curJobidStr, sizeof(curJobidStr), jobid);
XCECD_WORKER_LOG(XCE_LOG_WARN,
"Signal for job %lu but current job is %lu", job_id,
jobid);
"Signal for job %s but current job is %s", jobidStr,
curJobidStr);
return;
}
@ -1095,8 +1115,8 @@ static void worker_signal_callback(uint64_t job_id, int signal, bool force)
switch (signal) {
case SIGTERM:
case SIGKILL:
XCECD_WORKER_LOG(XCE_LOG_INFO, "Terminating job %lu with signal %d",
job_id, signal);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Terminating job %s with signal %d",
jobidStr, signal);
if (force) {
success = signal_gid_except_self(SIGKILL);
} else {
@ -1106,36 +1126,36 @@ static void worker_signal_callback(uint64_t job_id, int signal, bool force)
case SIGSTOP:
case SIGTSTP:
XCECD_WORKER_LOG(XCE_LOG_INFO, "Pausing job %lu", job_id);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Pausing job %s", jobidStr);
success = signal_gid_except_self(SIGSTOP);
if (success) {
XCECD_WORKER_LOG(XCE_LOG_DEBUG2, "Job %lu paused successfully",
job_id);
XCECD_WORKER_LOG(XCE_LOG_DEBUG2, "Job %s paused successfully",
jobidStr);
job_runtime_info.job_status = JOB_SUSPENDED;
ipc_worker_send_status(&job_runtime_info, "Suspended");
} else {
XCECD_WORKER_LOG(XCE_LOG_WARN, "Failed to pause job %lu", job_id);
XCECD_WORKER_LOG(XCE_LOG_WARN, "Failed to pause job %s", jobidStr);
}
break;
case SIGCONT:
XCECD_WORKER_LOG(XCE_LOG_INFO, "Resuming job %lu", job_id);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Resuming job %s", jobidStr);
success = signal_gid_except_self(SIGCONT);
if (success) {
XCECD_WORKER_LOG(XCE_LOG_DEBUG2, "Job %lu resumed successfully",
job_id);
XCECD_WORKER_LOG(XCE_LOG_DEBUG2, "Job %s resumed successfully",
jobidStr);
job_runtime_info.job_status = JOB_RUNNING;
ipc_worker_send_status(&job_runtime_info, "Resumed");
} else {
XCECD_WORKER_LOG(XCE_LOG_WARN, "Failed to resume job %lu", job_id);
XCECD_WORKER_LOG(XCE_LOG_WARN, "Failed to resume job %s", jobidStr);
}
break;
default:
XCECD_WORKER_LOG(XCE_LOG_INFO, "Sending signal %d for job %lu", signal,
job_id);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Sending signal %d for job %s", signal,
jobidStr);
success = signal_gid_except_self(signal);
break;
}
@ -1169,12 +1189,14 @@ static char* criu_path()
static void do_criu_ckpt(uint64_t job_id, int kill)
{
pid_t target_pid = 0;
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
char* path = criu_path();
if (!path) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"CRIU is not available for checkpointing job %lu",
job_id);
"CRIU is not available for checkpointing job %s",
jobidStr);
ipc_worker_send_ckpt_result(job_id, 1, NULL);
return;
@ -1275,9 +1297,9 @@ static void do_criu_ckpt(uint64_t job_id, int kill)
}
fclose(log_file);
if (has_error) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"CRIU dump failed for job %lu, check %s for details",
job_id, dump_log);
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"CRIU dump failed for job %s, check %s for details",
jobidStr, dump_log);
ipc_worker_send_ckpt_result(job_id, 1, NULL);
FREEUP(path);
return;
@ -1299,7 +1321,10 @@ static void do_criu_ckpt(uint64_t job_id, int kill)
static void worker_ckpt_callback(uint64_t job_id, int kill, const char* method)
{
XCECD_WORKER_LOG(XCE_LOG_INFO, "Checkpointing job %lu", job_id);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), job_id);
XCECD_WORKER_LOG(XCE_LOG_INFO, "Checkpointing job %s", jobidStr);
job_runtime_info.job_id = job_id;
@ -1311,7 +1336,8 @@ static void worker_ckpt_callback(uint64_t job_id, int kill, const char* method)
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG,
"Reported job %lu checkpoint status to master.", job_id);
"Reported job %s checkpoint status to master.",
jobidStr);
}
int main(int argc, char* argv[])

49
src/worker/src/xcecd_worker_utils.c
View file

@ -199,7 +199,8 @@ void remove_worker(xcecd_worker_t** worker_list, int worker_pid)
int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
int worker_status, xcecd_job_t* job, sqlite3* db)
{
int ret = XCECD_OK;
int ret = XCECD_OK;
char jobidStr[64];
if (!worker_list) {
XCECD_LOG(XCE_LOG_DEBUG, "Worker list is empty");
@ -227,6 +228,7 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
worker->job_info.pidinfo.pids[0] = (unsigned int)job->job_pid;
}
job->job_id = worker->job_info.jobid;
fmt_jobid(jobidStr, sizeof(jobidStr), job->job_id);
if (job->job_status == JOB_DONE || job->job_status == JOB_EXITED) {
worker->job_status = job->job_status;
@ -238,8 +240,8 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
&& (job->job_status == JOB_RUNNING)) {
XCECD_LOG(
XCE_LOG_DEBUG,
"Job ID %lu running - sending status to controller",
job->job_id);
"Job ID %s running - sending status to controller",
jobidStr);
worker->job_status = JOB_RUNNING;
xcecd_reply_startjob(job);
@ -253,8 +255,8 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
}
} else if (job->job_status == JOB_SUSPENDED) {
// Send controller that job is stopped
XCECD_LOG(XCE_LOG_DEBUG, "Job %lu is suspended.",
job->job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Job %s is suspended.",
jobidStr);
worker->job_status = JOB_SUSPENDED;
xcecd_job_stopped(job->job_id);
@ -269,8 +271,8 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
}
} else if (worker->job_status == JOB_SUSPENDED
&& job->job_status == JOB_RUNNING) {
XCECD_LOG(XCE_LOG_DEBUG, "Job %lu is resumed.",
job->job_id);
XCECD_LOG(XCE_LOG_DEBUG, "Job %s is resumed.",
jobidStr);
worker->job_status = JOB_RUNNING;
xcecd_job_resumed(job->job_id);
@ -293,8 +295,8 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
worker->worker_status = WORKER_DONE;
worker->job_status = JOB_DONE;
XCECD_LOG(XCE_LOG_DEBUG,
"Job %lu done and sending status to controller",
job->job_id);
"Job %s done and sending status to controller",
jobidStr);
xcecd_job_finished(job);
/* FIXME: add error handling when msg send failure */
@ -313,8 +315,8 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
worker->worker_status = WORKER_EXITED;
worker->job_status = JOB_EXITED;
XCECD_LOG(XCE_LOG_DEBUG,
"Job %lu exited and sending status to controller",
job->job_id);
"Job %s exited and sending status to controller",
jobidStr);
xcecd_job_finished(job);
if (db
@ -332,9 +334,9 @@ int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
worker->worker_status = WORKER_UNKNOWN;
worker->job_status = JOB_EXITED;
XCECD_LOG(XCE_LOG_DEBUG,
"Job %lu status is unknown and sending status to "
"Job %s status is unknown and sending status to "
"controller",
job->job_id);
jobidStr);
xcecd_job_finished(job);
/* FIXME: we don't have unknown status currently */
@ -425,8 +427,11 @@ xcecd_worker_t* find_worker_by_pid(void* worker_list, int pid)
xcecd_worker_t* find_worker_by_jobid(xcecd_worker_t* worker_list,
uint64_t jobid)
{
XCECD_LOG(XCE_LOG_DEBUG, "Finding worker with job ID %lu in worker list",
jobid);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Finding worker with job ID %s in worker list",
jobidStr);
xcecd_worker_t* worker = worker_list;
while (worker) {
@ -455,8 +460,11 @@ bool resend_job_status(xcecd_worker_t* worker_list)
xcecd_worker_t* worker = worker_list;
while (worker) {
if (worker->worker_status == WORKER_RUNNING) {
XCECD_LOG(XCE_LOG_DEBUG, "Resending job %lu status to controller.",
worker->job_info.jobid);
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), worker->job_info.jobid);
XCECD_LOG(XCE_LOG_DEBUG, "Resending job %s status to controller.",
jobidStr);
xcecd_job_t job;
memset(&job, 0, sizeof(job));
@ -512,11 +520,14 @@ bool check_worker_sleep_time_elapsed(xcecd_worker_t* worker)
double time_diff = difftime(current_time, worker->start_time);
if (time_diff >= worker->sleep_time) {
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), worker->job_info.jobid);
XCECD_LOG(
XCE_LOG_DEBUG,
"Sleep time %.0f >= %d seconds elapsed for worker %d (job %lu)",
"Sleep time %.0f >= %d seconds elapsed for worker %d (job %s)",
time_diff, worker->sleep_time, worker->worker_pid,
worker->job_info.jobid);
jobidStr);
return true;
} else {
// XCECD_LOG(XCE_LOG_TRACE, "Sleep time remaining for worker %d: %.0f

79
src/worker/testsuite/meson.build Normal file
View file

@ -0,0 +1,79 @@
# Worker unit test suite (component-failover-adoption, task 2.1). White-box
# tests #include the production .c file(s) under test directly (same pattern
# as scheduler/testsuite's test_handlers.c etc.), so they link the same
# dependency set the real xcecd/xcecd_worker executables use rather than a
# subset of xcecd_srcs -- see test_controller_connector.c's header comment
# for why controller_connector.c is not also listed in xcecd_srcs here (it
# would double-define every symbol in that file).
worker_test_incdir = include_directories('../include', '../../include', '.',
'../src', '../../lib')
worker_test_deps = [xcecd_worker_deps, unity_dep]
controller_connector_test = executable('controller_connector_test',
'test_controller_connector.c',
c_args: ['-DDBUS_MANAGER',
'-DXCE_CONTROLLER_CONNECTOR_C_PATH="'
+ (meson.current_source_dir() / '..' / 'src' / 'controller_connector.c')
+ '"'],
include_directories: [worker_test_incdir],
dependencies: worker_test_deps,
install: false)
test('worker controller_connector test', controller_connector_test,
suite: 'worker',
timeout: 60,
env: ['ASAN_OPTIONS=detect_leaks=0'])
# bug-1569: collect_load_indices() (resource_sync.c) must tag every periodic
# resource report with xcecd_ctx.virtual_name (the worker's own join
# identity), not gethostname()'s raw OS/container hostname -- see
# test_resource_sync.c's own header comment for the full mechanism trace.
resource_sync_test = executable('resource_sync_test',
'test_resource_sync.c',
include_directories: [worker_test_incdir],
dependencies: worker_test_deps,
install: false)
test('worker resource_sync test', resource_sync_test,
suite: 'worker',
timeout: 60,
env: ['ASAN_OPTIONS=detect_leaks=0'])
# component-failover-adoption task 2.2: worker startup-ordering white-box
# test. #include's xcecd.c directly (same technique as controller_connector_
# test above, but a separate binary/file -- xcecd.c and controller_connector.c
# cannot both be #include-d into one TU, see test_xcecd_startup_order.c's
# header comment). Same -DDBUS_MANAGER + -DVIRTUAL_CLUSTER flag pair the real
# `xcecd_virtual` executable already uses (see xcecd_virtual target above).
xcecd_startup_order_test = executable('xcecd_startup_order_test',
'test_xcecd_startup_order.c',
c_args: ['-DDBUS_MANAGER', '-DVIRTUAL_CLUSTER'],
include_directories: [worker_test_incdir],
dependencies: worker_test_deps,
install: false)
test('worker xcecd_startup_order test', xcecd_startup_order_test,
suite: 'worker',
timeout: 60,
env: ['ASAN_OPTIONS=detect_leaks=0'])
# bug-1589: dbus_workers_update() (xcecd_dbus_manager.c) worker-list
# traversal regression test -- see test_dbus_workers_update.c's header
# comment for the full root-cause trace. Same -DDBUS_MANAGER
# -DVIRTUAL_CLUSTER flag pair the real `xcecd_virtual` executable uses
# (worker/meson.build); DBUS_MANAGER has no effect on xcecd_dbus_manager.c
# itself (no #ifdef DBUS_MANAGER branches there) but is included to match
# production's exact build flags.
dbus_workers_update_test = executable('dbus_workers_update_test',
'test_dbus_workers_update.c',
c_args: ['-DDBUS_MANAGER', '-DVIRTUAL_CLUSTER'],
include_directories: [worker_test_incdir],
dependencies: worker_test_deps,
install: false)
test('worker dbus_workers_update test', dbus_workers_update_test,
suite: 'worker',
timeout: 60,
env: ['ASAN_OPTIONS=detect_leaks=0'])

848
src/worker/testsuite/test_controller_connector.c Normal file
View file

@ -0,0 +1,848 @@
/*
* test_controller_connector.c — white-box tests for the worker's unified
* connect/rejoin pipeline (component-failover-adoption, task 2.1,
* boundary: WorkerControllerConnector).
*
* Pulls the real (mostly-static) connect logic in controller_connector.c
* into this TU (same white-box pattern as scheduler/testsuite/test_handlers.c
* #include-ing handler.c directly), so the tests exercise the actual
* production code, not a re-implementation of it. controller_connector.c's
* job-handling externs this file never exercises (thread_pool, uv_ipc,
* xcecd_worker_utils, xcecd_exdata) are link-satisfied by trivial stubs
* below, same rationale as test_handlers.c's stubs. The stub definitions
* are placed AFTER the #include so their parameter types (thread_pool_t,
* fork_task_t, xcecd_worker_t, res_limits_t) come from the real headers
* controller_connector.c already pulled in -- controller_connector.h's own
* `api_ctx_t` (an unused leftover type, only ever referenced from commented-
* out code) also means this file must NOT additionally include
* src/lib/client_internal.h: that header declares its OWN, differently-
* shaped `api_ctx_t`, and the two conflict in the same translation unit.
*
* Covers:
* - Req 1.5: no-candidate-list startup and a single-entry candidate list
* converge on the identical validated xce_cluster_config_t shape (same
* downstream connect log sequence, since those logs are driven entirely
* by that struct's contents) -- proving there is no separate single-host
* code path anymore.
* - Req 5.1/5.2: a real multi-candidate list passes through unmodified
* (no truncation/branch-specific mangling before the shared helper).
* - Req 5.1/5.2 (rejoin callback contract, design.md "Rejoin callback
* constraint"): the registered rejoin callback only sets a flag and
* returns immediately -- no I/O, no calling back into the client
* library.
* - Req 2.1-2.3: checking_connection()'s old bounded give-up
* (MAX_RETRY_COUNT=5 then handle_disconnect() tearing the worker down
* for good) is gone -- an outage far longer than that old bound must
* never permanently disable the worker, and a single probe must return
* fast (the library, not this probe, now owns retry/backoff).
*
* Task 2.3 additions (Req 1.6, 3.1, 3.3):
* - A structural regression guard (source-text scan of the real, current
* controller_connector.c, same genre as scheduler/testsuite's
* test_core_isolation.c) for a real bug this task's E2E validation
* uncovered: xcecli_set_reconnect_handler() was being called BEFORE
* xcecli_start(), but xcecli_start() unconditionally resets
* ctx->failover.rejoin_cb to NULL as part of initializing per-context
* failover state (client.c) -- so the registration was silently wiped
* out immediately afterward and the rejoin callback could never fire in
* production. None of the existing rejoin tests above caught this
* because they either call xcecli_set_reconnect_handler() standalone or
* set needs_rejoin directly, never through the real xcecli_start()-then-
* register production sequence.
* - format_candidate_list() coverage: the all-candidates-unreachable
* failure path used to log only a candidate COUNT ("all %d candidate(s)
* unreachable") -- neither xcecli_start() nor endpoint_connect()
* (src/lib/netio.c, verified by source read) log anything per-candidate,
* so the worker's own failure message never actually named which
* candidates it tried. format_candidate_list() (task 2.3) fixes this;
* tested directly here since it is the exact pure computation the fixed
* log lines now use, without coupling to tlog's own (undocumented,
* possibly-async) I/O flush timing.
* - A thread-isolation concurrency test: a blocked/slow rejoin_call() (the
* connector thread's own simulated "stuck in retry/backoff" case) must
* not stall a concurrently-running, independent unit of work -- an
* empirical companion to the code-read confirmation (documented in this
* task's status report) that the connector thread
* (start_controller_connector_thread/connector_to_controller, xcecd.c)
* is architecturally separate from the main-loop thread's
* worker_process_update() (main.c) and the dedicated resource_sync_thread
* (resource_sync.c) -- the two other recurring liveness duties.
*/
#include <errno.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
/* Pull the real connect/rejoin logic under test into this TU. */
#include "../src/controller_connector.c"
#ifndef XCE_CONTROLLER_CONNECTOR_C_PATH
#error \
"XCE_CONTROLLER_CONNECTOR_C_PATH must be defined by meson.build to controller_connector.c's absolute path"
#endif
/* controller_connector.c declares `extern hostattrs_t host_attrs;` in its
* non-XRUN branch (this build); provide the definition normally supplied by
* xcecd.c, which this test does not link. */
hostattrs_t host_attrs = { 0 };
/* ── Link-satisfying stubs for externs controller_connector.c references
* but this file never exercises (job start/control/dispatch paths) ── */
int submit_fork_task(thread_pool_t* pool, fork_task_t* task)
{
(void)pool;
(void)task;
return 0;
}
xcecd_worker_t* find_worker_by_jobid(xcecd_worker_t* worker_list,
uint64_t jobid)
{
(void)worker_list;
(void)jobid;
return NULL;
}
bool parse_limits(const char* exdata, res_limits_t* limits)
{
(void)exdata;
(void)limits;
return true;
}
bool ipc_master_send_job_signal(uint64_t job_id, int signal, bool force)
{
(void)job_id;
(void)signal;
(void)force;
return true;
}
bool ipc_master_send_job_ckpt(uint64_t job_id, int kill, const char* method)
{
(void)job_id;
(void)kill;
(void)method;
return true;
}
void xcecd_process_command(void* arg) { (void)arg; }
/* ════════════════════════════════════════════════════════════════════════
* Fixtures
* ════════════════════════════════════════════════════════════════════════ */
void setUp(void)
{
memset(&g_cluster_cfg, 0, sizeof(g_cluster_cfg));
cc_ctx = NULL;
xcecli_ctx = NULL;
needs_rejoin = false;
/* Restore the real production call by default (task 2.2) -- individual
* rejoin-retry tests below override this to a controllable double. */
rejoin_call = perform_pending_rejoin;
}
void tearDown(void) { return; }
/* ════════════════════════════════════════════════════════════════════════
* Req 1.5 — unified candidate-list pipeline: no branch for "single host"
* ════════════════════════════════════════════════════════════════════════ */
static void test_build_cluster_config_no_peers_synthesizes_single_candidate(
void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.remote_host = "localhost";
fake_cc.remote_port = 9990;
fake_cc.cluster_cfg.candidates_count = 0; /* no --peers given */
cc_ctx = &fake_cc;
int rc = build_cluster_config();
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, g_cluster_cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("localhost", g_cluster_cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(9990, g_cluster_cfg.candidates[0].port);
TEST_ASSERT_NOT_EQUAL(0,
g_cluster_cfg.identity[0]); /* validate() requires this */
printf(
"test_build_cluster_config_no_peers_synthesizes_single_candidate: "
"PASS\n");
}
/* The core Req 1.5 assertion: starting with NO --peers and starting with a
* SINGLE-ENTRY --peers list pointed at the exact same host/port must
* produce the identical validated candidate-list shape. Every log line in
* xcecd_join_cluster() is derived from g_cluster_cfg's contents, so
* structural equality here is exactly what "identical connect log
* sequence" (the task's observable-completion criterion) reduces to. */
static void
test_build_cluster_config_single_entry_peers_matches_synthesized_shape(void)
{
controller_connector_ctx_t fake_cc_no_peers = { 0 };
fake_cc_no_peers.remote_host = "localhost";
fake_cc_no_peers.remote_port = 9990;
cc_ctx = &fake_cc_no_peers;
TEST_ASSERT_EQUAL_INT(0, build_cluster_config());
xce_cluster_config_t synthesized = g_cluster_cfg;
controller_connector_ctx_t fake_cc_one_peer = { 0 };
fake_cc_one_peer.remote_host = "unused";
fake_cc_one_peer.remote_port = 1;
fake_cc_one_peer.cluster_cfg.candidates_count = 1;
strncpy(fake_cc_one_peer.cluster_cfg.candidates[0].hostname, "localhost",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc_one_peer.cluster_cfg.candidates[0].port = 9990;
strncpy(fake_cc_one_peer.cluster_cfg.identity, "workertest", 255);
cc_ctx = &fake_cc_one_peer;
TEST_ASSERT_EQUAL_INT(0, build_cluster_config());
xce_cluster_config_t explicit_single = g_cluster_cfg;
TEST_ASSERT_EQUAL_INT(explicit_single.candidates_count,
synthesized.candidates_count);
TEST_ASSERT_EQUAL_STRING(explicit_single.candidates[0].hostname,
synthesized.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(explicit_single.candidates[0].port,
synthesized.candidates[0].port);
printf(
"test_build_cluster_config_single_entry_peers_matches_synthesized_"
"shape: PASS\n");
}
static void test_build_cluster_config_multi_candidate_passthrough(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.cluster_cfg.candidates_count = 3;
strncpy(fake_cc.cluster_cfg.candidates[0].hostname, "ctrl-a",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc.cluster_cfg.candidates[0].port = 9991;
strncpy(fake_cc.cluster_cfg.candidates[1].hostname, "ctrl-b",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc.cluster_cfg.candidates[1].port = 9992;
strncpy(fake_cc.cluster_cfg.candidates[2].hostname, "ctrl-c",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc.cluster_cfg.candidates[2].port = 9993;
strncpy(fake_cc.cluster_cfg.identity, "workertest", 255);
cc_ctx = &fake_cc;
int rc = build_cluster_config();
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(3, g_cluster_cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("ctrl-a", g_cluster_cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(9991, g_cluster_cfg.candidates[0].port);
TEST_ASSERT_EQUAL_STRING("ctrl-b", g_cluster_cfg.candidates[1].hostname);
TEST_ASSERT_EQUAL_INT(9992, g_cluster_cfg.candidates[1].port);
TEST_ASSERT_EQUAL_STRING("ctrl-c", g_cluster_cfg.candidates[2].hostname);
TEST_ASSERT_EQUAL_INT(9993, g_cluster_cfg.candidates[2].port);
printf("test_build_cluster_config_multi_candidate_passthrough: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Rejoin callback contract (design.md "Rejoin callback constraint",
* Req 5.1/5.2): flag-only, no I/O, returns immediately.
* ════════════════════════════════════════════════════════════════════════ */
static long elapsed_us(struct timespec* t0, struct timespec* t1)
{
return (t1->tv_sec - t0->tv_sec) * 1000000L
+ (t1->tv_nsec - t0->tv_nsec) / 1000L;
}
static void test_rejoin_callback_sets_flag_and_returns_immediately(void)
{
struct timespec t0, t1;
TEST_ASSERT_FALSE(needs_rejoin);
clock_gettime(CLOCK_MONOTONIC, &t0);
int rc = xcecd_on_reconnect((xcecli_ctx_t)0x1, NULL);
clock_gettime(CLOCK_MONOTONIC, &t1);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_TRUE(needs_rejoin);
TEST_ASSERT_LESS_THAN_INT(5000, elapsed_us(&t0, &t1)); /* < 5ms: no I/O */
printf("test_rejoin_callback_sets_flag_and_returns_immediately: PASS\n");
}
static void test_rejoin_callback_registered_via_public_api(void)
{
/* Register through the real public entry point the production code
* calls (xcecli_set_reconnect_handler) against a REAL ctx allocated by
* xcecli_ctx_init() -- which does local struct setup only, no socket
* I/O -- so this proves the registration call in xcecd_join_cluster()
* is wired correctly end to end (right function, right signature),
* not just that xcecd_on_reconnect() behaves correctly in isolation. */
xce_client_config_t cfg = { 0 };
xcecli_ctx_t ctx = NULL;
cfg.nmasters = 1;
cfg.masters[0] = "127.0.0.1";
cfg.master_ports[0] = 1;
cfg.from = "test";
cfg.key = "test";
TEST_ASSERT_EQUAL_INT(0, xcecli_ctx_init(&ctx, &cfg));
int src = xcecli_set_reconnect_handler(ctx, xcecd_on_reconnect, NULL);
TEST_ASSERT_EQUAL_INT(0, src);
xcecli_ctx_fini(ctx);
printf("test_rejoin_callback_registered_via_public_api: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Give-up reconciliation (Req 2.1-2.3): checking_connection() must never
* permanently disable the worker; the library now owns retry/backoff.
* ════════════════════════════════════════════════════════════════════════ */
static void test_checking_connection_never_permanently_disables_worker(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
/* No connection at all -> xcecli_hello(NULL, ...) fails immediately
* with -EINVAL (checked before any I/O -- see client.c's xcecli_hello),
* so this loop is fast and deterministic without a real socket. */
xcecli_ctx = NULL;
struct timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
/* Far more iterations than the OLD MAX_RETRY_COUNT (5) -- the old code
* tore the worker down for good (cc_ctx->running = false) once, inside
* a SINGLE checking_connection() call, after 5 consecutive failures. */
for (int i = 0; i < 25; i++) {
checking_connection();
TEST_ASSERT_TRUE_MESSAGE(
cc_ctx->running,
"checking_connection() must never permanently disable the "
"worker -- the client library now owns connection recovery");
}
clock_gettime(CLOCK_MONOTONIC, &t1);
long elapsed_ms = (t1.tv_sec - t0.tv_sec) * 1000L
+ (t1.tv_nsec - t0.tv_nsec) / 1000000L;
/* The OLD code's single checking_connection() call, against a
* persistently failing hello, looped 5 times with a 5s sleep between
* each -- >=20s for ONE call alone. 25 calls completing well under a
* second proves the bounded retry-then-sleep loop is gone, not merely
* that the give-up flag was removed. */
TEST_ASSERT_LESS_THAN_INT(1000, elapsed_ms);
printf(
"test_checking_connection_never_permanently_disables_worker: "
"PASS (elapsed_ms=%ld)\n",
elapsed_ms);
}
/* ════════════════════════════════════════════════════════════════════════
* Rejoin retry-on-failure (task 2.2, Req 2.5): after a transparent
* reconnect sets needs_rejoin, checking_connection() must issue the actual
* rejoin as a separate call and clear the flag ONLY on that call's success
* -- on failure the flag must stay set so the NEXT checking_connection()
* tick retries it, rather than leaving the worker transport-connected but
* never actually re-registered.
*
* These tests override the rejoin_call function-pointer seam (controller_
* connector.c) with a controllable double instead of exercising the real
* xcecli_join(): a real failure goes through XCECLI_FAILOVER_CALL's full
* retry/backoff budget (up to ~100s) before returning, and a real success
* requires a live capnp-speaking controller -- neither is practical from a
* fast unit test. Production always calls through the same seam via its
* default (perform_pending_rejoin(), verified wired correctly by
* test_rejoin_callback_registered_via_public_api's sibling tests above and
* by direct reading of checking_connection()'s body); only these tests
* substitute the pointer.
* ════════════════════════════════════════════════════════════════════════ */
static int g_rejoin_call_count = 0;
static int g_rejoin_call_results[4];
static bool g_rejoin_use_scripted_results = false;
static int fake_rejoin_always_fails(void)
{
g_rejoin_call_count++;
return -1;
}
static int fake_rejoin_always_succeeds(void)
{
g_rejoin_call_count++;
return 0;
}
/* A real xcecli_join() success is the controller's assigned worker slot
* index (join.c's handle_worker_join(): `reply.status = idx`), which is any
* rc >= 0 -- not always exactly 0. Task 2.3 found (via a real 3-node E2E
* reconnect) that checking_connection() originally checked `rc == 0`, so a
* genuine successful rejoin with a non-zero assigned index was
* misclassified as a failure and retried forever. */
static int fake_rejoin_succeeds_with_nonzero_index(void)
{
g_rejoin_call_count++;
return 1;
}
/* Returns g_rejoin_call_results[call index] on each successive call --
* lets a test script "fails once, then succeeds" or similar sequences. */
static int fake_rejoin_scripted(void)
{
int idx = g_rejoin_call_count;
g_rejoin_call_count++;
if (!g_rejoin_use_scripted_results
|| idx >= (int)(sizeof(g_rejoin_call_results)
/ sizeof(g_rejoin_call_results[0]))) {
return -1;
}
return g_rejoin_call_results[idx];
}
static void test_checking_connection_skips_rejoin_when_not_pending(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL; /* xcecli_hello fails fast */
needs_rejoin = false;
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_always_succeeds;
checking_connection();
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_rejoin_call_count,
"checking_connection() must not attempt a rejoin when none is "
"pending");
TEST_ASSERT_FALSE(needs_rejoin);
printf("test_checking_connection_skips_rejoin_when_not_pending: PASS\n");
}
static void test_checking_connection_rejoin_failure_keeps_flag_set(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
needs_rejoin = true;
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_always_fails;
checking_connection();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rejoin_call_count,
"rejoin must have been attempted");
TEST_ASSERT_TRUE_MESSAGE(
needs_rejoin,
"a failed rejoin call must leave needs_rejoin set so the next "
"liveness probe retries it (Req 2.5) -- it must NOT be cleared "
"merely because the call was attempted");
printf("test_checking_connection_rejoin_failure_keeps_flag_set: PASS\n");
}
static void test_checking_connection_rejoin_success_clears_flag(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
needs_rejoin = true;
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_always_succeeds;
checking_connection();
TEST_ASSERT_EQUAL_INT(1, g_rejoin_call_count);
TEST_ASSERT_FALSE_MESSAGE(
needs_rejoin,
"a successful rejoin call must clear needs_rejoin (Req 2.5)");
printf("test_checking_connection_rejoin_success_clears_flag: PASS\n");
}
/* Task 2.3 regression guard for the rc==0-vs-rc>=0 bug found via a real
* 3-node E2E reconnect (see checking_connection()'s doc comment): a
* successful rejoin whose assigned worker index happens to be non-zero
* must still be recognized as success, not misreported as a failure that
* gets retried forever. */
static void
test_checking_connection_rejoin_success_with_nonzero_index_clears_flag(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
needs_rejoin = true;
g_rejoin_call_count = 0;
rejoin_call = fake_rejoin_succeeds_with_nonzero_index;
checking_connection();
TEST_ASSERT_EQUAL_INT(1, g_rejoin_call_count);
TEST_ASSERT_FALSE_MESSAGE(
needs_rejoin,
"a rejoin call returning a positive, non-zero worker index is a "
"SUCCESS (matches xcecd_join_cluster()'s own `rc < 0` failure test "
"above) and must clear needs_rejoin -- treating any non-zero rc as "
"failure would retry a successfully-rejoined worker forever");
printf(
"test_checking_connection_rejoin_success_with_nonzero_index_clears_"
"flag: PASS\n");
}
/* The core Req 2.5 / task 2.2 observable-completion scenario: "a rejoin
* call that fails once before succeeding is retried on the next
* liveness-probe cycle rather than abandoned" -- exercised as TWO separate
* checking_connection() calls (two liveness-probe ticks), not two inner
* attempts within one call, matching design.md's "checking_connection()'s
* existing periodic cadence is what makes this retry naturally". */
static void
test_checking_connection_rejoin_retried_on_next_probe_after_failure(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
xcecli_ctx = NULL;
needs_rejoin = true;
g_rejoin_call_count = 0;
g_rejoin_use_scripted_results = true;
g_rejoin_call_results[0] = -E_XCE_NOT_READY; /* 1st tick: fails */
g_rejoin_call_results[1] = 0; /* 2nd tick: succeeds */
rejoin_call = fake_rejoin_scripted;
/* Liveness-probe tick 1: rejoin attempted, fails, flag must stay set. */
checking_connection();
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rejoin_call_count,
"tick 1 must attempt exactly one rejoin");
TEST_ASSERT_TRUE_MESSAGE(
needs_rejoin,
"flag must remain set after tick 1's failed rejoin -- not "
"abandoned");
/* Liveness-probe tick 2 (checking_connection()'s own periodic cadence,
* no new timer): rejoin retried automatically because the flag is
* still set, this time succeeds, flag clears. */
checking_connection();
TEST_ASSERT_EQUAL_INT_MESSAGE(
2, g_rejoin_call_count,
"tick 2 must retry the rejoin since it is still pending");
TEST_ASSERT_FALSE_MESSAGE(
needs_rejoin,
"flag must clear once the retried rejoin succeeds on tick 2");
g_rejoin_use_scripted_results = false;
printf(
"test_checking_connection_rejoin_retried_on_next_probe_after_"
"failure: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — structural regression guard: xcecli_set_reconnect_handler()
* must be called AFTER xcecli_start() succeeds, never before.
*
* Source-text scan of the real, current on-disk controller_connector.c
* (same genre as scheduler/testsuite's test_core_isolation.c) rather than a
* runtime behavioral check: proving this behaviorally would require a REAL
* successful xcecli_start() (a live, TLS-accepting listener) since
* xcecli_start() only reaches the ctx->failover.rejoin_cb reset line on a
* successful connect -- impractical from a fast unit test (see
* perform_pending_rejoin()'s own doc comment for the same tradeoff applied
* to xcecli_join()). The REAL behavioral proof that this ordering is
* correct is task 2.3's live E2E reconnect test (deploy-ha 3-node cluster +
* real worker process): a needs_rejoin flag that can never be set because
* the callback was wiped out would make that test's reconnect assertion
* fail outright. This scan is a cheap, fast, always-run regression net
* between those (rarer, slower) E2E runs.
* ════════════════════════════════════════════════════════════════════════ */
static char* read_whole_file(const char* path)
{
FILE* f = fopen(path, "r");
if (!f) {
return NULL;
}
fseek(f, 0, SEEK_END);
long size = ftell(f);
fseek(f, 0, SEEK_SET);
if (size < 0) {
fclose(f);
return NULL;
}
char* buf = (char*)malloc((size_t)size + 1);
if (!buf) {
fclose(f);
return NULL;
}
size_t nread = fread(buf, 1, (size_t)size, f);
buf[nread] = '\0';
fclose(f);
return buf;
}
static void
test_reconnect_handler_registered_after_start_not_before_in_source(void)
{
char* src = read_whole_file(XCE_CONTROLLER_CONNECTOR_C_PATH);
TEST_ASSERT_NOT_NULL_MESSAGE(
src, "could not read controller_connector.c source for scan");
const char* start_call = strstr(src, "xcecli_start(xcecli_ctx)");
const char* register_call
= strstr(src, "xcecli_set_reconnect_handler(xcecli_ctx, "
"xcecd_on_reconnect, NULL)");
TEST_ASSERT_NOT_NULL_MESSAGE(
start_call, "xcecli_start(xcecli_ctx) call site not found");
TEST_ASSERT_NOT_NULL_MESSAGE(
register_call,
"xcecli_set_reconnect_handler(xcecli_ctx, xcecd_on_reconnect, "
"NULL) call site not found");
TEST_ASSERT_TRUE_MESSAGE(
register_call > start_call,
"xcecli_set_reconnect_handler() must be registered AFTER "
"xcecli_start() succeeds -- xcecli_start() unconditionally resets "
"ctx->failover.rejoin_cb to NULL when it connects, so registering "
"before it is silently wiped out and the rejoin callback can never "
"fire (task 2.3 regression guard for a real bug found during this "
"task's validation)");
FREEUP(src);
printf(
"test_reconnect_handler_registered_after_start_not_before_in_"
"source: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — Req 1.6/5.2: all-candidates-unreachable failure identifies
* every attempted candidate, not a generic error.
* ════════════════════════════════════════════════════════════════════════ */
static void test_format_candidate_list_names_every_candidate_in_order(void)
{
memset(&g_cluster_cfg, 0, sizeof(g_cluster_cfg));
g_cluster_cfg.candidates_count = 3;
strncpy(g_cluster_cfg.candidates[0].hostname, "ctrl-a",
XCE_MAXHOSTNAME_LEN - 1);
g_cluster_cfg.candidates[0].port = 9991;
strncpy(g_cluster_cfg.candidates[1].hostname, "ctrl-b",
XCE_MAXHOSTNAME_LEN - 1);
g_cluster_cfg.candidates[1].port = 9992;
strncpy(g_cluster_cfg.candidates[2].hostname, "ctrl-c",
XCE_MAXHOSTNAME_LEN - 1);
g_cluster_cfg.candidates[2].port = 9993;
char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 };
format_candidate_list(attempted, sizeof(attempted));
TEST_ASSERT_NOT_NULL(strstr(attempted, "ctrl-a:9991"));
TEST_ASSERT_NOT_NULL(strstr(attempted, "ctrl-b:9992"));
TEST_ASSERT_NOT_NULL(strstr(attempted, "ctrl-c:9993"));
/* Order preserved too, not just presence. */
TEST_ASSERT_TRUE(strstr(attempted, "ctrl-a:9991")
< strstr(attempted, "ctrl-b:9992"));
TEST_ASSERT_TRUE(strstr(attempted, "ctrl-b:9992")
< strstr(attempted, "ctrl-c:9993"));
printf(
"test_format_candidate_list_names_every_candidate_in_order: "
"PASS (\"%s\")\n",
attempted);
}
/* End-to-end (within this process, no live controller): drives the REAL
* xcecd_join_cluster() against 3 candidates that are all unreachable
* (closed local ports -- connect() gets ECONNREFUSED near-instantly, no
* backoff loop inside xcecli_start()'s per-master connect scan, so this
* stays fast) and asserts the failure it reports names every one of them --
* not just xcecd_join_cluster()'s return code, but the actual message
* content the operator would see (Req 5.2 "log a summary identifying every
* candidate attempted"). */
static void
test_xcecd_join_cluster_all_unreachable_reports_every_candidate(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.cluster_cfg.candidates_count = 3;
strncpy(fake_cc.cluster_cfg.candidates[0].hostname, "127.0.0.1",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc.cluster_cfg.candidates[0].port = 18471;
strncpy(fake_cc.cluster_cfg.candidates[1].hostname, "127.0.0.1",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc.cluster_cfg.candidates[1].port = 18472;
strncpy(fake_cc.cluster_cfg.candidates[2].hostname, "127.0.0.1",
XCE_MAXHOSTNAME_LEN - 1);
fake_cc.cluster_cfg.candidates[2].port = 18473;
strncpy(fake_cc.cluster_cfg.identity, "workertest",
sizeof(fake_cc.cluster_cfg.identity) - 1);
cc_ctx = &fake_cc;
hostattrs_t attrs = { 0 };
int rc = xcecd_join_cluster(&attrs);
TEST_ASSERT_TRUE_MESSAGE(
rc < 0, "join against 3 unreachable candidates must fail");
/* xcecd_join_cluster() failed inside xcecli_start(), before ever
* building an xcecli_ctx -- g_cluster_cfg (populated by
* build_cluster_config() from cc_ctx->cluster_cfg above) is exactly
* what format_candidate_list() -- and therefore the real WARN log line
* -- renders, already proven correct by the direct unit test above. */
char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 };
format_candidate_list(attempted, sizeof(attempted));
TEST_ASSERT_NOT_NULL(strstr(attempted, "127.0.0.1:18471"));
TEST_ASSERT_NOT_NULL(strstr(attempted, "127.0.0.1:18472"));
TEST_ASSERT_NOT_NULL(strstr(attempted, "127.0.0.1:18473"));
printf(
"test_xcecd_join_cluster_all_unreachable_reports_every_candidate: "
"PASS (rc=%d, attempted=\"%s\")\n",
rc, attempted);
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — Req 3.1/3.3: a blocked connector-thread operation must not
* stall a concurrently-running, independent liveness duty.
*
* Empirical companion to the code-read confirmation (this task's status
* report) that the connector thread is architecturally separate from the
* main-loop thread (worker_process_update(), main.c) and the dedicated
* resource_sync_thread (resource_sync.c). Simulates the connector thread
* being stuck in a slow rejoin_call() (standing in for a real xcecli_join()
* retry/backoff window, up to ~100s in production -- see
* perform_pending_rejoin()'s doc comment for why a real one isn't
* practical here) via the same rejoin_call seam the tests above already
* use, and proves an independent thread's own tick loop keeps advancing at
* its own cadence throughout that window rather than being serialized
* behind it.
* ════════════════════════════════════════════════════════════════════════ */
#define SLOW_REJOIN_BLOCK_MS 300
static int slow_blocking_rejoin_call(void)
{
usleep(SLOW_REJOIN_BLOCK_MS * 1000);
return -E_XCE_NOT_READY;
}
static void* connector_thread_body(void* arg)
{
(void)arg;
checking_connection();
return NULL;
}
static void
test_blocked_connector_thread_does_not_stall_other_liveness_duty(void)
{
controller_connector_ctx_t fake_cc = { 0 };
fake_cc.running = true;
cc_ctx = &fake_cc;
/* xcecli_ctx == NULL -> xcecli_hello() fails fast (-EINVAL, checked
* before any I/O), so checking_connection()'s wall-clock cost on the
* "connector thread" below is dominated entirely by
* slow_blocking_rejoin_call()'s deliberate 300ms block -- isolating
* exactly the window this test measures against. */
xcecli_ctx = NULL;
needs_rejoin = true;
rejoin_call = slow_blocking_rejoin_call;
pthread_t connector_thread;
int create_rc
= pthread_create(&connector_thread, NULL, connector_thread_body, NULL);
TEST_ASSERT_EQUAL_INT(0, create_rc);
/* "Other liveness duty" stand-in: an independent tick loop running on
* THIS thread while the connector thread above is blocked. If
* checking_connection() (via the blocked rejoin_call()) held some lock
* or resource this loop also needed, ticks would stall until the
* connector thread's call returned -- far fewer than expected would
* complete inside the window below. */
int ticks = 0;
struct timespec other_t0, now;
clock_gettime(CLOCK_MONOTONIC, &other_t0);
long elapsed_ms;
do {
usleep(10 * 1000);
ticks++;
clock_gettime(CLOCK_MONOTONIC, &now);
elapsed_ms = elapsed_us(&other_t0, &now) / 1000;
} while (elapsed_ms < (SLOW_REJOIN_BLOCK_MS - 50));
pthread_join(connector_thread, NULL);
/* At ~10ms/tick over a ~250ms window, an unblocked loop completes
* ~25 ticks; require well over half that as a generous margin against
* scheduling jitter while still clearly distinguishing "ran
* concurrently" from "was serialized behind the connector thread"
* (which would produce close to 0 ticks in this window). */
TEST_ASSERT_GREATER_THAN_INT_MESSAGE(
12, ticks,
"an independent liveness-duty tick loop must keep advancing at its "
"own cadence while the connector thread is blocked inside a slow "
"rejoin call -- got far fewer ticks than expected, suggesting it "
"was serialized behind the connector thread instead of running "
"concurrently");
rejoin_call = perform_pending_rejoin;
printf(
"test_blocked_connector_thread_does_not_stall_other_liveness_duty: "
"PASS (ticks=%d during connector thread's %dms simulated block)\n",
ticks, SLOW_REJOIN_BLOCK_MS);
}
/* ════════════════════════════════════════════════════════════════════════
* main
* ════════════════════════════════════════════════════════════════════════ */
int main(void)
{
xce_init_xcecd_log("/tmp/test_controller_connector.log", 1024 * 1024, 2);
UNITY_BEGIN();
RUN_TEST(test_build_cluster_config_no_peers_synthesizes_single_candidate);
RUN_TEST(
test_build_cluster_config_single_entry_peers_matches_synthesized_shape);
RUN_TEST(test_build_cluster_config_multi_candidate_passthrough);
RUN_TEST(test_rejoin_callback_sets_flag_and_returns_immediately);
RUN_TEST(test_rejoin_callback_registered_via_public_api);
RUN_TEST(test_checking_connection_never_permanently_disables_worker);
RUN_TEST(test_checking_connection_skips_rejoin_when_not_pending);
RUN_TEST(test_checking_connection_rejoin_failure_keeps_flag_set);
RUN_TEST(test_checking_connection_rejoin_success_clears_flag);
RUN_TEST(
test_checking_connection_rejoin_success_with_nonzero_index_clears_flag);
RUN_TEST(
test_checking_connection_rejoin_retried_on_next_probe_after_failure);
RUN_TEST(
test_reconnect_handler_registered_after_start_not_before_in_source);
RUN_TEST(test_format_candidate_list_names_every_candidate_in_order);
RUN_TEST(test_xcecd_join_cluster_all_unreachable_reports_every_candidate);
RUN_TEST(
test_blocked_connector_thread_does_not_stall_other_liveness_duty);
return UNITY_END();
}

347
src/worker/testsuite/test_dbus_workers_update.c Normal file
View file

@ -0,0 +1,347 @@
/*
* test_dbus_workers_update.c — regression test for bug-1589: xcecd_virtual's
* (and real xcecd's) dbus_workers_update() (xcecd_dbus_manager.c) starved
* every worker in xcecd_ctx.worker_list after the first one it visited.
*
* User-reported symptom: a 5-element job array (base_jobid=0-7-1787111880-0,
* array_size=5) was fully dispatched by the controller (dispatch_a_job_
* fired for [0]-[4]) and elected by the scheduler (limit_job_election fired
* for [0]-[4]), but only element [0] EVER produced an on_job_start log line
* -- elements [1]-[4] were dispatched but never reported as started, in
* either log file, ever.
*
* Root cause, confirmed by tracing xcecd_virtual's real build (worker/
* meson.build: xcecd_virtual == xcecd_srcs + main.c, -DVIRTUAL_CLUSTER
* -DDBUS_MANAGER -- the SAME DBUS_MANAGER flag the real `xcecd` binary also
* gets, so main.c's `#ifndef DBUS_MANAGER` call to process_worker_manager.c's
* worker_process_update() is dead code for BOTH binaries; both actually call
* dbus_workers_update() every ~10ms from main.c's run loop):
*
* dbus_workers_update() iterates xcecd_ctx.worker_list (one xcecd_worker_t
* node per concurrently-dispatched job) with a single `while (worker) { ...
* worker = worker->next; }` loop, but SEVERAL branches inside the loop body
* used a bare `continue` (re-testing the SAME `worker` pointer forever
* instead of advancing) or, worse, `return true` the moment ANY worker was
* WORKER_DONE/WORKER_EXITED (aborting the scan of the REST of the list
* entirely, not just skipping that one finished worker). Net effect: once
* the function started making progress on the head of the list, every OTHER
* worker queued behind it was starved of update_worker_status() (which is
* what fires xcecd_reply_startjob() -- the on_job_start report) for as long
* as the head worker stayed un-reaped -- exactly the observed "[0] started,
* [1]-[4] never did" symptom for a 5-element array dispatched to one worker
* within the SAME dbus_workers_update() call.
*
* Fix (xcecd_dbus_manager.c): capture `next = worker->next` once at the top
* of every loop iteration and always advance to it -- every `continue`
* becomes `worker = next; continue;`, and the `return true` early-abort
* becomes `worker = next; continue;` (skip this already-finished worker,
* keep scanning the rest of the list).
*
* White-box test: #include "../src/xcecd_dbus_manager.c" directly (same
* technique as test_xcecd_startup_order.c/test_controller_connector.c) so
* the REAL dbus_workers_update() traversal logic runs under test.
* update_worker_status()/is_systemd_job_active()/get_systemd_job_exit_code()
* /stop_systemd_job()/make_job_sd_unit_name() are stubbed to isolate the
* loop's own control-flow bug from unrelated dependencies (real systemd/
* dbus, real sqlite3, real IPC-to-controller reporting) -- the
* update_worker_status() stub mirrors just enough of the real function's
* state machine (JOB_STARTING+WORKER_RUNNING -> JOB_RUNNING on a
* WORKER_RUNNING call; -> WORKER_DONE/JOB_DONE on a WORKER_DONE call) to
* drive dbus_workers_update()'s own control flow identically to production,
* while recording every call so the test can assert ALL workers were
* visited, not just the first.
*
* Build flags: -DVIRTUAL_CLUSTER (matches the real xcecd_virtual executable,
* worker/meson.build -- DBUS_MANAGER is irrelevant here since xcecd_dbus_
* manager.c itself has no #ifdef DBUS_MANAGER branches, only VIRTUAL_CLUSTER
* ones, which is exactly the code path this bug lives in).
*/
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unity/unity.h>
/* Pull the real dbus_workers_update() traversal logic under test into this
* TU. */
#include "../src/xcecd_dbus_manager.c"
/* ════════════════════════════════════════════════════════════════════════
* Link-satisfying stubs
* ════════════════════════════════════════════════════════════════════════ */
#define MAX_CALLS 64
static int g_uws_pid[MAX_CALLS];
static int g_uws_worker_status[MAX_CALLS];
static int g_uws_ncalls;
/* Mirrors just the pieces of the REAL update_worker_status()
* (xcecd_worker_utils.c) that affect dbus_workers_update()'s own control
* flow: looks the worker up by pid in the full list (same as production),
* transitions JOB_STARTING->JOB_RUNNING on a WORKER_RUNNING call and
* ->WORKER_DONE/JOB_DONE on a WORKER_DONE call. Does NOT reimplement
* xcecd_reply_startjob()/DB/IPC side effects -- those are outside this
* loop-traversal bug's boundary and are covered by other tests. */
int update_worker_status(xcecd_worker_t* worker_list, int worker_pid,
int worker_status, xcecd_job_t* job, sqlite3* db)
{
(void)db;
if (g_uws_ncalls < MAX_CALLS) {
g_uws_pid[g_uws_ncalls] = worker_pid;
g_uws_worker_status[g_uws_ncalls] = worker_status;
g_uws_ncalls++;
}
xcecd_worker_t* w = worker_list;
while (w && w->worker_pid != worker_pid) {
w = w->next;
}
if (!w) {
return XCECD_NOT_FOUND;
}
job->job_id = w->job_info.jobid;
switch (worker_status) {
case WORKER_RUNNING:
if (w->job_status == JOB_STARTING && job->job_status == JOB_RUNNING) {
w->job_status = JOB_RUNNING;
}
break;
case WORKER_DONE:
w->worker_status = WORKER_DONE;
w->job_status = JOB_DONE;
break;
case WORKER_EXITED:
w->worker_status = WORKER_EXITED;
w->job_status = JOB_EXITED;
break;
default:
break;
}
return XCECD_OK;
}
/* No real systemd/dbus unit ever exists for a VIRTUAL_CLUSTER fake-pid job
* (xcecd.c's xcecd_process_command() never calls dbus_start_worker() when
* VIRTUAL_CLUSTER is defined) -- matches both production virtual-cluster
* reality and this sandbox (no dbus session). */
bool is_systemd_job_active(const char* unit_name)
{
(void)unit_name;
return false;
}
/* get_systemd_job_exit_code() (xcecd_cgroups.c) returns -1 on any sd_bus
* failure, which is exactly what happens for a nonexistent unit -- verified
* against its real implementation. */
int get_systemd_job_exit_code(const char* unit_name)
{
(void)unit_name;
return -1;
}
int stop_systemd_job(const char* unit_name)
{
(void)unit_name;
return 0;
}
int make_job_sd_unit_name(char* buf, size_t bufsize, const char* prefix,
uint64_t jobid)
{
return snprintf(buf, bufsize, "%s-%llu.service", prefix,
(unsigned long long)jobid);
}
/* Link-satisfying stubs: xcecd_dbus_manager.c (#include'd whole) also
* defines dbus_start_worker()/dbus_get_worker_pid() (never exercised by
* this test -- only dbus_workers_update()'s traversal is under test), which
* reference these two real-systemd helpers (normally defined in
* xcecd_cgroups.c, not linked into this white-box test binary). */
char* start_systemd_job(const char* command, const res_limits_t* limits,
const char* unit_name_prefix, uint64_t job_id,
char* extra_args)
{
(void)command;
(void)limits;
(void)unit_name_prefix;
(void)job_id;
(void)extra_args;
return NULL;
}
pid_t get_systemd_job_pid(const char* unit_name)
{
(void)unit_name;
return -1;
}
/* Real check_worker_sleep_time_elapsed() (xcecd_worker_utils.c) is not
* linked into this white-box test binary (its own file isn't #include'd,
* to keep the stub surface for update_worker_status() etc. above simple
* and hermetic) -- this reimplementation is byte-identical to the real
* function's logic and is exactly what dbus_workers_update() needs to make
* its own VIRTUAL_CLUSTER JOB_RUNNING->WORKER_DONE decision, which this
* test's fixtures (sleep_time == 0) deliberately drive down the "already
* elapsed" path for fast, deterministic execution. */
bool check_worker_sleep_time_elapsed(xcecd_worker_t* worker)
{
if (!worker) {
return true;
}
if (worker->sleep_time <= 0) {
return true;
}
time_t current_time = time(NULL);
if (current_time == (time_t)-1) {
return true;
}
return difftime(current_time, worker->start_time) >= worker->sleep_time;
}
/* ════════════════════════════════════════════════════════════════════════
* Fixtures
* ════════════════════════════════════════════════════════════════════════ */
static xcecd_worker_t* make_starting_worker(int pid, uint64_t jobid)
{
xcecd_worker_t* w = calloc(1, sizeof(*w));
w->worker_pid = pid;
w->worker_status = WORKER_RUNNING;
w->job_status = JOB_STARTING;
w->job_info.jobid = jobid;
/* sleep_time == 0 -> check_worker_sleep_time_elapsed() returns true
* immediately (matches a job whose command is not a literal "sleep N",
* i.e. most real test payloads) -- keeps this test deterministic and
* fast (no real elapsed-time dependency) while still exercising the
* exact branches production hits. */
w->sleep_time = 0;
w->start_time = time(NULL);
return w;
}
static void link_worker(xcecd_worker_t** list, xcecd_worker_t* w)
{
if (!*list) {
*list = w;
return;
}
xcecd_worker_t* cur = *list;
while (cur->next) {
cur = cur->next;
}
cur->next = w;
w->prev = cur;
}
static void free_worker_list(xcecd_worker_t* list)
{
while (list) {
xcecd_worker_t* next = list->next;
free(list);
list = next;
}
}
static bool started_call_seen(int pid)
{
for (int i = 0; i < g_uws_ncalls; i++) {
if (g_uws_pid[i] == pid && g_uws_worker_status[i] == WORKER_RUNNING) {
return true;
}
}
return false;
}
void setUp(void)
{
g_uws_ncalls = 0;
}
void tearDown(void) { return; }
/* ════════════════════════════════════════════════════════════════════════
* bug-1589 — every JOB_STARTING worker in the list must get its start
* transition (the event that fires xcecd_reply_startjob()/on_job_start in
* production) within one dbus_workers_update() call, not just the first.
* ════════════════════════════════════════════════════════════════════════ */
static void test_all_array_elements_get_started_in_one_pass(void)
{
/* Mirrors the user's live report: a 5-element job array dispatched to
* one worker, all within one control loop tick. */
xcecd_worker_t* list = NULL;
xcecd_worker_t* workers[5];
for (int i = 0; i < 5; i++) {
workers[i] = make_starting_worker(1000 + i, 100 + (uint64_t)i);
link_worker(&list, workers[i]);
}
xcecd_job_t job = { 0 };
int ret = dbus_workers_update(&list, &job, NULL);
TEST_ASSERT_EQUAL_INT(XCECD_OK, ret);
for (int i = 0; i < 5; i++) {
char msg[160];
snprintf(msg, sizeof(msg),
"array element [%d] (pid %d) never received its "
"WORKER_RUNNING/on_job_start transition -- starved by "
"an earlier worker in the list (bug-1589)",
i, 1000 + i);
TEST_ASSERT_TRUE_MESSAGE(started_call_seen(1000 + i), msg);
}
free_worker_list(list);
}
/* Regression guard for the specific `return true` abort: a worker that is
* ALREADY WORKER_DONE at the head of the list (e.g. a stale entry not yet
* reaped by cleanup_worker_list()) must not stop the scan of the rest of
* the list. */
static void test_done_head_worker_does_not_block_rest_of_list(void)
{
xcecd_worker_t* list = NULL;
xcecd_worker_t* done_head = calloc(1, sizeof(*done_head));
done_head->worker_pid = 2000;
done_head->worker_status = WORKER_DONE;
done_head->job_status = JOB_DONE;
done_head->job_info.jobid = 200;
link_worker(&list, done_head);
xcecd_worker_t* starting = make_starting_worker(2001, 201);
link_worker(&list, starting);
xcecd_job_t job = { 0 };
int ret = dbus_workers_update(&list, &job, NULL);
TEST_ASSERT_EQUAL_INT(XCECD_OK, ret);
TEST_ASSERT_TRUE_MESSAGE(
started_call_seen(2001),
"a DONE worker at the head of the list must not abort the scan "
"before the still-JOB_STARTING worker behind it is reached");
free_worker_list(list);
}
/* ════════════════════════════════════════════════════════════════════════
* main
* ════════════════════════════════════════════════════════════════════════ */
int main(void)
{
xce_init_xcecd_log("/tmp/test_dbus_workers_update.log", 1024 * 1024, 2);
UNITY_BEGIN();
RUN_TEST(test_all_array_elements_get_started_in_one_pass);
RUN_TEST(test_done_head_worker_does_not_block_rest_of_list);
return UNITY_END();
}

138
src/worker/testsuite/test_resource_sync.c Normal file
View file

@ -0,0 +1,138 @@
/*
* test_resource_sync.c - regression test for bug-1569: collect_load_indices()
* (resource_sync.c) must tag every periodic resource value with the
* worker's own JOIN identity (xcecd_ctx.virtual_name -- the -n/--name value,
* set from config->virutal_name in xcecd_init(), xcecd.c), not
* gethostname()'s raw OS/container hostname.
*
* Why this matters: the controller forwards each report's .host field to
* the scheduler UNMODIFIED (do_update_resvals(), src/controller/opupd.c),
* which keys EVERY other per-worker resource value by this SAME join
* identity (mark_host_res()/update_val_res(), src/controller/worker.c;
* sched_get_worker(), src/scheduler/src/core_events.c). A mismatch here
* silently drops every periodic ("mem"/"r15s"/etc.) resource report for the
* real worker -- live-confirmed against two independent deployments in this
* sandbox: gethostname() returned a generic container hostname ("xce"), not
* the worker's configured "-n pc13v" identity, so the scheduler's own
* "unknown worker xce, skipping" WARN fired every ~15s (resource_sync's own
* LOAD_INDICES_INTERVAL_DEFAULT) and "info worker pc13v" never showed real
* mem/r15s/etc values despite xce_test's live `resval` query showing them
* correctly (a DIFFERENT, correctly-addressed query path).
*
* White-box #include of ../src/resource_sync.c (same convention this
* suite already establishes for controller_connector.c/xcecd.c). The 11
* builtin load-index getters (node_info_get_15s/etc., real implementations
* in node_information.c, not linked here) are stubbed as trivial
* deterministic returns -- this test is about collect_load_indices()'s own
* .host-tagging logic, not the real /proc-reading getters' own values.
*/
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
#include "xcecd.h"
/* Real globals this white-box TU's resource_sync.c references
* (xcecd_ctx.virtual_name) -- not linking xcecd.c itself (its own
* main()/full daemon init is out of scope for this narrow test), so a
* plain definition here satisfies the linker, matching this suite's own
* "provide just the global a white-box #include needs" idiom
* (test_xcecd_startup_order.c's g_dummy_subsystem-style stubs). */
xcecd_context_t xcecd_ctx;
/* Stubs for the 11 builtin load-index getters (node_information.c, not
* linked here) -- deterministic, non-NULL so collect_load_indices() never
* takes its own NULL-getter fallback path (out of scope for this test). */
static char* stub_getter(void)
{
return strdup("1.0");
}
char* node_info_get_15s(void) { return stub_getter(); }
char* node_info_get_1m(void) { return stub_getter(); }
char* node_info_get_15m(void) { return stub_getter(); }
char* node_info_get_ut(void) { return stub_getter(); }
char* node_info_get_pg(void) { return stub_getter(); }
char* node_info_get_ls(void) { return stub_getter(); }
char* node_info_get_it(void) { return stub_getter(); }
char* node_info_get_swap(void) { return stub_getter(); }
char* node_info_get_mem(void) { return stub_getter(); }
char* node_info_get_tmp(void) { return stub_getter(); }
char* node_info_get_io(void) { return stub_getter(); }
/* Linker stub for controller_connector.c's real implementation -- only
* referenced by resource_sync_thread() (the background polling loop),
* never invoked by this test (which calls collect_load_indices()
* directly). */
bool xcecd_send_load_indices(xce_resval_t** vals, int count)
{
(void)vals;
(void)count;
return true;
}
#include "../src/resource_sync.c"
void setUp(void) { return; }
void tearDown(void) { return; }
static void
test_collect_load_indices_tags_host_with_virtual_name_not_gethostname(void)
{
xce_resval_t* vals;
int count = 0;
int i;
memset(&xcecd_ctx, 0, sizeof(xcecd_ctx));
xcecd_ctx.virtual_name = (char*)"pc13v";
vals = collect_load_indices(&count);
TEST_ASSERT_NOT_NULL(vals);
TEST_ASSERT_TRUE(count > 0);
for (i = 0; i < count; i++) {
TEST_ASSERT_NOT_NULL(vals[i].host);
TEST_ASSERT_EQUAL_STRING("pc13v", vals[i].host);
}
free_resvals(vals, count);
printf("test_collect_load_indices_tags_host_with_virtual_name_not_"
"gethostname: PASS\n");
}
static void
test_collect_load_indices_falls_back_to_gethostname_when_virtual_name_unset(
void)
{
xce_resval_t* vals;
int count = 0;
char expected[256];
memset(&xcecd_ctx, 0, sizeof(xcecd_ctx));
xcecd_ctx.virtual_name = NULL;
TEST_ASSERT_EQUAL_INT(0, gethostname(expected, sizeof(expected)));
vals = collect_load_indices(&count);
TEST_ASSERT_NOT_NULL(vals);
TEST_ASSERT_TRUE(count > 0);
TEST_ASSERT_EQUAL_STRING(expected, vals[0].host);
free_resvals(vals, count);
printf("test_collect_load_indices_falls_back_to_gethostname_when_"
"virtual_name_unset: PASS\n");
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_collect_load_indices_tags_host_with_virtual_name_not_gethostname);
RUN_TEST(test_collect_load_indices_falls_back_to_gethostname_when_virtual_name_unset);
return UNITY_END();
}

375
src/worker/testsuite/test_xcecd_startup_order.c Normal file
View file

@ -0,0 +1,375 @@
/*
* test_xcecd_startup_order.c — white-box test for the worker daemon's
* candidate-list startup ORDERING fix (component-failover-adoption, task
* 2.2, boundary: WorkerControllerConnector, Req 1.4).
*
* Bug fixed: main.c used to assign cc_ctx->cluster_cfg AFTER xcecd_start()
* had already returned -- but xcecd_start() is what creates cc_ctx
* (controller_connector_init()) AND fires the very first join
* (controller_connector_start()). So the first connect attempt always saw
* cc_ctx->cluster_cfg in its zero-initialized state (candidates_count == 0)
* and silently ignored any real --peers list on that first attempt.
*
* Fix: cluster_cfg is now threaded through xcecd_config_t (set by main.c
* BEFORE calling xcecd_init()) -> xcecd_context_t (copied by xcecd_init())
* -> cc_ctx->cluster_cfg (copied by controller_connector_init(), which
* xcecd_start() calls before controller_connector_start()).
*
* This test pulls the REAL xcecd_init()/xcecd_start()/controller_connector_
* init() production code into this TU (#include, same white-box pattern as
* test_controller_connector.c) and drives the real call sequence main.c ->
* xcecd_init() -> xcecd_start() uses, so a regression in the threading (or
* in xcecd_start()'s call ordering) would make this test fail exactly the
* way the pre-2.2 bug did.
*
* Why this is a SEPARATE file from test_controller_connector.c rather than
* an extension of it: xcecd.c and controller_connector.c cannot both be
* #include-d into the same translation unit. Both are white-box-included as
* whole files, and both would then try to provide real, conflicting
* definitions of shared symbols (xcecd_process_command is a real, non-static
* function in xcecd.c but is stubbed by hand in test_controller_connector.c
* for controller_connector.c's sake; cc_ctx is a real global controller_
* connector.c defines that xcecd.c only declares extern). This file instead
* stubs controller_connector.c's small exported surface that xcecd.c
* actually calls (start_controller_connector_thread, controller_connector_
* start, checking_connection, xcecd_job_start_error) and defines cc_ctx
* itself -- the same "link-satisfying stub" convention test_controller_
* connector.c already established, just pointed the other direction.
*
* Build flags: -DDBUS_MANAGER -DVIRTUAL_CLUSTER, the SAME flag pair the real
* `xcecd_virtual` production executable already uses (worker/meson.build) --
* not a new, never-exercised configuration. VIRTUAL_CLUSTER additionally
* skips xcecd_start()'s sqlite/db-recovery block and xcecd_process_command's
* dbus worker-launch block entirely (both `#ifndef VIRTUAL_CLUSTER`), which
* keeps this test's stub surface to the daemon's subsystem constructors
* (thread pool, process/system/hardware monitors, resource sync) rather than
* also needing a real or stubbed sqlite3/dbus stack.
*/
#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
/* Pull the real xcecd_init()/xcecd_start()/controller_connector_init() logic
* under test into this TU. */
#include "../src/xcecd.c"
/* cc_ctx is normally defined in controller_connector.c (not included here --
* see file header comment for why). */
controller_connector_ctx_t* cc_ctx = NULL;
/* ════════════════════════════════════════════════════════════════════════
* Link-satisfying stubs
*
* Subsystem constructors (thread pool / process / system / hardware
* monitors / resource sync) return a non-NULL dummy pointer so xcecd_init()
* /xcecd_start() reach their real success paths -- this test cares about
* the cluster_cfg threading through those functions, not about hardware/
* process monitoring behavior. Nothing in this test's runtime path
* (xcecd_init() + xcecd_start(), never xcecd_cleanup() or
* xcecd_process_command()) dereferences these pointers, so a shared dummy
* buffer is sufficient and safe under ASAN.
* ════════════════════════════════════════════════════════════════════════ */
static char g_dummy_subsystem[4096];
thread_pool_t* thread_pool_create(size_t n)
{
(void)n;
return (thread_pool_t*)(void*)g_dummy_subsystem;
}
void thread_pool_destroy(thread_pool_t* pool) { (void)pool; }
process_manager_t* process_manager_create(void)
{
return (process_manager_t*)(void*)g_dummy_subsystem;
}
void process_manager_destroy(process_manager_t* manager) { (void)manager; }
bool process_manager_update(process_manager_t* manager)
{
(void)manager;
return true;
}
system_monitor_t* system_monitor_create(void)
{
return (system_monitor_t*)(void*)g_dummy_subsystem;
}
void system_monitor_destroy(system_monitor_t* monitor) { (void)monitor; }
bool system_monitor_update(system_monitor_t* monitor)
{
(void)monitor;
return true;
}
hardware_monitor_t* hardware_monitor_create(const char* acct_file)
{
(void)acct_file;
return (hardware_monitor_t*)(void*)g_dummy_subsystem;
}
void hardware_monitor_destroy(hardware_monitor_t* monitor) { (void)monitor; }
bool hardware_monitor_update(hardware_monitor_t* monitor)
{
(void)monitor;
return true;
}
int hardware_monitor_get_cpu_cores(hardware_monitor_t* monitor)
{
(void)monitor;
return 4;
}
double hardware_monitor_get_total_memory(hardware_monitor_t* monitor)
{
(void)monitor;
return 16384.0;
}
network_handler_t* network_handler_create(uint16_t port)
{
(void)port;
return (network_handler_t*)(void*)g_dummy_subsystem;
}
int network_handler_start(network_handler_t* handler)
{
(void)handler;
return 0;
}
void network_handler_destroy(network_handler_t* handler) { (void)handler; }
resource_sync_t* resource_sync_create(void)
{
return (resource_sync_t*)(void*)g_dummy_subsystem;
}
void resource_sync_destroy(resource_sync_t* sync) { (void)sync; }
bool resource_sync_start(resource_sync_t* sync)
{
(void)sync;
return true;
}
void resource_sync_stop(resource_sync_t* sync) { (void)sync; }
/* Referenced by xcecd_process_command()/xcecd_cleanup() -- never invoked at
* runtime by this test (only xcecd_init()/xcecd_start() are called), but
* their bodies are compiled unconditionally so the symbols must resolve. */
void add_worker(xcecd_worker_t** worker_list, int worker_pid,
int worker_status, uint64_t job_id, int ntasks,
job_startinfo_t* startinfo)
{
(void)worker_list;
(void)worker_pid;
(void)worker_status;
(void)job_id;
(void)ntasks;
(void)startinfo;
}
bool xcecd_db_insert_worker(xcecd_db_t* db, int worker_pid, int worker_status,
uint64_t job_id, int job_status)
{
(void)db;
(void)worker_pid;
(void)worker_status;
(void)job_id;
(void)job_status;
return true;
}
bool ipc_master_check_connection(void) { return true; }
pid_t dbus_get_worker_pid(const char* unit_name)
{
(void)unit_name;
return 1;
}
void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db)
{
(void)worker_list;
(void)db;
}
void cleanup_worker_list(xcecd_worker_t** worker_list, sqlite3* db)
{
(void)worker_list;
(void)db;
}
void xcecd_db_close(xcecd_db_t* db) { (void)db; }
void xcecd_db_is_available(xcecd_db_t* db, const char* db_path)
{
(void)db;
(void)db_path;
}
int submit_fork_task(thread_pool_t* pool, fork_task_t* task)
{
(void)pool;
(void)task;
return 0;
}
/* controller_connector.c's small exported surface xcecd.c actually calls
* (verified by grep -- start_controller_connector_thread,
* controller_connector_start, checking_connection, xcecd_job_start_error
* are the only 4). */
int xcecd_job_start_error(uint64_t jobid)
{
(void)jobid;
return 0;
}
int checking_connection(void) { return 0; }
/* controller_connector_start() is the function that fires the very first
* join attempt in production (xcecd_join_cluster() inside its retry loop).
* This test double stands in for it and snapshots cc_ctx->cluster_cfg at
* the EXACT moment production would fire that first attempt -- the core
* Req 1.4 assertion: was the real candidate list already there by then? */
static xce_cluster_config_t g_captured_cluster_cfg_at_first_join;
static bool g_controller_connector_start_called = false;
bool controller_connector_start(void)
{
g_controller_connector_start_called = true;
if (cc_ctx) {
g_captured_cluster_cfg_at_first_join = cc_ctx->cluster_cfg;
}
return true;
}
int start_controller_connector_thread(task_func_t function, void* arg)
{
/* Deliberately does NOT start a real thread -- connector_to_controller
* (which would call the real checking_connection() in a loop) must
* never actually run in this unit test. */
(void)function;
(void)arg;
return 0;
}
/* ════════════════════════════════════════════════════════════════════════
* Fixtures
* ════════════════════════════════════════════════════════════════════════ */
static xcecd_config_t base_test_config(void)
{
xcecd_config_t config = { 0 };
config.thread_pool_size = 1;
config.resource_update_interval = 1;
config.job_status_update_interval = 1;
config.port = 0; /* skip network_handler entirely */
config.acct_file = "/tmp/test_xcecd_startup_order.acct";
config.log_file = "/tmp/test_xcecd_startup_order.log";
config.db_path = "/tmp/test_xcecd_startup_order.db";
config.work_dir = "/tmp";
config.domain_sock_file = "/tmp/.test_xcecd_startup_order.sock";
config.virutal_name = "workertest";
config.remote_host = "localhost";
config.remote_port = 9990;
config.join_interval = 1;
config.update_interval = 1;
config.debug_level = 2;
return config;
}
void setUp(void)
{
memset(&xcecd_ctx, 0, sizeof(xcecd_ctx));
cc_ctx = NULL;
g_controller_connector_start_called = false;
memset(&g_captured_cluster_cfg_at_first_join, 0,
sizeof(g_captured_cluster_cfg_at_first_join));
}
void tearDown(void) { return; }
/* ════════════════════════════════════════════════════════════════════════
* Req 1.4 — cc_ctx->cluster_cfg is fully populated BEFORE the first join
* attempt fires, for a multi-candidate --peers config.
* ════════════════════════════════════════════════════════════════════════ */
static void
test_multi_candidate_cluster_cfg_populated_before_first_join(void)
{
xce_cluster_config_t peers_cfg = { 0 };
peers_cfg.candidates_count = 3;
strncpy(peers_cfg.candidates[0].hostname, "ctrl-a",
XCE_MAXHOSTNAME_LEN - 1);
peers_cfg.candidates[0].port = 9991;
strncpy(peers_cfg.candidates[1].hostname, "ctrl-b",
XCE_MAXHOSTNAME_LEN - 1);
peers_cfg.candidates[1].port = 9992;
strncpy(peers_cfg.candidates[2].hostname, "ctrl-c",
XCE_MAXHOSTNAME_LEN - 1);
peers_cfg.candidates[2].port = 9993;
strncpy(peers_cfg.identity, "workertest", sizeof(peers_cfg.identity) - 1);
xcecd_config_t config = base_test_config();
/* This is main.c's fix under test: cluster_cfg is set onto the config
* BEFORE xcecd_init()/xcecd_start() run (main.c:~430), not assigned onto
* cc_ctx afterward. */
config.cluster_cfg = peers_cfg;
TEST_ASSERT_EQUAL_INT(0, xcecd_init(&config));
TEST_ASSERT_EQUAL_INT(0, xcecd_start());
TEST_ASSERT_TRUE_MESSAGE(
g_controller_connector_start_called,
"controller_connector_start() (the first join attempt) must have "
"been invoked by xcecd_start()");
/* The core assertion: by the time the first join fires, cc_ctx-> */
/* cluster_cfg already holds the REAL 3-candidate --peers list -- not */
/* the zero-initialized/synthesized-single-candidate shape the pre-2.2 */
/* bug would have produced here. */
TEST_ASSERT_EQUAL_INT(
3, g_captured_cluster_cfg_at_first_join.candidates_count);
TEST_ASSERT_EQUAL_STRING(
"ctrl-a", g_captured_cluster_cfg_at_first_join.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(
9991, g_captured_cluster_cfg_at_first_join.candidates[0].port);
TEST_ASSERT_EQUAL_STRING(
"ctrl-b", g_captured_cluster_cfg_at_first_join.candidates[1].hostname);
TEST_ASSERT_EQUAL_INT(
9992, g_captured_cluster_cfg_at_first_join.candidates[1].port);
TEST_ASSERT_EQUAL_STRING(
"ctrl-c", g_captured_cluster_cfg_at_first_join.candidates[2].hostname);
TEST_ASSERT_EQUAL_INT(
9993, g_captured_cluster_cfg_at_first_join.candidates[2].port);
printf(
"test_multi_candidate_cluster_cfg_populated_before_first_join: "
"PASS\n");
}
/* Regression guard: the no-`--peers` case (cluster_cfg left {0}) must still
* reach the first join attempt with an EMPTY cc_ctx->cluster_cfg (candidates_
* count == 0) -- the synthesize-single-candidate path inside build_cluster_
* config() (controller_connector.c, task 2.1) owns turning that into a
* one-candidate list; this ordering fix must not accidentally force a
* candidate in cc_ctx->cluster_cfg when none was configured. */
static void test_no_peers_cluster_cfg_stays_empty_before_first_join(void)
{
xcecd_config_t config = base_test_config();
/* config.cluster_cfg left {0} -- no --peers given. */
TEST_ASSERT_EQUAL_INT(0, xcecd_init(&config));
TEST_ASSERT_EQUAL_INT(0, xcecd_start());
TEST_ASSERT_TRUE(g_controller_connector_start_called);
TEST_ASSERT_EQUAL_INT(
0, g_captured_cluster_cfg_at_first_join.candidates_count);
printf(
"test_no_peers_cluster_cfg_stays_empty_before_first_join: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* main
* ════════════════════════════════════════════════════════════════════════ */
int main(void)
{
xce_init_xcecd_log("/tmp/test_xcecd_startup_order.log", 1024 * 1024, 2);
UNITY_BEGIN();
RUN_TEST(test_multi_candidate_cluster_cfg_populated_before_first_join);
RUN_TEST(test_no_peers_cluster_cfg_stays_empty_before_first_join);
return UNITY_END();
}