myxce/src/worker/testsuite/test_xcecd_startup_order.c

/*
 * test_xcecd_startup_order.c — white-box test for the worker daemon's
 * candidate-list startup ORDERING fix (component-failover-adoption, task
 * 2.2, boundary: WorkerControllerConnector, Req 1.4).
 *
 * Bug fixed: main.c used to assign cc_ctx->cluster_cfg AFTER xcecd_start()
 * had already returned -- but xcecd_start() is what creates cc_ctx
 * (controller_connector_init()) AND fires the very first join
 * (controller_connector_start()). So the first connect attempt always saw
 * cc_ctx->cluster_cfg in its zero-initialized state (candidates_count == 0)
 * and silently ignored any real --peers list on that first attempt.
 *
 * Fix: cluster_cfg is now threaded through xcecd_config_t (set by main.c
 * BEFORE calling xcecd_init()) -> xcecd_context_t (copied by xcecd_init())
 * -> cc_ctx->cluster_cfg (copied by controller_connector_init(), which
 * xcecd_start() calls before controller_connector_start()).
 *
 * This test pulls the REAL xcecd_init()/xcecd_start()/controller_connector_
 * init() production code into this TU (#include, same white-box pattern as
 * test_controller_connector.c) and drives the real call sequence main.c ->
 * xcecd_init() -> xcecd_start() uses, so a regression in the threading (or
 * in xcecd_start()'s call ordering) would make this test fail exactly the
 * way the pre-2.2 bug did.
 *
 * Why this is a SEPARATE file from test_controller_connector.c rather than
 * an extension of it: xcecd.c and controller_connector.c cannot both be
 * #include-d into the same translation unit. Both are white-box-included as
 * whole files, and both would then try to provide real, conflicting
 * definitions of shared symbols (xcecd_process_command is a real, non-static
 * function in xcecd.c but is stubbed by hand in test_controller_connector.c
 * for controller_connector.c's sake; cc_ctx is a real global controller_
 * connector.c defines that xcecd.c only declares extern). This file instead
 * stubs controller_connector.c's small exported surface that xcecd.c
 * actually calls (start_controller_connector_thread, controller_connector_
 * start, checking_connection, xcecd_job_start_error) and defines cc_ctx
 * itself -- the same "link-satisfying stub" convention test_controller_
 * connector.c already established, just pointed the other direction.
 *
 * Build flags: -DDBUS_MANAGER -DVIRTUAL_CLUSTER, the SAME flag pair the real
 * `xcecd_virtual` production executable already uses (worker/meson.build) --
 * not a new, never-exercised configuration. VIRTUAL_CLUSTER additionally
 * skips xcecd_start()'s sqlite/db-recovery block and xcecd_process_command's
 * dbus worker-launch block entirely (both `#ifndef VIRTUAL_CLUSTER`), which
 * keeps this test's stub surface to the daemon's subsystem constructors
 * (thread pool, process/system/hardware monitors, resource sync) rather than
 * also needing a real or stubbed sqlite3/dbus stack.
 */
#include <errno.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <unity/unity.h>

/* Pull the real xcecd_init()/xcecd_start()/controller_connector_init() logic
 * under test into this TU. */
#include "../src/xcecd.c"

/* cc_ctx is normally defined in controller_connector.c (not included here --
 * see file header comment for why). */
controller_connector_ctx_t* cc_ctx = NULL;

/* ════════════════════════════════════════════════════════════════════════
 * Link-satisfying stubs
 *
 * Subsystem constructors (thread pool / process / system / hardware
 * monitors / resource sync) return a non-NULL dummy pointer so xcecd_init()
 * /xcecd_start() reach their real success paths -- this test cares about
 * the cluster_cfg threading through those functions, not about hardware/
 * process monitoring behavior. Nothing in this test's runtime path
 * (xcecd_init() + xcecd_start(), never xcecd_cleanup() or
 * xcecd_process_command()) dereferences these pointers, so a shared dummy
 * buffer is sufficient and safe under ASAN.
 * ════════════════════════════════════════════════════════════════════════ */
static char g_dummy_subsystem[4096];

thread_pool_t* thread_pool_create(size_t n)
{
    (void)n;
    return (thread_pool_t*)(void*)g_dummy_subsystem;
}
void thread_pool_destroy(thread_pool_t* pool) { (void)pool; }

process_manager_t* process_manager_create(void)
{
    return (process_manager_t*)(void*)g_dummy_subsystem;
}
void process_manager_destroy(process_manager_t* manager) { (void)manager; }
bool process_manager_update(process_manager_t* manager)
{
    (void)manager;
    return true;
}

system_monitor_t* system_monitor_create(void)
{
    return (system_monitor_t*)(void*)g_dummy_subsystem;
}
void system_monitor_destroy(system_monitor_t* monitor) { (void)monitor; }
bool system_monitor_update(system_monitor_t* monitor)
{
    (void)monitor;
    return true;
}

hardware_monitor_t* hardware_monitor_create(const char* acct_file)
{
    (void)acct_file;
    return (hardware_monitor_t*)(void*)g_dummy_subsystem;
}
void hardware_monitor_destroy(hardware_monitor_t* monitor) { (void)monitor; }
bool hardware_monitor_update(hardware_monitor_t* monitor)
{
    (void)monitor;
    return true;
}
int hardware_monitor_get_cpu_cores(hardware_monitor_t* monitor)
{
    (void)monitor;
    return 4;
}
double hardware_monitor_get_total_memory(hardware_monitor_t* monitor)
{
    (void)monitor;
    return 16384.0;
}
double hardware_monitor_get_total_swap(hardware_monitor_t* monitor)
{
    (void)monitor;
    return 4096.0;
}
bool hardware_monitor_get_disk_info(const char* path, int64_t* total_bytes)
{
    (void)path;
    *total_bytes = 8192;
    return true;
}

network_handler_t* network_handler_create(uint16_t port)
{
    (void)port;
    return (network_handler_t*)(void*)g_dummy_subsystem;
}
int network_handler_start(network_handler_t* handler)
{
    (void)handler;
    return 0;
}
void network_handler_destroy(network_handler_t* handler) { (void)handler; }

resource_sync_t* resource_sync_create(void)
{
    return (resource_sync_t*)(void*)g_dummy_subsystem;
}
void resource_sync_destroy(resource_sync_t* sync) { (void)sync; }
bool resource_sync_start(resource_sync_t* sync)
{
    (void)sync;
    return true;
}
void resource_sync_stop(resource_sync_t* sync) { (void)sync; }

/* Referenced by xcecd_process_command()/xcecd_cleanup() -- never invoked at
 * runtime by this test (only xcecd_init()/xcecd_start() are called), but
 * their bodies are compiled unconditionally so the symbols must resolve. */
void add_worker(xcecd_worker_t** worker_list, int worker_pid,
                int worker_status, uint64_t job_id, int ntasks,
                job_startinfo_t* startinfo)
{
    (void)worker_list;
    (void)worker_pid;
    (void)worker_status;
    (void)job_id;
    (void)ntasks;
    (void)startinfo;
}
bool xcecd_db_insert_worker(xcecd_db_t* db, int worker_pid, int worker_status,
                            uint64_t job_id, int job_status)
{
    (void)db;
    (void)worker_pid;
    (void)worker_status;
    (void)job_id;
    (void)job_status;
    return true;
}
bool ipc_master_check_connection(void) { return true; }
pid_t dbus_get_worker_pid(const char* unit_name)
{
    (void)unit_name;
    return 1;
}
void dbus_stop_all_workers(xcecd_worker_t** worker_list, sqlite3* db)
{
    (void)worker_list;
    (void)db;
}
void cleanup_worker_list(xcecd_worker_t** worker_list, sqlite3* db)
{
    (void)worker_list;
    (void)db;
}
void xcecd_db_close(xcecd_db_t* db) { (void)db; }
void xcecd_db_is_available(xcecd_db_t* db, const char* db_path)
{
    (void)db;
    (void)db_path;
}
int submit_fork_task(thread_pool_t* pool, fork_task_t* task)
{
    (void)pool;
    (void)task;
    return 0;
}

/* controller_connector.c's small exported surface xcecd.c actually calls
 * (verified by grep -- start_controller_connector_thread,
 * controller_connector_start, checking_connection, xcecd_job_start_error
 * are the only 4). */
int xcecd_job_start_error(uint64_t jobid)
{
    (void)jobid;
    return 0;
}
int checking_connection(void) { return 0; }

/* controller_connector_start() is the function that fires the very first
 * join attempt in production (xcecd_join_cluster() inside its retry loop).
 * This test double stands in for it and snapshots cc_ctx->cluster_cfg at
 * the EXACT moment production would fire that first attempt -- the core
 * Req 1.4 assertion: was the real candidate list already there by then? */
static xce_cluster_config_t g_captured_cluster_cfg_at_first_join;
static bool                 g_controller_connector_start_called = false;

bool controller_connector_start(void)
{
    g_controller_connector_start_called = true;
    if (cc_ctx) {
        g_captured_cluster_cfg_at_first_join = cc_ctx->cluster_cfg;
    }
    return true;
}

int start_controller_connector_thread(task_func_t function, void* arg)
{
    /* Deliberately does NOT start a real thread -- connector_to_controller
     * (which would call the real checking_connection() in a loop) must
     * never actually run in this unit test. */
    (void)function;
    (void)arg;
    return 0;
}

/* ════════════════════════════════════════════════════════════════════════
 * Fixtures
 * ════════════════════════════════════════════════════════════════════════ */

static xcecd_config_t base_test_config(void)
{
    xcecd_config_t config = { 0 };
    config.thread_pool_size           = 1;
    config.resource_update_interval   = 1;
    config.job_status_update_interval = 1;
    config.port                       = 0; /* skip network_handler entirely */
    config.acct_file                  = "/tmp/test_xcecd_startup_order.acct";
    config.log_file                   = "/tmp/test_xcecd_startup_order.log";
    config.db_path                    = "/tmp/test_xcecd_startup_order.db";
    config.work_dir                   = "/tmp";
    config.domain_sock_file = "/tmp/.test_xcecd_startup_order.sock";
    config.virutal_name     = "workertest";
    config.remote_host      = "localhost";
    config.remote_port      = 9990;
    config.join_interval    = 1;
    config.update_interval  = 1;
    config.debug_level      = 2;
    return config;
}

void setUp(void)
{
    memset(&xcecd_ctx, 0, sizeof(xcecd_ctx));
    cc_ctx                               = NULL;
    g_controller_connector_start_called  = false;
    memset(&g_captured_cluster_cfg_at_first_join, 0,
          sizeof(g_captured_cluster_cfg_at_first_join));
}

void tearDown(void) { return; }

/* ════════════════════════════════════════════════════════════════════════
 * Req 1.4 — cc_ctx->cluster_cfg is fully populated BEFORE the first join
 * attempt fires, for a multi-candidate --peers config.
 * ════════════════════════════════════════════════════════════════════════ */

static void
test_multi_candidate_cluster_cfg_populated_before_first_join(void)
{
    xce_cluster_config_t peers_cfg = { 0 };
    peers_cfg.candidates_count     = 3;
    strncpy(peers_cfg.candidates[0].hostname, "ctrl-a",
            XCE_MAXHOSTNAME_LEN - 1);
    peers_cfg.candidates[0].port = 9991;
    strncpy(peers_cfg.candidates[1].hostname, "ctrl-b",
            XCE_MAXHOSTNAME_LEN - 1);
    peers_cfg.candidates[1].port = 9992;
    strncpy(peers_cfg.candidates[2].hostname, "ctrl-c",
            XCE_MAXHOSTNAME_LEN - 1);
    peers_cfg.candidates[2].port = 9993;
    strncpy(peers_cfg.identity, "workertest", sizeof(peers_cfg.identity) - 1);

    xcecd_config_t config = base_test_config();
    /* This is main.c's fix under test: cluster_cfg is set onto the config
     * BEFORE xcecd_init()/xcecd_start() run (main.c:~430), not assigned onto
     * cc_ctx afterward. */
    config.cluster_cfg = peers_cfg;

    TEST_ASSERT_EQUAL_INT(0, xcecd_init(&config));
    TEST_ASSERT_EQUAL_INT(0, xcecd_start());

    TEST_ASSERT_TRUE_MESSAGE(
        g_controller_connector_start_called,
        "controller_connector_start() (the first join attempt) must have "
        "been invoked by xcecd_start()");

    /* The core assertion: by the time the first join fires, cc_ctx-> */
    /* cluster_cfg already holds the REAL 3-candidate --peers list -- not */
    /* the zero-initialized/synthesized-single-candidate shape the pre-2.2 */
    /* bug would have produced here. */
    TEST_ASSERT_EQUAL_INT(
        3, g_captured_cluster_cfg_at_first_join.candidates_count);
    TEST_ASSERT_EQUAL_STRING(
        "ctrl-a", g_captured_cluster_cfg_at_first_join.candidates[0].hostname);
    TEST_ASSERT_EQUAL_INT(
        9991, g_captured_cluster_cfg_at_first_join.candidates[0].port);
    TEST_ASSERT_EQUAL_STRING(
        "ctrl-b", g_captured_cluster_cfg_at_first_join.candidates[1].hostname);
    TEST_ASSERT_EQUAL_INT(
        9992, g_captured_cluster_cfg_at_first_join.candidates[1].port);
    TEST_ASSERT_EQUAL_STRING(
        "ctrl-c", g_captured_cluster_cfg_at_first_join.candidates[2].hostname);
    TEST_ASSERT_EQUAL_INT(
        9993, g_captured_cluster_cfg_at_first_join.candidates[2].port);

    printf(
        "test_multi_candidate_cluster_cfg_populated_before_first_join: "
        "PASS\n");
}

/* Regression guard: the no-`--peers` case (cluster_cfg left {0}) must still
 * reach the first join attempt with an EMPTY cc_ctx->cluster_cfg (candidates_
 * count == 0) -- the synthesize-single-candidate path inside build_cluster_
 * config() (controller_connector.c, task 2.1) owns turning that into a
 * one-candidate list; this ordering fix must not accidentally force a
 * candidate in cc_ctx->cluster_cfg when none was configured. */
static void test_no_peers_cluster_cfg_stays_empty_before_first_join(void)
{
    xcecd_config_t config = base_test_config();
    /* config.cluster_cfg left {0} -- no --peers given. */

    TEST_ASSERT_EQUAL_INT(0, xcecd_init(&config));
    TEST_ASSERT_EQUAL_INT(0, xcecd_start());

    TEST_ASSERT_TRUE(g_controller_connector_start_called);
    TEST_ASSERT_EQUAL_INT(
        0, g_captured_cluster_cfg_at_first_join.candidates_count);

    printf(
        "test_no_peers_cluster_cfg_stays_empty_before_first_join: PASS\n");
}

/* ════════════════════════════════════════════════════════════════════════
 * bug-1554: init_host_attrs() must source maxswap/maxtmp from real
 * detection (hardware_monitor_get_total_swap()/hardware_monitor_get_disk_
 * info(), both stubbed above) instead of the old hardcoded placeholder
 * constants (1024MB/2048MB) -- this test would have kept passing under the
 * old stub-constant code too (it doesn't assert the old magic numbers are
 * ABSENT), but it does pin that host_attrs picks up whatever these two
 * functions report, which the placeholder code structurally could not do.
 * ════════════════════════════════════════════════════════════════════════ */

static void test_init_host_attrs_uses_real_swap_and_tmp_detection(void)
{
    xcecd_config_t config = base_test_config();

    TEST_ASSERT_EQUAL_INT(0, xcecd_init(&config));
    TEST_ASSERT_EQUAL_INT(0, xcecd_start());

    TEST_ASSERT_EQUAL_FLOAT(4096.0f, (float)host_attrs.maxswap);
    TEST_ASSERT_EQUAL_FLOAT(8192.0f, (float)host_attrs.maxtmp);

    printf("test_init_host_attrs_uses_real_swap_and_tmp_detection: PASS\n");
}

/* ════════════════════════════════════════════════════════════════════════
 * main
 * ════════════════════════════════════════════════════════════════════════ */

int main(void)
{
    xce_init_xcecd_log("/tmp/test_xcecd_startup_order.log", 1024 * 1024, 2);

    UNITY_BEGIN();

    RUN_TEST(test_multi_candidate_cluster_cfg_populated_before_first_join);
    RUN_TEST(test_no_peers_cluster_cfg_stays_empty_before_first_join);
    RUN_TEST(test_init_host_attrs_uses_real_swap_and_tmp_detection);

    return UNITY_END();
}