/* * test_controller_connector.c — white-box tests for the worker's unified * connect/rejoin pipeline (component-failover-adoption, task 2.1, * boundary: WorkerControllerConnector). * * Pulls the real (mostly-static) connect logic in controller_connector.c * into this TU (same white-box pattern as scheduler/testsuite/test_handlers.c * #include-ing handler.c directly), so the tests exercise the actual * production code, not a re-implementation of it. controller_connector.c's * job-handling externs this file never exercises (thread_pool, uv_ipc, * xcecd_worker_utils, xcecd_exdata) are link-satisfied by trivial stubs * below, same rationale as test_handlers.c's stubs. The stub definitions * are placed AFTER the #include so their parameter types (thread_pool_t, * fork_task_t, xcecd_worker_t, res_limits_t) come from the real headers * controller_connector.c already pulled in -- controller_connector.h's own * `api_ctx_t` (an unused leftover type, only ever referenced from commented- * out code) also means this file must NOT additionally include * src/lib/client_internal.h: that header declares its OWN, differently- * shaped `api_ctx_t`, and the two conflict in the same translation unit. * * Covers: * - Req 1.5: no-candidate-list startup and a single-entry candidate list * converge on the identical validated xce_cluster_config_t shape (same * downstream connect log sequence, since those logs are driven entirely * by that struct's contents) -- proving there is no separate single-host * code path anymore. * - Req 5.1/5.2: a real multi-candidate list passes through unmodified * (no truncation/branch-specific mangling before the shared helper). * - Req 5.1/5.2 (rejoin callback contract, design.md "Rejoin callback * constraint"): the registered rejoin callback only sets a flag and * returns immediately -- no I/O, no calling back into the client * library. * - Req 2.1-2.3: checking_connection()'s old bounded give-up * (MAX_RETRY_COUNT=5 then handle_disconnect() tearing the worker down * for good) is gone -- an outage far longer than that old bound must * never permanently disable the worker, and a single probe must return * fast (the library, not this probe, now owns retry/backoff). * * Task 2.3 additions (Req 1.6, 3.1, 3.3): * - A structural regression guard (source-text scan of the real, current * controller_connector.c, same genre as scheduler/testsuite's * test_core_isolation.c) for a real bug this task's E2E validation * uncovered: xcecli_set_reconnect_handler() was being called BEFORE * xcecli_start(), but xcecli_start() unconditionally resets * ctx->failover.rejoin_cb to NULL as part of initializing per-context * failover state (client.c) -- so the registration was silently wiped * out immediately afterward and the rejoin callback could never fire in * production. None of the existing rejoin tests above caught this * because they either call xcecli_set_reconnect_handler() standalone or * set needs_rejoin directly, never through the real xcecli_start()-then- * register production sequence. * - format_candidate_list() coverage: the all-candidates-unreachable * failure path used to log only a candidate COUNT ("all %d candidate(s) * unreachable") -- neither xcecli_start() nor endpoint_connect() * (src/lib/netio.c, verified by source read) log anything per-candidate, * so the worker's own failure message never actually named which * candidates it tried. format_candidate_list() (task 2.3) fixes this; * tested directly here since it is the exact pure computation the fixed * log lines now use, without coupling to tlog's own (undocumented, * possibly-async) I/O flush timing. * - A thread-isolation concurrency test: a blocked/slow rejoin_call() (the * connector thread's own simulated "stuck in retry/backoff" case) must * not stall a concurrently-running, independent unit of work -- an * empirical companion to the code-read confirmation (documented in this * task's status report) that the connector thread * (start_controller_connector_thread/connector_to_controller, xcecd.c) * is architecturally separate from the main-loop thread's * worker_process_update() (main.c) and the dedicated resource_sync_thread * (resource_sync.c) -- the two other recurring liveness duties. */ #include #include #include #include #include #include #include #include #include /* Pull the real connect/rejoin logic under test into this TU. */ #include "../src/controller_connector.c" #ifndef XCE_CONTROLLER_CONNECTOR_C_PATH #error \ "XCE_CONTROLLER_CONNECTOR_C_PATH must be defined by meson.build to controller_connector.c's absolute path" #endif /* controller_connector.c declares `extern hostattrs_t host_attrs;` in its * non-XRUN branch (this build); provide the definition normally supplied by * xcecd.c, which this test does not link. */ hostattrs_t host_attrs = { 0 }; /* ── Link-satisfying stubs for externs controller_connector.c references * but this file never exercises (job start/control/dispatch paths) ── */ int submit_fork_task(thread_pool_t* pool, fork_task_t* task) { (void)pool; (void)task; return 0; } xcecd_worker_t* find_worker_by_jobid(xcecd_worker_t* worker_list, uint64_t jobid) { (void)worker_list; (void)jobid; return NULL; } bool parse_limits(const char* exdata, res_limits_t* limits) { (void)exdata; (void)limits; return true; } bool ipc_master_send_job_signal(uint64_t job_id, int signal, bool force) { (void)job_id; (void)signal; (void)force; return true; } bool ipc_master_send_job_ckpt(uint64_t job_id, int kill, const char* method) { (void)job_id; (void)kill; (void)method; return true; } void xcecd_process_command(void* arg) { (void)arg; } /* worker-rejoin-resync task 4.3 (Requirements 3.8, 8.1, 8.3): link- * satisfying stubs for xcecd_resync.c's public API -- this file's white-box * target is controller_connector.c only, it does not #include * xcecd_resync.c (that module's own resync_pending lifecycle and * multi-page query reassembly are covered against the real production * code by test_xcecd_resync.c). g_test_resync_pending lets * checking_connection()'s gate-check tests below control what the gate * reads without a real xcecd_resync.c backing it; the call-count/last-page * globals let the dispatch-wiring tests below observe that myreq_handler * actually reaches this function, without needing a real report to be * built and sent. */ static bool g_test_resync_pending = false; static int g_resync_handle_query_page_call_count = 0; static const xce_job_state_query_t* g_resync_handle_query_page_last_page = NULL; /* worker-rejoin-resync task 4.3 remediation (round 2, Requirements 3.8, * 6.1): lets the tests below observe that xcecd_on_reconnect() actually * calls xcecd_resync_reset() (the round-2 fix that bounds a stalled * multi-page query's worst case), and lets a deferred-rejoin test simulate * the gate clearing on its own, independent of checking_connection() ever * touching it directly. */ static int g_resync_reset_call_count = 0; /* worker-rejoin-resync task 6.1 (Requirement 3.7): link-satisfying spy stub * for xcecd_worker_utils.c's real retention-window setter -- this file's * white-box target is controller_connector.c only (does not #include * xcecd_worker_utils.c), same rationale as the xcecd_resync.c stubs above. * Lets the dispatch-wiring tests below observe that handle_pushconfig() * (myreq_handler's new OPCode_push case) actually reaches this function * with the decoded value, without needing a real xcecd_worker_utils.c * retention store backing it. */ static int g_retention_set_call_count; static uint64_t g_retention_window_ms_set; void xcecd_finished_job_retention_set_window_ms(uint64_t window_ms) { g_retention_set_call_count++; g_retention_window_ms_set = window_ms; } bool xcecd_resync_is_pending(void) { return g_test_resync_pending; } void xcecd_resync_reset(void) { g_resync_reset_call_count++; g_test_resync_pending = false; } int xcecd_resync_handle_query_page(xcecd_worker_t* worker_list, endpoint_t* ep, xce_peerinfo_t* from, const xce_job_state_query_t* page, uint64_t now_ms, int pktlimit) { (void)worker_list; (void)ep; (void)from; (void)now_ms; (void)pktlimit; g_resync_handle_query_page_call_count++; g_resync_handle_query_page_last_page = page; return (page != NULL) ? 1 : -EINVAL; } /* ════════════════════════════════════════════════════════════════════════ * Fixtures * ════════════════════════════════════════════════════════════════════════ */ void setUp(void) { memset(&g_cluster_cfg, 0, sizeof(g_cluster_cfg)); cc_ctx = NULL; xcecli_ctx = NULL; needs_rejoin = false; /* Restore the real production call by default (task 2.2) -- individual * rejoin-retry tests below override this to a controllable double. */ rejoin_call = perform_pending_rejoin; /* worker-rejoin-resync task 4.3: default the resync gate to "not * pending" and reset the dispatch-wiring spy globals. */ g_test_resync_pending = false; g_resync_handle_query_page_call_count = 0; g_resync_handle_query_page_last_page = NULL; g_resync_reset_call_count = 0; /* worker-rejoin-resync task 6.1: reset the retention-window setter spy. */ g_retention_set_call_count = 0; g_retention_window_ms_set = 0; } void tearDown(void) { return; } /* ════════════════════════════════════════════════════════════════════════ * Req 1.5 — unified candidate-list pipeline: no branch for "single host" * ════════════════════════════════════════════════════════════════════════ */ static void test_build_cluster_config_no_peers_synthesizes_single_candidate( void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.remote_host = "localhost"; fake_cc.remote_port = 9990; fake_cc.cluster_cfg.candidates_count = 0; /* no --peers given */ cc_ctx = &fake_cc; int rc = build_cluster_config(); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, g_cluster_cfg.candidates_count); TEST_ASSERT_EQUAL_STRING("localhost", g_cluster_cfg.candidates[0].hostname); TEST_ASSERT_EQUAL_INT(9990, g_cluster_cfg.candidates[0].port); TEST_ASSERT_NOT_EQUAL(0, g_cluster_cfg.identity[0]); /* validate() requires this */ printf( "test_build_cluster_config_no_peers_synthesizes_single_candidate: " "PASS\n"); } /* The core Req 1.5 assertion: starting with NO --peers and starting with a * SINGLE-ENTRY --peers list pointed at the exact same host/port must * produce the identical validated candidate-list shape. Every log line in * xcecd_join_cluster() is derived from g_cluster_cfg's contents, so * structural equality here is exactly what "identical connect log * sequence" (the task's observable-completion criterion) reduces to. */ static void test_build_cluster_config_single_entry_peers_matches_synthesized_shape(void) { controller_connector_ctx_t fake_cc_no_peers = { 0 }; fake_cc_no_peers.remote_host = "localhost"; fake_cc_no_peers.remote_port = 9990; cc_ctx = &fake_cc_no_peers; TEST_ASSERT_EQUAL_INT(0, build_cluster_config()); xce_cluster_config_t synthesized = g_cluster_cfg; controller_connector_ctx_t fake_cc_one_peer = { 0 }; fake_cc_one_peer.remote_host = "unused"; fake_cc_one_peer.remote_port = 1; fake_cc_one_peer.cluster_cfg.candidates_count = 1; strncpy(fake_cc_one_peer.cluster_cfg.candidates[0].hostname, "localhost", XCE_MAXHOSTNAME_LEN - 1); fake_cc_one_peer.cluster_cfg.candidates[0].port = 9990; strncpy(fake_cc_one_peer.cluster_cfg.identity, "workertest", 255); cc_ctx = &fake_cc_one_peer; TEST_ASSERT_EQUAL_INT(0, build_cluster_config()); xce_cluster_config_t explicit_single = g_cluster_cfg; TEST_ASSERT_EQUAL_INT(explicit_single.candidates_count, synthesized.candidates_count); TEST_ASSERT_EQUAL_STRING(explicit_single.candidates[0].hostname, synthesized.candidates[0].hostname); TEST_ASSERT_EQUAL_INT(explicit_single.candidates[0].port, synthesized.candidates[0].port); printf( "test_build_cluster_config_single_entry_peers_matches_synthesized_" "shape: PASS\n"); } static void test_build_cluster_config_multi_candidate_passthrough(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.cluster_cfg.candidates_count = 3; strncpy(fake_cc.cluster_cfg.candidates[0].hostname, "ctrl-a", XCE_MAXHOSTNAME_LEN - 1); fake_cc.cluster_cfg.candidates[0].port = 9991; strncpy(fake_cc.cluster_cfg.candidates[1].hostname, "ctrl-b", XCE_MAXHOSTNAME_LEN - 1); fake_cc.cluster_cfg.candidates[1].port = 9992; strncpy(fake_cc.cluster_cfg.candidates[2].hostname, "ctrl-c", XCE_MAXHOSTNAME_LEN - 1); fake_cc.cluster_cfg.candidates[2].port = 9993; strncpy(fake_cc.cluster_cfg.identity, "workertest", 255); cc_ctx = &fake_cc; int rc = build_cluster_config(); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(3, g_cluster_cfg.candidates_count); TEST_ASSERT_EQUAL_STRING("ctrl-a", g_cluster_cfg.candidates[0].hostname); TEST_ASSERT_EQUAL_INT(9991, g_cluster_cfg.candidates[0].port); TEST_ASSERT_EQUAL_STRING("ctrl-b", g_cluster_cfg.candidates[1].hostname); TEST_ASSERT_EQUAL_INT(9992, g_cluster_cfg.candidates[1].port); TEST_ASSERT_EQUAL_STRING("ctrl-c", g_cluster_cfg.candidates[2].hostname); TEST_ASSERT_EQUAL_INT(9993, g_cluster_cfg.candidates[2].port); printf("test_build_cluster_config_multi_candidate_passthrough: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Rejoin callback contract (design.md "Rejoin callback constraint", * Req 5.1/5.2): flag-only, no I/O, returns immediately. * ════════════════════════════════════════════════════════════════════════ */ static long elapsed_us(struct timespec* t0, struct timespec* t1) { return (t1->tv_sec - t0->tv_sec) * 1000000L + (t1->tv_nsec - t0->tv_nsec) / 1000L; } static void test_rejoin_callback_sets_flag_and_returns_immediately(void) { struct timespec t0, t1; TEST_ASSERT_FALSE(needs_rejoin); clock_gettime(CLOCK_MONOTONIC, &t0); int rc = xcecd_on_reconnect((xcecli_ctx_t)0x1, NULL); clock_gettime(CLOCK_MONOTONIC, &t1); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_TRUE(needs_rejoin); TEST_ASSERT_LESS_THAN_INT(5000, elapsed_us(&t0, &t1)); /* < 5ms: no I/O */ printf("test_rejoin_callback_sets_flag_and_returns_immediately: PASS\n"); } /* worker-rejoin-resync task 4.3 remediation (round 2, Requirements 3.8, * 6.1): the reconnect callback must also reset xcecd_resync.c's gate, not * just set needs_rejoin -- this is what bounds a stalled multi-page * query's worst case (see xcecd_resync_reset()'s own doc comment). Without * this call, a query whose connection dropped mid-reassembly would leave * resync_pending stuck true forever with nothing left to clear it. */ static void test_rejoin_callback_also_resets_resync_gate(void) { g_test_resync_pending = true; g_resync_reset_call_count = 0; int rc = xcecd_on_reconnect((xcecli_ctx_t)0x1, NULL); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_TRUE(needs_rejoin); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, g_resync_reset_call_count, "xcecd_on_reconnect() must call xcecd_resync_reset() exactly once " "so a resync gate left stuck by a stalled multi-page query is " "always cleared the instant the worker detects it has " "reconnected"); TEST_ASSERT_FALSE_MESSAGE( g_test_resync_pending, "the resync gate must actually be cleared, not merely have its " "reset function called"); printf("test_rejoin_callback_also_resets_resync_gate: PASS\n"); } static void test_rejoin_callback_registered_via_public_api(void) { /* Register through the real public entry point the production code * calls (xcecli_set_reconnect_handler) against a REAL ctx allocated by * xcecli_ctx_init() -- which does local struct setup only, no socket * I/O -- so this proves the registration call in xcecd_join_cluster() * is wired correctly end to end (right function, right signature), * not just that xcecd_on_reconnect() behaves correctly in isolation. */ xce_client_config_t cfg = { 0 }; xcecli_ctx_t ctx = NULL; cfg.nmasters = 1; cfg.masters[0] = "127.0.0.1"; cfg.master_ports[0] = 1; cfg.from = "test"; cfg.key = "test"; TEST_ASSERT_EQUAL_INT(0, xcecli_ctx_init(&ctx, &cfg)); int src = xcecli_set_reconnect_handler(ctx, xcecd_on_reconnect, NULL); TEST_ASSERT_EQUAL_INT(0, src); xcecli_ctx_fini(ctx); printf("test_rejoin_callback_registered_via_public_api: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Give-up reconciliation (Req 2.1-2.3): checking_connection() must never * permanently disable the worker; the library now owns retry/backoff. * ════════════════════════════════════════════════════════════════════════ */ static void test_checking_connection_never_permanently_disables_worker(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; /* No connection at all -> xcecli_hello(NULL, ...) fails immediately * with -EINVAL (checked before any I/O -- see client.c's xcecli_hello), * so this loop is fast and deterministic without a real socket. */ xcecli_ctx = NULL; struct timespec t0, t1; clock_gettime(CLOCK_MONOTONIC, &t0); /* Far more iterations than the OLD MAX_RETRY_COUNT (5) -- the old code * tore the worker down for good (cc_ctx->running = false) once, inside * a SINGLE checking_connection() call, after 5 consecutive failures. */ for (int i = 0; i < 25; i++) { checking_connection(); TEST_ASSERT_TRUE_MESSAGE( cc_ctx->running, "checking_connection() must never permanently disable the " "worker -- the client library now owns connection recovery"); } clock_gettime(CLOCK_MONOTONIC, &t1); long elapsed_ms = (t1.tv_sec - t0.tv_sec) * 1000L + (t1.tv_nsec - t0.tv_nsec) / 1000000L; /* The OLD code's single checking_connection() call, against a * persistently failing hello, looped 5 times with a 5s sleep between * each -- >=20s for ONE call alone. 25 calls completing well under a * second proves the bounded retry-then-sleep loop is gone, not merely * that the give-up flag was removed. */ TEST_ASSERT_LESS_THAN_INT(1000, elapsed_ms); printf( "test_checking_connection_never_permanently_disables_worker: " "PASS (elapsed_ms=%ld)\n", elapsed_ms); } /* ════════════════════════════════════════════════════════════════════════ * Rejoin retry-on-failure (task 2.2, Req 2.5): after a transparent * reconnect sets needs_rejoin, checking_connection() must issue the actual * rejoin as a separate call and clear the flag ONLY on that call's success * -- on failure the flag must stay set so the NEXT checking_connection() * tick retries it, rather than leaving the worker transport-connected but * never actually re-registered. * * These tests override the rejoin_call function-pointer seam (controller_ * connector.c) with a controllable double instead of exercising the real * xcecli_join(): a real failure goes through XCECLI_FAILOVER_CALL's full * retry/backoff budget (up to ~100s) before returning, and a real success * requires a live capnp-speaking controller -- neither is practical from a * fast unit test. Production always calls through the same seam via its * default (perform_pending_rejoin(), verified wired correctly by * test_rejoin_callback_registered_via_public_api's sibling tests above and * by direct reading of checking_connection()'s body); only these tests * substitute the pointer. * ════════════════════════════════════════════════════════════════════════ */ static int g_rejoin_call_count = 0; static int g_rejoin_call_results[4]; static bool g_rejoin_use_scripted_results = false; static int fake_rejoin_always_fails(void) { g_rejoin_call_count++; return -1; } static int fake_rejoin_always_succeeds(void) { g_rejoin_call_count++; return 0; } /* A real xcecli_join() success is the controller's assigned worker slot * index (join.c's handle_worker_join(): `reply.status = idx`), which is any * rc >= 0 -- not always exactly 0. Task 2.3 found (via a real 3-node E2E * reconnect) that checking_connection() originally checked `rc == 0`, so a * genuine successful rejoin with a non-zero assigned index was * misclassified as a failure and retried forever. */ static int fake_rejoin_succeeds_with_nonzero_index(void) { g_rejoin_call_count++; return 1; } /* Returns g_rejoin_call_results[call index] on each successive call -- * lets a test script "fails once, then succeeds" or similar sequences. */ static int fake_rejoin_scripted(void) { int idx = g_rejoin_call_count; g_rejoin_call_count++; if (!g_rejoin_use_scripted_results || idx >= (int)(sizeof(g_rejoin_call_results) / sizeof(g_rejoin_call_results[0]))) { return -1; } return g_rejoin_call_results[idx]; } static void test_checking_connection_skips_rejoin_when_not_pending(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; /* xcecli_hello fails fast */ needs_rejoin = false; g_rejoin_call_count = 0; rejoin_call = fake_rejoin_always_succeeds; checking_connection(); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, g_rejoin_call_count, "checking_connection() must not attempt a rejoin when none is " "pending"); TEST_ASSERT_FALSE(needs_rejoin); printf("test_checking_connection_skips_rejoin_when_not_pending: PASS\n"); } static void test_checking_connection_rejoin_failure_keeps_flag_set(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; needs_rejoin = true; g_rejoin_call_count = 0; rejoin_call = fake_rejoin_always_fails; checking_connection(); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rejoin_call_count, "rejoin must have been attempted"); TEST_ASSERT_TRUE_MESSAGE( needs_rejoin, "a failed rejoin call must leave needs_rejoin set so the next " "liveness probe retries it (Req 2.5) -- it must NOT be cleared " "merely because the call was attempted"); printf("test_checking_connection_rejoin_failure_keeps_flag_set: PASS\n"); } static void test_checking_connection_rejoin_success_clears_flag(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; needs_rejoin = true; g_rejoin_call_count = 0; rejoin_call = fake_rejoin_always_succeeds; checking_connection(); TEST_ASSERT_EQUAL_INT(1, g_rejoin_call_count); TEST_ASSERT_FALSE_MESSAGE( needs_rejoin, "a successful rejoin call must clear needs_rejoin (Req 2.5)"); printf("test_checking_connection_rejoin_success_clears_flag: PASS\n"); } /* Task 2.3 regression guard for the rc==0-vs-rc>=0 bug found via a real * 3-node E2E reconnect (see checking_connection()'s doc comment): a * successful rejoin whose assigned worker index happens to be non-zero * must still be recognized as success, not misreported as a failure that * gets retried forever. */ static void test_checking_connection_rejoin_success_with_nonzero_index_clears_flag(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; needs_rejoin = true; g_rejoin_call_count = 0; rejoin_call = fake_rejoin_succeeds_with_nonzero_index; checking_connection(); TEST_ASSERT_EQUAL_INT(1, g_rejoin_call_count); TEST_ASSERT_FALSE_MESSAGE( needs_rejoin, "a rejoin call returning a positive, non-zero worker index is a " "SUCCESS (matches xcecd_join_cluster()'s own `rc < 0` failure test " "above) and must clear needs_rejoin -- treating any non-zero rc as " "failure would retry a successfully-rejoined worker forever"); printf( "test_checking_connection_rejoin_success_with_nonzero_index_clears_" "flag: PASS\n"); } /* The core Req 2.5 / task 2.2 observable-completion scenario: "a rejoin * call that fails once before succeeding is retried on the next * liveness-probe cycle rather than abandoned" -- exercised as TWO separate * checking_connection() calls (two liveness-probe ticks), not two inner * attempts within one call, matching design.md's "checking_connection()'s * existing periodic cadence is what makes this retry naturally". */ static void test_checking_connection_rejoin_retried_on_next_probe_after_failure(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; needs_rejoin = true; g_rejoin_call_count = 0; g_rejoin_use_scripted_results = true; g_rejoin_call_results[0] = -E_XCE_NOT_READY; /* 1st tick: fails */ g_rejoin_call_results[1] = 0; /* 2nd tick: succeeds */ rejoin_call = fake_rejoin_scripted; /* Liveness-probe tick 1: rejoin attempted, fails, flag must stay set. */ checking_connection(); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rejoin_call_count, "tick 1 must attempt exactly one rejoin"); TEST_ASSERT_TRUE_MESSAGE( needs_rejoin, "flag must remain set after tick 1's failed rejoin -- not " "abandoned"); /* Liveness-probe tick 2 (checking_connection()'s own periodic cadence, * no new timer): rejoin retried automatically because the flag is * still set, this time succeeds, flag clears. */ checking_connection(); TEST_ASSERT_EQUAL_INT_MESSAGE( 2, g_rejoin_call_count, "tick 2 must retry the rejoin since it is still pending"); TEST_ASSERT_FALSE_MESSAGE( needs_rejoin, "flag must clear once the retried rejoin succeeds on tick 2"); g_rejoin_use_scripted_results = false; printf( "test_checking_connection_rejoin_retried_on_next_probe_after_" "failure: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 4.3 (worker-rejoin-resync, Requirements 3.8, 8.1, 8.3) — gating the * needs_rejoin-triggered rejoin action on xcecd_resync.c's resync_pending * flag (the "resume normal post-join operation" step Requirement 3.8 * actually names), and wiring OPCode_jobStateQuery's real receipt. * * Round-2 remediation note: round 1 gated checking_connection()'s ENTRY on * resync_pending, which also silently skipped xcecli_hello() and the * needs_rejoin check itself -- confirmed to permanently disable the * worker's own connection-recovery mechanism whenever resync_pending got * stuck true (a stalled multi-page query, e.g.). The gate now sits INSIDE * the needs_rejoin branch, guarding only rejoin_call(); see * test_checking_connection_survives_permanently_stuck_resync_pending() and * test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_ * source() further below for the regression coverage of that distinction. * * What THIS file proves is the two things that are genuinely * controller_connector.c's own: (1) checking_connection() actually * consults the gate before attempting the rejoin -- the same way it * already consults needs_rejoin -- and (2) myreq_handler's dispatch switch * actually routes OPCode_jobStateQuery to the resync module at all, the * exact gap task 4.2 left unclosed (confirmed by inspection: no case * existed for this opcode before this task). xcecd_resync.c's own * resync_pending set/clear lifecycle and multi-page query reassembly are * covered end-to-end, against the real production code and real wire * encode/decode, by test_xcecd_resync.c -- not duplicated here. * ════════════════════════════════════════════════════════════════════════ */ static void test_checking_connection_defers_rejoin_action_when_resync_pending(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; needs_rejoin = true; /* would normally trigger a rejoin attempt */ g_rejoin_call_count = 0; rejoin_call = fake_rejoin_always_succeeds; g_test_resync_pending = true; checking_connection(); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, g_rejoin_call_count, "checking_connection() must not attempt a rejoin while a job-state " "resync report is still being sent (Req 3.8) -- the worker's " "resumption of normal post-join operation must not begin or " "interleave with it"); TEST_ASSERT_TRUE_MESSAGE( needs_rejoin, "a deferred tick must leave needs_rejoin untouched, not clear it " "as if a rejoin had been attempted"); printf( "test_checking_connection_defers_rejoin_action_when_resync_pending: " "PASS\n"); } static void test_checking_connection_resumes_once_resync_no_longer_pending(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; xcecli_ctx = NULL; needs_rejoin = true; g_rejoin_call_count = 0; rejoin_call = fake_rejoin_always_succeeds; g_test_resync_pending = false; /* report already finished sending */ checking_connection(); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, g_rejoin_call_count, "once the gate clears, checking_connection() must resume its " "normal periodic activity on its very next tick"); TEST_ASSERT_FALSE(needs_rejoin); printf("test_checking_connection_resumes_once_resync_no_longer_pending: " "PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Round-2 remediation regression guard (Requirements 3.8, 6.1): FINDING 1 * of the round-1 review -- gating checking_connection()'s ENTRY on * resync_pending permanently disabled the worker's own connection-recovery * mechanism whenever a multi-page job-state query's final page could never * arrive (e.g. the worker's own connection to the controller dropped after * a non-final page). resync_pending must now guard ONLY the * needs_rejoin-triggered rejoin action -- xcecli_hello() (the worker's * liveness probe / reconnect-detection trigger) and the needs_rejoin check * itself must keep running every single tick regardless, extending * test_checking_connection_never_permanently_disables_worker's exact * pattern (many ticks, must never get permanently stuck) into this new * adversarial condition: resync_pending genuinely stuck true (simulating a * stalled multi-page query whose final page never arrives) for the whole * loop, never cleared by anything inside checking_connection() itself. * ════════════════════════════════════════════════════════════════════════ */ static void test_checking_connection_survives_permanently_stuck_resync_pending(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; /* No connection at all -> xcecli_hello(NULL, ...) fails immediately * with -EINVAL, same fast/deterministic setup as * test_checking_connection_never_permanently_disables_worker. */ xcecli_ctx = NULL; needs_rejoin = true; /* a rejoin is genuinely pending throughout */ g_rejoin_call_count = 0; rejoin_call = fake_rejoin_always_succeeds; /* Simulates a multi-page job-state query that stalled after a * non-final page (the worker's own connection dropped mid-query, so no * further page -- and therefore no call into * xcecd_resync_handle_query_page() that could ever clear the gate -- * will ever arrive). Left true for the entire loop below: nothing in * this test ever clears it, matching the round-1 defect's worst case * exactly. */ g_test_resync_pending = true; struct timespec t0, t1; clock_gettime(CLOCK_MONOTONIC, &t0); /* Far more iterations than the old MAX_RETRY_COUNT (5) -- see * test_checking_connection_never_permanently_disables_worker. Every * single tick must still leave the worker able to detect/recover its * own connection (cc_ctx->running never forced false), and the pending * rejoin must never be silently dropped (needs_rejoin stays set, exactly * because it is legitimately still deferred by the still-pending * resync, not because the worker gave up). */ for (int i = 0; i < 25; i++) { checking_connection(); TEST_ASSERT_TRUE_MESSAGE( cc_ctx->running, "a permanently-stuck resync_pending must never permanently " "disable the worker -- checking_connection()'s liveness probe " "and needs_rejoin check must keep running every tick " "regardless (Requirement 6.1)"); TEST_ASSERT_TRUE_MESSAGE( needs_rejoin, "the rejoin action stays correctly deferred tick after tick " "while resync_pending is genuinely stuck true -- it must " "never be silently cleared without actually being attempted"); } clock_gettime(CLOCK_MONOTONIC, &t1); long elapsed_ms = (t1.tv_sec - t0.tv_sec) * 1000L + (t1.tv_nsec - t0.tv_nsec) / 1000000L; TEST_ASSERT_EQUAL_INT_MESSAGE( 0, g_rejoin_call_count, "the rejoin action itself must stay deferred for as long as " "resync_pending is genuinely (if pathologically) stuck true -- " "narrowing the gate must not mean it stops applying"); /* 25 ticks completing well under a second proves this loop never fell * into any blocking wait -- same bound as the sibling invariant test. */ TEST_ASSERT_LESS_THAN_INT(1000, elapsed_ms); printf( "test_checking_connection_survives_permanently_stuck_resync_" "pending: PASS (elapsed_ms=%ld)\n", elapsed_ms); } /* A dummy backing buffer large enough for the real api_ctx_t * (client_internal.h) that xcecli_get_my_endpoint()/xcecli_get_myinfo() * (real functions, src/lib/client.c) cast xcecli_ctx to internally -- * this file must not include client_internal.h itself (its api_ctx_t * conflicts with controller_connector.h's own unused same-named type, see * this file's top-of-file comment), but neither accessor dereferences * anything through the returned pointer in THIS translation unit, since * xcecd_resync_handle_query_page() is a stub here that never dereferences * its ep/from parameters either -- only their addresses need to be * computable, which pure pointer arithmetic over a large enough zeroed * buffer safely provides regardless of the real struct's exact layout. */ static uint8_t g_dummy_api_ctx[8192]; /* Req 8.1/8.3: myreq_handler's OPCode_jobStateQuery case must route an * incoming query page to xcecd_resync_handle_query_page(). */ static void test_myreq_handler_routes_jobstatequery_to_resync_module(void) { controller_connector_ctx_t fake_cc = { 0 }; void* worker_list_head = NULL; fake_cc.running = true; fake_cc.worker_list = &worker_list_head; cc_ctx = &fake_cc; memset(g_dummy_api_ctx, 0, sizeof(g_dummy_api_ctx)); xcecli_ctx = (xcecli_ctx_t)g_dummy_api_ctx; uint64_t jobids[2] = { 42, 43 }; xce_job_state_query_t page = { 0 }; page.n_total = 2; page.pos = 0; page.count = 2; page.jobids = jobids; xce_request_t request = { 0 }; request.opcode = OPCode_jobStateQuery; request.body_which = Request_body_jobstatequerymsg; request.body.jobstatequerymsg = &page; int rc = myreq_handler(&request, NULL); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, g_resync_handle_query_page_call_count, "OPCode_jobStateQuery must reach xcecd_resync_handle_query_page() " "exactly once per incoming page -- this is the real dispatch " "wiring task 4.2 left out (no case existed for this opcode before " "this task)"); TEST_ASSERT_EQUAL_PTR_MESSAGE( &page, g_resync_handle_query_page_last_page, "the decoded query page must be passed through unchanged"); xcecli_ctx = NULL; printf("test_myreq_handler_routes_jobstatequery_to_resync_module: PASS\n"); } /* Without an active controller connection (xcecli_ctx == NULL), the * dispatch case must fail fast rather than call into the resync module * with a NULL endpoint/peerinfo. */ static void test_myreq_handler_jobstatequery_without_connection_is_rejected(void) { controller_connector_ctx_t fake_cc = { 0 }; void* worker_list_head = NULL; fake_cc.running = true; fake_cc.worker_list = &worker_list_head; cc_ctx = &fake_cc; xcecli_ctx = NULL; uint64_t jobids[1] = { 1 }; xce_job_state_query_t page = { 0 }; page.n_total = 1; page.pos = 0; page.count = 1; page.jobids = jobids; xce_request_t request = { 0 }; request.opcode = OPCode_jobStateQuery; request.body_which = Request_body_jobstatequerymsg; request.body.jobstatequerymsg = &page; int rc = myreq_handler(&request, NULL); TEST_ASSERT_TRUE(rc < 0); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, g_resync_handle_query_page_call_count, "without an active controller connection, the resync module must " "never be called"); printf("test_myreq_handler_jobstatequery_without_connection_is_rejected: " "PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * worker-rejoin-resync task 6.1 (Requirement 3.7): myreq_handler's new * OPCode_push case (handle_pushconfig) must apply a received * workerResyncWindowMs value to the retention window before the push * completes. * * Before this task, myreq_handler had NO case for OPCode_push at all -- * confirmed by inspection (this file's own white-box #include of the real, * current controller_connector.c) -- so a config push fell through to * `default: break;` and was silently discarded in its entirety, even for * the pre-existing clustername field. xcecd_finished_job_retention_set_ * window_ms()'s own doc comment (xcecd_worker_utils.h, task 4.1) * independently confirms the same gap. This is the first test coverage of * that dispatch case existing at all. * ════════════════════════════════════════════════════════════════════════ */ /* Builds a real, wire-encoded OPCode_push carrying a Conf4Worker payload * with the given workerResyncWindowMs value, pops it back off the queue, * and decodes it with the real xce_decode() -- exercising the actual * capnp encode/decode round trip (controller.capnp.c, generated), not just * a directly-constructed xce_request_t. Mirrors src/lib/testsuite/ * test_job_state_wire.c's own real-endpoint round-trip pattern. Caller * must xce_free_request(out) and endpoint_fini()/free() the endpoint. */ static endpoint_t* build_and_decode_pushconfig(uint32_t worker_resync_window_ms, xce_request_t* out) { xce_conf4worker_t conf4worker = { 0 }; conf4worker.clustername = (char*)"test-cluster"; conf4worker.worker_resync_window_ms = worker_resync_window_ms; xce_configdata_t configdata = { 0 }; configdata.data_which = ConfigData_data_workerconfig; configdata.data.conf4worker = &conf4worker; char nm[] = "test-controller"; char ky[] = "test-key"; xce_peerinfo_t from = { .name = nm, .key = ky }; endpoint_t* ep = endpoint_get(); TEST_ASSERT_NOT_NULL(ep); TEST_ASSERT_EQUAL_INT(0, xce_pushconfig_req(ep, &from, &configdata)); data_item_t* it = endpoint_deq(ep); TEST_ASSERT_NOT_NULL(it); io_packet_t* pkt = ITPTR_TO_PACKET(it); memset(out, 0, sizeof(*out)); TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); return ep; } /* A config push carrying a non-default workerResyncWindowMs value must be * applied to the retention window -- synchronously, within myreq_handler's * own call, i.e. before the push "completes" from the caller's point of * view (Requirement 3.7's own observable completion criterion). */ static void test_myreq_handler_routes_pushconfig_workerconfig_to_retention_setter(void) { xce_request_t decoded; endpoint_t* ep = build_and_decode_pushconfig(4242, &decoded); int rc = myreq_handler(&decoded, NULL); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, g_retention_set_call_count, "OPCode_push carrying a Conf4Worker payload must reach " "xcecd_finished_job_retention_set_window_ms() exactly once -- the " "dispatch case task 6.1 adds (no case existed for OPCode_push " "before this task)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 4242, g_retention_window_ms_set, "the applied value must be the real value decoded off the wire, " "not a hardcoded/default one"); xce_free_request(&decoded); endpoint_fini(ep); free(ep); printf("test_myreq_handler_routes_pushconfig_workerconfig_to_retention_" "setter: PASS\n"); } /* A second, different value must also be applied -- confirms the setter is * called with the field genuinely read from each push, not a fixed * constant from the first call. */ static void test_myreq_handler_routes_pushconfig_second_value_also_applied(void) { xce_request_t decoded; endpoint_t* ep = build_and_decode_pushconfig(7, &decoded); int rc = myreq_handler(&decoded, NULL); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, g_retention_set_call_count); TEST_ASSERT_EQUAL_UINT64(7, g_retention_window_ms_set); xce_free_request(&decoded); endpoint_fini(ep); free(ep); printf("test_myreq_handler_routes_pushconfig_second_value_also_applied: " "PASS\n"); } /* A config push for a DIFFERENT ConfigData union arm (e.g. a scheduler's * Conf4Sched, or the raw ClusterConfig arm) must NOT be misread as a * Conf4Worker payload -- the retention setter must not fire at all. */ static void test_myreq_handler_pushconfig_ignores_non_workerconfig_data(void) { cluster_config_t clusterconfig = { 0 }; clusterconfig.clustername = (char*)"test-cluster"; xce_configdata_t configdata = { 0 }; configdata.data_which = ConfigData_data_clusterconfig; configdata.data.clusterconfig = &clusterconfig; xce_request_t request = { 0 }; request.opcode = OPCode_push; request.body_which = Request_body_configpushmsg; request.body.configpushmsg = &configdata; int rc = myreq_handler(&request, NULL); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, g_retention_set_call_count, "a non-workerconfig ConfigData arm must never reach the retention " "setter"); printf("test_myreq_handler_pushconfig_ignores_non_workerconfig_data: " "PASS\n"); } /* A NULL configpushmsg (defensive: should never happen for a genuinely * decoded OPCode_push, but the handler must not crash) must also be a * silent no-op. */ static void test_myreq_handler_pushconfig_ignores_null_payload(void) { xce_request_t request = { 0 }; request.opcode = OPCode_push; request.body_which = Request_body_configpushmsg; request.body.configpushmsg = NULL; int rc = myreq_handler(&request, NULL); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(0, g_retention_set_call_count); printf("test_myreq_handler_pushconfig_ignores_null_payload: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.3 — structural regression guard: xcecli_set_reconnect_handler() * must be called AFTER xcecli_start() succeeds, never before. * * Source-text scan of the real, current on-disk controller_connector.c * (same genre as scheduler/testsuite's test_core_isolation.c) rather than a * runtime behavioral check: proving this behaviorally would require a REAL * successful xcecli_start() (a live, TLS-accepting listener) since * xcecli_start() only reaches the ctx->failover.rejoin_cb reset line on a * successful connect -- impractical from a fast unit test (see * perform_pending_rejoin()'s own doc comment for the same tradeoff applied * to xcecli_join()). The REAL behavioral proof that this ordering is * correct is task 2.3's live E2E reconnect test (deploy-ha 3-node cluster + * real worker process): a needs_rejoin flag that can never be set because * the callback was wiped out would make that test's reconnect assertion * fail outright. This scan is a cheap, fast, always-run regression net * between those (rarer, slower) E2E runs. * ════════════════════════════════════════════════════════════════════════ */ static char* read_whole_file(const char* path) { FILE* f = fopen(path, "r"); if (!f) { return NULL; } fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET); if (size < 0) { fclose(f); return NULL; } char* buf = (char*)malloc((size_t)size + 1); if (!buf) { fclose(f); return NULL; } size_t nread = fread(buf, 1, (size_t)size, f); buf[nread] = '\0'; fclose(f); return buf; } static void test_reconnect_handler_registered_after_start_not_before_in_source(void) { char* src = read_whole_file(XCE_CONTROLLER_CONNECTOR_C_PATH); TEST_ASSERT_NOT_NULL_MESSAGE( src, "could not read controller_connector.c source for scan"); const char* start_call = strstr(src, "xcecli_start(xcecli_ctx)"); const char* register_call = strstr(src, "xcecli_set_reconnect_handler(xcecli_ctx, " "xcecd_on_reconnect, NULL)"); TEST_ASSERT_NOT_NULL_MESSAGE( start_call, "xcecli_start(xcecli_ctx) call site not found"); TEST_ASSERT_NOT_NULL_MESSAGE( register_call, "xcecli_set_reconnect_handler(xcecli_ctx, xcecd_on_reconnect, " "NULL) call site not found"); TEST_ASSERT_TRUE_MESSAGE( register_call > start_call, "xcecli_set_reconnect_handler() must be registered AFTER " "xcecli_start() succeeds -- xcecli_start() unconditionally resets " "ctx->failover.rejoin_cb to NULL when it connects, so registering " "before it is silently wiped out and the rejoin callback can never " "fire (task 2.3 regression guard for a real bug found during this " "task's validation)"); FREEUP(src); printf( "test_reconnect_handler_registered_after_start_not_before_in_" "source: PASS\n"); } /* Structural companion to * test_checking_connection_survives_permanently_stuck_resync_pending() * above: proves WHY checking_connection() can never again become * permanently inert the way round 1's entry-guard placement did. Same * source-text-scan genre as the xcecli_start()-before- * xcecli_set_reconnect_handler() guard just above, because the behavioral * test cannot, by itself, distinguish "the gate is checked first and * returns early" from "the gate is checked last and defers only the * rejoin action" when needs_rejoin/resync_pending start out already true * in both cases -- both produce the same rejoin_call_count/needs_rejoin * observations. The distinction is textual: xcecli_hello() and the * needs_rejoin check must appear in checking_connection()'s body strictly * BEFORE any xcecd_resync_is_pending() call, never gated behind it. */ static void test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_source( void) { char* src = read_whole_file(XCE_CONTROLLER_CONNECTOR_C_PATH); TEST_ASSERT_NOT_NULL_MESSAGE( src, "could not read controller_connector.c source for scan"); const char* fn_start = strstr(src, "int checking_connection()"); TEST_ASSERT_NOT_NULL_MESSAGE(fn_start, "checking_connection() definition not found"); /* Search only from the function definition onward, so an unrelated, * earlier mention of any of these tokens elsewhere in the file (e.g. in * a doc comment) cannot produce a false pass. */ const char* hello_call = strstr(fn_start, "xcecli_hello(xcecli_ctx"); const char* needs_rejoin_check = strstr(fn_start, "if (needs_rejoin)"); const char* pending_check = strstr(fn_start, "xcecd_resync_is_pending()"); TEST_ASSERT_NOT_NULL_MESSAGE(hello_call, "xcecli_hello(xcecli_ctx, ...) call site " "not found in checking_connection()"); TEST_ASSERT_NOT_NULL_MESSAGE( needs_rejoin_check, "if (needs_rejoin) check not found in checking_connection()"); TEST_ASSERT_NOT_NULL_MESSAGE( pending_check, "xcecd_resync_is_pending() call not found in checking_connection()"); TEST_ASSERT_TRUE_MESSAGE( hello_call < needs_rejoin_check, "xcecli_hello() must run unconditionally, before the needs_rejoin " "check -- never gated behind resync_pending (Requirement 6.1's " "self-healing liveness probe)"); TEST_ASSERT_TRUE_MESSAGE( needs_rejoin_check < pending_check, "the resync_pending gate must be checked INSIDE the needs_rejoin " "branch (guarding only the rejoin action itself), not before it -- " "an entry-guard placement (round 1's confirmed defect) would put " "xcecd_resync_is_pending() before both xcecli_hello() and the " "needs_rejoin check, permanently disabling the worker's own " "connection-recovery mechanism whenever resync_pending gets stuck"); FREEUP(src); printf( "test_checking_connection_hello_and_needs_rejoin_precede_resync_" "gate_in_source: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.3 — Req 1.6/5.2: all-candidates-unreachable failure identifies * every attempted candidate, not a generic error. * ════════════════════════════════════════════════════════════════════════ */ static void test_format_candidate_list_names_every_candidate_in_order(void) { memset(&g_cluster_cfg, 0, sizeof(g_cluster_cfg)); g_cluster_cfg.candidates_count = 3; strncpy(g_cluster_cfg.candidates[0].hostname, "ctrl-a", XCE_MAXHOSTNAME_LEN - 1); g_cluster_cfg.candidates[0].port = 9991; strncpy(g_cluster_cfg.candidates[1].hostname, "ctrl-b", XCE_MAXHOSTNAME_LEN - 1); g_cluster_cfg.candidates[1].port = 9992; strncpy(g_cluster_cfg.candidates[2].hostname, "ctrl-c", XCE_MAXHOSTNAME_LEN - 1); g_cluster_cfg.candidates[2].port = 9993; char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 }; format_candidate_list(attempted, sizeof(attempted)); TEST_ASSERT_NOT_NULL(strstr(attempted, "ctrl-a:9991")); TEST_ASSERT_NOT_NULL(strstr(attempted, "ctrl-b:9992")); TEST_ASSERT_NOT_NULL(strstr(attempted, "ctrl-c:9993")); /* Order preserved too, not just presence. */ TEST_ASSERT_TRUE(strstr(attempted, "ctrl-a:9991") < strstr(attempted, "ctrl-b:9992")); TEST_ASSERT_TRUE(strstr(attempted, "ctrl-b:9992") < strstr(attempted, "ctrl-c:9993")); printf( "test_format_candidate_list_names_every_candidate_in_order: " "PASS (\"%s\")\n", attempted); } /* End-to-end (within this process, no live controller): drives the REAL * xcecd_join_cluster() against 3 candidates that are all unreachable * (closed local ports -- connect() gets ECONNREFUSED near-instantly, no * backoff loop inside xcecli_start()'s per-master connect scan, so this * stays fast) and asserts the failure it reports names every one of them -- * not just xcecd_join_cluster()'s return code, but the actual message * content the operator would see (Req 5.2 "log a summary identifying every * candidate attempted"). */ static void test_xcecd_join_cluster_all_unreachable_reports_every_candidate(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.cluster_cfg.candidates_count = 3; strncpy(fake_cc.cluster_cfg.candidates[0].hostname, "127.0.0.1", XCE_MAXHOSTNAME_LEN - 1); fake_cc.cluster_cfg.candidates[0].port = 18471; strncpy(fake_cc.cluster_cfg.candidates[1].hostname, "127.0.0.1", XCE_MAXHOSTNAME_LEN - 1); fake_cc.cluster_cfg.candidates[1].port = 18472; strncpy(fake_cc.cluster_cfg.candidates[2].hostname, "127.0.0.1", XCE_MAXHOSTNAME_LEN - 1); fake_cc.cluster_cfg.candidates[2].port = 18473; strncpy(fake_cc.cluster_cfg.identity, "workertest", sizeof(fake_cc.cluster_cfg.identity) - 1); cc_ctx = &fake_cc; hostattrs_t attrs = { 0 }; int rc = xcecd_join_cluster(&attrs); TEST_ASSERT_TRUE_MESSAGE( rc < 0, "join against 3 unreachable candidates must fail"); /* xcecd_join_cluster() failed inside xcecli_start(), before ever * building an xcecli_ctx -- g_cluster_cfg (populated by * build_cluster_config() from cc_ctx->cluster_cfg above) is exactly * what format_candidate_list() -- and therefore the real WARN log line * -- renders, already proven correct by the direct unit test above. */ char attempted[CANDIDATE_LIST_BUF_SIZE] = { 0 }; format_candidate_list(attempted, sizeof(attempted)); TEST_ASSERT_NOT_NULL(strstr(attempted, "127.0.0.1:18471")); TEST_ASSERT_NOT_NULL(strstr(attempted, "127.0.0.1:18472")); TEST_ASSERT_NOT_NULL(strstr(attempted, "127.0.0.1:18473")); printf( "test_xcecd_join_cluster_all_unreachable_reports_every_candidate: " "PASS (rc=%d, attempted=\"%s\")\n", rc, attempted); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.3 — Req 3.1/3.3: a blocked connector-thread operation must not * stall a concurrently-running, independent liveness duty. * * Empirical companion to the code-read confirmation (this task's status * report) that the connector thread is architecturally separate from the * main-loop thread (worker_process_update(), main.c) and the dedicated * resource_sync_thread (resource_sync.c). Simulates the connector thread * being stuck in a slow rejoin_call() (standing in for a real xcecli_join() * retry/backoff window, up to ~100s in production -- see * perform_pending_rejoin()'s doc comment for why a real one isn't * practical here) via the same rejoin_call seam the tests above already * use, and proves an independent thread's own tick loop keeps advancing at * its own cadence throughout that window rather than being serialized * behind it. * ════════════════════════════════════════════════════════════════════════ */ #define SLOW_REJOIN_BLOCK_MS 300 static int slow_blocking_rejoin_call(void) { usleep(SLOW_REJOIN_BLOCK_MS * 1000); return -E_XCE_NOT_READY; } static void* connector_thread_body(void* arg) { (void)arg; checking_connection(); return NULL; } static void test_blocked_connector_thread_does_not_stall_other_liveness_duty(void) { controller_connector_ctx_t fake_cc = { 0 }; fake_cc.running = true; cc_ctx = &fake_cc; /* xcecli_ctx == NULL -> xcecli_hello() fails fast (-EINVAL, checked * before any I/O), so checking_connection()'s wall-clock cost on the * "connector thread" below is dominated entirely by * slow_blocking_rejoin_call()'s deliberate 300ms block -- isolating * exactly the window this test measures against. */ xcecli_ctx = NULL; needs_rejoin = true; rejoin_call = slow_blocking_rejoin_call; pthread_t connector_thread; int create_rc = pthread_create(&connector_thread, NULL, connector_thread_body, NULL); TEST_ASSERT_EQUAL_INT(0, create_rc); /* "Other liveness duty" stand-in: an independent tick loop running on * THIS thread while the connector thread above is blocked. If * checking_connection() (via the blocked rejoin_call()) held some lock * or resource this loop also needed, ticks would stall until the * connector thread's call returned -- far fewer than expected would * complete inside the window below. */ int ticks = 0; struct timespec other_t0, now; clock_gettime(CLOCK_MONOTONIC, &other_t0); long elapsed_ms; do { usleep(10 * 1000); ticks++; clock_gettime(CLOCK_MONOTONIC, &now); elapsed_ms = elapsed_us(&other_t0, &now) / 1000; } while (elapsed_ms < (SLOW_REJOIN_BLOCK_MS - 50)); pthread_join(connector_thread, NULL); /* At ~10ms/tick over a ~250ms window, an unblocked loop completes * ~25 ticks; require well over half that as a generous margin against * scheduling jitter while still clearly distinguishing "ran * concurrently" from "was serialized behind the connector thread" * (which would produce close to 0 ticks in this window). */ TEST_ASSERT_GREATER_THAN_INT_MESSAGE( 12, ticks, "an independent liveness-duty tick loop must keep advancing at its " "own cadence while the connector thread is blocked inside a slow " "rejoin call -- got far fewer ticks than expected, suggesting it " "was serialized behind the connector thread instead of running " "concurrently"); rejoin_call = perform_pending_rejoin; printf( "test_blocked_connector_thread_does_not_stall_other_liveness_duty: " "PASS (ticks=%d during connector thread's %dms simulated block)\n", ticks, SLOW_REJOIN_BLOCK_MS); } /* ════════════════════════════════════════════════════════════════════════ * main * ════════════════════════════════════════════════════════════════════════ */ int main(void) { xce_init_xcecd_log("/tmp/test_controller_connector.log", 1024 * 1024, 2); UNITY_BEGIN(); RUN_TEST(test_build_cluster_config_no_peers_synthesizes_single_candidate); RUN_TEST( test_build_cluster_config_single_entry_peers_matches_synthesized_shape); RUN_TEST(test_build_cluster_config_multi_candidate_passthrough); RUN_TEST(test_rejoin_callback_sets_flag_and_returns_immediately); RUN_TEST(test_rejoin_callback_also_resets_resync_gate); RUN_TEST(test_rejoin_callback_registered_via_public_api); RUN_TEST(test_checking_connection_never_permanently_disables_worker); RUN_TEST(test_checking_connection_skips_rejoin_when_not_pending); RUN_TEST(test_checking_connection_rejoin_failure_keeps_flag_set); RUN_TEST(test_checking_connection_rejoin_success_clears_flag); RUN_TEST( test_checking_connection_rejoin_success_with_nonzero_index_clears_flag); RUN_TEST( test_checking_connection_rejoin_retried_on_next_probe_after_failure); RUN_TEST( test_checking_connection_defers_rejoin_action_when_resync_pending); RUN_TEST(test_checking_connection_resumes_once_resync_no_longer_pending); RUN_TEST( test_checking_connection_survives_permanently_stuck_resync_pending); RUN_TEST(test_myreq_handler_routes_jobstatequery_to_resync_module); RUN_TEST(test_myreq_handler_jobstatequery_without_connection_is_rejected); RUN_TEST( test_myreq_handler_routes_pushconfig_workerconfig_to_retention_setter); RUN_TEST(test_myreq_handler_routes_pushconfig_second_value_also_applied); RUN_TEST(test_myreq_handler_pushconfig_ignores_non_workerconfig_data); RUN_TEST(test_myreq_handler_pushconfig_ignores_null_payload); RUN_TEST( test_reconnect_handler_registered_after_start_not_before_in_source); RUN_TEST( test_checking_connection_hello_and_needs_rejoin_precede_resync_gate_in_source); RUN_TEST(test_format_candidate_list_names_every_candidate_in_order); RUN_TEST(test_xcecd_join_cluster_all_unreachable_reports_every_candidate); RUN_TEST( test_blocked_connector_thread_does_not_stall_other_liveness_duty); return UNITY_END(); }