/* * test_scheduler_connect.c — tests for the scheduler's unified * candidate-list connect/rejoin pipeline (component-failover-adoption, * task 3.1, boundary: SchedulerControllerClient). * * Links the REAL production scheduler_srcs (everything sched_init() and its * dependency graph need -- core.c, schedobj.c, resource.c, plugin.c, the * picker/allocator plugins, etc. -- see meson.build) and calls the REAL, * public sched_init()/sched_fini() entry points, rather than * #include-ing scheduler.c directly: scheduler.c's own dependency surface * is large enough that whole-file inclusion was already ruled out by this * codebase's own precedent (see test_picker_core_loop_boundary_test's and * test_picker_wiring.c's header comments) -- object-linking the real * scheduler_srcs sidesteps that without needing dozens of hand-written * stubs, at the cost of the two new helper functions this task adds * (sched_build_cluster_config()/sched_on_reconnect()) staying `static` and * therefore untestable directly. Their externally-observable effects are * covered here two ways instead: * * - Req 1.5 (no-candidate-list and single-entry-candidate-list converge on * the identical validated xce_cluster_config_t shape) and Req 1.1-1.3 * (multi-candidate passthrough, all-candidates-unreachable failure) are * exercised BEHAVIORALLY, by calling the real sched_init() against * closed local ports (connect() gets ECONNREFUSED near-instantly, same * "no backoff loop inside xcecli_start()'s per-master connect scan" * property worker's task 2.3 test relies on) and inspecting the * resulting sched->cluster_cfg. * - Req 1 AC6 / Req 5 AC2's "identifies which candidates were attempted" * (component-failover-adoption, task 3.3) is proven by genuine RUNTIME * LOG CAPTURE (dup2() redirect of STDOUT_FILENO for the duration of the * sched_init() call under test -- same mechanism * test_scheduler_decision_publish_retry.c's own capture_stdout_during() * established for this boundary, see * capture_stdout_during_sched_init()'s own header comment below), not * inferred from sched->cluster_cfg's contents: proving what sched_init() * actually PRINTS is a materially stronger check than proving the struct * it derives the message from, since sched_format_candidate_list() * (scheduler.c) -- the function that turns that struct into the printed * "host:port, host:port, ..." text -- is itself `static` and otherwise * has no direct call site in this file to exercise. * - The rejoin callback's "no I/O, sets a flag, returns immediately" * contract (Req 2.1, Req 5.1-5.2, design.md's "Rejoin callback * constraint") and the connect pipeline's single-path structure (Req * 1.5) are exercised via source-text scans of the real, current * scheduler.c -- same genre as test_picker_core_loop_boundary_test and * worker's test_reconnect_handler_registered_after_start_not_before_in_ * source: proving the timing/no-I-O property behaviorally would require * a real transparent reconnect mid-XCECLI_FAILOVER_CALL (a live, * TLS-accepting controller that drops the connection mid-retry), which * is exactly the impractical-for-a-fast-unit-test case those precedents * already established. * * "A three-candidate config where the first is a live standby joins * correctly" (this task's remaining observable-completion item) needs a * real capnp-speaking controller cluster to answer the join handshake and * issue a standby redirect -- validated manually against a live * xce-deploy-ha cluster as part of this task's own verification (see the * task's status report), not committed here as an automated test; the * committed, repeatable version of that scenario is task 3.3's job * (dedicated E2E coverage, same split worker used between its own tasks * 2.1 and 2.3). */ #include #include #include #include #include #include #include #include "client.h" #include "scheduler.h" #include "xce/cluster.h" #ifndef XCE_SCHEDULER_C_PATH #error "XCE_SCHEDULER_C_PATH must be defined by meson.build to scheduler.c's absolute path" #endif extern int scheduler_req_handler(xce_request_t*, void*); void setUp(void) { return; } void tearDown(void) { return; } /* ════════════════════════════════════════════════════════════════════════ * Helpers * ════════════════════════════════════════════════════════════════════════ */ static long elapsed_ms(struct timespec* t0, struct timespec* t1) { return (t1->tv_sec - t0->tv_sec) * 1000L + (t1->tv_nsec - t0->tv_nsec) / 1000000L; } static void init_legacy_config(xce_client_config_t* config, int port) { memset(config, 0, sizeof(*config)); config->nmasters = 1; config->masters[0] = "127.0.0.1"; config->master_ports[0] = port; config->from = "xce_scheduler_test"; config->key = "keepsecret$"; } static void init_cluster_cfg_n(xce_cluster_config_t* cfg, int first_port, int count) { memset(cfg, 0, sizeof(*cfg)); cfg->candidates_count = count; for (int i = 0; i < count; i++) { strncpy(cfg->candidates[i].hostname, "127.0.0.1", XCE_MAXHOSTNAME_LEN - 1); cfg->candidates[i].port = first_port + i; } strncpy(cfg->identity, "xce_scheduler_test", sizeof(cfg->identity) - 1); } /* Ports deliberately unlikely to have a real listener bound in CI/dev * sandboxes -- connect() against a closed local port fails fast * (ECONNREFUSED), so every sched_init() call below returns quickly without * needing a live controller, matching worker's own task 2.3 precedent * (test_xcecd_join_cluster_all_unreachable_reports_every_candidate). */ #define TEST_PORT_BASE 18581 /* ════════════════════════════════════════════════════════════════════════ * Req 1.5 — unified candidate-list pipeline: no branch for "single host" * ════════════════════════════════════════════════════════════════════════ */ static void test_no_peers_and_single_entry_peers_produce_identical_cluster_cfg_shape( void) { xce_scheduler_t sched_no_peers; xce_client_config_t config_no_peers; xce_scheduler_t sched_one_peer; xce_client_config_t config_one_peer; xce_cluster_config_t cluster_cfg_one_peer; /* (a) No --peers: cluster_cfg is NULL, legacy config carries the * default host/port -- sched_init() must synthesize a single-candidate * xce_cluster_config_t from it. */ init_legacy_config(&config_no_peers, TEST_PORT_BASE); sched_init(&sched_no_peers, &config_no_peers, NULL, scheduler_req_handler); /* (b) A single-entry --peers list pointed at the exact same host/port. */ init_legacy_config(&config_one_peer, 1 /* unused: cluster_cfg wins */); init_cluster_cfg_n(&cluster_cfg_one_peer, TEST_PORT_BASE, 1); sched_init(&sched_one_peer, &config_one_peer, &cluster_cfg_one_peer, scheduler_req_handler); TEST_ASSERT_EQUAL_INT(sched_one_peer.cluster_cfg.candidates_count, sched_no_peers.cluster_cfg.candidates_count); TEST_ASSERT_EQUAL_STRING(sched_one_peer.cluster_cfg.candidates[0].hostname, sched_no_peers.cluster_cfg.candidates[0].hostname); TEST_ASSERT_EQUAL_INT(sched_one_peer.cluster_cfg.candidates[0].port, sched_no_peers.cluster_cfg.candidates[0].port); printf( "test_no_peers_and_single_entry_peers_produce_identical_cluster_" "cfg_shape: PASS\n"); } static void test_multi_candidate_cluster_cfg_passthrough(void) { xce_scheduler_t sched; xce_client_config_t config; xce_cluster_config_t cluster_cfg; init_legacy_config(&config, 1 /* unused: cluster_cfg wins */); init_cluster_cfg_n(&cluster_cfg, TEST_PORT_BASE, 3); sched_init(&sched, &config, &cluster_cfg, scheduler_req_handler); TEST_ASSERT_EQUAL_INT(3, sched.cluster_cfg.candidates_count); for (int i = 0; i < 3; i++) { TEST_ASSERT_EQUAL_STRING("127.0.0.1", sched.cluster_cfg.candidates[i].hostname); TEST_ASSERT_EQUAL_INT(TEST_PORT_BASE + i, sched.cluster_cfg.candidates[i].port); } printf("test_multi_candidate_cluster_cfg_passthrough: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Req 1.6 / Req 2.1-2.3 — connect failure across every candidate is * reported (not silently hung, not treated as success) * ════════════════════════════════════════════════════════════════════════ */ static void test_sched_init_all_candidates_unreachable_fails_fast(void) { xce_scheduler_t sched; xce_client_config_t config; xce_cluster_config_t cluster_cfg; struct timespec t0, t1; int rc; init_legacy_config(&config, 1 /* unused: cluster_cfg wins */); init_cluster_cfg_n(&cluster_cfg, TEST_PORT_BASE + 10, 3); clock_gettime(CLOCK_MONOTONIC, &t0); rc = sched_init(&sched, &config, &cluster_cfg, scheduler_req_handler); clock_gettime(CLOCK_MONOTONIC, &t1); TEST_ASSERT_TRUE_MESSAGE( rc < 0, "sched_init() against 3 unreachable candidates must fail"); /* Generous bound: closed-port ECONNREFUSED is near-instant, well under * a second even across 3 candidates -- proves this is NOT the deleted * per-candidate loop silently retrying/blocking somewhere. */ TEST_ASSERT_LESS_THAN_INT(5000, elapsed_ms(&t0, &t1)); printf( "test_sched_init_all_candidates_unreachable_fails_fast: PASS " "(rc=%d, elapsed_ms=%ld)\n", rc, elapsed_ms(&t0, &t1)); } /* * Captures STDOUT_FILENO for the duration of a sched_init() call, returning * the captured text as a malloc'd, NUL-terminated buffer the caller must * free(). Same dup()/dup2() mechanism as * test_scheduler_decision_publish_retry.c's own capture_stdout_during() * (component-failover-adoption task 3.2) and, before that, * test_apply_correctness_verification.c's * test_apply_divergent_loud_fence_log_omits_false_peer_hwm_clause: this test * binary never calls mylog_init()/tlog_init() (only the real daemon main()s * do), so tlog's internal root log handle stays NULL and every WARN()/ * ERROR() call falls through tlog's early-print path, which writes the * fully formatted line directly to STDOUT_FILENO via a raw write() * (vendored tinylog, tlog.c) -- genuine runtime log capture, not a * source-text scan, so this proves what sched_init() actually PRINTS, not * merely what its source contains. */ static char* capture_stdout_during_sched_init(xce_scheduler_t* sched, xce_client_config_t* config, xce_cluster_config_t* cluster_cfg, int* rc_out) { char tmp[] = "/tmp/xce_sched_connect_logcap_XXXXXX"; int fd = mkstemp(tmp); TEST_ASSERT_TRUE_MESSAGE( fd >= 0, "fixture: mkstemp for stdout capture must succeed"); fflush(stdout); int saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0, "fixture: dup(STDOUT_FILENO) must succeed"); TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0, "fixture: dup2 redirect of STDOUT_FILENO must " "succeed"); *rc_out = sched_init(sched, config, cluster_cfg, scheduler_req_handler); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); off_t len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE_MESSAGE(len >= 0, "fixture: lseek on capture file must " "succeed"); char* buf = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(buf); if (len > 0) { TEST_ASSERT_TRUE_MESSAGE(lseek(fd, 0, SEEK_SET) == 0, "fixture: rewind of capture file must " "succeed"); ssize_t nread = read(fd, buf, (size_t)len); TEST_ASSERT_TRUE_MESSAGE(nread == len, "fixture: reading back captured buffer " "must succeed"); } close(fd); unlink(tmp); return buf; } /* * test_all_candidates_unreachable_log_names_every_candidate_host_port * (component-failover-adoption, task 3.3, Req 1 AC6 / Req 5 AC2) -- * behavioral proof that sched_init()'s all-candidates-unreachable failure * path actually PRINTS every attempted candidate's host:port, not merely * that rc<0 (test_sched_init_all_candidates_unreachable_fails_fast above * already covers the rc<0/fails-fast half; this is the missing "identifying * which candidates were attempted" half). sched_format_candidate_list() * itself (scheduler.c) is `static` and this file object-links scheduler_srcs * rather than #include-ing scheduler.c (see file header comment), so it * cannot be called directly -- captures the real WARN() output sched_init() * emits instead, same genre as this file's own header comment already * establishes for the rejoin-callback/single-connect-sequence structural * checks, but behavioral rather than source-text here since the actual * runtime output is exactly what "produces a failure message naming each * attempted candidate" (this task's own Observable completion wording) * means. */ static void test_all_candidates_unreachable_log_names_every_candidate_host_port(void) { xce_scheduler_t sched; xce_client_config_t config; xce_cluster_config_t cluster_cfg; int rc; char* captured; init_legacy_config(&config, 1 /* unused: cluster_cfg wins */); init_cluster_cfg_n(&cluster_cfg, TEST_PORT_BASE + 40, 3); captured = capture_stdout_during_sched_init(&sched, &config, &cluster_cfg, &rc); TEST_ASSERT_TRUE_MESSAGE( rc < 0, "sched_init() against 3 unreachable candidates must fail"); /* Every configured candidate's "host:port" must appear in the captured * failure log, in the exact format sched_format_candidate_list() * produces -- not just a candidate count. */ for (int i = 0; i < 3; i++) { char expected[64]; snprintf(expected, sizeof(expected), "127.0.0.1:%d", TEST_PORT_BASE + 40 + i); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, expected), "all-candidates-unreachable log must name every attempted " "candidate's host:port -- one is missing"); } free(captured); printf("test_all_candidates_unreachable_log_names_every_candidate_host_" "port: PASS\n"); } static void test_sched_init_no_peers_default_also_fails_cleanly(void) { xce_scheduler_t sched; xce_client_config_t config; int rc; /* No --peers, and the legacy default host/port also happens to be * unreachable in this sandbox -- confirms the synthesized single- * candidate path fails the same clean way (rc < 0), never hangs, never * mis-reports success. */ init_legacy_config(&config, TEST_PORT_BASE + 20); rc = sched_init(&sched, &config, NULL, scheduler_req_handler); TEST_ASSERT_TRUE(rc < 0); TEST_ASSERT_EQUAL_INT(1, sched.cluster_cfg.candidates_count); printf("test_sched_init_no_peers_default_also_fails_cleanly: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Structural regression guards (source-text scan of the real, current * on-disk scheduler.c -- same genre as test_picker_core_loop_boundary_test * and worker's test_reconnect_handler_registered_after_start_not_before_ * in_source; see this file's header comment for why these two properties * are checked this way instead of behaviorally). * ════════════════════════════════════════════════════════════════════════ */ static char* read_whole_file(const char* path) { FILE* f = fopen(path, "r"); if (!f) { return NULL; } fseek(f, 0, SEEK_END); long size = ftell(f); fseek(f, 0, SEEK_SET); if (size < 0) { fclose(f); return NULL; } char* buf = (char*)malloc((size_t)size + 1); if (!buf) { fclose(f); return NULL; } size_t nread = fread(buf, 1, (size_t)size, f); buf[nread] = '\0'; fclose(f); return buf; } static void test_reconnect_handler_registered_after_start_not_before(void) { char* src = read_whole_file(XCE_SCHEDULER_C_PATH); TEST_ASSERT_NOT_NULL_MESSAGE(src, "could not read scheduler.c source for scan"); const char* start_call = strstr(src, "xcecli_start(sched->apictx)"); const char* register_call = strstr(src, "xcecli_set_reconnect_handler(sched->apictx, " "sched_on_reconnect, NULL)"); TEST_ASSERT_NOT_NULL_MESSAGE( start_call, "xcecli_start(sched->apictx) call site not found"); TEST_ASSERT_NOT_NULL_MESSAGE( register_call, "xcecli_set_reconnect_handler(sched->apictx, sched_on_reconnect, " "NULL) call site not found"); TEST_ASSERT_TRUE_MESSAGE( register_call > start_call, "xcecli_set_reconnect_handler() must be registered AFTER " "xcecli_start() succeeds -- xcecli_start() unconditionally resets " "ctx->failover.rejoin_cb to NULL when it connects, so registering " "before it is silently wiped out and the rejoin callback can never " "fire (the exact production bug worker's task 2.3 found and fixed " "for the identical ordering)."); free(src); printf( "test_reconnect_handler_registered_after_start_not_before: " "PASS\n"); } /* Extracts the body of a named function (from its opening '{' to the * matching closing '}', simple brace-depth counting -- sufficient here * since neither target function contains string/char literals with braces) * from src. Returns a malloc'd copy, or NULL if the function is not found. */ static char* extract_function_body(const char* src, const char* fn_name) { const char* p = strstr(src, fn_name); if (!p) { return NULL; } const char* brace = strchr(p, '{'); if (!brace) { return NULL; } int depth = 0; const char* q = brace; do { if (*q == '{') { depth++; } else if (*q == '}') { depth--; } q++; } while (depth > 0 && *q != '\0'); size_t len = (size_t)(q - brace); char* body = (char*)malloc(len + 1); if (!body) { return NULL; } memcpy(body, brace, len); body[len] = '\0'; return body; } static void test_rejoin_callback_body_has_no_io_or_xcecli_calls(void) { char* src = read_whole_file(XCE_SCHEDULER_C_PATH); TEST_ASSERT_NOT_NULL_MESSAGE(src, "could not read scheduler.c source for scan"); char* body = extract_function_body(src, "static int sched_on_reconnect("); TEST_ASSERT_NOT_NULL_MESSAGE(body, "sched_on_reconnect() definition not found"); TEST_ASSERT_NULL_MESSAGE( strstr(body, "xcecli_"), "sched_on_reconnect() must not call any xcecli_* function -- the " "library invokes it while still holding its own per-ctx lock " "(design.md's Rejoin callback constraint); doing any I/O here " "self-deadlocks the calling thread."); free(body); free(src); printf("test_rejoin_callback_body_has_no_io_or_xcecli_calls: PASS\n"); } static void test_sched_init_has_single_connect_sequence_not_a_loop(void) { char* src = read_whole_file(XCE_SCHEDULER_C_PATH); TEST_ASSERT_NOT_NULL_MESSAGE(src, "could not read scheduler.c source for scan"); char* body = extract_function_body(src, "int sched_init(xce_scheduler_t"); TEST_ASSERT_NOT_NULL_MESSAGE(body, "sched_init() definition not found"); /* Exactly one xcecli_join() call site: the former per-candidate loop * called it once per iteration (a real loop, not a single call), and * the former single-host `else` branch duplicated it a second time in * the same function. A single call site proves both are gone. */ const char* first_join = strstr(body, "xcecli_join(sched->apictx,"); TEST_ASSERT_NOT_NULL_MESSAGE(first_join, "xcecli_join(sched->apictx, ...) call site " "not found in sched_init()"); const char* second_join = strstr(first_join + 1, "xcecli_join(sched->apictx,"); TEST_ASSERT_NULL_MESSAGE( second_join, "sched_init() must contain exactly ONE xcecli_join() call site -- " "more than one means the per-candidate loop or the separate " "single-host branch this task deletes is still present"); /* Same check for xcecli_ctx_init(): the deleted per-candidate loop * called it once per iteration PLUS once more unconditionally before * the loop (the "later discarded" initial init) -- a single call site * proves the whole dual-path structure is gone. */ const char* first_init = strstr(body, "xcecli_ctx_init(&(sched->apictx)"); TEST_ASSERT_NOT_NULL_MESSAGE(first_init, "xcecli_ctx_init(&(sched->apictx), ...) " "call site not found in sched_init()"); const char* second_init = strstr(first_init + 1, "xcecli_ctx_init(&(sched->apictx)"); TEST_ASSERT_NULL_MESSAGE( second_init, "sched_init() must contain exactly ONE xcecli_ctx_init() call " "site -- more than one means the discarded initial init or the " "per-candidate loop's re-init is still present"); /* The shared conversion helper (task 1.2) must actually be in the * single-sequence path -- proves the pipeline runs through the shared * validate/convert helper rather than hand-rolling nmasters/masters[] * assignment (the deleted loop's own pattern) again. */ TEST_ASSERT_NOT_NULL_MESSAGE( strstr(body, "xce_cluster_config_to_client_config("), "sched_init() must call the shared " "xce_cluster_config_to_client_config() helper"); free(body); free(src); printf("test_sched_init_has_single_connect_sequence_not_a_loop: PASS\n"); } /* * test_no_periodic_heartbeat_call_in_source (component-failover-adoption, * task 3.3, Req 3 AC3) - a source-text-scan structural regression guard, * same genre as this file's other structural tests above. * * Confirmed by source read (grep -rn "xcecli_hello\|heartbeat\|liveness" * src/scheduler/src/*.c returns zero matches): the scheduler makes no * periodic liveness/heartbeat call of its own to a connected controller * today, unlike worker's checking_connection()/xcecli_hello() * (controller_connector.c) or apiserver. This means Req 3 AC3's "reconnect * retries shall not delay or prevent the component's own heartbeat * responses" is satisfied BY CONSTRUCTION for the scheduler -- there is no * heartbeat call for a stuck retry to ever delay -- not by any runtime * behavior this task adds. A behavioral test would have nothing to * exercise; this cheap net instead fails loudly if a future change * introduces a periodic xcecli_hello-style call into scheduler.c, which * would silently reopen exactly the liveness hazard Req 3 AC3 rules out. */ static void test_no_periodic_heartbeat_call_in_source(void) { char* src = read_whole_file(XCE_SCHEDULER_C_PATH); TEST_ASSERT_NOT_NULL_MESSAGE(src, "could not read scheduler.c source for scan"); TEST_ASSERT_NULL_MESSAGE( strstr(src, "xcecli_hello"), "Req 3 AC3: the scheduler must make no periodic liveness/heartbeat " "call of its own to a connected controller -- introducing one " "(e.g. a checking_connection()-style poll loop calling " "xcecli_hello(), mirroring worker's controller_connector.c) would " "give a stalled reconnect retry a heartbeat call it could delay, " "reopening the exact liveness hazard this requirement rules out."); TEST_ASSERT_NULL_MESSAGE( strstr(src, "heartbeat"), "no heartbeat-labeled logic of any kind is expected in scheduler.c " "-- see this test's header comment"); free(src); printf("test_no_periodic_heartbeat_call_in_source: PASS\n"); } int main(void) { UNITY_BEGIN(); RUN_TEST(test_no_peers_and_single_entry_peers_produce_identical_cluster_cfg_shape); RUN_TEST(test_multi_candidate_cluster_cfg_passthrough); RUN_TEST(test_sched_init_all_candidates_unreachable_fails_fast); RUN_TEST(test_all_candidates_unreachable_log_names_every_candidate_host_port); RUN_TEST(test_sched_init_no_peers_default_also_fails_cleanly); RUN_TEST(test_reconnect_handler_registered_after_start_not_before); RUN_TEST(test_no_periodic_heartbeat_call_in_source); RUN_TEST(test_rejoin_callback_body_has_no_io_or_xcecli_calls); RUN_TEST(test_sched_init_has_single_connect_sequence_not_a_loop); return UNITY_END(); }