feat(component-failover-adoption): validate scheduler backpressure, fix candidate-name diagnostics

Adds a worst-case-occupancy test proving the scheduler's picker/allocator
pipeline buffers briefly past a stalled publish (decisions cbuf cap 5,
jbuf cap 1) then genuinely blocks until connectivity recovers, with zero
job loss (Req 3 AC2). Adds a source-scan regression guard confirming the
scheduler makes no periodic heartbeat call of its own (Req 3 AC3).

Also fixes a real gap found along the way: sched_init()'s
all-candidates-unreachable and join-failure log lines named only a
candidate count, never which candidates were attempted (same class of
bug worker's task 2.3 fixed). Added sched_format_candidate_list(),
covered by a real dup2()-based stdout capture around a live sched_init()
failure path (round-1 review caught this diagnostic shipping with zero
test coverage; fixed in round 2).

Multi-candidate join and kill-primary/reconnect were validated manually
against a live 3-node cluster rather than committed as an automated
test, following worker's task 2.3 precedent -- no automated live-deploy
E2E harness exists for any C daemon in this repo.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NNB4zGPCsHyDUCDJyMbiQN

src/scheduler/testsuite/test_scheduler_connect.c

@@ -24,10 +24,20 @@
  *    closed local ports (connect() gets ECONNREFUSED near-instantly, same
  *    "no backoff loop inside xcecli_start()'s per-master connect scan"
  *    property worker's task 2.3 test relies on) and inspecting the
- *    resulting sched->cluster_cfg -- every connect log line sched_init()
- *    emits is derived from that same struct's contents, so structural
- *    equality here is exactly what "identical connect log sequence" (this
- *    task's own observable-completion wording) reduces to.
+ *    resulting sched->cluster_cfg.
+ *  - Req 1 AC6 / Req 5 AC2's "identifies which candidates were attempted"
+ *    (component-failover-adoption, task 3.3) is proven by genuine RUNTIME
+ *    LOG CAPTURE (dup2() redirect of STDOUT_FILENO for the duration of the
+ *    sched_init() call under test -- same mechanism
+ *    test_scheduler_decision_publish_retry.c's own capture_stdout_during()
+ *    established for this boundary, see
+ *    capture_stdout_during_sched_init()'s own header comment below), not
+ *    inferred from sched->cluster_cfg's contents: proving what sched_init()
+ *    actually PRINTS is a materially stronger check than proving the struct
+ *    it derives the message from, since sched_format_candidate_list()
+ *    (scheduler.c) -- the function that turns that struct into the printed
+ *    "host:port, host:port, ..." text -- is itself `static` and otherwise
+ *    has no direct call site in this file to exercise.
  *  - The rejoin callback's "no I/O, sets a flag, returns immediately"
  *    contract (Req 2.1, Req 5.1-5.2, design.md's "Rejoin callback
  *    constraint") and the connect pipeline's single-path structure (Req
@@ -55,6 +65,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <time.h>
+#include <unistd.h>
 
 #include <unity/unity.h>
 
@@ -204,6 +215,120 @@ static void test_sched_init_all_candidates_unreachable_fails_fast(void)
         rc, elapsed_ms(&t0, &t1));
 }
 
+/*
+ * Captures STDOUT_FILENO for the duration of a sched_init() call, returning
+ * the captured text as a malloc'd, NUL-terminated buffer the caller must
+ * free(). Same dup()/dup2() mechanism as
+ * test_scheduler_decision_publish_retry.c's own capture_stdout_during()
+ * (component-failover-adoption task 3.2) and, before that,
+ * test_apply_correctness_verification.c's
+ * test_apply_divergent_loud_fence_log_omits_false_peer_hwm_clause: this test
+ * binary never calls mylog_init()/tlog_init() (only the real daemon main()s
+ * do), so tlog's internal root log handle stays NULL and every WARN()/
+ * ERROR() call falls through tlog's early-print path, which writes the
+ * fully formatted line directly to STDOUT_FILENO via a raw write()
+ * (vendored tinylog, tlog.c) -- genuine runtime log capture, not a
+ * source-text scan, so this proves what sched_init() actually PRINTS, not
+ * merely what its source contains. */
+static char* capture_stdout_during_sched_init(xce_scheduler_t*      sched,
+                                              xce_client_config_t*  config,
+                                              xce_cluster_config_t* cluster_cfg,
+                                              int*                  rc_out)
+{
+    char tmp[] = "/tmp/xce_sched_connect_logcap_XXXXXX";
+    int  fd    = mkstemp(tmp);
+    TEST_ASSERT_TRUE_MESSAGE(
+        fd >= 0, "fixture: mkstemp for stdout capture must succeed");
+
+    fflush(stdout);
+    int saved_stdout = dup(STDOUT_FILENO);
+    TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0,
+                             "fixture: dup(STDOUT_FILENO) must succeed");
+    TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0,
+                             "fixture: dup2 redirect of STDOUT_FILENO must "
+                             "succeed");
+
+    *rc_out = sched_init(sched, config, cluster_cfg, scheduler_req_handler);
+
+    fflush(stdout);
+    dup2(saved_stdout, STDOUT_FILENO);
+    close(saved_stdout);
+
+    off_t len = lseek(fd, 0, SEEK_CUR);
+    TEST_ASSERT_TRUE_MESSAGE(len >= 0, "fixture: lseek on capture file must "
+                                       "succeed");
+
+    char* buf = (char*)calloc(1, (size_t)len + 1);
+    TEST_ASSERT_NOT_NULL(buf);
+    if (len > 0) {
+        TEST_ASSERT_TRUE_MESSAGE(lseek(fd, 0, SEEK_SET) == 0,
+                                 "fixture: rewind of capture file must "
+                                 "succeed");
+        ssize_t nread = read(fd, buf, (size_t)len);
+        TEST_ASSERT_TRUE_MESSAGE(nread == len,
+                                 "fixture: reading back captured buffer "
+                                 "must succeed");
+    }
+    close(fd);
+    unlink(tmp);
+
+    return buf;
+}
+
+/*
+ * test_all_candidates_unreachable_log_names_every_candidate_host_port
+ * (component-failover-adoption, task 3.3, Req 1 AC6 / Req 5 AC2) --
+ * behavioral proof that sched_init()'s all-candidates-unreachable failure
+ * path actually PRINTS every attempted candidate's host:port, not merely
+ * that rc<0 (test_sched_init_all_candidates_unreachable_fails_fast above
+ * already covers the rc<0/fails-fast half; this is the missing "identifying
+ * which candidates were attempted" half). sched_format_candidate_list()
+ * itself (scheduler.c) is `static` and this file object-links scheduler_srcs
+ * rather than #include-ing scheduler.c (see file header comment), so it
+ * cannot be called directly -- captures the real WARN() output sched_init()
+ * emits instead, same genre as this file's own header comment already
+ * establishes for the rejoin-callback/single-connect-sequence structural
+ * checks, but behavioral rather than source-text here since the actual
+ * runtime output is exactly what "produces a failure message naming each
+ * attempted candidate" (this task's own Observable completion wording)
+ * means.
+ */
+static void
+test_all_candidates_unreachable_log_names_every_candidate_host_port(void)
+{
+    xce_scheduler_t      sched;
+    xce_client_config_t  config;
+    xce_cluster_config_t cluster_cfg;
+    int                  rc;
+    char*                captured;
+
+    init_legacy_config(&config, 1 /* unused: cluster_cfg wins */);
+    init_cluster_cfg_n(&cluster_cfg, TEST_PORT_BASE + 40, 3);
+
+    captured
+        = capture_stdout_during_sched_init(&sched, &config, &cluster_cfg, &rc);
+
+    TEST_ASSERT_TRUE_MESSAGE(
+        rc < 0, "sched_init() against 3 unreachable candidates must fail");
+
+    /* Every configured candidate's "host:port" must appear in the captured
+     * failure log, in the exact format sched_format_candidate_list()
+     * produces -- not just a candidate count. */
+    for (int i = 0; i < 3; i++) {
+        char expected[64];
+        snprintf(expected, sizeof(expected), "127.0.0.1:%d",
+                 TEST_PORT_BASE + 40 + i);
+        TEST_ASSERT_NOT_NULL_MESSAGE(
+            strstr(captured, expected),
+            "all-candidates-unreachable log must name every attempted "
+            "candidate's host:port -- one is missing");
+    }
+
+    free(captured);
+    printf("test_all_candidates_unreachable_log_names_every_candidate_host_"
+           "port: PASS\n");
+}
+
 static void test_sched_init_no_peers_default_also_fails_cleanly(void)
 {
     xce_scheduler_t     sched;
@@ -404,14 +529,57 @@ static void test_sched_init_has_single_connect_sequence_not_a_loop(void)
     printf("test_sched_init_has_single_connect_sequence_not_a_loop: PASS\n");
 }
 
+/*
+ * test_no_periodic_heartbeat_call_in_source (component-failover-adoption,
+ * task 3.3, Req 3 AC3) - a source-text-scan structural regression guard,
+ * same genre as this file's other structural tests above.
+ *
+ * Confirmed by source read (grep -rn "xcecli_hello\|heartbeat\|liveness"
+ * src/scheduler/src/*.c returns zero matches): the scheduler makes no
+ * periodic liveness/heartbeat call of its own to a connected controller
+ * today, unlike worker's checking_connection()/xcecli_hello()
+ * (controller_connector.c) or apiserver. This means Req 3 AC3's "reconnect
+ * retries shall not delay or prevent the component's own heartbeat
+ * responses" is satisfied BY CONSTRUCTION for the scheduler -- there is no
+ * heartbeat call for a stuck retry to ever delay -- not by any runtime
+ * behavior this task adds. A behavioral test would have nothing to
+ * exercise; this cheap net instead fails loudly if a future change
+ * introduces a periodic xcecli_hello-style call into scheduler.c, which
+ * would silently reopen exactly the liveness hazard Req 3 AC3 rules out.
+ */
+static void test_no_periodic_heartbeat_call_in_source(void)
+{
+    char* src = read_whole_file(XCE_SCHEDULER_C_PATH);
+    TEST_ASSERT_NOT_NULL_MESSAGE(src,
+                                 "could not read scheduler.c source for scan");
+
+    TEST_ASSERT_NULL_MESSAGE(
+        strstr(src, "xcecli_hello"),
+        "Req 3 AC3: the scheduler must make no periodic liveness/heartbeat "
+        "call of its own to a connected controller -- introducing one "
+        "(e.g. a checking_connection()-style poll loop calling "
+        "xcecli_hello(), mirroring worker's controller_connector.c) would "
+        "give a stalled reconnect retry a heartbeat call it could delay, "
+        "reopening the exact liveness hazard this requirement rules out.");
+    TEST_ASSERT_NULL_MESSAGE(
+        strstr(src, "heartbeat"),
+        "no heartbeat-labeled logic of any kind is expected in scheduler.c "
+        "-- see this test's header comment");
+
+    free(src);
+    printf("test_no_periodic_heartbeat_call_in_source: PASS\n");
+}
+
 int main(void)
 {
     UNITY_BEGIN();
     RUN_TEST(test_no_peers_and_single_entry_peers_produce_identical_cluster_cfg_shape);
     RUN_TEST(test_multi_candidate_cluster_cfg_passthrough);
     RUN_TEST(test_sched_init_all_candidates_unreachable_fails_fast);
+    RUN_TEST(test_all_candidates_unreachable_log_names_every_candidate_host_port);
     RUN_TEST(test_sched_init_no_peers_default_also_fails_cleanly);
     RUN_TEST(test_reconnect_handler_registered_after_start_not_before);
+    RUN_TEST(test_no_periodic_heartbeat_call_in_source);
     RUN_TEST(test_rejoin_callback_body_has_no_io_or_xcecli_calls);
     RUN_TEST(test_sched_init_has_single_connect_sequence_not_a_loop);
     return UNITY_END();