fix(component-failover-adoption): Phase 6 -- close apiserver's idle-period rejoin gap (bug-1548/bug-1549)

XCECLI_FF_CALL (backing xcecli_heartbeat) can silently rotate+reconnect
the transport entirely inside a single call and still return success, so
heartbeat.go's failure-to-success transition detector could never observe
a reconnect that completes within one heartbeat tick -- the common case,
since elections finish in single-digit seconds against a default 30s
interval. apiserver's application-level rejoin never fired on a real
idle-period failover, and job pushes silently stopped reaching it.

Adds a monotonic reconnect_gen counter to endpoint_t (netio.h), bumped by
xce_ctx_reconnect() (xcectx.c) on every successful reconnect regardless of
trigger path. heartbeat.go now compares this generation every tick instead
of tracking success/failure transitions. Re-validated live against task
5.2's exact scenario (apiserver alone, zero write traffic): rejoin fired
22s after a primary kill, 12s after election.

Reopens and closes out component-failover-adoption's own NO-GO -> GO
validation cycle from 2026-08-15.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/lib/meson.build

@@ -178,6 +178,7 @@ libxce_test_src = ['testsuite/test.c',
                    'testsuite/test_libxce_retry.c',
                    'testsuite/test_netio_on_connected.c',
                    'testsuite/test_netio_on_fini.c',
+                   'testsuite/test_netio_reconnect_gen.c',
                    'testsuite/test_netio_arm_failure.c',
                    'testsuite/test_netio_close_propagation.c',
                    'testsuite/test_netio_bounded_connect.c',

src/lib/testsuite/test.c

@@ -41,6 +41,7 @@ extern void test_cluster_parse(void);
 extern void test_libxce_retry(void);
 extern void test_netio_on_connected(void);
 extern void test_netio_on_fini(void);
+extern void test_netio_reconnect_gen(void);
 extern void test_netio_arm_failure(void);
 extern void test_netio_close_propagation(void);
 extern void test_netio_bounded_connect(void);
@@ -160,6 +161,7 @@ int main(int argc, char* argv[])
     TEST(test_libxce_retry);
     TEST(test_netio_on_connected);
     TEST(test_netio_on_fini);
+    TEST(test_netio_reconnect_gen);
     TEST(test_netio_arm_failure);
     TEST(test_netio_close_propagation);
     TEST(test_netio_bounded_connect);

src/lib/testsuite/test_netio_reconnect_gen.c

@@ -0,0 +1,90 @@
+/*
+ * test_netio_reconnect_gen.c — TDD tests for endpoint_t.reconnect_gen.
+ *
+ * component-failover-adoption Phase 6 (bug-1548): a monotonic counter bumped
+ * by xce_ctx_reconnect() (xcectx.c) on every successful transport reconnect,
+ * so a consumer (apiserver's heartbeat prober) can detect a reconnect even
+ * when it completed silently inside a single XCECLI_FF_CALL and the call's
+ * own return code never surfaced a failure.
+ *
+ * Mirrors test_netio_on_fini.c's structure for the same class of field
+ * (a passive observation point added to endpoint_t). These tests exercise
+ * only the field's lifecycle guarantees at the netio.c level — they do NOT
+ * exercise xce_ctx_reconnect() itself (that requires a live socket/SSL
+ * handshake) and are not a substitute for the required live E2E re-run of
+ * task 5.2's scenario. Tests verify:
+ *   1. A freshly heap-allocated endpoint_t starts with reconnect_gen == 0
+ *   2. endpoint_init() does not reset a previously-bumped value
+ *   3. endpoint_fini() does not reset a previously-bumped value
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+
+#include <unity/unity.h>
+#include "netio.h"
+
+/* ── Test 1: fresh endpoint_t starts at zero ─────────────────────── */
+
+static void test_reconnect_gen_starts_at_zero(void)
+{
+    endpoint_t *ep = endpoint_get();
+    TEST_ASSERT_NOT_NULL(ep);
+
+    TEST_ASSERT_EQUAL_UINT64(0, ep->reconnect_gen);
+
+    free(ep);
+    printf("test_reconnect_gen_starts_at_zero: PASS\n");
+}
+
+/* ── Test 2: endpoint_init() does not reset a bumped value ───────── */
+
+static void test_reconnect_gen_survives_endpoint_init(void)
+{
+    endpoint_t *ep = endpoint_get();
+    TEST_ASSERT_NOT_NULL(ep);
+
+    ep->reconnect_gen = 7;
+
+    int fd = endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
+    TEST_ASSERT_TRUE(fd >= 0);
+
+    TEST_ASSERT_EQUAL_UINT64(7, ep->reconnect_gen);
+
+    endpoint_fini(ep);
+    free(ep);
+    printf("test_reconnect_gen_survives_endpoint_init: PASS\n");
+}
+
+/* ── Test 3: endpoint_fini() does not reset a bumped value ───────── */
+
+static void test_reconnect_gen_survives_endpoint_fini(void)
+{
+    endpoint_t *ep = endpoint_get();
+    TEST_ASSERT_NOT_NULL(ep);
+
+    int fd = endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
+    TEST_ASSERT_TRUE(fd >= 0);
+
+    ep->reconnect_gen = 3;
+
+    endpoint_fini(ep);
+
+    TEST_ASSERT_EQUAL_UINT64(3, ep->reconnect_gen);
+
+    free(ep);
+    printf("test_reconnect_gen_survives_endpoint_fini: PASS\n");
+}
+
+/* ── Test suite entry point ───────────────────────────────────────── */
+
+void test_netio_reconnect_gen(void)
+{
+    printf("\n>> netio reconnect_gen field lifecycle tests\n\n");
+
+    RUN_TEST(test_reconnect_gen_starts_at_zero);
+    RUN_TEST(test_reconnect_gen_survives_endpoint_init);
+    RUN_TEST(test_reconnect_gen_survives_endpoint_fini);
+}

src/lib/xcectx.c

@@ -393,6 +393,14 @@ int xce_ctx_reconnect(xce_ctx_t* ctx, char* host, int port)
         return rc;
     }
 
+    /* bug-1548: mark that a reconnect actually completed, so a consumer
+     * polling ep->reconnect_gen (e.g. apiserver's heartbeat prober) can
+     * detect it even when this whole reconnect happened silently inside a
+     * single XCECLI_FF_CALL and the caller's own return code never surfaced
+     * a failure. Only bumped on this success path — a failed attempt above
+     * already returned early without reaching here. */
+    ctx->ep->reconnect_gen++;
+
     return 0;
 }