fix(component-failover-adoption): Phase 6 -- close apiserver's idle-period rejoin gap (bug-1548/bug-1549)

XCECLI_FF_CALL (backing xcecli_heartbeat) can silently rotate+reconnect
the transport entirely inside a single call and still return success, so
heartbeat.go's failure-to-success transition detector could never observe
a reconnect that completes within one heartbeat tick -- the common case,
since elections finish in single-digit seconds against a default 30s
interval. apiserver's application-level rejoin never fired on a real
idle-period failover, and job pushes silently stopped reaching it.

Adds a monotonic reconnect_gen counter to endpoint_t (netio.h), bumped by
xce_ctx_reconnect() (xcectx.c) on every successful reconnect regardless of
trigger path. heartbeat.go now compares this generation every tick instead
of tracking success/failure transitions. Re-validated live against task
5.2's exact scenario (apiserver alone, zero write traffic): rejoin fired
22s after a primary kill, 12s after election.

Reopens and closes out component-failover-adoption's own NO-GO -> GO
validation cycle from 2026-08-15.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/frontend/apiserver/backend/heartbeat_rejoin_test.go

@@ -11,13 +11,18 @@ import (
 
 // heartbeat_rejoin_test.go exercises runHeartbeat wired to reconnect.go's
 // REAL setNeedsRejoin/checkAndPerformRejoin (component-failover-adoption
-// task 4.4, boundary: ApiserverBackendBridge, Req 2 AC4/AC5) -- proving the
-// idle-period rejoin trigger end-to-end at real tick granularity, the same
-// way clib.go's StartHeartbeat wires them in production. heartbeat_test.go
-// covers runHeartbeat's own tick/transition-tracking logic via generic
-// injected callbacks, without CGO; this file proves those callbacks are
-// wired to the right production functions and that the composition
-// actually clears/retains needsRejoin correctly across real ticks.
+// task 4.4/Phase 6, boundary: ApiserverBackendBridge, Req 2 AC4/AC5) --
+// proving the idle-period rejoin trigger end-to-end at real tick
+// granularity, the same way clib.go's StartHeartbeat wires them in
+// production (currentReconnectGen itself -- the real
+// C.xcecli_get_my_endpoint(gCtx.ctx) read -- is NOT exercised here; these
+// tests inject a scripted currentReconnectGen double, same as
+// heartbeat_test.go, since gCtx has no live connection in a unit test).
+// heartbeat_test.go covers runHeartbeat's own tick/generation-tracking
+// logic via generic injected callbacks, without CGO; this file proves
+// those callbacks are wired to the right production functions and that the
+// composition actually clears/retains needsRejoin correctly across real
+// ticks.
 //
 // Kept //go:build cgo since needsRejoin/setNeedsRejoin/checkAndPerformRejoin/
 // performRejoinFn only exist in the CGO build (reconnect.go, task 4.3) --
@@ -28,10 +33,24 @@ import (
 // performRejoinFn (reconnect.go's existing test seam, task 4.3) is
 // overridden here to script the actual xcecli_join outcome without a live
 // controller -- same technique reconnect_test.go's own retry-on-failure
-// tests use. scriptedHeartbeatFn (heartbeat_test.go) scripts the heartbeat
-// send outcomes.
+// tests use. scriptedHeartbeatFn/scriptedReconnectGen (heartbeat_test.go)
+// script the heartbeat send outcomes and the reconnect-generation reads.
 
-func TestRunHeartbeat_IdlePeriodRecovery_SetsNeedsRejoinAndRetriesUntilJoinSucceeds(t *testing.T) {
+// This is the CGO-level counterpart to
+// TestRunHeartbeat_ReconnectDetected_FiresOnGenChangeEvenWhenAllTicksSucceed
+// in heartbeat_test.go, and is the actual proof of closure for bug-1548 at
+// the level production code runs at: every heartbeat tick's own fn() call
+// reports SUCCESS (the literal shape XCECLI_FF_CALL's silent inline
+// reconnect+recover produces -- no failing tick is ever observed), yet a
+// reconnect genuinely happened (the generation counter changed). The old
+// failure-to-success transition detector could never set needsRejoin in
+// this scenario; this test proves the replacement generation-based
+// detector does. Still only proof of the DETECTOR's logic wired to the
+// real production callbacks -- whether currentReconnectGen's real
+// C.xcecli_get_my_endpoint read is itself reachable/reliable against a real
+// XCECLI_FF_CALL reconnect is proven only by the required live E2E re-run
+// of task 5.2's scenario (Phase 6, task 6.2), not by any unit test.
+func TestRunHeartbeat_IdlePeriodRecovery_GenChangeAloneSetsNeedsRejoinEvenWhenAllTicksSucceed(t *testing.T) {
 	captureLog(t) // checkAndPerformRejoin logs via utils.Log; nil otherwise
 	resetNeedsRejoinForTest()
 	atomic.StoreInt32(&heartbeatIntervalSecs, 1)
@@ -39,6 +58,53 @@ func TestRunHeartbeat_IdlePeriodRecovery_SetsNeedsRejoinAndRetriesUntilJoinSucce
 	origPerformRejoinFn := performRejoinFn
 	defer func() { performRejoinFn = origPerformRejoinFn }()
 
+	var joinCalls int32
+	joinDone := make(chan struct{}, 8)
+	performRejoinFn = func() int {
+		atomic.AddInt32(&joinCalls, 1)
+		joinDone <- struct{}{}
+		return 0 // succeeds on the first attempt
+	}
+
+	// Every heartbeat tick this test observes reports success -- no failure
+	// ever surfaces to Go, matching XCECLI_FF_CALL's silent self-heal.
+	fn := scriptedHeartbeatFn([]int{0, 0, 0})
+	// Generation changes on the second tick (pre-loop baseline=5, tick1=5
+	// unchanged, tick2=6 changed) -- a reconnect happened even though the
+	// caller never saw a failing tick.
+	gens := scriptedReconnectGen([]uint64{5, 5, 6, 6})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		runHeartbeat(ctx, fn, gens, setNeedsRejoin, checkAndPerformRejoin)
+		close(done)
+	}()
+
+	select {
+	case <-joinDone:
+	case <-time.After(6 * time.Second):
+		t.Fatal("timed out waiting for the rejoin attempt triggered by the generation change")
+	}
+	cancel()
+	<-done
+
+	if got := atomic.LoadInt32(&joinCalls); got != 1 {
+		t.Errorf("performRejoinFn called %d times, want exactly 1", got)
+	}
+	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
+		t.Error("expected needsRejoin to clear once the rejoin succeeds")
+	}
+}
+
+func TestRunHeartbeat_RejoinRetriesOnFailureUntilItSucceeds(t *testing.T) {
+	captureLog(t)
+	resetNeedsRejoinForTest()
+	atomic.StoreInt32(&heartbeatIntervalSecs, 1)
+
+	origPerformRejoinFn := performRejoinFn
+	defer func() { performRejoinFn = origPerformRejoinFn }()
+
 	var joinCalls int32
 	joinDone := make(chan struct{}, 8)
 	performRejoinFn = func() int {
@@ -50,22 +116,20 @@ func TestRunHeartbeat_IdlePeriodRecovery_SetsNeedsRejoinAndRetriesUntilJoinSucce
 		return 0 // succeeds on the second, separate attempt
 	}
 
-	// Heartbeat transport outcomes: fail (primary down), fail, succeed
-	// (transparent reconnect -- fires the transition), succeed (retry
-	// tick, while the first rejoin attempt is still pending).
-	fn := scriptedHeartbeatFn([]int{-1, -1, 0, 0})
+	fn := scriptedHeartbeatFn([]int{0, 0, 0})
+	// Generation changes once, on tick 1 (baseline=1, tick1=2 changed,
+	// stays 2 thereafter) -- one reconnect, but the resulting rejoin must
+	// be retried across multiple successful ticks until it succeeds (Req 2
+	// AC5), not abandoned after its first failed attempt.
+	gens := scriptedReconnectGen([]uint64{1, 2, 2, 2})
 
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() {
-		runHeartbeat(ctx, fn, setNeedsRejoin, checkAndPerformRejoin)
+		runHeartbeat(ctx, fn, gens, setNeedsRejoin, checkAndPerformRejoin)
 		close(done)
 	}()
 
-	// Wait for both scripted rejoin attempts: one triggered by the
-	// transition tick (fails), one retried on the next successful tick
-	// (succeeds) -- proving the failed rejoin is retried, not abandoned
-	// (Req 2 AC5), at real heartbeat-tick granularity.
 	for i := 0; i < 2; i++ {
 		select {
 		case <-joinDone:
@@ -85,7 +149,7 @@ func TestRunHeartbeat_IdlePeriodRecovery_SetsNeedsRejoinAndRetriesUntilJoinSucce
 	}
 }
 
-func TestRunHeartbeat_AllHeartbeatsSucceed_NeverSetsNeedsRejoin(t *testing.T) {
+func TestRunHeartbeat_NoGenChange_NeverSetsNeedsRejoin(t *testing.T) {
 	resetNeedsRejoinForTest()
 	atomic.StoreInt32(&heartbeatIntervalSecs, 1)
 
@@ -101,13 +165,13 @@ func TestRunHeartbeat_AllHeartbeatsSucceed_NeverSetsNeedsRejoin(t *testing.T) {
 	tickEvents := make(chan struct{}, 8)
 	fn := func(_ int) int {
 		tickEvents <- struct{}{}
-		return 0 // always succeeds -- no failure ever observed
+		return 0 // always succeeds
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() {
-		runHeartbeat(ctx, fn, setNeedsRejoin, checkAndPerformRejoin)
+		runHeartbeat(ctx, fn, constantReconnectGen(9), setNeedsRejoin, checkAndPerformRejoin)
 		close(done)
 	}()
 
@@ -122,8 +186,8 @@ func TestRunHeartbeat_AllHeartbeatsSucceed_NeverSetsNeedsRejoin(t *testing.T) {
 	<-done
 
 	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
-		t.Error("needsRejoin must stay clear when every heartbeat tick succeeds -- " +
-			"no failure ever occurred to transition from")
+		t.Error("needsRejoin must stay clear when the reconnect generation never changes -- " +
+			"no reconnect ever occurred")
 	}
 	if got := atomic.LoadInt32(&joinCalls); got != 0 {
 		t.Errorf("performRejoinFn called %d times, want 0 (checkAndPerformRejoin "+