feat(component-failover-adoption): trigger apiserver's rejoin from the idle-period heartbeat path

xcecli_heartbeat() is XCECLI_FF_CALL-wrapped -- a deliberately
non-retrying fire-and-forget path that does its own best-effort
one-shot reconnect but never calls xcecli_invoke_rejoin() (that only
fires from the write-call retry path). Left as-is, a controller
failover with no in-flight write request would silently repair the
transport connection without ever re-joining, permanently starving
push notifications.

runHeartbeat() gains two callback hooks fired at the right points --
onFailureToSuccess on a genuine transition, onSuccessTick on every
successful tick thereafter -- wired directly to task 4.3's existing
setNeedsRejoin()/checkAndPerformRejoin() from StartHeartbeat(). No
second rejoin mechanism: checkAndPerformRejoin() is a cheap no-op when
nothing is pending, so calling it unconditionally on every successful
tick is safe and matches the requirement that retry isn't gated on the
transition edge alone.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NNB4zGPCsHyDUCDJyMbiQN

src/frontend/apiserver/backend/heartbeat_rejoin_test.go

@@ -0,0 +1,132 @@
+//go:build cgo
+
+package backend
+
+import (
+	"context"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// heartbeat_rejoin_test.go exercises runHeartbeat wired to reconnect.go's
+// REAL setNeedsRejoin/checkAndPerformRejoin (component-failover-adoption
+// task 4.4, boundary: ApiserverBackendBridge, Req 2 AC4/AC5) -- proving the
+// idle-period rejoin trigger end-to-end at real tick granularity, the same
+// way clib.go's StartHeartbeat wires them in production. heartbeat_test.go
+// covers runHeartbeat's own tick/transition-tracking logic via generic
+// injected callbacks, without CGO; this file proves those callbacks are
+// wired to the right production functions and that the composition
+// actually clears/retains needsRejoin correctly across real ticks.
+//
+// Kept //go:build cgo since needsRejoin/setNeedsRejoin/checkAndPerformRejoin/
+// performRejoinFn only exist in the CGO build (reconnect.go, task 4.3) --
+// this file itself does not `import "C"` (a _test.go file can never do that
+// directly, golang/go#13938), matching the reconnect_test.go/candidates_test.go
+// precedent already established in this package.
+//
+// performRejoinFn (reconnect.go's existing test seam, task 4.3) is
+// overridden here to script the actual xcecli_join outcome without a live
+// controller -- same technique reconnect_test.go's own retry-on-failure
+// tests use. scriptedHeartbeatFn (heartbeat_test.go) scripts the heartbeat
+// send outcomes.
+
+func TestRunHeartbeat_IdlePeriodRecovery_SetsNeedsRejoinAndRetriesUntilJoinSucceeds(t *testing.T) {
+	captureLog(t) // checkAndPerformRejoin logs via utils.Log; nil otherwise
+	resetNeedsRejoinForTest()
+	atomic.StoreInt32(&heartbeatIntervalSecs, 1)
+
+	origPerformRejoinFn := performRejoinFn
+	defer func() { performRejoinFn = origPerformRejoinFn }()
+
+	var joinCalls int32
+	joinDone := make(chan struct{}, 8)
+	performRejoinFn = func() int {
+		n := atomic.AddInt32(&joinCalls, 1)
+		joinDone <- struct{}{}
+		if n == 1 {
+			return -1 // simulates -E_XCE_NOT_READY: this attempt's own retry budget exhausted
+		}
+		return 0 // succeeds on the second, separate attempt
+	}
+
+	// Heartbeat transport outcomes: fail (primary down), fail, succeed
+	// (transparent reconnect -- fires the transition), succeed (retry
+	// tick, while the first rejoin attempt is still pending).
+	fn := scriptedHeartbeatFn([]int{-1, -1, 0, 0})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		runHeartbeat(ctx, fn, setNeedsRejoin, checkAndPerformRejoin)
+		close(done)
+	}()
+
+	// Wait for both scripted rejoin attempts: one triggered by the
+	// transition tick (fails), one retried on the next successful tick
+	// (succeeds) -- proving the failed rejoin is retried, not abandoned
+	// (Req 2 AC5), at real heartbeat-tick granularity.
+	for i := 0; i < 2; i++ {
+		select {
+		case <-joinDone:
+		case <-time.After(6 * time.Second):
+			t.Fatalf("timed out waiting for rejoin attempt %d", i+1)
+		}
+	}
+	cancel()
+	<-done
+
+	if got := atomic.LoadInt32(&joinCalls); got != 2 {
+		t.Errorf("performRejoinFn called %d times, want exactly 2 (first attempt "+
+			"fails, retried on the next successful heartbeat tick)", got)
+	}
+	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
+		t.Error("expected needsRejoin to clear once the retried rejoin succeeds")
+	}
+}
+
+func TestRunHeartbeat_AllHeartbeatsSucceed_NeverSetsNeedsRejoin(t *testing.T) {
+	resetNeedsRejoinForTest()
+	atomic.StoreInt32(&heartbeatIntervalSecs, 1)
+
+	origPerformRejoinFn := performRejoinFn
+	defer func() { performRejoinFn = origPerformRejoinFn }()
+
+	var joinCalls int32
+	performRejoinFn = func() int {
+		atomic.AddInt32(&joinCalls, 1)
+		return 0
+	}
+
+	tickEvents := make(chan struct{}, 8)
+	fn := func(_ int) int {
+		tickEvents <- struct{}{}
+		return 0 // always succeeds -- no failure ever observed
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() {
+		runHeartbeat(ctx, fn, setNeedsRejoin, checkAndPerformRejoin)
+		close(done)
+	}()
+
+	for i := 0; i < 3; i++ {
+		select {
+		case <-tickEvents:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("timed out waiting for tick %d", i+1)
+		}
+	}
+	cancel()
+	<-done
+
+	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
+		t.Error("needsRejoin must stay clear when every heartbeat tick succeeds -- " +
+			"no failure ever occurred to transition from")
+	}
+	if got := atomic.LoadInt32(&joinCalls); got != 0 {
+		t.Errorf("performRejoinFn called %d times, want 0 (checkAndPerformRejoin "+
+			"must no-op when no rejoin is pending)", got)
+	}
+}