myxce/src/frontend/apiserver/backend/heartbeat_rejoin_test.go

//go:build cgo

package backend

import (
	"context"
	"sync/atomic"
	"testing"
	"time"
)

// heartbeat_rejoin_test.go exercises runHeartbeat wired to reconnect.go's
// REAL setNeedsRejoin/checkAndPerformRejoin (component-failover-adoption
// task 4.4/Phase 6, boundary: ApiserverBackendBridge, Req 2 AC4/AC5) --
// proving the idle-period rejoin trigger end-to-end at real tick
// granularity, the same way clib.go's StartHeartbeat wires them in
// production (currentReconnectGen itself -- the real
// C.xcecli_get_my_endpoint(gCtx.ctx) read -- is NOT exercised here; these
// tests inject a scripted currentReconnectGen double, same as
// heartbeat_test.go, since gCtx has no live connection in a unit test).
// heartbeat_test.go covers runHeartbeat's own tick/generation-tracking
// logic via generic injected callbacks, without CGO; this file proves
// those callbacks are wired to the right production functions and that the
// composition actually clears/retains needsRejoin correctly across real
// ticks.
//
// Kept //go:build cgo since needsRejoin/setNeedsRejoin/checkAndPerformRejoin/
// performRejoinFn only exist in the CGO build (reconnect.go, task 4.3) --
// this file itself does not `import "C"` (a _test.go file can never do that
// directly, golang/go#13938), matching the reconnect_test.go/candidates_test.go
// precedent already established in this package.
//
// performRejoinFn (reconnect.go's existing test seam, task 4.3) is
// overridden here to script the actual xcecli_join outcome without a live
// controller -- same technique reconnect_test.go's own retry-on-failure
// tests use. scriptedHeartbeatFn/scriptedReconnectGen (heartbeat_test.go)
// script the heartbeat send outcomes and the reconnect-generation reads.

// This is the CGO-level counterpart to
// TestRunHeartbeat_ReconnectDetected_FiresOnGenChangeEvenWhenAllTicksSucceed
// in heartbeat_test.go, and is the actual proof of closure for bug-1548 at
// the level production code runs at: every heartbeat tick's own fn() call
// reports SUCCESS (the literal shape XCECLI_FF_CALL's silent inline
// reconnect+recover produces -- no failing tick is ever observed), yet a
// reconnect genuinely happened (the generation counter changed). The old
// failure-to-success transition detector could never set needsRejoin in
// this scenario; this test proves the replacement generation-based
// detector does. Still only proof of the DETECTOR's logic wired to the
// real production callbacks -- whether currentReconnectGen's real
// C.xcecli_get_my_endpoint read is itself reachable/reliable against a real
// XCECLI_FF_CALL reconnect is proven only by the required live E2E re-run
// of task 5.2's scenario (Phase 6, task 6.2), not by any unit test.
func TestRunHeartbeat_IdlePeriodRecovery_GenChangeAloneSetsNeedsRejoinEvenWhenAllTicksSucceed(t *testing.T) {
	captureLog(t) // checkAndPerformRejoin logs via utils.Log; nil otherwise
	resetNeedsRejoinForTest()
	atomic.StoreInt32(&heartbeatIntervalSecs, 1)

	origPerformRejoinFn := performRejoinFn
	defer func() { performRejoinFn = origPerformRejoinFn }()

	var joinCalls int32
	joinDone := make(chan struct{}, 8)
	performRejoinFn = func() int {
		atomic.AddInt32(&joinCalls, 1)
		joinDone <- struct{}{}
		return 0 // succeeds on the first attempt
	}

	// Every heartbeat tick this test observes reports success -- no failure
	// ever surfaces to Go, matching XCECLI_FF_CALL's silent self-heal.
	fn := scriptedHeartbeatFn([]int{0, 0, 0})
	// Generation changes on the second tick (pre-loop baseline=5, tick1=5
	// unchanged, tick2=6 changed) -- a reconnect happened even though the
	// caller never saw a failing tick.
	gens := scriptedReconnectGen([]uint64{5, 5, 6, 6})

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() {
		runHeartbeat(ctx, fn, gens, setNeedsRejoin, checkAndPerformRejoin)
		close(done)
	}()

	select {
	case <-joinDone:
	case <-time.After(6 * time.Second):
		t.Fatal("timed out waiting for the rejoin attempt triggered by the generation change")
	}
	cancel()
	<-done

	if got := atomic.LoadInt32(&joinCalls); got != 1 {
		t.Errorf("performRejoinFn called %d times, want exactly 1", got)
	}
	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
		t.Error("expected needsRejoin to clear once the rejoin succeeds")
	}
}

func TestRunHeartbeat_RejoinRetriesOnFailureUntilItSucceeds(t *testing.T) {
	captureLog(t)
	resetNeedsRejoinForTest()
	atomic.StoreInt32(&heartbeatIntervalSecs, 1)

	origPerformRejoinFn := performRejoinFn
	defer func() { performRejoinFn = origPerformRejoinFn }()

	var joinCalls int32
	joinDone := make(chan struct{}, 8)
	performRejoinFn = func() int {
		n := atomic.AddInt32(&joinCalls, 1)
		joinDone <- struct{}{}
		if n == 1 {
			return -1 // simulates -E_XCE_NOT_READY: this attempt's own retry budget exhausted
		}
		return 0 // succeeds on the second, separate attempt
	}

	fn := scriptedHeartbeatFn([]int{0, 0, 0})
	// Generation changes once, on tick 1 (baseline=1, tick1=2 changed,
	// stays 2 thereafter) -- one reconnect, but the resulting rejoin must
	// be retried across multiple successful ticks until it succeeds (Req 2
	// AC5), not abandoned after its first failed attempt.
	gens := scriptedReconnectGen([]uint64{1, 2, 2, 2})

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() {
		runHeartbeat(ctx, fn, gens, setNeedsRejoin, checkAndPerformRejoin)
		close(done)
	}()

	for i := 0; i < 2; i++ {
		select {
		case <-joinDone:
		case <-time.After(6 * time.Second):
			t.Fatalf("timed out waiting for rejoin attempt %d", i+1)
		}
	}
	cancel()
	<-done

	if got := atomic.LoadInt32(&joinCalls); got != 2 {
		t.Errorf("performRejoinFn called %d times, want exactly 2 (first attempt "+
			"fails, retried on the next successful heartbeat tick)", got)
	}
	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
		t.Error("expected needsRejoin to clear once the retried rejoin succeeds")
	}
}

func TestRunHeartbeat_NoGenChange_NeverSetsNeedsRejoin(t *testing.T) {
	resetNeedsRejoinForTest()
	atomic.StoreInt32(&heartbeatIntervalSecs, 1)

	origPerformRejoinFn := performRejoinFn
	defer func() { performRejoinFn = origPerformRejoinFn }()

	var joinCalls int32
	performRejoinFn = func() int {
		atomic.AddInt32(&joinCalls, 1)
		return 0
	}

	tickEvents := make(chan struct{}, 8)
	fn := func(_ int) int {
		tickEvents <- struct{}{}
		return 0 // always succeeds
	}

	ctx, cancel := context.WithCancel(context.Background())
	done := make(chan struct{})
	go func() {
		runHeartbeat(ctx, fn, constantReconnectGen(9), setNeedsRejoin, checkAndPerformRejoin)
		close(done)
	}()

	for i := 0; i < 3; i++ {
		select {
		case <-tickEvents:
		case <-time.After(5 * time.Second):
			t.Fatalf("timed out waiting for tick %d", i+1)
		}
	}
	cancel()
	<-done

	if got := atomic.LoadInt32(&needsRejoin); got != 0 {
		t.Error("needsRejoin must stay clear when the reconnect generation never changes -- " +
			"no reconnect ever occurred")
	}
	if got := atomic.LoadInt32(&joinCalls); got != 0 {
		t.Errorf("performRejoinFn called %d times, want 0 (checkAndPerformRejoin "+
			"must no-op when no rejoin is pending)", got)
	}
}