fix(component-failover-adoption): Phase 6 -- close apiserver's idle-period rejoin gap (bug-1548/bug-1549)
XCECLI_FF_CALL (backing xcecli_heartbeat) can silently rotate+reconnect the transport entirely inside a single call and still return success, so heartbeat.go's failure-to-success transition detector could never observe a reconnect that completes within one heartbeat tick -- the common case, since elections finish in single-digit seconds against a default 30s interval. apiserver's application-level rejoin never fired on a real idle-period failover, and job pushes silently stopped reaching it. Adds a monotonic reconnect_gen counter to endpoint_t (netio.h), bumped by xce_ctx_reconnect() (xcectx.c) on every successful reconnect regardless of trigger path. heartbeat.go now compares this generation every tick instead of tracking success/failure transitions. Re-validated live against task 5.2's exact scenario (apiserver alone, zero write traffic): rejoin fired 22s after a primary kill, 12s after election. Reopens and closes out component-failover-adoption's own NO-GO -> GO validation cycle from 2026-08-15. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
0cb5918517
commit
d479b70764
SSH key fingerprint:
SHA256:8Vf2pZ3h6rJa7EhGK+WaAQ9Mta/apZvteW6g9U/WKKk
11 changed files with 430 additions and 146 deletions
|
|
@ -31,10 +31,18 @@ func TestSetHeartbeatInterval_NegativeIgnored(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// constantReconnectGen returns a currentReconnectGen double that never
|
||||
// changes value -- used by tests that only care about runHeartbeat's
|
||||
// tick/cancellation mechanics or fn's own success/failure classification,
|
||||
// not its reconnect-detection behavior (see the Phase 6 tests below for
|
||||
// that coverage).
|
||||
func constantReconnectGen(gen uint64) func() uint64 {
|
||||
return func() uint64 { return gen }
|
||||
}
|
||||
|
||||
// noopRejoinHooks are used by tests that only care about runHeartbeat's
|
||||
// tick/cancellation mechanics, not its failure-to-success/pending-rejoin
|
||||
// callback behavior (see TestRunHeartbeat_FailureToSuccessTransition_* etc.
|
||||
// below for that coverage).
|
||||
// tick/cancellation mechanics, not its reconnect-detected/pending-rejoin
|
||||
// callback behavior (see the Phase 6 tests below for that coverage).
|
||||
func noopRejoinHooks() (func(), func()) {
|
||||
return func() {}, func() {}
|
||||
}
|
||||
|
|
@ -47,12 +55,12 @@ func TestRunHeartbeat_CallsFunction(t *testing.T) {
|
|||
atomic.AddInt64(&callCount, 1)
|
||||
return 0
|
||||
}
|
||||
onFailureToSuccess, onSuccessTick := noopRejoinHooks()
|
||||
onReconnectDetected, onSuccessTick := noopRejoinHooks()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runHeartbeat(ctx, fn, onFailureToSuccess, onSuccessTick)
|
||||
runHeartbeat(ctx, fn, constantReconnectGen(0), onReconnectDetected, onSuccessTick)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
|
|
@ -74,11 +82,11 @@ func TestRunHeartbeat_CancelsOnContext(t *testing.T) {
|
|||
atomic.AddInt64(&callCount, 1)
|
||||
return 0
|
||||
}
|
||||
onFailureToSuccess, onSuccessTick := noopRejoinHooks()
|
||||
onReconnectDetected, onSuccessTick := noopRejoinHooks()
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runHeartbeat(ctx, fn, onFailureToSuccess, onSuccessTick)
|
||||
runHeartbeat(ctx, fn, constantReconnectGen(0), onReconnectDetected, onSuccessTick)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
|
|
@ -111,80 +119,69 @@ func scriptedHeartbeatFn(results []int) func(int) int {
|
|||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════════════════
|
||||
// component-failover-adoption task 4.4 (Req 2 AC4/AC5) -- idle-period
|
||||
// rejoin trigger. These tests exercise runHeartbeat's own tick/transition-
|
||||
// tracking logic via generic injected callbacks, entirely without CGO (the
|
||||
// production wiring -- reconnect.go's real setNeedsRejoin/
|
||||
// checkAndPerformRejoin -- is exercised end-to-end in
|
||||
// heartbeat_rejoin_test.go, //go:build cgo).
|
||||
// ════════════════════════════════════════════════════════════════════════
|
||||
|
||||
func TestRunHeartbeat_FailureToSuccessTransition_FiresOnFailureToSuccessThenOnSuccessTick(t *testing.T) {
|
||||
atomic.StoreInt32(&heartbeatIntervalSecs, 1)
|
||||
|
||||
// fail, fail, succeed -- the transition happens on tick 3.
|
||||
fn := scriptedHeartbeatFn([]int{-1, -1, 0})
|
||||
|
||||
events := make(chan string, 8)
|
||||
onFailureToSuccess := func() { events <- "failure_to_success" }
|
||||
onSuccessTick := func() { events <- "success_tick" }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runHeartbeat(ctx, fn, onFailureToSuccess, onSuccessTick)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
want := []string{"failure_to_success", "success_tick"}
|
||||
got := make([]string, 0, len(want))
|
||||
for range want {
|
||||
select {
|
||||
case e := <-events:
|
||||
got = append(got, e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("timed out waiting for heartbeat callback events, got %v so far", got)
|
||||
}
|
||||
}
|
||||
// Cancel immediately once the expected events are observed, before any
|
||||
// further tick can fire -- keeps the assertion below exact regardless
|
||||
// of scheduling slop.
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("callback event %d = %q, want %q (full sequence got=%v want=%v) -- "+
|
||||
"onFailureToSuccess must fire exactly once, before onSuccessTick, "+
|
||||
"only on the fail-then-succeed transition tick",
|
||||
i, got[i], want[i], got, want)
|
||||
// scriptedReconnectGen returns a currentReconnectGen double that yields
|
||||
// gens[0], gens[1], ... on successive calls (one call per tick -- runHeartbeat
|
||||
// reads it exactly once per tick, after fn), clamping to the last entry for
|
||||
// any calls beyond len(gens). The very first call (the pre-loop baseline
|
||||
// read in runHeartbeat, before any tick) is NOT scripted from this slice --
|
||||
// callers should include their intended pre-loop baseline as gens[0] and
|
||||
// note the first in-loop read is gens[1] onward matching a real
|
||||
// endpoint_t.reconnect_gen starting at a fixed value and changing on later
|
||||
// reconnects.
|
||||
func scriptedReconnectGen(gens []uint64) func() uint64 {
|
||||
var idx int32
|
||||
return func() uint64 {
|
||||
i := atomic.AddInt32(&idx, 1) - 1
|
||||
if int(i) >= len(gens) {
|
||||
i = int32(len(gens) - 1)
|
||||
}
|
||||
return gens[i]
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHeartbeat_EverySuccessfulTick_RetriesWhilePending(t *testing.T) {
|
||||
// ════════════════════════════════════════════════════════════════════════
|
||||
// component-failover-adoption Phase 6 (bug-1548) -- reconnect-generation-
|
||||
// based idle-period rejoin trigger, replacing task 4.4's original failure-
|
||||
// to-success transition detector (which structurally could not observe a
|
||||
// reconnect that completed silently inside a single XCECLI_FF_CALL). These
|
||||
// tests exercise runHeartbeat's own tick/generation-tracking logic via
|
||||
// generic injected callbacks, entirely without CGO (the production wiring
|
||||
// -- reconnect.go's real setNeedsRejoin/checkAndPerformRejoin plus clib.go's
|
||||
// real currentReconnectGen reading C.xcecli_get_my_endpoint -- is exercised
|
||||
// end-to-end in heartbeat_rejoin_test.go, //go:build cgo).
|
||||
// ════════════════════════════════════════════════════════════════════════
|
||||
|
||||
// This is the literal shape of bug-1548: every heartbeat tick the Go loop
|
||||
// observes reports SUCCESS (fn never returns a failure -- XCECLI_FF_CALL's
|
||||
// silent inline reconnect+recover means the caller's own return code never
|
||||
// surfaces one), yet a reconnect genuinely happened (the generation counter
|
||||
// changed). The old failure-to-success transition detector could never fire
|
||||
// here since it requires an observed failing tick first. This test is the
|
||||
// actual proof the fix closes the gap the old detector's own green test
|
||||
// suite could not catch -- it is still, deliberately, only proof of the
|
||||
// DETECTOR's logic: whether the generation counter is reachable/reliable
|
||||
// against the real XCECLI_FF_CALL path is proven only by a live E2E re-run
|
||||
// (component-failover-adoption Phase 6, task 6.2), not by this test.
|
||||
func TestRunHeartbeat_ReconnectDetected_FiresOnGenChangeEvenWhenAllTicksSucceed(t *testing.T) {
|
||||
atomic.StoreInt32(&heartbeatIntervalSecs, 1)
|
||||
|
||||
// fail, then succeed 3 times in a row: onFailureToSuccess must fire
|
||||
// only on the first success (the transition), but onSuccessTick must
|
||||
// fire on ALL THREE successes -- proving the pending-rejoin check is
|
||||
// retried on every successful tick, not abandoned after the transition
|
||||
// tick (Req 2 AC5: "checked every tick, not only the transition edge").
|
||||
fn := scriptedHeartbeatFn([]int{-1, 0, 0, 0})
|
||||
fn := scriptedHeartbeatFn([]int{0, 0, 0}) // every tick succeeds
|
||||
gens := scriptedReconnectGen([]uint64{5, 5, 6, 6})
|
||||
|
||||
events := make(chan string, 8)
|
||||
onFailureToSuccess := func() { events <- "f2s" }
|
||||
onReconnectDetected := func() { events <- "reconnect_detected" }
|
||||
onSuccessTick := func() { events <- "tick" }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runHeartbeat(ctx, fn, onFailureToSuccess, onSuccessTick)
|
||||
runHeartbeat(ctx, fn, gens, onReconnectDetected, onSuccessTick)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
want := []string{"f2s", "tick", "tick", "tick"}
|
||||
// tick 1: gen 5->5 (no change), tick succeeds -> "tick"
|
||||
// tick 2: gen 5->6 (changed) -> "reconnect_detected", then tick succeeds -> "tick"
|
||||
want := []string{"tick", "reconnect_detected", "tick"}
|
||||
got := make([]string, 0, len(want))
|
||||
for range want {
|
||||
select {
|
||||
|
|
@ -199,19 +196,60 @@ func TestRunHeartbeat_EverySuccessfulTick_RetriesWhilePending(t *testing.T) {
|
|||
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("callback event %d = %q, want %q (full sequence got=%v want=%v)",
|
||||
t.Fatalf("callback event %d = %q, want %q (full sequence got=%v want=%v) -- "+
|
||||
"onReconnectDetected must fire on a reconnect_gen change even though "+
|
||||
"every tick's own fn() call reported success",
|
||||
i, got[i], want[i], got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHeartbeat_AllTicksSucceed_NeverFiresOnFailureToSuccess(t *testing.T) {
|
||||
// A reconnect can complete on a tick whose own send then fails for an
|
||||
// unrelated reason -- the generation check must not be gated on tick
|
||||
// success, unlike onSuccessTick.
|
||||
func TestRunHeartbeat_GenChangeOnFailedTick_StillFiresOnReconnectDetected(t *testing.T) {
|
||||
atomic.StoreInt32(&heartbeatIntervalSecs, 1)
|
||||
|
||||
fn := scriptedHeartbeatFn([]int{-1}) // the one tick this test observes fails
|
||||
gens := scriptedReconnectGen([]uint64{1, 2})
|
||||
|
||||
events := make(chan string, 8)
|
||||
onReconnectDetected := func() { events <- "reconnect_detected" }
|
||||
onSuccessTick := func() { events <- "tick" }
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runHeartbeat(ctx, fn, gens, onReconnectDetected, onSuccessTick)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case e := <-events:
|
||||
if e != "reconnect_detected" {
|
||||
t.Fatalf("first event = %q, want %q -- a gen change must fire "+
|
||||
"onReconnectDetected even on a tick whose own fn() call failed", e, "reconnect_detected")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for onReconnectDetected")
|
||||
}
|
||||
cancel()
|
||||
<-done
|
||||
|
||||
select {
|
||||
case e := <-events:
|
||||
t.Fatalf("unexpected extra event %q -- onSuccessTick must not fire on a failed tick", e)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHeartbeat_NoGenChange_NeverFiresOnReconnectDetected(t *testing.T) {
|
||||
atomic.StoreInt32(&heartbeatIntervalSecs, 1)
|
||||
|
||||
fn := scriptedHeartbeatFn([]int{0, 0, 0})
|
||||
|
||||
var f2sCalls int32
|
||||
onFailureToSuccess := func() { atomic.AddInt32(&f2sCalls, 1) }
|
||||
var reconnectCalls int32
|
||||
onReconnectDetected := func() { atomic.AddInt32(&reconnectCalls, 1) }
|
||||
|
||||
var tickCalls int32
|
||||
tickEvents := make(chan struct{}, 8)
|
||||
|
|
@ -223,7 +261,7 @@ func TestRunHeartbeat_AllTicksSucceed_NeverFiresOnFailureToSuccess(t *testing.T)
|
|||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
runHeartbeat(ctx, fn, onFailureToSuccess, onSuccessTick)
|
||||
runHeartbeat(ctx, fn, constantReconnectGen(42), onReconnectDetected, onSuccessTick)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
|
|
@ -237,9 +275,9 @@ func TestRunHeartbeat_AllTicksSucceed_NeverFiresOnFailureToSuccess(t *testing.T)
|
|||
cancel()
|
||||
<-done
|
||||
|
||||
if got := atomic.LoadInt32(&f2sCalls); got != 0 {
|
||||
t.Errorf("onFailureToSuccess called %d times, want 0 for an all-successful "+
|
||||
"run -- no failure was ever observed, so there is no transition to fire on", got)
|
||||
if got := atomic.LoadInt32(&reconnectCalls); got != 0 {
|
||||
t.Errorf("onReconnectDetected called %d times, want 0 -- the generation "+
|
||||
"never changed, so no reconnect ever happened", got)
|
||||
}
|
||||
if got := atomic.LoadInt32(&tickCalls); got < 3 {
|
||||
t.Errorf("onSuccessTick called %d times, want at least 3", got)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue