Adds an atomic degraded-until state (health.go: IsDegraded/MarkDegraded/ MarkHealthy) that lets request handlers fail fast with a clear unavailable error instead of blocking on xcecli's own ~100s retry budget when the backend is known-degraded. Consolidates what tasks 4.3/4.4 left as duplicated inline "call xcecli_*; checkAndPerformRejoin()" pairs across five wrapper functions into one shared callBackend() helper: checks IsDegraded() before every call (fail fast, skip the C call and the rejoin check entirely on the degraded path -- checking rejoin there would itself block up to ~100s, defeating the fail-fast purpose), marks degraded on -E_XCE_NOT_READY, marks healthy at checkAndPerformRejoin()'s existing success point (the single call site both the write path and the idle-period heartbeat path already funnel through). The circuit breaker's own TTL-based state is deliberately never read by the rejoin-retry mechanism -- checkAndPerformRejoin() has no IsDegraded() dependency, so a fast-fail window never suppresses retry-until-actually-joined; heartbeat.go's unconditional per-tick call keeps that cadence alive regardless of degraded state, and a 10s TTL (well under the ~100s retry budget) lets write traffic resume contributing once it clears. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01NNB4zGPCsHyDUCDJyMbiQN
144 lines
5.2 KiB
Go
144 lines
5.2 KiB
Go
package backend
|
|
|
|
import (
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// health_test.go exercises health.go's degraded/healthy state machine
|
|
// (component-failover-adoption task 4.5, boundary: ApiserverBackendHealth,
|
|
// Req 4.1-4.2). Deliberately no build tag, matching health.go itself: pure
|
|
// Go, no CGO -- runs under both CGO_ENABLED=0 and =1, so this file is the
|
|
// one this package's tests can run without any backend/nix CGO setup at
|
|
// all (see the task's own validation command: `CGO_ENABLED=0 go test
|
|
// ./backend/...`).
|
|
|
|
// resetHealthForTest restores degradedUntilNanos to its zero value (never
|
|
// degraded) between tests, mirroring reconnect_test.go's
|
|
// resetNeedsRejoinForTest for the sibling needsRejoin state.
|
|
func resetHealthForTest() {
|
|
atomic.StoreInt64(°radedUntilNanos, 0)
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// Req 4.1 -- basic degraded/healthy transitions
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
func TestIsDegraded_FreshState_ReportsHealthy(t *testing.T) {
|
|
resetHealthForTest()
|
|
|
|
if IsDegraded() {
|
|
t.Error("expected IsDegraded() to report false before MarkDegraded is ever called")
|
|
}
|
|
}
|
|
|
|
func TestMarkDegraded_ThenIsDegraded_ReportsTrue(t *testing.T) {
|
|
resetHealthForTest()
|
|
|
|
MarkDegraded(1 * time.Minute)
|
|
|
|
if !IsDegraded() {
|
|
t.Error("expected IsDegraded() to report true immediately after MarkDegraded")
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// design.md's own stated validation requirement: "unit tests cover
|
|
// degraded->healthy transitions via both the TTL expiry and explicit
|
|
// MarkHealthy()"
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
func TestMarkHealthy_ClearsDegradedStateExplicitly(t *testing.T) {
|
|
resetHealthForTest()
|
|
MarkDegraded(1 * time.Minute)
|
|
if !IsDegraded() {
|
|
t.Fatal("setup: expected degraded immediately after MarkDegraded")
|
|
}
|
|
|
|
MarkHealthy()
|
|
|
|
if IsDegraded() {
|
|
t.Error("expected IsDegraded() to report false immediately after MarkHealthy()")
|
|
}
|
|
}
|
|
|
|
func TestMarkDegraded_TTLExpiry_ClearsAutomaticallyWithoutMarkHealthy(t *testing.T) {
|
|
resetHealthForTest()
|
|
|
|
MarkDegraded(20 * time.Millisecond)
|
|
if !IsDegraded() {
|
|
t.Fatal("setup: expected degraded immediately after MarkDegraded")
|
|
}
|
|
|
|
time.Sleep(60 * time.Millisecond)
|
|
|
|
if IsDegraded() {
|
|
t.Error("expected IsDegraded() to report false once the TTL has elapsed, " +
|
|
"without any explicit MarkHealthy() call -- the half-open circuit " +
|
|
"breaker's own timeout must clear it")
|
|
}
|
|
}
|
|
|
|
func TestMarkDegraded_ExtendsAnExistingDegradedWindow(t *testing.T) {
|
|
resetHealthForTest()
|
|
|
|
MarkDegraded(20 * time.Millisecond)
|
|
time.Sleep(10 * time.Millisecond)
|
|
// A second call (e.g. a second call's own retry-budget exhaustion)
|
|
// resets the window from now, not additively -- confirms MarkDegraded
|
|
// always sets an absolute deadline, not a relative extension bug.
|
|
MarkDegraded(20 * time.Millisecond)
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
if !IsDegraded() {
|
|
t.Error("expected still degraded 10ms into a freshly-reset 20ms window")
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
// Concurrent-request safety (a go test -race concern, though -race itself
|
|
// is known not to work in this ASAN-linked CGO environment -- see
|
|
// tasks.md's task 4.2 Implementation Note. This proves race-freedom by
|
|
// construction: every access goes through sync/atomic, so this test's
|
|
// value is in exercising heavy concurrent read/write without a panic or
|
|
// data corruption, not in relying on -race actually running clean here).
|
|
// ════════════════════════════════════════════════════════════════════════
|
|
|
|
func TestHealthState_ConcurrentReadersAndWriters_NoDataCorruption(t *testing.T) {
|
|
resetHealthForTest()
|
|
|
|
const goroutines = 50
|
|
const iterations = 200
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(goroutines * 3)
|
|
|
|
for i := 0; i < goroutines; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := 0; j < iterations; j++ {
|
|
_ = IsDegraded()
|
|
}
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := 0; j < iterations; j++ {
|
|
MarkDegraded(time.Millisecond)
|
|
}
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
for j := 0; j < iterations; j++ {
|
|
MarkHealthy()
|
|
}
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
// No assertion on the final state (inherently racy which goroutine's
|
|
// write lands last) -- reaching here without a panic/deadlock, under
|
|
// concurrent atomic reads and writes from 150 goroutines, is the proof
|
|
// this test exists for.
|
|
}
|