feat(component-failover-adoption): add apiserver's backend-health circuit breaker

Adds an atomic degraded-until state (health.go: IsDegraded/MarkDegraded/
MarkHealthy) that lets request handlers fail fast with a clear
unavailable error instead of blocking on xcecli's own ~100s retry
budget when the backend is known-degraded.

Consolidates what tasks 4.3/4.4 left as duplicated inline
"call xcecli_*; checkAndPerformRejoin()" pairs across five wrapper
functions into one shared callBackend() helper: checks IsDegraded()
before every call (fail fast, skip the C call and the rejoin check
entirely on the degraded path -- checking rejoin there would itself
block up to ~100s, defeating the fail-fast purpose), marks degraded on
-E_XCE_NOT_READY, marks healthy at checkAndPerformRejoin()'s existing
success point (the single call site both the write path and the
idle-period heartbeat path already funnel through).

The circuit breaker's own TTL-based state is deliberately never read
by the rejoin-retry mechanism -- checkAndPerformRejoin() has no
IsDegraded() dependency, so a fast-fail window never suppresses
retry-until-actually-joined; heartbeat.go's unconditional per-tick
call keeps that cadence alive regardless of degraded state, and a
10s TTL (well under the ~100s retry budget) lets write traffic resume
contributing once it clears.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NNB4zGPCsHyDUCDJyMbiQN

src/frontend/apiserver/backend/health_test.go

@@ -0,0 +1,144 @@
+package backend
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+// health_test.go exercises health.go's degraded/healthy state machine
+// (component-failover-adoption task 4.5, boundary: ApiserverBackendHealth,
+// Req 4.1-4.2). Deliberately no build tag, matching health.go itself: pure
+// Go, no CGO -- runs under both CGO_ENABLED=0 and =1, so this file is the
+// one this package's tests can run without any backend/nix CGO setup at
+// all (see the task's own validation command: `CGO_ENABLED=0 go test
+// ./backend/...`).
+
+// resetHealthForTest restores degradedUntilNanos to its zero value (never
+// degraded) between tests, mirroring reconnect_test.go's
+// resetNeedsRejoinForTest for the sibling needsRejoin state.
+func resetHealthForTest() {
+	atomic.StoreInt64(&degradedUntilNanos, 0)
+}
+
+// ════════════════════════════════════════════════════════════════════════
+// Req 4.1 -- basic degraded/healthy transitions
+// ════════════════════════════════════════════════════════════════════════
+
+func TestIsDegraded_FreshState_ReportsHealthy(t *testing.T) {
+	resetHealthForTest()
+
+	if IsDegraded() {
+		t.Error("expected IsDegraded() to report false before MarkDegraded is ever called")
+	}
+}
+
+func TestMarkDegraded_ThenIsDegraded_ReportsTrue(t *testing.T) {
+	resetHealthForTest()
+
+	MarkDegraded(1 * time.Minute)
+
+	if !IsDegraded() {
+		t.Error("expected IsDegraded() to report true immediately after MarkDegraded")
+	}
+}
+
+// ════════════════════════════════════════════════════════════════════════
+// design.md's own stated validation requirement: "unit tests cover
+// degraded->healthy transitions via both the TTL expiry and explicit
+// MarkHealthy()"
+// ════════════════════════════════════════════════════════════════════════
+
+func TestMarkHealthy_ClearsDegradedStateExplicitly(t *testing.T) {
+	resetHealthForTest()
+	MarkDegraded(1 * time.Minute)
+	if !IsDegraded() {
+		t.Fatal("setup: expected degraded immediately after MarkDegraded")
+	}
+
+	MarkHealthy()
+
+	if IsDegraded() {
+		t.Error("expected IsDegraded() to report false immediately after MarkHealthy()")
+	}
+}
+
+func TestMarkDegraded_TTLExpiry_ClearsAutomaticallyWithoutMarkHealthy(t *testing.T) {
+	resetHealthForTest()
+
+	MarkDegraded(20 * time.Millisecond)
+	if !IsDegraded() {
+		t.Fatal("setup: expected degraded immediately after MarkDegraded")
+	}
+
+	time.Sleep(60 * time.Millisecond)
+
+	if IsDegraded() {
+		t.Error("expected IsDegraded() to report false once the TTL has elapsed, " +
+			"without any explicit MarkHealthy() call -- the half-open circuit " +
+			"breaker's own timeout must clear it")
+	}
+}
+
+func TestMarkDegraded_ExtendsAnExistingDegradedWindow(t *testing.T) {
+	resetHealthForTest()
+
+	MarkDegraded(20 * time.Millisecond)
+	time.Sleep(10 * time.Millisecond)
+	// A second call (e.g. a second call's own retry-budget exhaustion)
+	// resets the window from now, not additively -- confirms MarkDegraded
+	// always sets an absolute deadline, not a relative extension bug.
+	MarkDegraded(20 * time.Millisecond)
+	time.Sleep(10 * time.Millisecond)
+
+	if !IsDegraded() {
+		t.Error("expected still degraded 10ms into a freshly-reset 20ms window")
+	}
+}
+
+// ════════════════════════════════════════════════════════════════════════
+// Concurrent-request safety (a go test -race concern, though -race itself
+// is known not to work in this ASAN-linked CGO environment -- see
+// tasks.md's task 4.2 Implementation Note. This proves race-freedom by
+// construction: every access goes through sync/atomic, so this test's
+// value is in exercising heavy concurrent read/write without a panic or
+// data corruption, not in relying on -race actually running clean here).
+// ════════════════════════════════════════════════════════════════════════
+
+func TestHealthState_ConcurrentReadersAndWriters_NoDataCorruption(t *testing.T) {
+	resetHealthForTest()
+
+	const goroutines = 50
+	const iterations = 200
+
+	var wg sync.WaitGroup
+	wg.Add(goroutines * 3)
+
+	for i := 0; i < goroutines; i++ {
+		go func() {
+			defer wg.Done()
+			for j := 0; j < iterations; j++ {
+				_ = IsDegraded()
+			}
+		}()
+		go func() {
+			defer wg.Done()
+			for j := 0; j < iterations; j++ {
+				MarkDegraded(time.Millisecond)
+			}
+		}()
+		go func() {
+			defer wg.Done()
+			for j := 0; j < iterations; j++ {
+				MarkHealthy()
+			}
+		}()
+	}
+
+	wg.Wait()
+	// No assertion on the final state (inherently racy which goroutine's
+	// write lands last) -- reaching here without a panic/deadlock, under
+	// concurrent atomic reads and writes from 150 goroutines, is the proof
+	// this test exists for.
+}