fix(component-failover-adoption): unify apiserver's dual connect paths, fix connect-time UAF

Replaces apiserver's two connect mechanisms -- a per-peer Go loop that
hardcoded nmasters=1 per attempt, and a separate direct-construction
path for the legacy single-host config -- with one pipeline:
buildClusterConfig() parses (via task 4.1's C parser) or synthesizes a
single-candidate config, validates it, then ConfigConverter
(xce_cluster_config_to_client_config) produces the client config for
one xce_backend_init() call. A 3-candidate config now actually
exercises the C library's own failover/standby-redirect handling on
initial connect, matching worker's and scheduler's task 2.1/3.1 work,
instead of a Go-level per-candidate retry loop that never built a real
multi-master config.

Along the way: found and fixed a genuine use-after-free. The deleted
tryConnect() freed its C.CString-allocated host address immediately
after its single connect call returned, but xcecli_ctx_init() stores a
raw pointer to the config forever (client.c:38) and every later
failover rotation re-dereferences it (client_failover.c) -- so any
reconnect after the first would read freed memory. Fixed by giving the
cluster/client config structs package-level, process-lifetime storage
(gClusterCfg/gClientCfg), mirroring worker's and scheduler's file-scope
static pattern, with host/from/key written into fixed-size C arrays
instead of freed heap pointers.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NNB4zGPCsHyDUCDJyMbiQN

src/frontend/apiserver/backend/clib.go

@@ -12,6 +12,7 @@ package backend
 #include "idgen.h"
 #include "controller.capnp.h"
 #include "val_helpers.h"
+#include "xce/cluster.h"
 
 // backend_ctx_t wraps xcecli_ctx_t for use as the C handler's arg.
 typedef struct {
@@ -69,6 +70,43 @@ import (
 // gCtx holds the C client context for the lifetime of the process.
 var gCtx C.backend_ctx_t
 
+// gClusterCfg is the single validated xce_cluster_config_t RegisterToBackend
+// builds on every call -- either by parsing config.Peers via the shared C
+// parser (xce_parse_peers), or by synthesizing a one-candidate list from the
+// legacy config.Address/config.Port fields when Peers is empty (Req 1 AC5: a
+// single controller is the candidates_count=1 case of the identical
+// validate/convert/connect pipeline, not a separately-maintained code path
+// -- see buildClusterConfig() below).
+//
+// Package-level, not a buildClusterConfig()/RegisterToBackend()-local
+// variable, for the same reason gCtx and gIDCtx already are (process-lifetime
+// stable address): xce_cluster_config_to_client_config() (ConfigConverter,
+// src/lib/cluster.c) aliases gClientCfg.masters[i] directly at this struct's
+// own candidates[i].hostname fixed-char-array storage -- no strdup, no copy.
+// A caller that built the source xce_cluster_config_t on the stack (or any
+// Go-local variable) and then let gClientCfg outlive that scope would have a
+// use-after-scope bug: confirmed by reading src/lib/client.c's
+// xcecli_ctx_init() (`ctx->conf = cfg` -- stores a RAW pointer, never
+// copies) and src/lib/client_failover.c (`ac->conf->masters[next]` is
+// dereferenced on every failover rotation, not just at initial connect).
+// Worker's and scheduler's C implementations hit this identical hazard
+// (component-failover-adoption tasks 2.1/3.1) and solved it by keeping the
+// source config on a process-lifetime struct (worker: cc_ctx->cluster_cfg;
+// scheduler: sched->cluster_cfg) -- this package-level var is the direct Go
+// equivalent.
+var gClusterCfg C.xce_cluster_config_t
+
+// gClientCfg is the xce_client_config_t ConfigConverter populates from
+// gClusterCfg and xce_backend_init() connects with. Package-level for the
+// same reason as gClusterCfg above: xcecli_ctx_init() stores gCtx.ctx->conf
+// as a raw pointer to whichever struct address is passed to it, dereferenced
+// for the ctx's entire runtime life (every failover rotation reads
+// ctx->conf->masters[]/nmasters, not only the initial connect) -- so this
+// struct's address must be stable for the process lifetime, not a
+// RegisterToBackend-local variable that goes out of scope when the function
+// returns.
+var gClientCfg C.xce_client_config_t
+
 // gIDCtx is the C idgen context used to generate job IDs.
 var gIDCtx C.idgen_ctx_t
 var idCtxOnce sync.Once
@@ -77,34 +115,94 @@ var idCtxOnce sync.Once
 // Must be set via StartNotificationListener before RegisterToBackend.
 var gHandler NotificationHandler
 
-func tryConnect(host string, port int, from, key string) error {
-	cAddr := C.CString(host)
-	cFrom := C.CString(from)
-	cKey := C.CString(key)
-	defer func() {
-		C.free(unsafe.Pointer(cAddr))
-		C.free(unsafe.Pointer(cFrom))
-		C.free(unsafe.Pointer(cKey))
-	}()
-
-	var cConfig C.xce_client_config_t
-	cConfig.nmasters = 1
-	cConfig.masters[0] = cAddr
-	cConfig.master_ports[0] = C.int(port)
-	cConfig.from = cFrom
-	cConfig.key = cKey
-
-	rc := C.xce_backend_init(&cConfig, &gCtx)
-	if rc < 0 {
-		return fmt.Errorf("xce_backend_init failed on %s:%d: %d", host, port, int(rc))
+// writeCCharArray copies s into dst, a fixed-size C char array field of
+// gClusterCfg (hostname/identity/key), truncating to len(dst)-1 bytes and
+// always leaving the result NUL-terminated. Because these are fixed arrays
+// embedded directly in gClusterCfg (a package-level, process-lifetime
+// struct -- see its doc comment), populating them this way needs no
+// C.CString/C.free bookkeeping at all: xce_cluster_config_to_client_config()
+// later aliases pointers directly into this same storage.
+func writeCCharArray(dst []C.char, s string) {
+	n := len(s)
+	if n > len(dst)-1 {
+		n = len(dst) - 1
 	}
-	return nil
+	for i := 0; i < n; i++ {
+		dst[i] = C.char(s[i])
+	}
+	dst[n] = 0
 }
 
-// RegisterToBackend initialises the XCE client library and joins as an API
-// server.  StartNotificationListener should be called first so that push
-// events have a handler ready before the first push can arrive.
-func RegisterToBackend(config *BackendConfig) error {
+// formatCandidateList renders gClusterCfg's current candidates as
+// "host:port, host:port, ..." for diagnostic logging (Req 5.1, 5.2) -- the
+// same textual shape scheduler's sched_format_candidate_list() (scheduler.c)
+// produces, for cross-component log consistency.
+func formatCandidateList() string {
+	n := int(gClusterCfg.candidates_count)
+	parts := make([]string, 0, n)
+	for i := 0; i < n; i++ {
+		cand := &gClusterCfg.candidates[i]
+		parts = append(parts, fmt.Sprintf("%s:%d", C.GoString(&cand.hostname[0]), int(cand.port)))
+	}
+	return strings.Join(parts, ", ")
+}
+
+// currentClusterCandidates returns gClusterCfg's current candidates as a
+// Go-native slice. Exists so clib_test.go (a //go:build cgo test file that
+// cannot `import "C"` itself -- a Go-toolchain restriction, golang/go#13938)
+// can assert on buildClusterConfig()'s output shape without re-implementing
+// CGO field access in the test file, mirroring the candidates.go/
+// candidates_test.go split already established in this package.
+func currentClusterCandidates() []Candidate {
+	n := int(gClusterCfg.candidates_count)
+	out := make([]Candidate, n)
+	for i := 0; i < n; i++ {
+		cand := &gClusterCfg.candidates[i]
+		out[i] = Candidate{
+			Name: C.GoString(&cand.name[0]),
+			Host: C.GoString(&cand.hostname[0]),
+			Port: int(cand.port),
+		}
+	}
+	return out
+}
+
+// currentClientConfigNMasters exposes gClientCfg.nmasters as a plain int,
+// for the same test-file-cannot-import-C reason as currentClusterCandidates
+// above. Lets tests confirm xce_cluster_config_to_client_config() actually
+// propagated the full candidate count into the struct xce_backend_init()
+// connects with -- proving ONE call carries all N candidates, rather than N
+// separate single-candidate calls (the deleted per-peer Go loop's behavior,
+// which always hardcoded nmasters=1).
+func currentClientConfigNMasters() int {
+	return int(gClientCfg.nmasters)
+}
+
+// buildClusterConfig builds and validates the single xce_cluster_config_t
+// this connect attempt will use (Req 1 AC1-3/AC5), writing directly into the
+// package-level gClusterCfg (see its doc comment for why this must be
+// package-level rather than a local variable).
+//
+// A single controller is not a separately-maintained code path: when
+// config.Peers is non-empty it is parsed via the same shared C parser
+// (xce_parse_peers) worker and scheduler already call directly; otherwise a
+// one-candidate list is synthesized from config.Address/config.Port. Either
+// source is then run through the identical xce_cluster_config_validate()
+// call below -- no branch downstream of this function distinguishes the two
+// sources (Req 1 AC5), mirroring scheduler's sched_build_cluster_config()
+// (scheduler.c) and worker's equivalent (controller_connector.c).
+//
+// This intentionally does not reuse parseCandidates() (candidates.go):
+// parseCandidates() parses into its own function-local xce_cluster_config_t
+// and reconstructs a Go []Candidate slice from it -- exactly the
+// use-after-scope-prone, throwaway-C-struct pattern gClusterCfg's doc
+// comment above warns against for this call site. buildClusterConfig() must
+// parse directly into the package-level gClusterCfg so the very same struct
+// flows unmodified into xce_cluster_config_to_client_config() below.
+//
+// Does not call xce_cluster_config_to_client_config() or connect -- callers
+// (RegisterToBackend) do that separately.
+func buildClusterConfig(config *BackendConfig) error {
 	from := config.From
 	if from == "" {
 		from = "xce_apiserver"
@@ -114,39 +212,88 @@ func RegisterToBackend(config *BackendConfig) error {
 		key = "keepsecret$"
 	}
 
-	if len(config.Peers) > 0 {
-		// Parsing is routed through the same C parser (xce_parse_peers) worker
-		// and scheduler already call directly -- Req 6.1/6.2. A malformed
-		// entry rejects the whole candidate list up front (matching
-		// worker's/scheduler's own all-or-nothing --peers parsing) rather
-		// than being skipped with a warning as the old hand-written
-		// parsePeer()-per-entry loop did.
-		candidates, err := parseCandidates(strings.Join(config.Peers, ","))
-		if err != nil {
-			return err
-		}
+	gClusterCfg = C.xce_cluster_config_t{}
 
-		var lastErr error
-		for _, cand := range candidates {
-			if err := tryConnect(cand.Host, cand.Port, from, key); err != nil {
-				logger.Warn("Candidate unreachable, trying next",
-					zap.String("host", cand.Host), zap.Int("port", cand.Port), zap.Error(err))
-				lastErr = err
-				continue
-			}
-			logger.Info("Backend connection established (HA)",
-				zap.String("host", cand.Host), zap.Int("port", cand.Port))
-			return nil
+	if len(config.Peers) > 0 {
+		peersStr := strings.Join(config.Peers, ",")
+		cPeers := C.CString(peersStr)
+		rc := C.xce_parse_peers(cPeers, &gClusterCfg)
+		C.free(unsafe.Pointer(cPeers))
+		if rc != 0 {
+			return fmt.Errorf(
+				"apiserver: --peers parse error: expected \"[name=]host:port,...\" got %q",
+				peersStr)
 		}
-		return fmt.Errorf("all HA candidates failed: %w", lastErr)
+	} else {
+		gClusterCfg.candidates_count = 1
+		writeCCharArray(gClusterCfg.candidates[0].hostname[:], config.Address)
+		gClusterCfg.candidates[0].port = C.int(config.Port)
 	}
 
-	if err := tryConnect(config.Address, config.Port, from, key); err != nil {
+	writeCCharArray(gClusterCfg.identity[:], from)
+	writeCCharArray(gClusterCfg.key[:], key)
+
+	if rc := C.xce_cluster_config_validate(&gClusterCfg); rc != 0 {
+		return fmt.Errorf(
+			"apiserver: invalid controller candidate configuration (rc=%d): %s",
+			int(rc), formatCandidateList())
+	}
+
+	return nil
+}
+
+// RegisterToBackend initialises the XCE client library and joins as an API
+// server, using the full configured candidate list -- config.Peers if set,
+// otherwise a single-candidate list synthesized from config.Address/
+// config.Port (Req 1 AC5: the candidates_count=1 case of the identical
+// pipeline, not a separate code path -- see buildClusterConfig()). Builds
+// exactly ONE validated xce_cluster_config_t regardless of source, converts
+// it once via the shared ConfigConverter
+// (xce_cluster_config_to_client_config), and calls xce_backend_init()
+// exactly ONCE -- replacing the former dual-mechanism implementation (a
+// Go-level per-peer tryConnect() retry loop for the Peers case, and a
+// second, separately-coded tryConnect() call for the legacy Address/Port
+// case; both bypassed xce_cluster_config_t/xce_cluster_config_validate()/
+// xce_cluster_config_to_client_config() entirely, and both always
+// hardcoded nmasters=1).
+//
+// StartNotificationListener should be called first so that push events have
+// a handler ready before the first push can arrive.
+func RegisterToBackend(config *BackendConfig) error {
+	if err := buildClusterConfig(config); err != nil {
 		return err
 	}
+
+	logger.Info("Attempting backend connection",
+		zap.Int("candidates", int(gClusterCfg.candidates_count)),
+		zap.String("candidate_list", formatCandidateList()))
+
+	gClientCfg = C.xce_client_config_t{}
+	if rc := C.xce_cluster_config_to_client_config(&gClusterCfg, &gClientCfg); rc != 0 {
+		return fmt.Errorf("apiserver: candidate list conversion failed (rc=%d)", int(rc))
+	}
+	// ConfigConverter deliberately does not set from/key (caller-owned
+	// identity/auth, per its own doc comment) -- point them at gClusterCfg's
+	// own identity/key fixed arrays rather than allocating separate C
+	// strings: gClusterCfg is the same package-level, process-lifetime
+	// struct ConfigConverter already aliased gClientCfg.masters[i] into, so
+	// this needs no additional C.CString/C.free bookkeeping.
+	gClientCfg.from = &gClusterCfg.identity[0]
+	gClientCfg.key = &gClusterCfg.key[0]
+
+	rc := C.xce_backend_init(&gClientCfg, &gCtx)
+	if rc < 0 {
+		logger.Warn("All configured backend candidates failed",
+			zap.Int("candidates", int(gClusterCfg.candidates_count)),
+			zap.String("candidate_list", formatCandidateList()),
+			zap.Int("rc", int(rc)))
+		return fmt.Errorf("xce_backend_init failed across %d candidate(s) (rc=%d): %s",
+			int(gClusterCfg.candidates_count), int(rc), formatCandidateList())
+	}
+
 	logger.Info("Backend connection established",
-		zap.String("address", config.Address),
-		zap.Int("port", config.Port))
+		zap.Int("nmasters", int(gClientCfg.nmasters)),
+		zap.String("candidate_list", formatCandidateList()))
 	return nil
 }