Merge branch 'working' into job_peek_lchen

job_peek_lchen: Add fetchfile feature (job output/error peek proxy)
working: HA/Raft infrastructure (consensus, replication, config mgmt)

Conflict resolution summary:
- controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30
- cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK
- job.c: kept full fetchfile pending-request subsystem (~183 lines)
- request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS
- helper.c: added missing preVoteRequest/preVoteReply mappings
- All 15 new test registrations (test_cbuf.c + 14 HA tests)

src/frontend/apiserver/backend/clib.go

@@ -76,6 +76,30 @@ var idCtxOnce sync.Once
 // Must be set via StartNotificationListener before RegisterToBackend.
 var gHandler NotificationHandler
 
+func tryConnect(host string, port int, from, key string) error {
+	cAddr := C.CString(host)
+	cFrom := C.CString(from)
+	cKey := C.CString(key)
+	defer func() {
+		C.free(unsafe.Pointer(cAddr))
+		C.free(unsafe.Pointer(cFrom))
+		C.free(unsafe.Pointer(cKey))
+	}()
+
+	var cConfig C.xce_client_config_t
+	cConfig.nmasters = 1
+	cConfig.masters[0] = cAddr
+	cConfig.master_ports[0] = C.int(port)
+	cConfig.from = cFrom
+	cConfig.key = cKey
+
+	rc := C.xce_backend_init(&cConfig, &gCtx)
+	if rc < 0 {
+		return fmt.Errorf("xce_backend_init failed on %s:%d: %d", host, port, int(rc))
+	}
+	return nil
+}
+
 // RegisterToBackend initialises the XCE client library and joins as an API
 // server.  StartNotificationListener should be called first so that push
 // events have a handler ready before the first push can arrive.
@@ -89,27 +113,31 @@ func RegisterToBackend(config *BackendConfig) error {
 		key = "keepsecret$"
 	}
 
-	cAddr := C.CString(config.Address)
-	cFrom := C.CString(from)
-	cKey := C.CString(key)
-	defer func() {
-		C.free(unsafe.Pointer(cAddr))
-		C.free(unsafe.Pointer(cFrom))
-		C.free(unsafe.Pointer(cKey))
-	}()
-
-	var cConfig C.xce_client_config_t
-	cConfig.nmasters = 1
-	cConfig.masters[0] = cAddr
-	cConfig.master_ports[0] = C.int(config.Port)
-	cConfig.from = cFrom
-	cConfig.key = cKey
-
-	rc := C.xce_backend_init(&cConfig, &gCtx)
-	if rc < 0 {
-		return fmt.Errorf("xce_backend_init failed: %d", int(rc))
+	if len(config.Peers) > 0 {
+		var lastErr error
+		for _, peer := range config.Peers {
+			host, port, err := parsePeer(peer)
+			if err != nil {
+				logger.Warn("Skipping invalid peer", zap.String("peer", peer), zap.Error(err))
+				lastErr = err
+				continue
+			}
+			if err := tryConnect(host, port, from, key); err != nil {
+				logger.Warn("Candidate unreachable, trying next",
+					zap.String("host", host), zap.Int("port", port), zap.Error(err))
+				lastErr = err
+				continue
+			}
+			logger.Info("Backend connection established (HA)",
+				zap.String("host", host), zap.Int("port", port))
+			return nil
+		}
+		return fmt.Errorf("all HA candidates failed: %w", lastErr)
 	}
 
+	if err := tryConnect(config.Address, config.Port, from, key); err != nil {
+		return err
+	}
 	logger.Info("Backend connection established",
 		zap.String("address", config.Address),
 		zap.Int("port", config.Port))

src/frontend/apiserver/backend/config.go

@@ -7,6 +7,12 @@ type BackendConfig struct {
 	Port    int    // controller port (e.g. 9990)
 	From    string // identity sent to controller (default: "xce_apiserver")
 	Key     string // authentication key
+
+	// Peers lists additional controller candidates for HA mode in "host:port"
+	// format.  When non-empty, RegisterToBackend tries each candidate in order
+	// and connects to whichever accepts the join as primary.  Address+Port
+	// serve as a single-host fallback when Peers is empty.
+	Peers []string
 }
 
 // FetchFileResult holds the data returned by FetchFile.

src/frontend/apiserver/backend/convert.go

@@ -80,8 +80,9 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
 
 // cWorkerToModel extracts C xce_worker_t fields into a rawWorkerData and maps it to HostInfo.
 func cWorkerToModel(worker *C.xce_worker_t) *models.HostInfo {
+	workerName := goString(worker.name)
 	r := rawWorkerData{
-		name:   goString(worker.name),
+		name:   workerName,
 		status: int(worker.status),
 	}
 	if worker.attrs != nil {
@@ -96,6 +97,16 @@ func cWorkerToModel(worker *C.xce_worker_t) *models.HostInfo {
 		r.maxTmp = int(a.maxtmp)
 		r.nDisks = int(a.ndisks)
 	}
+	n := int(worker.nresvals_to_sched)
+	if n > 0 {
+		entries := unsafe.Slice(worker.resvals_to_sched, n)
+		r.resvals = make([]models.ResourceDistributionInfo, 0, n)
+		for _, nv := range entries {
+			if rd := cNameValToWorkerResval(workerName, nv); rd != nil {
+				r.resvals = append(r.resvals, *rd)
+			}
+		}
+	}
 	return rawWorkerToModel(r)
 }
 
@@ -189,7 +200,7 @@ func cClusterConfToClusterInfo(conf *C.cluster_config_t) *models.ClusterInfo {
 	return rawClusterToModel(rawClusterData{
 		name:                   name,
 		heartbeatInterval:      int(conf.heartbeat_interval),
-		vcpus:                  int(conf.vcpus),
+		vcpus:                  0, // vcpus moved to myconf_t (--vcpus CLI flag); not in cluster_config_t
 		inactiveHeartbeatCount: int(conf.inactive_heartbeat_count),
 	})
 }
@@ -267,6 +278,55 @@ func freeJobSubmit(js *C.xce_job_submit_t, sd *C.job_submit_data_t, si *C.job_st
 	}
 }
 
+// cNameValToWorkerResval converts a C xce_name_val_t pointer to a ResourceDistributionInfo.
+// workerName is set as Location. Returns nil if nv or nv.val is nil.
+func cNameValToWorkerResval(workerName string, nv *C.xce_name_val_t) *models.ResourceDistributionInfo {
+	if nv == nil {
+		return nil
+	}
+	rd := &models.ResourceDistributionInfo{
+		ResourceName: C.GoString(nv.name),
+		Location:     workerName,
+		IsDynamic:    true,
+	}
+	if nv.val == nil {
+		logger.Warn("worker resval with nil value, skipping",
+			zap.String("resource", rd.ResourceName),
+			zap.String("host", workerName))
+		return nil
+	}
+	switch C.xce_val_which(nv.val) {
+	case C.Value_u_ivalue32:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_i32(nv.val))
+	case C.Value_u_ivalue64:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_i64(nv.val))
+	case C.Value_u_uvalue32:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_u32(nv.val))
+	case C.Value_u_uvalue64:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_u64(nv.val))
+	case C.Value_u_fvalue:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_fval(nv.val))
+	case C.Value_u_svalue:
+		rd.ValueType = int(models.STRING)
+		rd.StringValue = C.GoString(C.xce_val_str(nv.val))
+	case C.Value_u_bvalue:
+		rd.ValueType = int(models.BOOLEAN)
+		rd.BooleanValue = C.xce_val_bool(nv.val) != 0
+	default:
+		logger.Warn("unknown xce_val_t discriminant in worker resval, skipping",
+			zap.Int("which", int(C.xce_val_which(nv.val))),
+			zap.String("resource", rd.ResourceName),
+			zap.String("host", workerName))
+		return nil
+	}
+	return rd
+}
+
 // cResvalToDistribution converts a C xce_resval_t pointer to a Go ResourceDistributionInfo.
 // Returns nil if vp.val is NULL (nil guard prevents CGO panic).
 // vp.val is a *C.xce_val_t pointer field — pass directly, no &.

src/frontend/apiserver/backend/convert_pure.go

@@ -4,7 +4,6 @@ import (
 	"apiserver/models"
 	"encoding/json"
 	"fmt"
-	"time"
 )
 
 // JobID bit layout constants — mirrors src/include/config.h.
@@ -155,6 +154,7 @@ type rawWorkerData struct {
 	maxSwap   int
 	maxTmp    int
 	nDisks    int
+	resvals   []models.ResourceDistributionInfo
 }
 
 // rawQueueData mirrors the C xce_queue_t fields used by the Go model.
@@ -207,18 +207,10 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
 			info.ExecUser = si.execUser
 		}
 	}
-	if r.subTime > 0 {
-		info.SubmitTime = time.Unix(r.subTime, 0)
-	}
-	if r.queueTime > 0 {
-		info.QueueTime = time.Unix(r.queueTime, 0)
-	}
-	if r.startTime > 0 {
-		info.StartTime = time.Unix(r.startTime, 0)
-	}
-	if r.finishTime > 0 {
-		info.EndTime = time.Unix(r.finishTime, 0)
-	}
+	info.SubmitTime = r.subTime
+	info.QueueTime  = r.queueTime
+	info.StartTime  = r.startTime
+	info.EndTime    = r.finishTime
 	if len(r.execHosts) > 0 {
 		type hostSlot struct {
 			HostName string `json:"host_name"`
@@ -241,8 +233,9 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
 // rawWorkerToModel maps a rawWorkerData to a models.HostInfo.
 func rawWorkerToModel(r rawWorkerData) *models.HostInfo {
 	info := &models.HostInfo{
-		HostName:   r.name,
-		HostStatus: r.status,
+		HostName:       r.name,
+		HostStatus:     r.status,
+		ResourceValues: r.resvals,
 	}
 	if r.hasAttrs {
 		info.HostType = r.hostType

src/frontend/apiserver/backend/convert_test.go

@@ -3,7 +3,6 @@ package backend
 import (
 	"apiserver/models"
 	"testing"
-	"time"
 )
 
 func TestRawJobToModel_BasicFields(t *testing.T) {
@@ -81,17 +80,17 @@ func TestRawJobToModel_Timestamps(t *testing.T) {
 
 	info := rawJobToModel(r)
 
-	if info.SubmitTime != time.Unix(1700000000, 0) {
-		t.Errorf("SubmitTime mismatch")
+	if info.SubmitTime != 1700000000 {
+		t.Errorf("SubmitTime: got %d, want 1700000000", info.SubmitTime)
 	}
-	if info.QueueTime != time.Unix(1700000010, 0) {
-		t.Errorf("QueueTime mismatch")
+	if info.QueueTime != 1700000010 {
+		t.Errorf("QueueTime: got %d, want 1700000010", info.QueueTime)
 	}
-	if info.StartTime != time.Unix(1700000020, 0) {
-		t.Errorf("StartTime mismatch")
+	if info.StartTime != 1700000020 {
+		t.Errorf("StartTime: got %d, want 1700000020", info.StartTime)
 	}
-	if info.EndTime != time.Unix(1700000030, 0) {
-		t.Errorf("EndTime mismatch")
+	if info.EndTime != 1700000030 {
+		t.Errorf("EndTime: got %d, want 1700000030", info.EndTime)
 	}
 }
 

src/frontend/apiserver/backend/peers.go

@@ -0,0 +1,24 @@
+package backend
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+)
+
+// parsePeer splits a "host:port" string. Uses net.SplitHostPort so IPv6
+// bracket notation (e.g. "[::1]:9990") is handled correctly.
+func parsePeer(peer string) (host string, port int, err error) {
+	h, portStr, err := net.SplitHostPort(peer)
+	if err != nil {
+		return "", 0, fmt.Errorf("invalid peer %q: %w", peer, err)
+	}
+	if h == "" {
+		return "", 0, fmt.Errorf("invalid peer %q: empty host", peer)
+	}
+	port, err = strconv.Atoi(portStr)
+	if err != nil || port <= 0 || port > 65535 {
+		return "", 0, fmt.Errorf("invalid port in peer %q", peer)
+	}
+	return h, port, nil
+}

src/frontend/apiserver/backend/peers_test.go

@@ -0,0 +1,95 @@
+package backend
+
+import "testing"
+
+func TestParsePeer(t *testing.T) {
+	tests := []struct {
+		name      string
+		peer      string
+		wantHost  string
+		wantPort  int
+		wantError bool
+	}{
+		{
+			name:     "valid simple host:port",
+			peer:     "ctrl01:9990",
+			wantHost: "ctrl01",
+			wantPort: 9990,
+		},
+		{
+			name:     "valid IPv4",
+			peer:     "192.168.1.10:9990",
+			wantHost: "192.168.1.10",
+			wantPort: 9990,
+		},
+		{
+			name:     "valid FQDN",
+			peer:     "ctrl01.cluster.local:9990",
+			wantHost: "ctrl01.cluster.local",
+			wantPort: 9990,
+		},
+		{
+			name:     "min valid port",
+			peer:     "host:1",
+			wantHost: "host",
+			wantPort: 1,
+		},
+		{
+			name:     "max valid port",
+			peer:     "host:65535",
+			wantHost: "host",
+			wantPort: 65535,
+		},
+		{
+			name:      "missing colon",
+			peer:      "ctrl019990",
+			wantError: true,
+		},
+		{
+			name:      "port zero",
+			peer:      "ctrl01:0",
+			wantError: true,
+		},
+		{
+			name:      "port out of range",
+			peer:      "ctrl01:65536",
+			wantError: true,
+		},
+		{
+			name:      "non-numeric port",
+			peer:      "ctrl01:abc",
+			wantError: true,
+		},
+		{
+			name:      "colon at start (empty host)",
+			peer:      ":9990",
+			wantError: true,
+		},
+		{
+			name:      "empty string",
+			peer:      "",
+			wantError: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			host, port, err := parsePeer(tc.peer)
+			if tc.wantError {
+				if err == nil {
+					t.Errorf("parsePeer(%q): expected error, got host=%q port=%d", tc.peer, host, port)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parsePeer(%q): unexpected error: %v", tc.peer, err)
+			}
+			if host != tc.wantHost {
+				t.Errorf("host: got %q, want %q", host, tc.wantHost)
+			}
+			if port != tc.wantPort {
+				t.Errorf("port: got %d, want %d", port, tc.wantPort)
+			}
+		})
+	}
+}

src/frontend/apiserver/backend/stub_funcs_test.go

@@ -29,4 +29,31 @@ func TestEvalSelectForHost_StubErrNoCGO(t *testing.T) {
 func TestCleanup_StubNoOp(t *testing.T) {
 	// Should not panic and completes immediately
 	Cleanup()
+}
+
+// TestRegisterToBackend_StubSingleHost verifies the stub returns errNoCGO
+// for a single-host BackendConfig (no Peers).
+func TestRegisterToBackend_StubSingleHost(t *testing.T) {
+	err := RegisterToBackend(&BackendConfig{
+		Address: "ctrl01",
+		Port:    9990,
+		From:    "xce_apiserver",
+		Key:     "keepsecret$",
+	})
+	if err != errNoCGO {
+		t.Errorf("expected errNoCGO, got %v", err)
+	}
+}
+
+// TestRegisterToBackend_StubWithPeers verifies the stub returns errNoCGO
+// even when Peers are configured (HA mode), since CGO is not available.
+func TestRegisterToBackend_StubWithPeers(t *testing.T) {
+	err := RegisterToBackend(&BackendConfig{
+		Peers: []string{"ctrl01:9990", "ctrl02:9990"},
+		From:  "xce_apiserver",
+		Key:   "keepsecret$",
+	})
+	if err != errNoCGO {
+		t.Errorf("expected errNoCGO with peers configured, got %v", err)
+	}
 }