Merge branch 'working' into job_peek_lchen

job_peek_lchen: Add fetchfile feature (job output/error peek proxy) working: HA/Raft infrastructure (consensus, replication, config mgmt) Conflict resolution summary: - controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30 - cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK - job.c: kept full fetchfile pending-request subsystem (~183 lines) - request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS - helper.c: added missing preVoteRequest/preVoteReply mappings - All 15 new test registrations (test_cbuf.c + 14 HA tests)
2026-07-22 13:57:25 +08:00 · 2026-07-22 13:57:25 +08:00 · 3a2094a461
commit 3a2094a461
parent d2b47e3985 295769d8ae
529 changed files with 345174 additions and 5048 deletions
--- a/src/frontend/CLAUDE.md
+++ b/src/frontend/CLAUDE.md
@ -35,37 +35,14 @@ config/      YAML configs and JSON schemas

 ## Conventions

-### Controllers
- Struct: `type {Resource}Controller struct{}`
- Handler signature: `func (c {Resource}Controller) {Action}{Resource}(ctx *gin.Context)`
- Always return via helpers — never write response directly:
-  ```go
-  ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, data)
-  ReturnError(ctx, common.ERROR_CODE, err.Error(), nil)
-  ```
+See `.kiro/steering/api-standards.md` (handler pattern, middleware chain, error codes, response format), `.kiro/steering/error-handling.md` (ReturnError early-return, zap logging), `.kiro/steering/security.md` (RBAC, JWT, auth backends), and `.kiro/steering/testing.md` (TDD, table-driven tests, state isolation).

-### Error Handling
- Early return on every error — no deferred error accumulation
- Log with zap fields before returning: `logger.Error("msg", zap.Error(err))`
- Use `preCheck{Resource}Request()` helpers for input validation before business logic
-
-### RBAC
- Protect routes with `controllers.RequirePermission("action:domain")` middleware
- Permission strings follow `action:scope` pattern (e.g., `job:view:self`, `queue:submit:all`)
-
-### Logging
- Import as: `logger "apiserver/utils"`
- Always include context: `zap.String("key", val)`, `zap.Error(err)`
-
-### Models
- JSON tags use snake_case: `json:"field_name"`
- Required fields: `binding:"required"`
- YAML tags for config structs: `yaml:"field_name,omitempty"`
-
-### Tests
- Use `t.TempDir()` for file fixtures
- Reset global state with a helper (e.g., `resetRBACState()` pattern in rbac_check_test.go)
- Table-driven tests for filter/validation logic
+### Quick reference (from steering)
+- Handler: `preCheck*() → service call → ReturnSuccess/ReturnError; return`
+- Middleware chain: WhitelistURL → WhitelistIP → RateLimit → JWT → CORS → RBAC
+- RBAC: `RequirePermission("action:scope")` — permissions in `config/rbac.yaml`
+- Logging: `logger.Error("msg", zap.Error(err))` — never `fmt.Println`
+- Tests: co-located `_test.go`, `t.TempDir()`, global state reset via `t.Cleanup`

 ## Key Files
 - `controllers/response.go` — ReturnSuccess / ReturnError helpers
--- a/src/frontend/apiserver/backend/clib.go
+++ b/src/frontend/apiserver/backend/clib.go
@ -76,6 +76,30 @@ var idCtxOnce sync.Once
 // Must be set via StartNotificationListener before RegisterToBackend.
 var gHandler NotificationHandler

+func tryConnect(host string, port int, from, key string) error {
+	cAddr := C.CString(host)
+	cFrom := C.CString(from)
+	cKey := C.CString(key)
+	defer func() {
+		C.free(unsafe.Pointer(cAddr))
+		C.free(unsafe.Pointer(cFrom))
+		C.free(unsafe.Pointer(cKey))
+	}()
+
+	var cConfig C.xce_client_config_t
+	cConfig.nmasters = 1
+	cConfig.masters[0] = cAddr
+	cConfig.master_ports[0] = C.int(port)
+	cConfig.from = cFrom
+	cConfig.key = cKey
+
+	rc := C.xce_backend_init(&cConfig, &gCtx)
+	if rc < 0 {
+		return fmt.Errorf("xce_backend_init failed on %s:%d: %d", host, port, int(rc))
+	}
+	return nil
+}
+
 // RegisterToBackend initialises the XCE client library and joins as an API
 // server.  StartNotificationListener should be called first so that push
 // events have a handler ready before the first push can arrive.
@ -89,27 +113,31 @@ func RegisterToBackend(config *BackendConfig) error {
 		key = "keepsecret$"
 	}

-	cAddr := C.CString(config.Address)
-	cFrom := C.CString(from)
-	cKey := C.CString(key)
-	defer func() {
-		C.free(unsafe.Pointer(cAddr))
-		C.free(unsafe.Pointer(cFrom))
-		C.free(unsafe.Pointer(cKey))
-	}()
-
-	var cConfig C.xce_client_config_t
-	cConfig.nmasters = 1
-	cConfig.masters[0] = cAddr
-	cConfig.master_ports[0] = C.int(config.Port)
-	cConfig.from = cFrom
-	cConfig.key = cKey
-
-	rc := C.xce_backend_init(&cConfig, &gCtx)
-	if rc < 0 {
-		return fmt.Errorf("xce_backend_init failed: %d", int(rc))
+	if len(config.Peers) > 0 {
+		var lastErr error
+		for _, peer := range config.Peers {
+			host, port, err := parsePeer(peer)
+			if err != nil {
+				logger.Warn("Skipping invalid peer", zap.String("peer", peer), zap.Error(err))
+				lastErr = err
+				continue
+			}
+			if err := tryConnect(host, port, from, key); err != nil {
+				logger.Warn("Candidate unreachable, trying next",
+					zap.String("host", host), zap.Int("port", port), zap.Error(err))
+				lastErr = err
+				continue
+			}
+			logger.Info("Backend connection established (HA)",
+				zap.String("host", host), zap.Int("port", port))
+			return nil
+		}
+		return fmt.Errorf("all HA candidates failed: %w", lastErr)
 	}

+	if err := tryConnect(config.Address, config.Port, from, key); err != nil {
+		return err
+	}
 	logger.Info("Backend connection established",
 		zap.String("address", config.Address),
 		zap.Int("port", config.Port))
--- a/src/frontend/apiserver/backend/config.go
+++ b/src/frontend/apiserver/backend/config.go
@ -7,6 +7,12 @@ type BackendConfig struct {
 	Port    int    // controller port (e.g. 9990)
 	From    string // identity sent to controller (default: "xce_apiserver")
 	Key     string // authentication key
+
+	// Peers lists additional controller candidates for HA mode in "host:port"
+	// format.  When non-empty, RegisterToBackend tries each candidate in order
+	// and connects to whichever accepts the join as primary.  Address+Port
+	// serve as a single-host fallback when Peers is empty.
+	Peers []string
 }

 // FetchFileResult holds the data returned by FetchFile.
--- a/src/frontend/apiserver/backend/convert.go
+++ b/src/frontend/apiserver/backend/convert.go
@ -80,8 +80,9 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {

 // cWorkerToModel extracts C xce_worker_t fields into a rawWorkerData and maps it to HostInfo.
 func cWorkerToModel(worker *C.xce_worker_t) *models.HostInfo {
+	workerName := goString(worker.name)
 	r := rawWorkerData{
-		name:   goString(worker.name),
+		name:   workerName,
 		status: int(worker.status),
 	}
 	if worker.attrs != nil {
@ -96,6 +97,16 @@ func cWorkerToModel(worker *C.xce_worker_t) *models.HostInfo {
 		r.maxTmp = int(a.maxtmp)
 		r.nDisks = int(a.ndisks)
 	}
+	n := int(worker.nresvals_to_sched)
+	if n > 0 {
+		entries := unsafe.Slice(worker.resvals_to_sched, n)
+		r.resvals = make([]models.ResourceDistributionInfo, 0, n)
+		for _, nv := range entries {
+			if rd := cNameValToWorkerResval(workerName, nv); rd != nil {
+				r.resvals = append(r.resvals, *rd)
+			}
+		}
+	}
 	return rawWorkerToModel(r)
 }

@ -189,7 +200,7 @@ func cClusterConfToClusterInfo(conf *C.cluster_config_t) *models.ClusterInfo {
 	return rawClusterToModel(rawClusterData{
 		name:                   name,
 		heartbeatInterval:      int(conf.heartbeat_interval),
-		vcpus:                  int(conf.vcpus),
+		vcpus:                  0, // vcpus moved to myconf_t (--vcpus CLI flag); not in cluster_config_t
 		inactiveHeartbeatCount: int(conf.inactive_heartbeat_count),
 	})
 }
@ -267,6 +278,55 @@ func freeJobSubmit(js *C.xce_job_submit_t, sd *C.job_submit_data_t, si *C.job_st
 	}
 }

+// cNameValToWorkerResval converts a C xce_name_val_t pointer to a ResourceDistributionInfo.
+// workerName is set as Location. Returns nil if nv or nv.val is nil.
+func cNameValToWorkerResval(workerName string, nv *C.xce_name_val_t) *models.ResourceDistributionInfo {
+	if nv == nil {
+		return nil
+	}
+	rd := &models.ResourceDistributionInfo{
+		ResourceName: C.GoString(nv.name),
+		Location:     workerName,
+		IsDynamic:    true,
+	}
+	if nv.val == nil {
+		logger.Warn("worker resval with nil value, skipping",
+			zap.String("resource", rd.ResourceName),
+			zap.String("host", workerName))
+		return nil
+	}
+	switch C.xce_val_which(nv.val) {
+	case C.Value_u_ivalue32:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_i32(nv.val))
+	case C.Value_u_ivalue64:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_i64(nv.val))
+	case C.Value_u_uvalue32:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_u32(nv.val))
+	case C.Value_u_uvalue64:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_u64(nv.val))
+	case C.Value_u_fvalue:
+		rd.ValueType = int(models.NUMERIC)
+		rd.NumericValue = float64(C.xce_val_fval(nv.val))
+	case C.Value_u_svalue:
+		rd.ValueType = int(models.STRING)
+		rd.StringValue = C.GoString(C.xce_val_str(nv.val))
+	case C.Value_u_bvalue:
+		rd.ValueType = int(models.BOOLEAN)
+		rd.BooleanValue = C.xce_val_bool(nv.val) != 0
+	default:
+		logger.Warn("unknown xce_val_t discriminant in worker resval, skipping",
+			zap.Int("which", int(C.xce_val_which(nv.val))),
+			zap.String("resource", rd.ResourceName),
+			zap.String("host", workerName))
+		return nil
+	}
+	return rd
+}
+
 // cResvalToDistribution converts a C xce_resval_t pointer to a Go ResourceDistributionInfo.
 // Returns nil if vp.val is NULL (nil guard prevents CGO panic).
 // vp.val is a *C.xce_val_t pointer field — pass directly, no &.
--- a/src/frontend/apiserver/backend/convert_pure.go
+++ b/src/frontend/apiserver/backend/convert_pure.go
@ -4,7 +4,6 @@ import (
 	"apiserver/models"
 	"encoding/json"
 	"fmt"
-	"time"
 )

 // JobID bit layout constants — mirrors src/include/config.h.
@ -155,6 +154,7 @@ type rawWorkerData struct {
 	maxSwap   int
 	maxTmp    int
 	nDisks    int
+	resvals   []models.ResourceDistributionInfo
 }

 // rawQueueData mirrors the C xce_queue_t fields used by the Go model.
@ -207,18 +207,10 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
 			info.ExecUser = si.execUser
 		}
 	}
-	if r.subTime > 0 {
-		info.SubmitTime = time.Unix(r.subTime, 0)
-	}
-	if r.queueTime > 0 {
-		info.QueueTime = time.Unix(r.queueTime, 0)
-	}
-	if r.startTime > 0 {
-		info.StartTime = time.Unix(r.startTime, 0)
-	}
-	if r.finishTime > 0 {
-		info.EndTime = time.Unix(r.finishTime, 0)
-	}
+	info.SubmitTime = r.subTime
+	info.QueueTime  = r.queueTime
+	info.StartTime  = r.startTime
+	info.EndTime    = r.finishTime
 	if len(r.execHosts) > 0 {
 		type hostSlot struct {
 			HostName string `json:"host_name"`
@ -241,8 +233,9 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
 // rawWorkerToModel maps a rawWorkerData to a models.HostInfo.
 func rawWorkerToModel(r rawWorkerData) *models.HostInfo {
 	info := &models.HostInfo{
-		HostName:   r.name,
-		HostStatus: r.status,
+		HostName:       r.name,
+		HostStatus:     r.status,
+		ResourceValues: r.resvals,
 	}
 	if r.hasAttrs {
 		info.HostType = r.hostType
--- a/src/frontend/apiserver/backend/convert_test.go
+++ b/src/frontend/apiserver/backend/convert_test.go
@ -3,7 +3,6 @@ package backend
 import (
 	"apiserver/models"
 	"testing"
-	"time"
 )

 func TestRawJobToModel_BasicFields(t *testing.T) {
@ -81,17 +80,17 @@ func TestRawJobToModel_Timestamps(t *testing.T) {

 	info := rawJobToModel(r)

-	if info.SubmitTime != time.Unix(1700000000, 0) {
-		t.Errorf("SubmitTime mismatch")
+	if info.SubmitTime != 1700000000 {
+		t.Errorf("SubmitTime: got %d, want 1700000000", info.SubmitTime)
 	}
-	if info.QueueTime != time.Unix(1700000010, 0) {
-		t.Errorf("QueueTime mismatch")
+	if info.QueueTime != 1700000010 {
+		t.Errorf("QueueTime: got %d, want 1700000010", info.QueueTime)
 	}
-	if info.StartTime != time.Unix(1700000020, 0) {
-		t.Errorf("StartTime mismatch")
+	if info.StartTime != 1700000020 {
+		t.Errorf("StartTime: got %d, want 1700000020", info.StartTime)
 	}
-	if info.EndTime != time.Unix(1700000030, 0) {
-		t.Errorf("EndTime mismatch")
+	if info.EndTime != 1700000030 {
+		t.Errorf("EndTime: got %d, want 1700000030", info.EndTime)
 	}
 }

--- a/src/frontend/apiserver/backend/peers.go
+++ b/src/frontend/apiserver/backend/peers.go
@ -0,0 +1,24 @@
+package backend
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+)
+
+// parsePeer splits a "host:port" string. Uses net.SplitHostPort so IPv6
+// bracket notation (e.g. "[::1]:9990") is handled correctly.
+func parsePeer(peer string) (host string, port int, err error) {
+	h, portStr, err := net.SplitHostPort(peer)
+	if err != nil {
+		return "", 0, fmt.Errorf("invalid peer %q: %w", peer, err)
+	}
+	if h == "" {
+		return "", 0, fmt.Errorf("invalid peer %q: empty host", peer)
+	}
+	port, err = strconv.Atoi(portStr)
+	if err != nil || port <= 0 || port > 65535 {
+		return "", 0, fmt.Errorf("invalid port in peer %q", peer)
+	}
+	return h, port, nil
+}
--- a/src/frontend/apiserver/backend/peers_test.go
+++ b/src/frontend/apiserver/backend/peers_test.go
@ -0,0 +1,95 @@
+package backend
+
+import "testing"
+
+func TestParsePeer(t *testing.T) {
+	tests := []struct {
+		name      string
+		peer      string
+		wantHost  string
+		wantPort  int
+		wantError bool
+	}{
+		{
+			name:     "valid simple host:port",
+			peer:     "ctrl01:9990",
+			wantHost: "ctrl01",
+			wantPort: 9990,
+		},
+		{
+			name:     "valid IPv4",
+			peer:     "192.168.1.10:9990",
+			wantHost: "192.168.1.10",
+			wantPort: 9990,
+		},
+		{
+			name:     "valid FQDN",
+			peer:     "ctrl01.cluster.local:9990",
+			wantHost: "ctrl01.cluster.local",
+			wantPort: 9990,
+		},
+		{
+			name:     "min valid port",
+			peer:     "host:1",
+			wantHost: "host",
+			wantPort: 1,
+		},
+		{
+			name:     "max valid port",
+			peer:     "host:65535",
+			wantHost: "host",
+			wantPort: 65535,
+		},
+		{
+			name:      "missing colon",
+			peer:      "ctrl019990",
+			wantError: true,
+		},
+		{
+			name:      "port zero",
+			peer:      "ctrl01:0",
+			wantError: true,
+		},
+		{
+			name:      "port out of range",
+			peer:      "ctrl01:65536",
+			wantError: true,
+		},
+		{
+			name:      "non-numeric port",
+			peer:      "ctrl01:abc",
+			wantError: true,
+		},
+		{
+			name:      "colon at start (empty host)",
+			peer:      ":9990",
+			wantError: true,
+		},
+		{
+			name:      "empty string",
+			peer:      "",
+			wantError: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			host, port, err := parsePeer(tc.peer)
+			if tc.wantError {
+				if err == nil {
+					t.Errorf("parsePeer(%q): expected error, got host=%q port=%d", tc.peer, host, port)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("parsePeer(%q): unexpected error: %v", tc.peer, err)
+			}
+			if host != tc.wantHost {
+				t.Errorf("host: got %q, want %q", host, tc.wantHost)
+			}
+			if port != tc.wantPort {
+				t.Errorf("port: got %d, want %d", port, tc.wantPort)
+			}
+		})
+	}
+}
--- a/src/frontend/apiserver/backend/stub_funcs_test.go
+++ b/src/frontend/apiserver/backend/stub_funcs_test.go
@ -29,4 +29,31 @@ func TestEvalSelectForHost_StubErrNoCGO(t *testing.T) {
 func TestCleanup_StubNoOp(t *testing.T) {
 	// Should not panic and completes immediately
 	Cleanup()
+}
+
+// TestRegisterToBackend_StubSingleHost verifies the stub returns errNoCGO
+// for a single-host BackendConfig (no Peers).
+func TestRegisterToBackend_StubSingleHost(t *testing.T) {
+	err := RegisterToBackend(&BackendConfig{
+		Address: "ctrl01",
+		Port:    9990,
+		From:    "xce_apiserver",
+		Key:     "keepsecret$",
+	})
+	if err != errNoCGO {
+		t.Errorf("expected errNoCGO, got %v", err)
+	}
+}
+
+// TestRegisterToBackend_StubWithPeers verifies the stub returns errNoCGO
+// even when Peers are configured (HA mode), since CGO is not available.
+func TestRegisterToBackend_StubWithPeers(t *testing.T) {
+	err := RegisterToBackend(&BackendConfig{
+		Peers: []string{"ctrl01:9990", "ctrl02:9990"},
+		From:  "xce_apiserver",
+		Key:   "keepsecret$",
+	})
+	if err != errNoCGO {
+		t.Errorf("expected errNoCGO with peers configured, got %v", err)
+	}
 }
--- a/src/frontend/apiserver/cmd/main.go
+++ b/src/frontend/apiserver/cmd/main.go
@ -66,6 +66,7 @@ func main() {
 			Port:    utils.BackendYamlConfig.Port,
 			From:    utils.BackendYamlConfig.From,
 			Key:     utils.BackendYamlConfig.Key,
+			Peers:   utils.BackendYamlConfig.Peers,
 		}
 		if err := backend.RegisterToBackend(bcfg); err != nil {
 			logger.Error("Failed to connect to backend", zap.Error(err))
--- a/src/frontend/apiserver/config/auth.yaml
+++ b/src/frontend/apiserver/config/auth.yaml
@ -11,6 +11,9 @@ config_file:
    - username: admin
      password: admin
      roles: [cluster_admin]
+    - username: rshen
+      password: Letmein
+      roles: [cluster_admin]
    - username: user1
      password: user1
      roles: [basic_user]
--- a/src/frontend/apiserver/controllers/hosts.go
+++ b/src/frontend/apiserver/controllers/hosts.go
@ -536,21 +536,16 @@ func validateHost(ctx *gin.Context, control int, hostnames []string) (successHos
 }

 func isHostControlActionAllowed(actionType int, status int) bool {
+	isClosed := (status&models.HOSTSTAT["CLOSED"]) != 0 ||
+		(status&models.HOSTSTAT["CLOSED_BY_ADMIN"]) != 0
 	switch actionType {
-	case 1: // open strictly
-		if status&models.HOSTSTAT["CLOSED"] != 0 {
-			return true
-		}
+	case 1: // open
+		return isClosed
 	case 2: // close
-		if (status & models.HOSTSTAT["CLOSED"]) != 0 {
-			return false
-		} else {
-			return true
-		}
+		return !isClosed
 	default:
 		return false
 	}
-	return false
 }
 func preCheckHostControlRequest(req *models.HostControlRequest) (bool, error) {
 	logger.Debug("Enter preCheckHostControlRequest")
--- a/src/frontend/apiserver/controllers/jobs.go
+++ b/src/frontend/apiserver/controllers/jobs.go
@ -1535,6 +1535,13 @@ func preCheckJobRequest(req *models.JobSubmitRequest) ([]int, error) {
 	if req.QueueName == "" {
 		req.QueueName = "normal"
 	} else {
+		// Reject submissions to the reserved internal recovery queue.
+		if req.QueueName == "lost_and_found" {
+			logger.Error("Job submission to reserved queue rejected",
+				zap.String("queue_name", req.QueueName))
+			return nil, fmt.Errorf(
+				"cannot submit jobs to 'lost_and_found' — this is an internal recovery queue")
+		}
 		//validate queue_name whether exists
 		queueInfo, ok := store.Queues.Get(req.QueueName)
 		if !ok {
--- a/src/frontend/apiserver/controllers/jobs_test.go
+++ b/src/frontend/apiserver/controllers/jobs_test.go
@ -0,0 +1,76 @@
+package controllers
+
+import (
+	"apiserver/models"
+	"apiserver/store"
+	"apiserver/utils"
+	"strings"
+	"testing"
+
+	"go.uber.org/zap"
+)
+
+func resetJobSubmitState() {
+	store.InitStores()
+	utils.Log = zap.NewNop()
+}
+
+// TestPreCheckJobRequest_LostAndFound verifies that submitting to the
+// "lost_and_found" queue is rejected with a clear error message.
+func TestPreCheckJobRequest_LostAndFound(t *testing.T) {
+	resetJobSubmitState()
+
+	req := &models.JobSubmitRequest{
+		QueueName: "lost_and_found",
+	}
+
+	_, err := preCheckJobRequest(req)
+	if err == nil {
+		t.Fatal("expected error when submitting to lost_and_found queue, got nil")
+	}
+	if !strings.Contains(err.Error(), "lost_and_found") {
+		t.Errorf("error message should mention 'lost_and_found', got: %v", err)
+	}
+	if !strings.Contains(err.Error(), "internal recovery queue") {
+		t.Errorf("error message should mention 'internal recovery queue', got: %v", err)
+	}
+}
+
+// TestPreCheckJobRequest_NormalQueue verifies that a normal (non-reserved)
+// queue name does not trigger the reserved-name rejection.
+func TestPreCheckJobRequest_NormalQueue(t *testing.T) {
+	resetJobSubmitState()
+
+	// Seed a valid open queue in the store.
+	store.Queues.Put("normal", models.QueueInfo{
+		QueueName:   "normal",
+		QueueStatus: 0, // open
+	})
+
+	req := &models.JobSubmitRequest{
+		QueueName: "normal",
+	}
+
+	_, err := preCheckJobRequest(req)
+	// Should not be rejected for the reserved-name check; may fail for other
+	// reasons (e.g. missing JobName validation), but not with our new error.
+	if err != nil && strings.Contains(err.Error(), "internal recovery queue") {
+		t.Errorf("normal queue should not be rejected with reserved-name error: %v", err)
+	}
+}
+
+// TestPreCheckJobRequest_EmptyQueue verifies that an empty queue name is
+// allowed (defaults to "normal") and is not confused with reserved names.
+func TestPreCheckJobRequest_EmptyQueue(t *testing.T) {
+	resetJobSubmitState()
+
+	req := &models.JobSubmitRequest{
+		QueueName: "",
+	}
+
+	// Empty queue name sets default — should not trigger reserved-name error.
+	_, err := preCheckJobRequest(req)
+	if err != nil && strings.Contains(err.Error(), "internal recovery queue") {
+		t.Errorf("empty queue should not trigger reserved-name error: %v", err)
+	}
+}
--- a/src/frontend/apiserver/models/common.go
+++ b/src/frontend/apiserver/models/common.go
@ -1,7 +1,5 @@
 package models

-import "time"
-
 const (
 	CONTROL_JOB     = 1
 	CONTROL_HOST    = 2
@ -11,9 +9,9 @@ const (
 )

 type ActionControl struct {
-	TargetType int       `json:"target_type"`
-	ActionType int       `json:"action_type"`
-	Content    Jsonb     `json:"content"`
-	SubmitTime time.Time `json:"submit_time"`
-	Submitter  string    `json:"submitter"`
+	TargetType int    `json:"target_type"`
+	ActionType int    `json:"action_type"`
+	Content    Jsonb  `json:"content"`
+	SubmitTime int64  `json:"submit_time"`
+	Submitter  string `json:"submitter"`
 }
--- a/src/frontend/apiserver/models/hosts.go
+++ b/src/frontend/apiserver/models/hosts.go
@ -10,16 +10,19 @@ const (
 var SupportedHostActions = []string{"open", "close"}

 var HOSTSTAT = map[string]int{
-	"UNAVAIL": 0x00010000,
-	"LOCKEDU": 0x00020000,
-	"LOCKEDW": 0x00040000,
-	"BUSY":    0x00080000,
-	"CLOSED":  0x00100000,
+	"UNAVAIL":         0x00010000,
+	"LOCKEDU":         0x00020000,
+	"LOCKEDW":         0x00040000,
+	"BUSY":            0x00080000,
+	"CLOSED":          0x00100000,
+	"CLOSED_BY_ADMIN": 0x00400000,
 }

 func HostStateStr(state int) string {
 	if (state & HOSTSTAT["UNAVAIL"]) != 0 {
 		return "UNAVAIL"
+	} else if (state & HOSTSTAT["CLOSED_BY_ADMIN"]) != 0 {
+		return "CLOSED_BY_ADMIN"
 	} else if (state & HOSTSTAT["CLOSED"]) != 0 {
 		return "CLOSED"
 	} else if (state & HOSTSTAT["BUSY"]) != 0 {
@ -86,8 +89,9 @@ type HostInfo struct {
 	NumUSusp    int     `json:"num_ususp"`
 	//NumRsv      int     `json:"num_rsv"`
 	NumDone   int    `json:"num_done"`
-	Topology  string `json:"topology"`   // JSONB
-	SchedStop string `json:"sched_stop"` // JSONB
+	Topology       string                    `json:"topology"`        // JSONB
+	SchedStop      string                    `json:"sched_stop"`      // JSONB
+	ResourceValues []ResourceDistributionInfo `json:"resource_values"` // per-worker resource values
 	//RealLoad  string `json:"real_load"`  // JSONB
 }

--- a/src/frontend/apiserver/models/hosts_test.go
+++ b/src/frontend/apiserver/models/hosts_test.go
@ -23,6 +23,11 @@ func TestHostStateStr(t *testing.T) {
 			state: HOSTSTAT["CLOSED"],
 			want:  "CLOSED",
 		},
+		{
+			name:  "CLOSED_BY_ADMIN flag",
+			state: HOSTSTAT["CLOSED_BY_ADMIN"],
+			want:  "CLOSED_BY_ADMIN",
+		},
 		{
 			name:  "BUSY flag",
 			state: HOSTSTAT["BUSY"],
@ -48,6 +53,11 @@ func TestHostStateStr(t *testing.T) {
 			state: HOSTSTAT["UNAVAIL"] | HOSTSTAT["CLOSED"],
 			want:  "UNAVAIL", // UNAVAIL takes priority
 		},
+		{
+			name:  "combined CLOSED+CLOSED_BY_ADMIN flags",
+			state: HOSTSTAT["CLOSED"] | HOSTSTAT["CLOSED_BY_ADMIN"],
+			want:  "CLOSED_BY_ADMIN", // CLOSED_BY_ADMIN overrides generic CLOSED
+		},
 	}

 	for _, tt := range tests {
--- a/src/frontend/apiserver/models/jobs.go
+++ b/src/frontend/apiserver/models/jobs.go
@ -61,18 +61,19 @@ var JobSignalMapTable = map[string]int{
 }

 var JOBSTAT = map[string]int{
-	"PEND":    0x01,
-	"RUN":     0x02,
-	"EXIT":    0x04,
-	"DONE":    0x08,
-	"PSUSP":   0x10,
-	"USUSP":   0x20,
-	"SSUSP":   0x40,
-	"PDONE":   0x80,
-	"PERR":    0x100,
-	"WAIT":    0x200,
-	"RUNKWN":  0x8000,
-	"UNKNOWN": 0x10000, // C JOB_STAT_UNKWN: worker lost contact with controller
+	"PEND":      0x01,
+	"RUN":       0x02,
+	"EXIT":      0x04,
+	"DONE":      0x08,
+	"PSUSP":     0x10,
+	"USUSP":     0x20,
+	"SSUSP":     0x40,
+	"PDONE":     0x80,
+	"PERR":      0x100,
+	"WAIT":      0x200,
+	"TRANSIENT": 0x400, // C JOB_STAT_TRANSIENT: job state uncertain after failover
+	"RUNKWN":    0x8000,
+	"UNKNOWN":   0x10000, // C JOB_STAT_UNKWN: worker lost contact with controller
 }

 var LAUNCHTYPE = map[string]int{
@ -81,7 +82,10 @@ var LAUNCHTYPE = map[string]int{
 }

 func JobStateStr(state int) string {
-	if (state & JOBSTAT["PEND"]) != 0 {
+	// Check TRANSIENT first — it takes priority after failover
+	if (state & JOBSTAT["TRANSIENT"]) != 0 {
+		return "TRANSIENT"
+	} else if (state & JOBSTAT["PEND"]) != 0 {
 		if (state & JOBSTAT["PSUSP"]) != 0 {
 			return "PSUSP"
 		}
@ -485,7 +489,7 @@ type JobInfo struct {
 	ArrayIdx           int        `json:"array_idx"` //new
 	SubmitUser         string     `json:"submit_user"`
 	FromHost           string     `json:"from_host"`
-	SubmitTime         time.Time  `json:"submit_time"`
+	SubmitTime         int64      `json:"submit_time"`
 	JobStatus          int        `json:"job_status"`
 	JobName            string     `json:"job_name"`
 	Command            string     `json:"command"`
@ -531,10 +535,10 @@ type JobInfo struct {
 	Tags               Jsonb      `json:"tags"`                 // new, JSONB as []byte
 	EffectiveResReq    string     `json:"effective_resreq"`     //new
 	EffectiveUserGroup string     `json:"effective_user_group"` //new
-	QueueTime          time.Time  `json:"queue_time"`
-	StartTime          time.Time  `json:"start_time"`
-	EndTime            time.Time  `json:"end_time"`
-	ReserveTime        time.Time  `json:"reserve_time"`
+	QueueTime          int64      `json:"queue_time"`
+	StartTime          int64      `json:"start_time"`
+	EndTime            int64      `json:"end_time"`
+	ReserveTime        int64      `json:"reserve_time"`
 	Duration           int        `json:"duration"`
 	CPUTime            int        `json:"cpu_time"`
 	ExitStatus         Jsonb      `json:"exit_status"` //change type,  JSONB as []byte
@ -585,12 +589,12 @@ type JobAdditionalFields struct {
 	JobID      int64     `json:"job_id"`
 	SubmitUser string    `json:"submit_user"`
 	JobStatus  int       `json:"job_status"`
-	SubmitTime time.Time `json:"submit_time"`
-	FromHost   string    `json:"from_host"`
-	ExecHosts  string    `json:"exec_hosts"`
+	SubmitTime int64  `json:"submit_time"`
+	FromHost   string `json:"from_host"`
+	ExecHosts  string `json:"exec_hosts"`

-	StartTime         time.Time `json:"start_time"`
-	EndTime           time.Time `json:"end_time"`
+	StartTime         int64 `json:"start_time"`
+	EndTime           int64 `json:"end_time"`
 	JobRusage         Jsonb     `json:"job_rusage"`
 	MaxMem            float64   `json:"max_mem"`
 	AvgMem            float64   `json:"avg_mem"`
--- a/src/frontend/apiserver/models/jobs_test.go
+++ b/src/frontend/apiserver/models/jobs_test.go
@ -58,6 +58,16 @@ func TestJobStateStr(t *testing.T) {
 			state: 0,
 			want:  "ERR",
 		},
+		{
+			name:  "TRANSIENT (0x400 — job uncertain after failover)",
+			state: 0x400,
+			want:  "TRANSIENT",
+		},
+		{
+			name:  "TRANSIENT with other bits set",
+			state: 0x400 | 0x01, // TRANSIENT + PEND
+			want:  "TRANSIENT",
+		},
 	}

 	for _, tt := range tests {
@ -73,18 +83,19 @@ func TestJobStateStr(t *testing.T) {
 func TestJobStatMapValues(t *testing.T) {
 	// Verify bitmask values match C JOB_STAT_* defines in src/include/xce/types.h.
 	expected := map[string]int{
-		"PEND":    0x01,
-		"RUN":     0x02,
-		"EXIT":    0x04,
-		"DONE":    0x08,
-		"PSUSP":   0x10,
-		"USUSP":   0x20,
-		"SSUSP":   0x40,
-		"PDONE":   0x80,
-		"PERR":    0x100,
-		"WAIT":    0x200,
-		"RUNKWN":  0x8000,
-		"UNKNOWN": 0x10000,
+		"PEND":      0x01,
+		"RUN":       0x02,
+		"EXIT":      0x04,
+		"DONE":      0x08,
+		"PSUSP":     0x10,
+		"USUSP":     0x20,
+		"SSUSP":     0x40,
+		"PDONE":     0x80,
+		"PERR":      0x100,
+		"WAIT":      0x200,
+		"TRANSIENT": 0x400,
+		"RUNKWN":    0x8000,
+		"UNKNOWN":   0x10000,
 	}
 	for key, want := range expected {
 		got, ok := JOBSTAT[key]
--- a/src/frontend/apiserver/utils/config.go
+++ b/src/frontend/apiserver/utils/config.go
@ -13,10 +13,15 @@ import (
 )

 type BackendConfig struct {
-	Address string `yaml:"address"`
-	Port    int    `yaml:"port"`
-	From    string `yaml:"from"`
-	Key     string `yaml:"key"`
+	Address string   `yaml:"address"`
+	Port    int      `yaml:"port"`
+	From    string   `yaml:"from"`
+	Key     string   `yaml:"key"`
+	// Peers lists additional controller candidates for HA mode in "host:port"
+	// format.  When non-empty, RegisterToBackend tries each candidate in order
+	// and connects to whichever accepts the join as primary.  Address+Port
+	// serve as a single-host fallback when Peers is empty.
+	Peers []string `yaml:"peers"`
 }

 type AppServerConfig struct {
@ -250,6 +255,7 @@ func InitConf() error {
 		Port:    viper.GetInt("backend.port"),
 		From:    viper.GetString("backend.from"),
 		Key:     viper.GetString("backend.key"),
+		Peers:   viper.GetStringSlice("backend.peers"),
 	}

 	return nil
--- a/src/frontend/xcecli/cmd/jobs/submit.go
+++ b/src/frontend/xcecli/cmd/jobs/submit.go
@ -11,6 +11,7 @@ import (
 	"runtime"
 	"strings"
 	"time"
+	"xcecli/common/format"
 	"xcecli/common/request"
 	"xcecli/common/utils"
 	"xcecli/config"
@ -488,7 +489,8 @@ func (js *JobSubmitter) RunJobsSubmit() error {
 		return err
 	}

-	fmt.Printf("%s\n", jobResp.Data.Message)
+	msg := strings.Replace(jobResp.Data.Message, fmt.Sprintf("<%d>", jobResp.Data.JobID), fmt.Sprintf("<%s>", format.FormatJobID(jobResp.Data.JobID)), 1)
+	fmt.Println(msg)
 	logger.Debug("job submitted successfully", zap.String("message", jobResp.Data.Message))
 	logger.Info("job submitted successfully", zap.String("message", jobResp.Data.Message))

--- a/src/frontend/xcecli/common/output/page_print_array_elements_jobs.go
+++ b/src/frontend/xcecli/common/output/page_print_array_elements_jobs.go
@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
-	"time"
 	jobfmt "xcecli/common/format"
 	"xcecli/models"
 )
@ -96,7 +95,7 @@ func (p *ArrayElementsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
 			case models.HEADER_JOB_NAME:
 				width = len(job.JobName)
 			case models.HEADER_JOB_SUBMIT_TIME:
-				width = len(job.SubmitTime)
+				width = len(fmtUnixTime(job.SubmitTime))
 			}

 			if !p.wideFormat && width > 15 {
@ -166,7 +165,7 @@ func (p *ArrayElementsPagePrinter) formatRows(jobs *models.JobsResponse, truncat
 				mainRow = appendOrDefault(mainRow, FormatStringField(job.FromHost, width, truncatFlag, true), "-")
 			case models.HEADER_JOB_EXEC_HOST:
 				var execHosts models.ExecHosts
-				if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 {
+				if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 && execHosts.HostSlots[0].HostName != "" {
 					firstHost := fmt.Sprintf("%d*%s", execHosts.HostSlots[0].NumSlots, execHosts.HostSlots[0].HostName)
 					mainRow = appendOrDefault(mainRow, FormatStringField(firstHost, width, truncatFlag, true), "-")

@ -190,21 +189,7 @@ func (p *ArrayElementsPagePrinter) formatRows(jobs *models.JobsResponse, truncat
 			case models.HEADER_JOB_NAME:
 				mainRow = appendOrDefault(mainRow, FormatStringField(job.JobName, width, truncatFlag, true), "-")
 			case models.HEADER_JOB_SUBMIT_TIME:
-				// remove the timezone part and parse the time
-				timeStr := strings.Split(job.SubmitTime, " ")[0:2]
-				timeStrWithoutTZ := strings.Join(timeStr, " ")
-
-				if submitTime, err := time.Parse("2006-01-02 15:04:05", timeStrWithoutTZ); err == nil {
-					month := submitTime.Format("Jan")
-					displayTime := fmt.Sprintf("%s %02d %02d:%02d",
-						month,
-						submitTime.Day(),
-						submitTime.Hour(),
-						submitTime.Minute())
-					mainRow = appendOrDefault(mainRow, FormatStringField(displayTime, width, truncatFlag, true), "-")
-				} else {
-					mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-")
-				}
+				mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.SubmitTime), width, truncatFlag, true), "-")
 			}
 		}
 		rows = append(rows, mainRow)
--- a/src/frontend/xcecli/common/output/page_print_hosts.go
+++ b/src/frontend/xcecli/common/output/page_print_hosts.go
@ -146,6 +146,7 @@ func (p *HostsPagePrinter) formatRows(hosts *models.HostsResponse, truncatFlag b
 		truncatFlag = false
 	}
 	for _, host := range hosts.Data.Rows {
+		isUnavail := host.HostStatus == "UNAVAIL"
 		row := make([]string, 0, len(p.fieldFormats))
 		for _, format := range p.fieldFormats {
 			width := p.columnWidths[format.name]
@ -156,17 +157,41 @@ func (p *HostsPagePrinter) formatRows(hosts *models.HostsResponse, truncatFlag b
 			case models.HEADER_HOST_STATUS:
 				row = appendOrDefault(row, FormatStringField(host.HostStatus, width, truncatFlag, true), "-")
 			case models.HEADER_HOST_HOST_TYPE:
-				row = appendOrDefault(row, FormatStringField(host.HostType, width, truncatFlag, true), "-")
+				if isUnavail {
+					row = append(row, FormatStringField("UNKNOWN", width, truncatFlag, true))
+				} else {
+					row = appendOrDefault(row, FormatStringField(host.HostType, width, truncatFlag, true), "-")
+				}
 			case models.HEADER_HOST_NCPUS:
-				row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumCPUs), "0")
+				if isUnavail {
+					row = append(row, FormatStringField("-", width, truncatFlag, true))
+				} else {
+					row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumCPUs), "0")
+				}
 			case models.HEADER_HOST_MAX_SLOT:
-				row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.MaxSlot), "0")
+				if isUnavail {
+					row = append(row, FormatStringField("-", width, truncatFlag, true))
+				} else {
+					row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.MaxSlot), "0")
+				}
 			case models.HEADER_HOST_NUM_RUN:
-				row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumRun), "0")
+				if isUnavail {
+					row = append(row, FormatStringField("-", width, truncatFlag, true))
+				} else {
+					row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumRun), "0")
+				}
 			case models.HEADER_HOST_NUM_SSUSP:
-				row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumSSusp), "0")
+				if isUnavail {
+					row = append(row, FormatStringField("-", width, truncatFlag, true))
+				} else {
+					row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumSSusp), "0")
+				}
 			case models.HEADER_HOST_NUM_USUSP:
-				row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumUSusp), "0")
+				if isUnavail {
+					row = append(row, FormatStringField("-", width, truncatFlag, true))
+				} else {
+					row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumUSusp), "0")
+				}
 			}
 		}
 		rows = append(rows, row)
@ -188,6 +213,7 @@ func (p *HostsPagePrinter) printJsonPage(hosts *models.HostsResponse) {

 func printHostsVerboseOutput(hosts *models.HostsResponse) {
 	for _, host := range hosts.Data.Rows {
+		isUnavail := host.HostStatus == "UNAVAIL"
 		fmt.Printf("HOST   %s\n", host.HostName)
 		// table 1
 		headers := models.HostDetailHeaders
@ -199,19 +225,43 @@ func printHostsVerboseOutput(hosts *models.HostsResponse) {
 			case models.HEADER_HOST_STATUS:
 				val = host.HostStatus
 			case models.HEADER_HOST_CPUF:
-				val = fmt.Sprintf("%.2f", host.CPUFactor)
+				if isUnavail {
+					val = "-"
+				} else {
+					val = fmt.Sprintf("%.2f", host.CPUFactor)
+				}
 			case models.HEADER_HOST_JLU:
 				val = "-"
 			case models.HEADER_HOST_MAX:
-				val = fmt.Sprintf("%d", host.MaxSlot)
+				if isUnavail {
+					val = "-"
+				} else {
+					val = fmt.Sprintf("%d", host.MaxSlot)
+				}
 			case models.HEADER_HOST_NJOBS:
-				val = fmt.Sprintf("%d", host.NumRun)
+				if isUnavail {
+					val = "-"
+				} else {
+					val = fmt.Sprintf("%d", host.NumRun)
+				}
 			case models.HEADER_HOST_RUN:
-				val = fmt.Sprintf("%d", host.NumRun)
+				if isUnavail {
+					val = "-"
+				} else {
+					val = fmt.Sprintf("%d", host.NumRun)
+				}
 			case models.HEADER_HOST_SSUSP:
-				val = fmt.Sprintf("%d", host.NumSSusp)
+				if isUnavail {
+					val = "-"
+				} else {
+					val = fmt.Sprintf("%d", host.NumSSusp)
+				}
 			case models.HEADER_HOST_USUSP:
-				val = fmt.Sprintf("%d", host.NumUSusp)
+				if isUnavail {
+					val = "-"
+				} else {
+					val = fmt.Sprintf("%d", host.NumUSusp)
+				}
 			case models.HEADER_HOST_RSV:
 				val = "-"
 			case models.HEADER_HOST_DISPT_WINDOW:
@ -234,8 +284,14 @@ func printHostsVerboseOutput(hosts *models.HostsResponse) {

 		// table2
 		headers2 := append([]string{""}, models.HostLoadSchedulingHeaders...)
-		totalVals := []string{"Total", "0.0", "0.0", "0.0", "1%", "0.0", "33", "1", "1137", "51G", "3.9G", "6.8G", "8", "0.0"}
-		reserveVals := []string{"Reserved", "0.0", "0.0", "0.0", "0%", "0.0", "0", "0", "0", "0M", "0M", "0M", "-", "-"}
+		var totalVals, reserveVals []string
+		if isUnavail {
+			totalVals = []string{"Total", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"}
+			reserveVals = []string{"Reserved", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"}
+		} else {
+			totalVals = []string{"Total", "0.0", "0.0", "0.0", "1%", "0.0", "33", "1", "1137", "51G", "3.9G", "6.8G", "8", "0.0"}
+			reserveVals = []string{"Reserved", "0.0", "0.0", "0.0", "0%", "0.0", "0", "0", "0", "0M", "0M", "0M", "-", "-"}
+		}

 		widths2 := make([]int, len(headers2))
 		for i := range headers2 {
--- a/src/frontend/xcecli/common/output/page_print_hosts_test.go
+++ b/src/frontend/xcecli/common/output/page_print_hosts_test.go
@ -0,0 +1,103 @@
+package output
+
+import (
+	"strings"
+	"testing"
+	"xcecli/models"
+)
+
+func makeHostsResponse(hosts ...models.XCEHost) *models.HostsResponse {
+	r := &models.HostsResponse{}
+	r.Data.Rows = hosts
+	r.Data.Count = len(hosts)
+	return r
+}
+
+func TestFormatRows_UnavailShowsDashes(t *testing.T) {
+	host := models.XCEHost{
+		HostName:   "pc13v",
+		HostStatus: "UNAVAIL",
+		NumCPUs:    16,
+		MaxSlot:    32,
+		NumRun:     5,
+		NumSSusp:   2,
+		NumUSusp:   1,
+		HostType:   "*6_64-linux-gnu",
+	}
+	resp := makeHostsResponse(host)
+
+	p := NewHostsPagePrinter()
+	p.PrepareForPrinting(resp)
+	rows := p.formatRows(resp, false)
+
+	if len(rows) != 1 {
+		t.Fatalf("expected 1 row, got %d", len(rows))
+	}
+	row := rows[0]
+
+	// Build a map from trimmed value by column header index.
+	headers := p.fieldFormats
+	vals := make(map[string]string, len(headers))
+	for i, f := range headers {
+		vals[f.name] = strings.TrimSpace(row[i])
+	}
+
+	for _, col := range []string{
+		models.HEADER_HOST_NCPUS,
+		models.HEADER_HOST_MAX_SLOT,
+		models.HEADER_HOST_NUM_RUN,
+		models.HEADER_HOST_NUM_SSUSP,
+		models.HEADER_HOST_NUM_USUSP,
+	} {
+		if vals[col] != "-" {
+			t.Errorf("UNAVAIL host: %s = %q, want \"-\"", col, vals[col])
+		}
+	}
+
+	if vals[models.HEADER_HOST_HOST_TYPE] != "UNKNOWN" {
+		t.Errorf("UNAVAIL host: HOST_TYPE = %q, want \"UNKNOWN\"", vals[models.HEADER_HOST_HOST_TYPE])
+	}
+}
+
+func TestFormatRows_AvailableShowsValues(t *testing.T) {
+	host := models.XCEHost{
+		HostName:   "pc14v",
+		HostStatus: "OK",
+		NumCPUs:    16,
+		MaxSlot:    32,
+		NumRun:     5,
+		NumSSusp:   2,
+		NumUSusp:   1,
+		HostType:   "linux",
+	}
+	resp := makeHostsResponse(host)
+
+	p := NewHostsPagePrinter()
+	p.PrepareForPrinting(resp)
+	rows := p.formatRows(resp, false)
+
+	if len(rows) != 1 {
+		t.Fatalf("expected 1 row, got %d", len(rows))
+	}
+	row := rows[0]
+
+	headers := p.fieldFormats
+	vals := make(map[string]string, len(headers))
+	for i, f := range headers {
+		vals[f.name] = strings.TrimSpace(row[i])
+	}
+
+	checks := map[string]string{
+		models.HEADER_HOST_NCPUS:     "16",
+		models.HEADER_HOST_MAX_SLOT:  "32",
+		models.HEADER_HOST_NUM_RUN:   "5",
+		models.HEADER_HOST_NUM_SSUSP: "2",
+		models.HEADER_HOST_NUM_USUSP: "1",
+		models.HEADER_HOST_HOST_TYPE: "linux",
+	}
+	for col, want := range checks {
+		if vals[col] != want {
+			t.Errorf("OK host: %s = %q, want %q", col, vals[col], want)
+		}
+	}
+}
--- a/src/frontend/xcecli/common/output/page_print_jobs.go
+++ b/src/frontend/xcecli/common/output/page_print_jobs.go
@ -98,7 +98,7 @@ func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
 			case models.HEADER_JOB_NAME:
 				width = len(job.JobName)
 			case models.HEADER_JOB_SUBMIT_TIME:
-				width = len(job.SubmitTime)
+				width = len(fmtUnixTime(job.SubmitTime))
 			case models.HEADER_JOB_CORE_LIMIT:
 				width = len(fmt.Sprintf("%f", job.CoreLimit))
 			case models.HEADER_JOB_MEM_LIMIT:
@ -179,7 +179,7 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool

 			case models.HEADER_JOB_EXEC_HOST:
 				var execHosts models.ExecHosts
-				if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 {
+				if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 && execHosts.HostSlots[0].HostName != "" {
 					firstHost := fmt.Sprintf("%d*%s", execHosts.HostSlots[0].NumSlots, execHosts.HostSlots[0].HostName)
 					mainRow = appendOrDefault(mainRow,
 						FormatStringField(firstHost, width, truncatFlag, false), "-")
@ -201,11 +201,7 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
 					mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-")
 				}
 			case models.HEADER_JOB_SUBMIT_TIME:
-				if submitTime, err := parseJobTime(job.SubmitTime); err == nil {
-					mainRow = appendOrDefault(mainRow, FormatStringField(formatJobTime(submitTime), width, truncatFlag, true), "-")
-				} else {
-					mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-")
-				}
+				mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.SubmitTime), width, truncatFlag, true), "-")

 			// normal fields
 			case models.HEADER_JOB_NAME:
@ -223,9 +219,9 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
 			case models.HEADER_JOB_PROJECT_NAME:
 				mainRow = appendOrDefault(mainRow, FormatStringField(job.ProjectName, width, truncatFlag, true), "-")
 			case models.HEADER_JOB_START_TIME:
-				mainRow = appendOrDefault(mainRow, FormatStringField(job.StartTime, width, truncatFlag, true), "-")
+				mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.StartTime), width, truncatFlag, true), "-")
 			case models.HEADER_JOB_END_TIME:
-				mainRow = appendOrDefault(mainRow, FormatStringField(job.EndTime, width, truncatFlag, true), "-")
+				mainRow = appendOrDefault(mainRow, FormatStringField(fmtUnixTime(job.EndTime), width, truncatFlag, true), "-")
 			case models.HEADER_JOB_CWD:
 				mainRow = appendOrDefault(mainRow, FormatStringField(job.CWD, width, truncatFlag, true), "-")
 			case models.HEADER_JOB_LAUNCH_TYPE:
@ -303,7 +299,7 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {

 		// (2) print the submit time info
 		submitLine := ""
-		if submitTime, err := parseJobTime(job.SubmitTime); err == nil {
+		if submitTime, ok := unixToTime(job.SubmitTime); ok {
 			submitLine = submitLine + fmt.Sprintf("%s %s %02d:%02d:%02d:  ",
 				submitTime.Format("Jan"), submitTime.Format("02"),
 				submitTime.Hour(), submitTime.Minute(), submitTime.Second())
@ -322,9 +318,20 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
 		}
 		PrintWrappedLine(w, submitLine, 80, 18)

-		// (3) print the start time info
+		// (3) print the pending reason for jobs waiting to be dispatched
+		if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
+			if job.PendingReason != "" {
+				fmt.Fprintf(w, " PENDING REASONS:\n")
+				fmt.Fprintf(w, " %s\n", job.PendingReason)
+			}
+		}
+
+		// (4) print the start time info — skip for jobs that haven't started yet
 		startLine := ""
-		if startTime, err := parseJobTime(job.StartTime); err == nil {
+		if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
+			goto printEndLine
+		}
+		if startTime, ok := unixToTime(job.StartTime); ok {
 			month := startTime.Format("Jan")
 			day := startTime.Format("02")
 			exec_hostname := ""
@ -380,10 +387,11 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
 		}
 		PrintWrappedLine(w, startLine, 80, 18)

+	printEndLine:
 		// (4) print end line
 		endLine := ""
 		if job.JobStatus == "DONE" || job.JobStatus == "EXIT" {
-			if endTime, err := parseJobTime(job.EndTime); err == nil {
+			if endTime, ok := unixToTime(job.EndTime); ok {
 				endLine = endLine + fmt.Sprintf("%s %s %02d:%02d:%02d:  ",
 					endTime.Format("Jan"), endTime.Format("02"),
 					endTime.Hour(), endTime.Minute(), endTime.Second())
--- a/src/frontend/xcecli/common/output/time.go
+++ b/src/frontend/xcecli/common/output/time.go
@ -5,17 +5,36 @@ import (
 	"time"
 )

-// parseJobTime parses a job time string supporting RFC3339 and space-separated formats.
+// parseJobTime parses a time string in RFC3339 or "2006-01-02 15:04:05" format.
 func parseJobTime(s string) (time.Time, error) {
 	if s == "" {
 		return time.Time{}, fmt.Errorf("empty time string")
 	}
-	for _, f := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02 15:04:05 MST"} {
-		if t, err := time.Parse(f, s); err == nil {
+	for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05"} {
+		if t, err := time.Parse(layout, s); err == nil {
 			return t, nil
 		}
 	}
-	return time.Time{}, fmt.Errorf("cannot parse time: %q", s)
+	return time.Time{}, fmt.Errorf("cannot parse time %q", s)
+}
+
+// unixToTime converts a Unix-second timestamp to time.Time.
+// Returns (time, true) for ts > 0; (zero, false) for unset/zero values.
+func unixToTime(ts int64) (time.Time, bool) {
+	if ts <= 0 {
+		return time.Time{}, false
+	}
+	return time.Unix(ts, 0), true
+}
+
+// fmtUnixTime formats a Unix timestamp as "Jan 02 15:04" for table display.
+// Returns "-" for zero/unset values.
+func fmtUnixTime(ts int64) string {
+	t, ok := unixToTime(ts)
+	if !ok {
+		return "-"
+	}
+	return formatJobTime(t)
 }

 // formatJobTime formats a time.Time as "Jan 02 15:04" for table display.
--- a/src/frontend/xcecli/models/jobs.go
+++ b/src/frontend/xcecli/models/jobs.go
@ -3,8 +3,6 @@ package models
 import (
 	"encoding/json"
 	"fmt"
-	"time"
-	
 )

 const (
@ -32,6 +30,7 @@ const (
 	HEADER_JOB_LAUNCH_TYPE     = "LAUNCH_TYPE"
 	HEADER_JOB_NUM_PROCESSORS  = "NUM_PROCESSORS"
 	HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL"
+	HEADER_JOB_PENDING_REASON  = "PENDING_REASON"

 	// second line
 	HEADER_JOB_R15S = "r15s"
@ -114,10 +113,10 @@ type QueryJob struct {
 	TermTime           string        `json:"term_time"`
 	UserPriority       int           `json:"user_priority"`
 	DependCond         string        `json:"depend_cond"`
-	SubmitTime         string        `json:"submit_time"`
-	ReserveTime        time.Time     `json:"reserve_time"`
-	StartTime          string        `json:"start_time"`
-	EndTime            string        `json:"end_time"`
+	SubmitTime         int64         `json:"submit_time"`
+	ReserveTime        int64         `json:"reserve_time"`
+	StartTime          int64         `json:"start_time"`
+	EndTime            int64         `json:"end_time"`
 	JobRusage          string        `json:"job_rusage"`
 	MaxMem             float64       `json:"max_mem"`
 	AvgMem             float64       `json:"avg_mem"`
@ -210,6 +209,7 @@ var JobFieldsMap = map[string]string{
 	HEADER_JOB_LAUNCH_TYPE:     "launch_type",
 	HEADER_JOB_NUM_PROCESSORS:  "num_processors",
 	HEADER_JOB_INTERACTIVE_URL: "interactive_job_url",
+	HEADER_JOB_PENDING_REASON:  "pending_reason",
 	//job_filter.HEADER_JOB_PRIORITY:     "job_priority",
 }

@ -258,16 +258,23 @@ func (e ExecHosts) String() string {

 // UnmarshalJSON customizes the JSON parsing method, compatible with both string "-" and structure formats
 func (e *ExecHosts) UnmarshalJSON(data []byte) error {
+	// JSON null means no allocation — treat as empty
+	if string(data) == "null" {
+		e.HostSlots = []HostSlot{}
+		e.NumHosts = 0
+		return nil
+	}
+
 	// First try to parse as a string
 	var str string
 	if err := json.Unmarshal(data, &str); err == nil {
-		// If the string is "-", return an empty ExecHosts
-		if str == "-" {
+		// Empty string or "-" both mean no hosts
+		if str == "" || str == "-" {
 			e.HostSlots = []HostSlot{}
 			e.NumHosts = 0
 			return nil
 		}
-		// If it's any other string, treat it as a single hostname
+		// Any other non-empty string is a single hostname
 		e.HostSlots = []HostSlot{{HostName: str, NumSlots: 1}}
 		e.NumHosts = 1
 		return nil