feat(apiserver): expose PEND reason end-to-end (scheduler → API)

The pending_reason field was declared on models.JobInfo but never populated:
scheduler computed PEND reasons and the controller stored them on
job->pend_reasons, yet the data stopped there. Two gaps closed:

1. controller/apiserver.c: apiserver_job_hook_fun now handles
   HOOK_DETAIL_SETPREASON by pushing the updated job to API servers (reusing
   apiserver_job_update_fun). Without this, a job whose PEND reasons change
   never reached the API server — only JOB_NEW/UPDATE/START/EXIT events did,
   and setpreason fired none of those.

2. backend/convert.go + convert_pure.go: cJobToModel / rawJobToModel now
   extract xce_job_t.pend_reasons (main_reason + host/res reasons) and
   marshal them into models.JobInfo.PendingReason as JSONB. models/jobs.go:
   pending_reason added to JOB_DEFAULT_COLUMN so default job queries return
   it (it was only in JSONBJFields, never selected).

Verified live on t1/t2/t3: a job with resreq "type == nosuchhost" stays
PEND and returns
  "pending_reason":{"main_reason":0,"host_reasons":[{"host":"t1","reason":3},
   {"host":"t2","reason":3},{"host":"t3","reason":3}]}
(main_reason=3 PREASON_MATCH_FAIL per host) via both /jobs list, /jobs/:id
and the SSE stream. 4 new unit tests (rawJobToModel pendReason marshal +
name mapping), all apiserver backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>

src/controller/apiserver.c

@@ -414,6 +414,14 @@ static int apiserver_job_hook_fun(int type, int situation, int detail,
         return apiserver_job_fail_fun(type, situation, data, ap);
     case HOOK_DETAIL_JOB_RUSAGE_UPDATE:
         return apiserver_job_rusage_update_fun(type, situation, data, ap);
+    case HOOK_DETAIL_SETPREASON:
+        /* The scheduler pushed new PEND reasons via setpreason and the
+         * controller just stored them on the job (set_preason). Push the
+         * updated job to the API servers the same way a JOB_UPDATE does —
+         * otherwise the API servers keep the job without pend_reasons for
+         * its whole PEND lifetime (the job's own state never changed, so no
+         * JOB_UPDATE fires) and the pending_reason field stays empty. */
+        return apiserver_job_update_fun(type, situation, data, ap);
     }
 
     return 0;

src/frontend/apiserver/backend/convert.go

@@ -9,6 +9,10 @@ package backend
 
 // Helper to index into xce_host_alloc_t** (CGO cannot dereference C pointer-to-pointer arrays).
 static xce_host_alloc_t *job_alloc_host(xce_job_alloc_t *a, int i) { return a->allocs[i]; }
+
+// Helpers to index into the reason pointer arrays of xce_job_preason_t.
+static xce_host_reason_t *job_host_reason(xce_job_preason_t *p, int i) { return p->host_based_reasons[i]; }
+static xce_res_reason_t *job_res_reason(xce_job_preason_t *p, int i) { return p->res_based_reasons[i]; }
 */
 import "C"
 
@@ -117,6 +121,30 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
 			}
 		}
 	}
+	// PEND reason: the scheduler pushes these via setpreason; the controller
+	// stores them on job->pend_reasons. Maps to models.JobInfo.PendingReason.
+	if job.pend_reasons != nil {
+		pr := job.pend_reasons
+		r.pendReason = &rawPendReason{
+			MainReason:     int(pr.main_reason),
+			MainReasonName: pendReasonName(int(pr.main_reason)),
+		}
+		for i := 0; i < int(pr.nhost_reasons); i++ {
+			hr := C.job_host_reason(pr, C.int(i))
+			if hr != nil && hr.host != nil {
+				r.pendReason.HostReasons = append(r.pendReason.HostReasons,
+					rawHostReason{Host: goString(hr.host), Reason: int(hr.reason)})
+			}
+		}
+		for i := 0; i < int(pr.nres_reasons); i++ {
+			rr := C.job_res_reason(pr, C.int(i))
+			if rr != nil && rr.resource != nil {
+				r.pendReason.ResReasons = append(r.pendReason.ResReasons,
+					rawResReason{Resource: goString(rr.resource),
+						Host: goString(rr.host), Reason: int(rr.reason)})
+			}
+		}
+	}
 	return rawJobToModel(r)
 }
 

src/frontend/apiserver/backend/convert_pure.go

@@ -159,6 +159,47 @@ type rawSubmitData struct {
 	startInfo   *rawStartInfo
 }
 
+// rawHostReason mirrors the C xce_host_reason_t (one host-level PEND reason).
+type rawHostReason struct {
+	Host   string `json:"host"`
+	Reason int    `json:"reason"`
+}
+
+// rawResReason mirrors the C xce_res_reason_t (one resource-level PEND reason).
+type rawResReason struct {
+	Resource string `json:"resource"`
+	Host     string `json:"host"`
+	Reason   int    `json:"reason"`
+}
+
+// rawPendReason mirrors the C xce_job_preason_t attached to a PEND job by the
+// controller (scheduler → setpreason → job->pend_reasons). JSON-tagged so the
+// marshaled value lands in models.JobInfo.PendingReason as a JSONB object —
+// processJobData's JSONBJFields pass then surfaces it in the API response.
+type rawPendReason struct {
+	MainReason   int             `json:"main_reason"`
+	MainReasonName string        `json:"main_reason_name"`
+	HostReasons  []rawHostReason `json:"host_reasons,omitempty"`
+	ResReasons   []rawResReason  `json:"res_reasons,omitempty"`
+}
+
+// pendReasonName maps a PREASON_* main_reason code to a readable name. The
+// codes are the PEND reason bitfield of xce_job_preason_t (PREASON_BAD_RESREQ/
+// NO_RESOURCE/MATCH_FAIL, src/include/preason.h). Unknown codes are reported
+// verbatim so a new scheduler reason never silently disappears.
+func pendReasonName(code int) string {
+	switch code {
+	case 1: // PREASON_BAD_RESREQ
+		return "bad_resreq"
+	case 2: // PREASON_NO_RESOURCE
+		return "no_resource"
+	case 3: // PREASON_MATCH_FAIL
+		return "match_fail"
+	default:
+		return fmt.Sprintf("unknown(%d)", code)
+	}
+}
+
 // rawJobData mirrors the C xce_job_t fields used by the Go model.
 type rawJobData struct {
 	jobID      int64
@@ -170,6 +211,7 @@ type rawJobData struct {
 	finishTime int64
 	submit     *rawSubmitData
 	execHosts  []string
+	pendReason *rawPendReason
 }
 
 // rawWorkerData mirrors the C xce_worker_t fields used by the Go model.
@@ -258,6 +300,11 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
 		data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)})
 		info.ExecHosts = models.Jsonb{String: string(data)}
 	}
+	if r.pendReason != nil {
+		if data, err := json.Marshal(r.pendReason); err == nil {
+			info.PendingReason = models.Jsonb{String: string(data)}
+		}
+	}
 	return info
 }
 

src/frontend/apiserver/backend/convert_test.go

@@ -2,6 +2,7 @@ package backend
 
 import (
 	"apiserver/models"
+	"encoding/json"
 	"testing"
 )
 
@@ -96,6 +97,80 @@ func TestRawJobToModel_NilSubmit(t *testing.T) {
 	}
 }
 
+// PEND reason: rawJobData.pendReason must marshal into
+// JobInfo.PendingReason as a JSONB string (processJobData parses it via
+// JSONBJFields). No pendReason → PendingReason stays empty.
+func TestRawJobToModel_PendReason(t *testing.T) {
+	r := rawJobData{
+		jobID: 7,
+		state: 1, // JOB_STAT_PEND
+		pendReason: &rawPendReason{
+			MainReason:     2, // PREASON_NO_RESOURCE
+			MainReasonName: "no_resource",
+			HostReasons:    []rawHostReason{{Host: "t1", Reason: 2}},
+			ResReasons:     []rawResReason{{Resource: "slots", Host: "t1", Reason: 2}},
+		},
+	}
+
+	info := rawJobToModel(r)
+
+	if info.PendingReason.String == "" {
+		t.Fatal("PendingReason should be populated for a PEND job, got empty")
+	}
+	var decoded struct {
+		MainReason      int    `json:"main_reason"`
+		MainReasonName  string `json:"main_reason_name"`
+		HostReasons     []struct {
+			Host   string `json:"host"`
+			Reason int    `json:"reason"`
+		} `json:"host_reasons"`
+		ResReasons []struct {
+			Resource string `json:"resource"`
+			Host     string `json:"host"`
+			Reason   int    `json:"reason"`
+		} `json:"res_reasons"`
+	}
+	if err := json.Unmarshal([]byte(info.PendingReason.String), &decoded); err != nil {
+		t.Fatalf("PendingReason should be valid JSON, got err: %v", err)
+	}
+	if decoded.MainReason != 2 || decoded.MainReasonName != "no_resource" {
+		t.Errorf("main_reason: got %d/%q, want 2/no_resource",
+			decoded.MainReason, decoded.MainReasonName)
+	}
+	if len(decoded.HostReasons) != 1 || decoded.HostReasons[0].Host != "t1" {
+		t.Errorf("host_reasons: got %+v, want [t1]", decoded.HostReasons)
+	}
+	if len(decoded.ResReasons) != 1 || decoded.ResReasons[0].Resource != "slots" {
+		t.Errorf("res_reasons: got %+v, want [slots]", decoded.ResReasons)
+	}
+}
+
+func TestRawJobToModel_PendReasonNil(t *testing.T) {
+	info := rawJobToModel(rawJobData{jobID: 8, state: 1})
+	if info.PendingReason.String != "" {
+		t.Errorf("PendingReason should be empty without a pendReason, got %q",
+			info.PendingReason.String)
+	}
+}
+
+// pendReasonName maps PREASON_* codes; unknown codes are surfaced verbatim.
+func TestPendReasonName(t *testing.T) {
+	cases := []struct {
+		code int
+		want string
+	}{
+		{1, "bad_resreq"},
+		{2, "no_resource"},
+		{3, "match_fail"},
+		{42, "unknown(42)"},
+	}
+	for _, c := range cases {
+		if got := pendReasonName(c.code); got != c.want {
+			t.Errorf("pendReasonName(%d): got %q, want %q", c.code, got, c.want)
+		}
+	}
+}
+
 func TestRawJobToModel_Timestamps(t *testing.T) {
 	r := rawJobData{
 		jobID:      1,

src/frontend/apiserver/models/jobs.go

@@ -206,6 +206,7 @@ var JOB_DEFAULT_COLUMN = []string{
 	"job_name",
 	"command",
 	"submit_time",
+	"pending_reason",
 }
 
 var ARRAY_JOB_DEFAULT_COLUMN = []string{