fix(frontend): job control no longer returns HTTP 500 (bug-1490)

Two stacked defects in the apiserver job-control link, both of which kept
hold/release/cancel from ever reaching the controller:

1. controllers/jobs.go: validateJobControl and foundJob asserted
   jobinfo.Data["array_idx"].(int64), but StoreToOrderMaps produces
   array_idx as int (models.JobInfo.ArrayIdx is declared int) -> panic
   -> Gin Recovery -> HTTP 500 before backend.ControlJob ran. Add
   normalizeJobInt64 / jobArrayIdx helpers and use them in both
   functions (job_status already had this fix via normalizeJobStatus;
   array_idx was missed).

2. backend/clib.go: backend.ControlJob set xce_jobctrl_t.action to the
   models.JOB_* constant (JOB_STOP=1 / JOB_RESUME=2 / JOB_KILL=3 /
   JOB_CHKPNT=8), but that field is the capnp JobControl union
   discriminant (JobControl_u_siginfo=0 / JobControl_u_ckptoption=1).
   The controller's job_ctrl() selected the wrong union branch (or
   none, since encode_JobControl drops the union pointer for
   non-0/1 discriminants) and rejected every control with -EINVAL.
   Add jobCtrlDiscriminant() and use it.

Also: resolveExecUser was defined in the cgo-only convert.go but is
exercised by a non-cgo test, so `make test` (CGO_ENABLED=0) failed to
build the backend package. Move it to convert_pure.go so it compiles in
both modes.

Tests: 3 controller regression tests (validateJobControl allowed /
not-allowed branches, foundJob array-child branch) + 1 backend
jobCtrlDiscriminant test. make test fully green. Cluster E2E verified:
hold -> PSUSP, release -> PEND, cancel -> EXIT on a PEND job, no 500,
no -EINVAL, all daemons healthy.

Co-Authored-By: Claude <noreply@anthropic.com>

src/frontend/apiserver/backend/clib.go

@@ -394,7 +394,12 @@ func SubmitJob(job *models.JobInfo, arraySize int) error {
 func ControlJob(jobid uint64, action int, content string) error {
 	var ctrl C.xce_jobctrl_t
 	ctrl.jobid = C.uint64_t(jobid)
-	ctrl.action = C.int(action)
+	// ctrl.action is the capnp JobControl union DISCRIMINANT (0=siginfo,
+	// 1=ckptoption), NOT the models.JOB_* constant. bug-1490: setting it to
+	// JOB_STOP/RESUME/KILL/CHKPNT (1/2/3/8) made the controller's job_ctrl()
+	// select the wrong union branch (or none) and reject every control with
+	// -EINVAL, so hold/release/cancel never reached the job.
+	ctrl.action = C.int(jobCtrlDiscriminant(action))
 
 	// For signal-based actions set signo; for checkpoint set ckpt options.
 	// xce_jobctrl_t.u is a C union — we use a static C helper to set the

src/frontend/apiserver/backend/convert.go

@@ -249,19 +249,6 @@ func cClusterConfToClusterInfo(conf *C.cluster_config_t) *models.ClusterInfo {
 
 // modelJobToSubmit converts a Go JobInfo to C xce_job_submit_t heap structs.
 // The caller must free all returned C memory by calling freeJobSubmit.
-// resolveExecUser returns the user a job should run as: an explicit exec_user
-// wins; otherwise the job runs as the submitting user. bug-1500: the submit
-// conversion previously dropped this entirely, so the worker received a NULL
-// execuser and ran every job as "nobody". Kept as a plain-Go function so it is
-// unit-testable without cgo (the Go toolchain rejects `import "C"` in _test.go).
-func resolveExecUser(job *models.JobInfo) string {
-	execUser := job.ExecUser
-	if execUser == "" {
-		execUser = job.SubmitUser
-	}
-	return execUser
-}
-
 func modelJobToSubmit(job *models.JobInfo) (*C.xce_job_submit_t, *C.job_submit_data_t, *C.job_startinfo_t) {
 	si := (*C.job_startinfo_t)(C.calloc(1, C.sizeof_job_startinfo_t))
 	si.cmdline = C.CString(job.Command)

src/frontend/apiserver/backend/convert_pure.go

@@ -7,6 +7,36 @@ import (
 	"fmt"
 )
 
+// resolveExecUser returns the user a job should run as: an explicit exec_user
+// wins; otherwise the job runs as the submitting user. bug-1500: the submit
+// conversion previously dropped this entirely, so the worker received a NULL
+// execuser and ran every job as "nobody". Plain Go so it is unit-testable
+// without cgo — keep it OUT of the cgo-only convert.go.
+func resolveExecUser(job *models.JobInfo) string {
+	execUser := job.ExecUser
+	if execUser == "" {
+		execUser = job.SubmitUser
+	}
+	return execUser
+}
+
+// jobCtrlDiscriminant maps a models.JOB_* action to the capnp JobControl union
+// discriminant carried in xce_jobctrl_t.action: JobControl_u_siginfo (0) for
+// signal-based actions (kill/stop/resume), JobControl_u_ckptoption (1) for
+// checkpoint. The models.JOB_* constants are a DIFFERENT enum — using them as
+// the discriminant made the controller's job_ctrl() select the wrong union
+// branch (or none) and reject every control request with -EINVAL (bug-1490).
+// Returns -1 for an unknown action (clib.go never calls ControlJob with one).
+func jobCtrlDiscriminant(action int) int {
+	switch action {
+	case models.JOB_STOP, models.JOB_RESUME, models.JOB_KILL:
+		return 0 // JobControl_u_siginfo
+	case models.JOB_CHKPNT:
+		return 1 // JobControl_u_ckptoption
+	}
+	return -1
+}
+
 // JobID bit layout constants — mirrors src/include/config.h.
 //
 //	|63     62|61        56|55       52|51         20|19      15|14          0|

src/frontend/apiserver/backend/convert_test.go

@@ -5,6 +5,33 @@ import (
 	"testing"
 )
 
+// TestJobCtrlDiscriminant is a regression test for bug-1490: backend.ControlJob
+// used to set xce_jobctrl_t.action to the models.JOB_* constant
+// (JOB_STOP=1/JOB_RESUME=2/JOB_KILL=3/JOB_CHKPNT=8), but that field is the
+// capnp JobControl union DISCRIMINANT (JobControl_u_siginfo=0,
+// JobControl_u_ckptoption=1). The controller's job_ctrl() interpreted the wrong
+// union branch (or none) and rejected every control request with -EINVAL, so
+// hold/release/cancel never reached the job.
+func TestJobCtrlDiscriminant(t *testing.T) {
+	cases := []struct {
+		name   string
+		action int
+		want   int
+	}{
+		{"stop is a signal action", models.JOB_STOP, 0},   // JobControl_u_siginfo
+		{"resume is a signal action", models.JOB_RESUME, 0},
+		{"kill is a signal action", models.JOB_KILL, 0},
+		{"checkpoint is a ckptoption action", models.JOB_CHKPNT, 1}, // JobControl_u_ckptoption
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := jobCtrlDiscriminant(tc.action); got != tc.want {
+				t.Errorf("jobCtrlDiscriminant(%d) = %d, want %d", tc.action, got, tc.want)
+			}
+		})
+	}
+}
+
 func TestRawJobToModel_BasicFields(t *testing.T) {
 	r := rawJobData{
 		jobID:   123,