myxce/src/frontend/apiserver/controllers/jobs_handler_test.go

//go:build !cgo

// Handler-level tests for the job endpoints (CreateJob, GetJobsInfo,
// GetJobByIDs, ControlJobByID and the filter-based ControlJob), driven
// through the test seams in jobs.go (generateJobID, generateArrayJobID,
// submitJob, controlJob) so no C backend is needed.
//
// This file is !cgo because it reuses the JWT/RBAC fixtures (signTestJWT,
// setupRBACTest) from integration_test.go, which is !cgo-only.
package controllers

import (
	"apiserver/backend"
	"apiserver/models"
	"apiserver/store"
	"apiserver/utils"
	"bytes"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/gin-gonic/gin"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
)

// handlerTestUser is the user every handler fixture request is signed as.
const handlerTestUser = "handler_user"

// handlerTestRBAC grants handlerTestUser every permission the job handlers
// check, over the "all" domain. The role is deliberately not cluster_admin:
// the handlers are registered without RequirePermission, and each handler
// does its own job:control / queue:control / queue:submit checks.
const handlerTestRBAC = `
permissions:
  - job:view
  - job:control
  - queue:submit
  - queue:control
  - queue:view
roles:
  handler_tester:
    permissions:
      - action: job:view
        domain:
          - type: all
      - action: job:control
        domain:
          - type: all
      - action: queue:submit
        domain:
          - type: all
      - action: queue:control
        domain:
          - type: all
      - action: queue:view
        domain:
          - type: all
users:
  handler_user:
    roles:
      - name: handler_tester
        domain:
          - type: all
`

// handlerTestOrdinaryBase is the first ID the generateJobID replacement
// hands out: tag 0, a non-zero timestamp field, index bits clear.
const handlerTestOrdinaryBase = uint64(1)<<40 | uint64(1)<<20

// handlerTestArrayBase is the first array key the generateArrayJobID
// replacement hands out (array-capable: tag 0, low 15 bits clear).
const handlerTestArrayBase = uint64(2)<<40 | uint64(1)<<20

// submittedJob is one call the submitJob replacement saw.
type submittedJob struct {
	Job       models.JobInfo
	ArraySize int
}

// dispatchedControl is one call the controlJob replacement saw.
type dispatchedControl struct {
	UniqueJobID uint64
	Action      int
	Content     string
}

// handlerFixture is the shared fixture of the job handler tests: a gin
// router with the job handlers, a signed token, seeded stores and recording
// replacements of the backend seams.
type handlerFixture struct {
	t      *testing.T
	router *gin.Engine
	token  string

	mu sync.Mutex
	// generateCalls records the arraySize argument of every generateJobID call.
	generateCalls []int
	// arrayCalls records the n argument of every generateArrayJobID call.
	arrayCalls []int
	submitted  []submittedJob
	dispatched []dispatchedControl
	nextID     uint64
	nextArray  uint64

	// Errors the replacements return (nil = success).
	generateErr error
	submitErr   error
	controlErr  error
}

// newHandlerFixture builds the fixture. Order matters: setupRBACTest calls
// resetIntegrationState -> store.InitStores, which replaces every store, so
// the original store pointers are saved first, RBAC is set up next, and
// seeding (seedJob/seedQueue) happens afterwards. Everything replaced here is
// restored in t.Cleanup.
func newHandlerFixture(t *testing.T) *handlerFixture {
	t.Helper()
	return newHandlerFixtureAs(t, handlerTestRBAC, handlerTestUser, "handler_tester")
}

// newHandlerFixtureAs is newHandlerFixture with the RBAC config and the
// signed user and role given (bug-1934: tests of narrower job scopes).
func newHandlerFixtureAs(t *testing.T, rbac, user, role string) *handlerFixture {
	t.Helper()

	// Save every global this fixture (or setupRBACTest) replaces.
	origJobs, origArrayJobs, origQueues, origHosts := store.Jobs, store.ArrayJobs, store.Queues, store.Hosts
	origResources, origResLocs, origClusters := store.Resources, store.ResourceLocations, store.Clusters
	origExecHosts, origRoles, origUserRoles := store.JobExecHosts, store.Roles, store.UserRoles
	origGenerate, origGenerateArray := generateJobID, generateArrayJobID
	origSubmit, origControl := submitJob, controlJob
	origJobCfg, origLog, origAuthCfg := utils.JobYamlConfig, utils.Log, utils.AuthenticationYamlConfig
	origRBAC := rbacConfig
	origUserGroups := store.UserGroupsSnapshot()

	t.Cleanup(func() {
		store.Jobs, store.ArrayJobs, store.Queues, store.Hosts = origJobs, origArrayJobs, origQueues, origHosts
		store.Resources, store.ResourceLocations, store.Clusters = origResources, origResLocs, origClusters
		store.JobExecHosts, store.Roles, store.UserRoles = origExecHosts, origRoles, origUserRoles
		generateJobID, generateArrayJobID = origGenerate, origGenerateArray
		submitJob, controlJob = origSubmit, origControl
		utils.JobYamlConfig, utils.Log, utils.AuthenticationYamlConfig = origJobCfg, origLog, origAuthCfg
		rbacConfig = origRBAC
		store.SetUserGroups(origUserGroups)
	})

	// Auth/RBAC (replaces the stores with empty ones), then config.
	setupRBACTest(t, rbac)
	utils.JobYamlConfig = &utils.JobConfig{MaxArraySize: 1000}

	f := &handlerFixture{
		t:         t,
		token:     signTestJWT(user, role, time.Now().Add(time.Hour)),
		nextID:    handlerTestOrdinaryBase,
		nextArray: handlerTestArrayBase,
	}

	generateJobID = func(_, _ int, arraySize int) (uint64, error) {
		f.mu.Lock()
		defer f.mu.Unlock()
		f.generateCalls = append(f.generateCalls, arraySize)
		if f.generateErr != nil {
			return 0, f.generateErr
		}
		id := f.nextID
		f.nextID++
		return id, nil
	}
	generateArrayJobID = func(_, _ int, n int) (uint64, error) {
		f.mu.Lock()
		defer f.mu.Unlock()
		f.arrayCalls = append(f.arrayCalls, n)
		if f.generateErr != nil {
			return 0, f.generateErr
		}
		key := f.nextArray
		f.nextArray += uint64(1) << 15 // next array-capable key
		return backend.MakeElementID(key, 1), nil
	}
	submitJob = func(job *models.JobInfo, arraySize int) error {
		f.mu.Lock()
		defer f.mu.Unlock()
		if f.submitErr != nil {
			return f.submitErr
		}
		f.submitted = append(f.submitted, submittedJob{Job: *job, ArraySize: arraySize})
		return nil
	}
	controlJob = func(jobid uint64, action int, content string) error {
		f.mu.Lock()
		defer f.mu.Unlock()
		if f.controlErr != nil {
			return f.controlErr
		}
		f.dispatched = append(f.dispatched, dispatchedControl{UniqueJobID: jobid, Action: action, Content: content})
		return nil
	}

	jc := JobController{}
	r := newTestRouter()
	g := r.Group("/xce/v1/jobs")
	g.GET("", jc.GetJobsInfo)
	g.GET("/:jobids", jc.GetJobByIDs)
	g.GET("/:jobids/output", jc.GetJobOutput)
	g.POST("/create", jc.CreateJob)
	g.POST("/:jobids/control", jc.ControlJobByID)
	g.POST("/:jobids/restart", jc.RestartJobByID)
	g.POST("/control", jc.ControlJob)
	f.router = r
	return f
}

// seedQueue adds an open queue to store.Queues (idempotent).
func (f *handlerFixture) seedQueue(name string) {
	f.t.Helper()
	if !store.Queues.Has(name) {
		store.Queues.Put(name, models.QueueInfo{QueueName: name})
	}
}

// seedJob stores job under its Unique_JobID and seeds its queue, so the
// handlers' queue:control check (store.GetAllQueueNames) admits it. An empty
// QueueName defaults to "normal" and an empty SubmitUser to handlerTestUser.
func (f *handlerFixture) seedJob(job models.JobInfo) models.JobInfo {
	f.t.Helper()
	if job.QueueName == "" {
		job.QueueName = "normal"
	}
	if job.SubmitUser == "" {
		job.SubmitUser = handlerTestUser
	}
	f.seedQueue(job.QueueName)
	store.Jobs.Put(uint64(job.Unique_JobID), job)
	return job
}

// seedOrdinaryJob seeds an ordinary (non-array) job with the given status
// name (a models.JOBSTAT key, e.g. "RUN").
func (f *handlerFixture) seedOrdinaryJob(id uint64, name, status string) models.JobInfo {
	f.t.Helper()
	st, ok := models.JOBSTAT[status]
	if !ok {
		f.t.Fatalf("seedOrdinaryJob: unknown status %q", status)
	}
	return f.seedJob(models.JobInfo{
		Unique_JobID: int64(id),
		JobID:        int64(id),
		ArrayIdx:     0,
		JobName:      name,
		JobStatus:    st,
	})
}

// seedArrayElements seeds elements idx... of the array identified by key
// (an array-capable key), all with the given name and status.
func (f *handlerFixture) seedArrayElements(key uint64, name, status string, idx ...int) []models.JobInfo {
	f.t.Helper()
	if !backend.IsArrayCapableKey(key) {
		f.t.Fatalf("seedArrayElements: %#x is not an array-capable key", key)
	}
	st, ok := models.JOBSTAT[status]
	if !ok {
		f.t.Fatalf("seedArrayElements: unknown status %q", status)
	}
	out := make([]models.JobInfo, 0, len(idx))
	for _, k := range idx {
		elem := backend.MakeElementID(key, k)
		out = append(out, f.seedJob(models.JobInfo{
			Unique_JobID: int64(elem),
			JobID:        int64(backend.ExtractArrayBaseID(elem)),
			ArrayIdx:     backend.ExtractArrayIdx(elem),
			JobName:      name,
			JobStatus:    st,
		}))
	}
	return out
}

// do sends a request with an optional JSON body, signed with the fixture
// token, and returns the recorded response.
func (f *handlerFixture) do(method, path string, body interface{}) *httptest.ResponseRecorder {
	f.t.Helper()
	var buf bytes.Buffer
	if body != nil {
		if err := json.NewEncoder(&buf).Encode(body); err != nil {
			f.t.Fatalf("encode request body: %v", err)
		}
	}
	req := httptest.NewRequest(method, path, &buf)
	req.RemoteAddr = "127.0.0.1:0"
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", "Bearer "+f.token)
	w := httptest.NewRecorder()
	f.router.ServeHTTP(w, req)
	return w
}

// submit posts a submit body to CreateJob. The body is a JSON object as a
// client sends it (models.JobSubmitRequest is not used directly: its zero
// CustomTime fields do not marshal into a form the strict binder accepts).
func (f *handlerFixture) submit(req map[string]interface{}) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.do(http.MethodPost, "/xce/v1/jobs/create", req)
}

// controlByID posts a control action to ControlJobByID for jobids (the
// path segment as written, e.g. "123" or "123[1-3]").
func (f *handlerFixture) controlByID(jobids, action string) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.do(http.MethodPost, "/xce/v1/jobs/"+jobids+"/control",
		models.JobControlJobIDRequest{Action: action})
}

// controlByFilter posts a control request to the filter-based ControlJob.
func (f *handlerFixture) controlByFilter(req models.JobControlRequest) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.do(http.MethodPost, "/xce/v1/jobs/control", req)
}

// dispatchedIDs returns the unique_job_id of every controlJob call, in order.
func (f *handlerFixture) dispatchedIDs() []uint64 {
	f.mu.Lock()
	defer f.mu.Unlock()
	ids := make([]uint64, 0, len(f.dispatched))
	for _, d := range f.dispatched {
		ids = append(ids, d.UniqueJobID)
	}
	return ids
}

// submissions returns a copy of every submitJob call.
func (f *handlerFixture) submissions() []submittedJob {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]submittedJob(nil), f.submitted...)
}

// jobIDString renders a job ID as the decimal path segment the REST API takes.
func jobIDString(id uint64) string { return strconv.FormatUint(id, 10) }

// handlerResponse is the envelope ReturnSuccess/ReturnError write.
type handlerResponse struct {
	Code    int             `json:"code"`
	Msg     interface{}     `json:"msg"`
	Data    json.RawMessage `json:"data"`
	Details json.RawMessage `json:"details"`
}

// decodeHandlerResponse decodes the response envelope.
func decodeHandlerResponse(t *testing.T, w *httptest.ResponseRecorder) handlerResponse {
	t.Helper()
	var r handlerResponse
	if err := json.Unmarshal(w.Body.Bytes(), &r); err != nil {
		t.Fatalf("decode response: %v; body: %s", err, w.Body.String())
	}
	return r
}

// ========== Smoke tests (task 5.4) ==========

// TestJobHandler_SmokeSubmitOrdinaryJob submits an ordinary job through the
// generateJobID/submitJob seams.
func TestJobHandler_SmokeSubmitOrdinaryJob(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	w := f.submit(map[string]interface{}{"job_name": "smoke", "command": "sleep 1", "queue_name": "normal"})
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	resp := decodeHandlerResponse(t, w)
	if resp.Code != 201 {
		t.Fatalf("code = %d, want 201; body: %s", resp.Code, w.Body.String())
	}
	var data models.JobSubmitResponse
	if err := json.Unmarshal(resp.Data, &data); err != nil {
		t.Fatalf("decode data: %v", err)
	}
	if uint64(data.JobID) != handlerTestOrdinaryBase {
		t.Errorf("jobid = %d, want %d", data.JobID, handlerTestOrdinaryBase)
	}

	if len(f.generateCalls) != 1 || f.generateCalls[0] != 1 {
		t.Errorf("generateJobID calls = %v, want [1]", f.generateCalls)
	}
	if len(f.arrayCalls) != 0 {
		t.Errorf("generateArrayJobID calls = %v, want none", f.arrayCalls)
	}
	subs := f.submissions()
	if len(subs) != 1 {
		t.Fatalf("submitJob calls = %d, want 1", len(subs))
	}
	if got := uint64(subs[0].Job.Unique_JobID); got != handlerTestOrdinaryBase {
		t.Errorf("submitted unique_job_id = %d, want %d", got, handlerTestOrdinaryBase)
	}
	if subs[0].ArraySize != 1 || subs[0].Job.JobName != "smoke" || subs[0].Job.ArrayIdx != 0 {
		t.Errorf("submitted = {size %d, name %q, idx %d}, want {1, smoke, 0}",
			subs[0].ArraySize, subs[0].Job.JobName, subs[0].Job.ArrayIdx)
	}
}

// TestJobHandler_SmokeSubmitErrorIs500 checks a submitJob failure surfaces
// as a 500 and nothing is recorded.
func TestJobHandler_SmokeSubmitErrorIs500(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal") // the default queue must exist (bug-1960)
	f.submitErr = errors.New("backend down")

	w := f.submit(map[string]interface{}{"job_name": "smoke", "command": "sleep 1"})
	if w.Code != http.StatusInternalServerError {
		t.Fatalf("status = %d, want 500; body: %s", w.Code, w.Body.String())
	}
	if !strings.Contains(w.Body.String(), "backend down") {
		t.Errorf("body does not carry the submit error: %s", w.Body.String())
	}
	if len(f.submissions()) != 0 {
		t.Errorf("submissions recorded despite error: %v", f.submissions())
	}
}

// TestJobHandler_SubmitInteractiveAttachExdata checks the submit handler
// packs interactive/attach into the job's ExtendData: attach=1 only for the
// terminal-attach TTY kind (the worker reaps ttyd when no client ever
// connects), never for --browse or other interactive modes.
func TestJobHandler_SubmitInteractiveAttachExdata(t *testing.T) {
	cases := []struct {
		name       string
		req        map[string]interface{}
		wantAttach bool
	}{
		{"tty attach", map[string]interface{}{
			"job_name": "a", "command": "sleep 1", "interactive": 1, "attach": 1,
		}, true},
		{"tty web view (no attach)", map[string]interface{}{
			"job_name": "a", "command": "sleep 1", "interactive": 1,
		}, false},
		{"xjob with attach ignored worker-side but passed", map[string]interface{}{
			"job_name": "a", "command": "sleep 1", "interactive": 2, "attach": 1,
		}, true},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			f := newHandlerFixture(t)
			f.seedQueue("normal")

			w := f.submit(tc.req)
			if w.Code != http.StatusOK {
				t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
			}
			subs := f.submissions()
			if len(subs) != 1 {
				t.Fatalf("submitJob calls = %d, want 1", len(subs))
			}
			var ex map[string]interface{}
			if raw := subs[0].Job.ExtendData.String; raw != "" {
				if err := json.Unmarshal([]byte(raw), &ex); err != nil {
					t.Fatalf("ExtendData %q is not JSON: %v", raw, err)
				}
			}
			if got, ok := ex["interactive"]; !ok || int(got.(float64)) != tc.req["interactive"].(int) {
				t.Errorf("ExtendData interactive = %v (%v), want %d", got, ok, tc.req["interactive"])
			}
			_, hasAttach := ex["attach"]
			if hasAttach != tc.wantAttach {
				t.Errorf("ExtendData attach present = %v, want %v (exdata: %s)", hasAttach, tc.wantAttach, subs[0].Job.ExtendData.String)
			}
		})
	}
}

// TestJobHandler_SubmitX11Exdata checks the submit handler packs x11 into
// the job's ExtendData for -X jobs: the key selects the X11-forwarding
// backend worker-side (DISPLAY to the forwarding port instead of VNC).
func TestJobHandler_SubmitX11Exdata(t *testing.T) {
	cases := []struct {
		name    string
		req     map[string]interface{}
		wantX11 bool
	}{
		{"x11 job", map[string]interface{}{
			"job_name": "a", "command": "xclock", "interactive": 2, "x11": 1,
		}, true},
		{"plain xjob (VNC backend)", map[string]interface{}{
			"job_name": "a", "command": "xclock", "interactive": 2,
		}, false},
		{"tty job without x11", map[string]interface{}{
			"job_name": "a", "command": "sleep 1", "interactive": 1,
		}, false},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			f := newHandlerFixture(t)
			f.seedQueue("normal")

			w := f.submit(tc.req)
			if w.Code != http.StatusOK {
				t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
			}
			subs := f.submissions()
			if len(subs) != 1 {
				t.Fatalf("submitJob calls = %d, want 1", len(subs))
			}
			var ex map[string]interface{}
			if raw := subs[0].Job.ExtendData.String; raw != "" {
				if err := json.Unmarshal([]byte(raw), &ex); err != nil {
					t.Fatalf("ExtendData %q is not JSON: %v", raw, err)
				}
			}
			_, hasX11 := ex["x11"]
			if hasX11 != tc.wantX11 {
				t.Errorf("ExtendData x11 present = %v, want %v (exdata: %s)", hasX11, tc.wantX11, subs[0].Job.ExtendData.String)
			}
		})
	}
}

// TestJobHandler_SmokeControlByID controls one seeded ordinary job by ID
// and checks the dispatch is recorded once with its unique_job_id.
func TestJobHandler_SmokeControlByID(t *testing.T) {
	f := newHandlerFixture(t)
	const id = uint64(1)<<40 | 7
	f.seedOrdinaryJob(id, "victim", "RUN")
	f.seedOrdinaryJob(id+1, "bystander", "RUN")

	w := f.controlByID(jobIDString(id), "kill")
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	if resp := decodeHandlerResponse(t, w); resp.Code != 202 {
		t.Fatalf("code = %d, want 202; body: %s", resp.Code, w.Body.String())
	}
	got := f.dispatchedIDs()
	if len(got) != 1 || got[0] != id {
		t.Fatalf("dispatched = %v, want [%d]", got, id)
	}
	if f.dispatched[0].Action != models.JOB_KILL {
		t.Errorf("dispatched action = %d, want JOB_KILL (%d)", f.dispatched[0].Action, models.JOB_KILL)
	}
}

// TestJobHandler_SmokeControlByFilter controls one seeded job through the
// filter-based ControlJob and checks the dispatch is recorded.
func TestJobHandler_SmokeControlByFilter(t *testing.T) {
	f := newHandlerFixture(t)
	const id = uint64(1)<<40 | 9
	f.seedOrdinaryJob(id, "filtered", "RUN")
	f.seedOrdinaryJob(id+1, "other", "RUN")

	w := f.controlByFilter(models.JobControlRequest{
		Filter: models.JobControlFilter{JobName: "filtered"},
		Action: "kill",
	})
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	got := f.dispatchedIDs()
	if len(got) != 1 || got[0] != id {
		t.Fatalf("dispatched = %v, want [%d]", got, id)
	}
}

// TestJobHandler_SmokeControlErrorIs500 checks a controlJob failure is a 500.
func TestJobHandler_SmokeControlErrorIs500(t *testing.T) {
	f := newHandlerFixture(t)
	const id = uint64(1)<<40 | 11
	f.seedOrdinaryJob(id, "victim", "RUN")
	f.controlErr = errors.New("backend down")

	w := f.controlByID(jobIDString(id), "kill")
	if w.Code != http.StatusInternalServerError {
		t.Fatalf("status = %d, want 500; body: %s", w.Code, w.Body.String())
	}
	if !strings.Contains(w.Body.String(), "backend down") {
		t.Errorf("body does not carry the control error: %s", w.Body.String())
	}
}

// ========== Array submission (task 6.2) ==========

// submitData decodes the JobSubmitResponse of a successful submit.
func submitData(t *testing.T, w *httptest.ResponseRecorder) models.JobSubmitResponse {
	t.Helper()
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	resp := decodeHandlerResponse(t, w)
	if resp.Code != 201 {
		t.Fatalf("code = %d, want 201; body: %s", resp.Code, w.Body.String())
	}
	var data models.JobSubmitResponse
	if err := json.Unmarshal(resp.Data, &data); err != nil {
		t.Fatalf("decode data: %v", err)
	}
	return data
}

// assertNothingCreated checks no ID was allocated and nothing submitted (2.10).
func (f *handlerFixture) assertNothingCreated() {
	f.t.Helper()
	if len(f.generateCalls) != 0 || len(f.arrayCalls) != 0 {
		f.t.Errorf("ID allocation called: generateJobID %v, generateArrayJobID %v; want none",
			f.generateCalls, f.arrayCalls)
	}
	if subs := f.submissions(); len(subs) != 0 {
		f.t.Errorf("submitJob called %d times, want 0", len(subs))
	}
}

// TestJobHandler_SubmitArrayRange: arr[1-3] creates one 3-element array
// through generateArrayJobID, stores the name "arr" and reports the array
// identity and size (1.1, 1.3, 3.1, 3.2).
func TestJobHandler_SubmitArrayRange(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	data := submitData(t, f.submit(map[string]interface{}{
		"job_name": "arr[1-3]", "command": "sleep 1", "queue_name": "normal"}))

	first := backend.MakeElementID(handlerTestArrayBase, 1)
	if len(f.arrayCalls) != 1 || f.arrayCalls[0] != 3 {
		t.Fatalf("generateArrayJobID calls = %v, want [3]", f.arrayCalls)
	}
	if len(f.generateCalls) != 0 {
		t.Errorf("generateJobID calls = %v, want none", f.generateCalls)
	}
	subs := f.submissions()
	if len(subs) != 1 {
		t.Fatalf("submitJob calls = %d, want 1", len(subs))
	}
	s := subs[0]
	if s.ArraySize != 3 {
		t.Errorf("submitted array size = %d, want 3", s.ArraySize)
	}
	if s.Job.JobName != "arr" {
		t.Errorf("stored job name = %q, want %q", s.Job.JobName, "arr")
	}
	if uint64(s.Job.Unique_JobID) != first || uint64(s.Job.JobID) != handlerTestArrayBase || s.Job.ArrayIdx != 1 {
		t.Errorf("submitted ids = {unique %#x, job %#x, idx %d}, want {%#x, %#x, 1}",
			s.Job.Unique_JobID, s.Job.JobID, s.Job.ArrayIdx, first, handlerTestArrayBase)
	}
	// Each element gets a distinct identity: first + i (1.3).
	for i := 0; i < s.ArraySize; i++ {
		if backend.ExtractArrayIdx(first+uint64(i)) != i+1 ||
			backend.ExtractArrayBaseID(first+uint64(i)) != handlerTestArrayBase {
			t.Errorf("element %d identity %#x does not decode to [%d] of the array", i, first+uint64(i), i+1)
		}
	}

	if uint64(data.JobID) != handlerTestArrayBase {
		t.Errorf("response jobid = %d, want array id %d", data.JobID, handlerTestArrayBase)
	}
	if data.ArraySize != 3 {
		t.Errorf("response array_size = %d, want 3", data.ArraySize)
	}
	wantMsg := "Job <" + jobIDString(handlerTestArrayBase) + "> is submitted to queue <normal> (3 elements)."
	if data.Message != wantMsg {
		t.Errorf("response message = %q, want %q", data.Message, wantMsg)
	}
}

// TestJobHandler_SubmitOneElementArray: name[1] is a one-element array
// (1.2, 7.8) through generateArrayJobID with n = 1.
func TestJobHandler_SubmitOneElementArray(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	data := submitData(t, f.submit(map[string]interface{}{
		"job_name": "name[1]", "command": "sleep 1", "queue_name": "normal"}))

	if len(f.arrayCalls) != 1 || f.arrayCalls[0] != 1 {
		t.Fatalf("generateArrayJobID calls = %v, want [1]", f.arrayCalls)
	}
	if len(f.generateCalls) != 0 {
		t.Errorf("generateJobID calls = %v, want none", f.generateCalls)
	}
	subs := f.submissions()
	if len(subs) != 1 || subs[0].ArraySize != 1 || subs[0].Job.JobName != "name" || subs[0].Job.ArrayIdx != 1 {
		t.Fatalf("submissions = %+v, want one {size 1, name \"name\", idx 1}", subs)
	}
	if data.ArraySize != 1 {
		t.Errorf("response array_size = %d, want 1", data.ArraySize)
	}
	if uint64(data.JobID) != handlerTestArrayBase {
		t.Errorf("response jobid = %d, want array id %d", data.JobID, handlerTestArrayBase)
	}
}

// TestJobHandler_SubmitArrayExprRejected: index-expression errors are 400
// with the parser's message unchanged, and nothing is created (2.x, 2.10).
func TestJobHandler_SubmitArrayExprRejected(t *testing.T) {
	cases := []struct {
		name        string
		jobName     string
		interactive int
		wantMsg     string
	}{
		{"2.5 start 0", "arr[0-3]", 0, "job name 'arr[0-3]': element numbers start at 1"},
		{"2.1 interactive", "arr[1-3]", 1, "job name 'arr[1-3]': interactive jobs cannot be arrays"},
		{"2.3 list", "arr[1,2]", 0, ""},
		{"2.7 start above 1", "arr[5]", 0, ""},
		{"2.8 above limit", "arr[1-1001]", 0, ""},
		{"2.2 malformed", "arr]", 0, ""},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			f := newHandlerFixture(t)
			f.seedQueue("normal")
			want := tc.wantMsg
			if want == "" {
				_, _, _, ae := ParseSubmitIndexExpr(tc.jobName, tc.interactive != 0, 1000)
				if ae == nil {
					t.Fatalf("parser accepted %q", tc.jobName)
				}
				want = ae.Msg
			}
			w := f.submit(map[string]interface{}{"job_name": tc.jobName, "command": "sleep 1",
				"queue_name": "normal", "interactive": tc.interactive})
			if w.Code != http.StatusBadRequest {
				t.Fatalf("status = %d, want 400; body: %s", w.Code, w.Body.String())
			}
			resp := decodeHandlerResponse(t, w)
			if resp.Code != 400 || resp.Msg != want {
				t.Errorf("response = {%d, %v}, want {400, %q}", resp.Code, resp.Msg, want)
			}
			f.assertNothingCreated()
		})
	}
}

// TestJobHandler_SubmitPlainNamesUnchanged: names without '[' or ']' keep
// today's errors and 500 status and never create an array (1.4).
func TestJobHandler_SubmitPlainNamesUnchanged(t *testing.T) {
	cases := []struct{ jobName, wantMsg string }{
		{"my job", "job array list format is invalid: arrayName 'my job' contains invalid characters. Allowed characters are [a-zA-Z0-9._-]"},
		{"foo%5", "job array list format is invalid: arrayName 'foo%5' contains invalid characters. Allowed characters are [a-zA-Z0-9._-]"},
		{"foo%", "job array list format is invalid: invalid slot limit value '': must be a positive integer"},
	}
	for _, tc := range cases {
		t.Run(tc.jobName, func(t *testing.T) {
			f := newHandlerFixture(t)
			f.seedQueue("normal")
			w := f.submit(map[string]interface{}{"job_name": tc.jobName, "command": "sleep 1", "queue_name": "normal"})
			if w.Code != http.StatusInternalServerError {
				t.Fatalf("status = %d, want 500; body: %s", w.Code, w.Body.String())
			}
			if resp := decodeHandlerResponse(t, w); resp.Code != 500 || resp.Msg != tc.wantMsg {
				t.Errorf("response = {%d, %v}, want {500, %q}", resp.Code, resp.Msg, tc.wantMsg)
			}
			f.assertNothingCreated()
		})
	}
}

// TestJobHandler_SubmitOrdinaryUnchanged: an ordinary job is accepted (no
// typed-nil error leaks out of the pre-check), reported as before and its
// response carries no array_size (3.3).
func TestJobHandler_SubmitOrdinaryUnchanged(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	w := f.submit(map[string]interface{}{"job_name": "plain", "command": "sleep 1", "queue_name": "normal"})
	data := submitData(t, w)
	if strings.Contains(w.Body.String(), "array_size") {
		t.Errorf("ordinary submit response carries array_size: %s", w.Body.String())
	}
	wantMsg := "Job <" + jobIDString(handlerTestOrdinaryBase) + "> is submitted to default queue <normal>."
	if data.Message != wantMsg || uint64(data.JobID) != handlerTestOrdinaryBase {
		t.Errorf("response = {%d, %q}, want {%d, %q}", data.JobID, data.Message, handlerTestOrdinaryBase, wantMsg)
	}
	if len(f.generateCalls) != 1 || f.generateCalls[0] != 1 || len(f.arrayCalls) != 0 {
		t.Errorf("allocation = generateJobID %v, generateArrayJobID %v; want [1], none", f.generateCalls, f.arrayCalls)
	}

	// Unnamed job: still ordinary.
	w = f.submit(map[string]interface{}{"command": "sleep 2", "queue_name": "normal"})
	submitData(t, w)
	if len(f.arrayCalls) != 0 {
		t.Errorf("unnamed job allocated an array: %v", f.arrayCalls)
	}
}

// TestJobHandler_SubmitArrayFileNameSubstitution: at submission a new
// array's path names get the first element's values, %I = 1 and
// %J = <array-id>[1] (9.4). Checked on cwd: out_file/err_file are refused
// since bug-1936 (the worker never writes them), cwd takes the same
// substitution.
func TestJobHandler_SubmitArrayFileNameSubstitution(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	submitData(t, f.submit(map[string]interface{}{
		"job_name": "arr[1-3]", "command": "sleep 1", "queue_name": "normal",
		"cwd": "/w/out.%J.%I"}))

	subs := f.submissions()
	if len(subs) != 1 {
		t.Fatalf("submitJob calls = %d, want 1", len(subs))
	}
	// %J is the first element's identity, written <array-id>[1].
	elem := backend.FormatJobID(int64(handlerTestArrayBase)) + "[1]"
	if got := backend.FormatJobID(int64(backend.MakeElementID(handlerTestArrayBase, 1))); got != elem {
		t.Fatalf("first element formats as %q, want %q", got, elem)
	}
	if want := "/w/out." + elem + ".1"; subs[0].Job.CWD != want {
		t.Errorf("cwd = %q, want %q", subs[0].Job.CWD, want)
	}
}

// TestJobHandler_SubmitArrayLogsArrayIdentity: CreateJob logs a new array
// by its array identity and size, not by the raw element ID (5.9).
func TestJobHandler_SubmitArrayLogsArrayIdentity(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")
	core, logs := observer.New(zapcore.InfoLevel)
	utils.Log = zap.New(core)

	submitData(t, f.submit(map[string]interface{}{
		"job_name": "arr[1-3]", "command": "sleep 1", "queue_name": "normal"}))

	wantID := backend.FormatJobID(int64(handlerTestArrayBase))
	for _, msg := range []string{"Generated job ID", "Job submission successfully from "} {
		entries := logs.FilterMessage(msg).All()
		if len(entries) != 1 {
			t.Fatalf("log %q: %d entries, want 1", msg, len(entries))
		}
		fields := entries[0].ContextMap()
		if fields["array_jobid"] != wantID {
			t.Errorf("log %q: array_jobid = %v, want %q", msg, fields["array_jobid"], wantID)
		}
		if fields["array_size"] != int64(3) {
			t.Errorf("log %q: array_size = %v (%T), want 3", msg, fields["array_size"], fields["array_size"])
		}
		if _, ok := fields["jobID"]; ok {
			t.Errorf("log %q: carries the raw jobID for an array: %v", msg, fields)
		}
	}
}

// ========== Query by selection (task 7.3, bug-1898) ==========

// querySeed is the store content of the query tests: arrays X ("foo",
// elements 1-4), Y ("foobar", elements 1-2), Z (unnamed, element 1) and W
// ("baz", elements 1-2), and the ordinary job O ("bar").
type querySeed struct {
	X, Y, Z, W, O uint64
}

func (s querySeed) elem(key uint64, k int) uint64 { return backend.MakeElementID(key, k) }

func seedQueryJobs(f *handlerFixture) querySeed {
	f.t.Helper()
	s := querySeed{
		X: handlerTestArrayBase + 10<<15,
		Y: handlerTestArrayBase + 11<<15,
		Z: handlerTestArrayBase + 12<<15,
		W: handlerTestArrayBase + 13<<15,
		// array-capable (low 15 bits clear) but ordinary: element-only
		// matching must not select it for O[0] (5.11)
		O: handlerTestOrdinaryBase + 20<<15,
	}
	f.seedArrayElements(s.X, "foo", "RUN", 1, 2, 3, 4)
	f.seedArrayElements(s.Y, "foobar", "RUN", 1, 2)
	f.seedArrayElements(s.Z, "", "RUN", 1)
	f.seedArrayElements(s.W, "baz", "RUN", 1, 2)
	f.seedOrdinaryJob(s.O, "bar", "RUN")
	return s
}

// allQueryIDs is every unique_job_id seedQueryJobs stores.
func (s querySeed) all() []uint64 {
	return []uint64{
		s.elem(s.X, 1), s.elem(s.X, 2), s.elem(s.X, 3), s.elem(s.X, 4),
		s.elem(s.Y, 1), s.elem(s.Y, 2), s.elem(s.Z, 1),
		s.elem(s.W, 1), s.elem(s.W, 2), s.O,
	}
}

// listJobs calls GetJobsInfo with filter (URL-escaped here).
func (f *handlerFixture) listJobs(filter string) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.do(http.MethodGet, "/xce/v1/jobs?filter="+url.QueryEscape(filter), nil)
}

// getJobs calls GetJobByIDs with the path segment jobids (URL-escaped here).
func (f *handlerFixture) getJobs(jobids string) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.do(http.MethodGet, "/xce/v1/jobs/"+url.PathEscape(jobids), nil)
}

// rowIDs returns the sorted unique_job_id of every row of a 200 list reply.
func rowIDs(t *testing.T, w *httptest.ResponseRecorder) []uint64 {
	t.Helper()
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	dec := json.NewDecoder(bytes.NewReader(w.Body.Bytes()))
	dec.UseNumber()
	var raw struct {
		Data struct {
			Count int `json:"count"`
			Rows  []struct {
				Data map[string]interface{} `json:"Data"`
			} `json:"rows"`
		} `json:"data"`
	}
	if err := dec.Decode(&raw); err != nil {
		t.Fatalf("decode list reply: %v; body: %s", err, w.Body.String())
	}
	ids := make([]uint64, 0, len(raw.Data.Rows))
	for _, r := range raw.Data.Rows {
		n, ok := r.Data["unique_job_id"].(json.Number)
		if !ok {
			t.Fatalf("row without unique_job_id: %v", r.Data)
		}
		id, err := strconv.ParseUint(n.String(), 10, 64)
		if err != nil {
			t.Fatalf("unique_job_id %q: %v", n, err)
		}
		ids = append(ids, id)
	}
	if raw.Data.Count != len(ids) {
		t.Errorf("count = %d, rows = %d", raw.Data.Count, len(ids))
	}
	sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
	return ids
}

func sortedIDs(ids ...uint64) []uint64 {
	out := append([]uint64{}, ids...)
	sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
	return out
}

func assertIDs(t *testing.T, name string, got, want []uint64) {
	t.Helper()
	want = sortedIDs(want...)
	if len(got) != len(want) {
		t.Errorf("%s: rows %v, want %v", name, got, want)
		return
	}
	for i := range got {
		if got[i] != want[i] {
			t.Errorf("%s: rows %v, want %v", name, got, want)
			return
		}
	}
}

// assert400 checks a 400 reply whose message contains want.
func assert400(t *testing.T, name string, w *httptest.ResponseRecorder, want string) {
	t.Helper()
	if w.Code != http.StatusBadRequest {
		t.Errorf("%s: status = %d, want 400; body: %s", name, w.Code, w.Body.String())
		return
	}
	r := decodeHandlerResponse(t, w)
	msg, _ := r.Msg.(string)
	if !strings.Contains(msg, want) {
		t.Errorf("%s: msg = %q, want it to contain %q", name, msg, want)
	}
	if len(r.Data) != 0 && string(r.Data) != "null" {
		t.Errorf("%s: a rejected query carries data: %s", name, r.Data)
	}
}

// arrayStatsRows decodes the rows of a type=array_stats list reply.
func arrayStatsRows(t *testing.T, w *httptest.ResponseRecorder) []map[string]interface{} {
	t.Helper()
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	dec := json.NewDecoder(bytes.NewReader(w.Body.Bytes()))
	dec.UseNumber()
	var raw struct {
		Data struct {
			Count int `json:"count"`
			Rows  []struct {
				Data map[string]interface{} `json:"Data"`
			} `json:"rows"`
		} `json:"data"`
	}
	if err := dec.Decode(&raw); err != nil {
		t.Fatalf("decode array_stats reply: %v; body: %s", err, w.Body.String())
	}
	rows := make([]map[string]interface{}, 0, len(raw.Data.Rows))
	for _, r := range raw.Data.Rows {
		rows = append(rows, r.Data)
	}
	if raw.Data.Count != len(rows) {
		t.Errorf("count = %d, rows = %d", raw.Data.Count, len(rows))
	}
	return rows
}

// TestJobHandler_ArrayStatsDerivedFromJobStore covers `xce jobs list -A`
// (type=array_stats): rows are derived from the live job store, one per
// array, with counters matching the elements' states and a job_id_human
// annotation (store.ArrayJobs has no production writer).
func TestJobHandler_ArrayStatsDerivedFromJobStore(t *testing.T) {
	f := newHandlerFixture(t)
	s := seedQueryJobs(f)
	x, y := s.X, s.Y
	X := backend.FormatJobID(int64(x))

	// Move element 2 of X to DONE so its counters must shift accordingly
	// (seedArrayElements stored all four as RUN).
	elem2 := backend.MakeElementID(x, 2)
	got, _ := store.Jobs.Get(elem2)
	got.JobStatus = models.JOBSTAT["DONE"]
	store.Jobs.Put(elem2, got)

	// GetJobsInfo?type=array_stats lists every seeded array with counters.
	w := f.do(http.MethodGet, "/xce/v1/jobs?type=array_stats", nil)
	rows := arrayStatsRows(t, w)
	if len(rows) != 4 {
		t.Fatalf("array_stats rows = %d, want 4 (arrays X, Y, Z, W)", len(rows))
	}

	byJobID := map[uint64]map[string]interface{}{}
	for _, row := range rows {
		n, ok := row["job_id"].(json.Number)
		if !ok {
			t.Fatalf("stats row without job_id: %v", row)
		}
		id, _ := strconv.ParseUint(n.String(), 10, 64)
		byJobID[id] = row
	}

	xRow := byJobID[x]
	if xRow == nil {
		t.Fatalf("array X missing from stats: %v", rows)
	}
	// X has elements 1,2,3,4: 3 RUN + 1 DONE after the overwrite.
	for field, want := range map[string]float64{
		"num_jobs": 4, "num_run": 3, "num_done": 1,
		"num_pend": 0, "num_exit": 0,
	} {
		n, _ := xRow[field].(json.Number)
		if got, _ := n.Float64(); got != want {
			t.Errorf("X.%s = %v, want %v", field, n.String(), want)
		}
	}
	if human, _ := xRow["job_id_human"].(string); human != X {
		t.Errorf("X.job_id_human = %q, want %q", human, X)
	}

	yRow := byJobID[y]
	if yRow == nil {
		t.Fatalf("array Y missing from stats: %v", rows)
	}
	if n, _ := yRow["num_jobs"].(json.Number); n.String() != "2" {
		t.Errorf("Y.num_jobs = %v, want 2", yRow["num_jobs"])
	}

	// job_id filter (the ARRAY_JOB_SUPPORT_FILTERS path) narrows to one row.
	w = f.do(http.MethodGet, "/xce/v1/jobs?type=array_stats&filter="+
		url.QueryEscape("{job_id:eq:"+X+"}"), nil)
	rows = arrayStatsRows(t, w)
	if len(rows) != 1 {
		t.Fatalf("filtered array_stats rows = %d, want 1", len(rows))
	}

	// GetJobByIDs?type=array_stats answers the same derived row.
	w = f.do(http.MethodGet, "/xce/v1/jobs/"+url.PathEscape(X)+"?type=array_stats", nil)
	rows = arrayStatsRows(t, w)
	if len(rows) != 1 {
		t.Fatalf("GetJobByIDs array_stats rows = %d, want 1; body: %s", len(rows), w.Body.String())
	}
}

// TestJobHandler_QuerySelections covers the GetJobsInfo rows of the design's
// Testing Strategy (Query).
func TestJobHandler_QuerySelections(t *testing.T) {
	f := newHandlerFixture(t)
	s := seedQueryJobs(f)
	x, y, z, w := s.X, s.Y, s.Z, s.W
	X := jobIDString(x)
	Y := jobIDString(y)
	O := jobIDString(s.O)
	writtenX := backend.FormatJobID(int64(x))

	cases := []struct {
		name   string
		filter string
		want   []uint64
	}{
		{"eq single element (bug-1898)", "{job_id:eq:" + X + "[2]}", []uint64{s.elem(x, 2)}},
		{"eq range", "{job_id:eq:" + X + "[2-3]}", []uint64{s.elem(x, 2), s.elem(x, 3)}},
		{"ordinary-id[0] lists nothing (5.11)", "{job_id:eq:" + O + "[0]}", nil},
		{"array-id[0] without element 0 lists nothing (5.11)", "{job_id:eq:" + X + "[0]}", nil},
		{"unnamed: eq [1] lists nothing", "{job_name:eq:[1]}", nil},
		{"unnamed: contains [1] lists element 1 of every array",
			"{job_name:contains:[1]}", []uint64{s.elem(x, 1), s.elem(y, 1), s.elem(z, 1), s.elem(w, 1)}},
		{"notcontains foo[1]", "{job_name:notcontains:foo[1]}", []uint64{s.elem(z, 1), s.elem(w, 1)}},
		{"in + ne entries on the same field are AND'ed",
			"{job_id:in:(" + X + "[1]," + O + ")},{job_id:ne:" + O + "}", []uint64{s.elem(x, 1)}},
		{"job_id and job_name selections intersect",
			"{job_id:in:(" + X + "[1]," + X + "[2])},{job_name:eq:foo[2]}", []uint64{s.elem(x, 2)}},
		{"in with a bare and a bracketed value matches both",
			"{job_id:in:(" + O + "," + X + "[1])}", []uint64{s.O, s.elem(x, 1)}},
		{"notin excludes the element", "{job_id:notin:(" + X + "[1])}",
			[]uint64{s.elem(x, 2), s.elem(x, 3), s.elem(x, 4), s.elem(y, 1), s.elem(y, 2),
				s.elem(z, 1), s.elem(w, 1), s.elem(w, 2), s.O}},
		{"startwith foo[1]", "{job_name:startwith:foo[1]}", []uint64{s.elem(x, 1), s.elem(y, 1)}},
		{"written bare key alone (5.6)", "{job_id:eq:" + writtenX + "}",
			[]uint64{s.elem(x, 1), s.elem(x, 2), s.elem(x, 3), s.elem(x, 4)}},
		{"written bare key mixed with a bracketed value (5.6)",
			"{job_id:in:(" + writtenX + "," + Y + "[1])}",
			[]uint64{s.elem(x, 1), s.elem(x, 2), s.elem(x, 3), s.elem(x, 4), s.elem(y, 1)}},
		{"between on job_id evaluates like in", "{job_id:between:(" + X + "[1]," + Y + "[2])}",
			[]uint64{s.elem(x, 1), s.elem(y, 2)}},
	}
	for _, c := range cases {
		assertIDs(t, c.name, rowIDs(t, f.listJobs(c.filter)), c.want)
	}

	// sanity: an unfiltered (default submit_user) listing sees every seeded row
	assertIDs(t, "all rows", rowIDs(t, f.listJobs("{submit_user:eq:"+handlerTestUser+"}")), s.all())
}

// TestJobHandler_QuerySelectionRejections: rejected selections reject the
// whole query with 400 and list nothing.
func TestJobHandler_QuerySelectionRejections(t *testing.T) {
	f := newHandlerFixture(t)
	s := seedQueryJobs(f)
	X := jobIDString(s.X)
	O := jobIDString(s.O)
	// a host exists (so GetHostList accepts it) but runs no job, so the
	// exec_hosts lookup alone would answer count 0
	store.Hosts.Put("h1", models.HostInfo{HostName: "h1"})

	cases := []struct {
		name, filter, want string
	}{
		{"5.7 with an exec_hosts filter matching no job",
			"{exec_hosts:eq:h1},{job_id:eq:" + X + "[40000]}", "invalid"},
		{"null operator on a bracketed name", "{job_name:notnull:foo[1]}", "operator doesn't support"},
		{"ne on a bracketed identity", "{job_id:ne:" + X + "[1]}", "operator doesn't support"},
		{"valid selection followed by an invalid one",
			"{job_id:eq:" + X + "[1]},{job_name:eq:foo[5-3]}", "reversed"},
		{"5.8: element of a non-array identity", "{job_id:eq:" + jobIDString(s.O+1) + "[1]}", "not an array identity"},
		{"5.16 syntax", "{job_id:eq:" + X + "[a]}", "invalid job identity or element selection"},
		{"5.17 marked decimal without index",
			"{job_id:eq:" + jobIDString(s.elem(s.X, 1)) + "}", "must be named as"},
		{"5.14 too-wide written key", "{job_id:in:(" + O + ",64-0-1-0[1])}", "wider than allowed"},
	}
	for _, c := range cases {
		assert400(t, c.name, f.listJobs(c.filter), c.want)
	}

	// the exec_hosts row's message names the selection (5.7) and is not
	// replaced by "bad job array index"
	w := f.listJobs("{exec_hosts:eq:h1},{job_id:eq:" + X + "[40000]}")
	if strings.Contains(w.Body.String(), "bad job array index") {
		t.Errorf("message replaced by the old text: %s", w.Body.String())
	}
	if !strings.Contains(w.Body.String(), X+"[40000]") {
		t.Errorf("message does not name the selection: %s", w.Body.String())
	}
}

// TestJobHandler_GetJobByIDsSelections covers GET /jobs/{jobids}.
func TestJobHandler_GetJobByIDsSelections(t *testing.T) {
	f := newHandlerFixture(t)
	s := seedQueryJobs(f)
	X := jobIDString(s.X)
	O := jobIDString(s.O)

	assertIDs(t, "X[1-3]", rowIDs(t, f.getJobs(X+"[1-3]")),
		[]uint64{s.elem(s.X, 1), s.elem(s.X, 2), s.elem(s.X, 3)})
	assertIDs(t, "X[2],Y[1]", rowIDs(t, f.getJobs(X+"[2],"+jobIDString(s.Y)+"[1]")),
		[]uint64{s.elem(s.X, 2), s.elem(s.Y, 1)})
	assertIDs(t, "bare X lists every element (5.6)", rowIDs(t, f.getJobs(X)),
		[]uint64{s.elem(s.X, 1), s.elem(s.X, 2), s.elem(s.X, 3), s.elem(s.X, 4)})
	assertIDs(t, "written bare O", rowIDs(t, f.getJobs(backend.FormatJobID(int64(s.O)))), []uint64{s.O})
	assertIDs(t, "O[0] lists nothing (5.11)", rowIDs(t, f.getJobs(O+"[0]")), nil)

	assert400(t, "valid then invalid", f.getJobs(X+"[1],"+X+"[40000]"), X+"[40000]")
	assert400(t, "5.17", f.getJobs(jobIDString(s.elem(s.X, 1))), "must be named as")
	assert400(t, "5.14", f.getJobs("64-0-1-0"), "wider than allowed")
	assert400(t, "5.5", f.getJobs(X+"[3-1]"), "reversed")
}

// ========== Control by identity (task 7.4, bug-1899) ==========

// controlItem is one accepted or failed item of a control reply.
type controlItem struct {
	UniqueJobID uint64
	HasUnique   bool
	JobID       uint64
	ArrayIdx    int
	Msg         string
}

// controlReply is the data of a 202 control reply.
type controlReply struct {
	Status   string
	Accepted []controlItem
	Failed   []controlItem
}

// decodeControlReply decodes a successful (202) control reply.
func decodeControlReply(t *testing.T, w *httptest.ResponseRecorder) controlReply {
	t.Helper()
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	dec := json.NewDecoder(bytes.NewReader(w.Body.Bytes()))
	dec.UseNumber()
	var raw struct {
		Code int `json:"code"`
		Data struct {
			Status   string                   `json:"status"`
			Accepted []map[string]interface{} `json:"accepted"`
			Failed   []map[string]interface{} `json:"failed"`
		} `json:"data"`
	}
	if err := dec.Decode(&raw); err != nil {
		t.Fatalf("decode control reply: %v; body: %s", err, w.Body.String())
	}
	if raw.Code != 202 {
		t.Fatalf("code = %d, want 202; body: %s", raw.Code, w.Body.String())
	}
	num := func(m map[string]interface{}, k string) (uint64, bool) {
		n, ok := m[k].(json.Number)
		if !ok {
			return 0, false
		}
		v, err := strconv.ParseUint(n.String(), 10, 64)
		if err != nil {
			// job_id/unique_job_id are int64 on the wire; keep the bit pattern
			s, err2 := strconv.ParseInt(n.String(), 10, 64)
			if err2 != nil {
				t.Fatalf("%s %q: %v", k, n, err)
			}
			v = uint64(s)
		}
		return v, true
	}
	conv := func(in []map[string]interface{}) []controlItem {
		out := make([]controlItem, 0, len(in))
		for _, m := range in {
			var it controlItem
			it.UniqueJobID, it.HasUnique = num(m, "unique_job_id")
			it.JobID, _ = num(m, "job_id")
			idx, _ := num(m, "array_idx")
			it.ArrayIdx = int(idx)
			it.Msg, _ = m["msg"].(string)
			out = append(out, it)
		}
		return out
	}
	return controlReply{Status: raw.Data.Status, Accepted: conv(raw.Data.Accepted), Failed: conv(raw.Data.Failed)}
}

// itemIDs returns the unique_job_id of every item, in reply order.
func itemIDs(t *testing.T, items []controlItem) []uint64 {
	t.Helper()
	ids := make([]uint64, 0, len(items))
	for _, it := range items {
		if !it.HasUnique {
			t.Errorf("result item without unique_job_id: %+v", it)
		}
		ids = append(ids, it.UniqueJobID)
	}
	return ids
}

// assertExactIDs checks got equals want element by element (order kept).
func assertExactIDs(t *testing.T, name string, got, want []uint64) {
	t.Helper()
	if len(got) != len(want) {
		t.Errorf("%s: %v, want %v", name, got, want)
		return
	}
	for i := range got {
		if got[i] != want[i] {
			t.Errorf("%s: %v, want %v", name, got, want)
			return
		}
	}
}

// assertSeeded fails the test unless every id is in store.Jobs, so an
// act-on-no-job assertion cannot pass on an empty store.
func assertSeeded(t *testing.T, ids ...uint64) {
	t.Helper()
	for _, id := range ids {
		if !store.Jobs.Has(id) {
			t.Fatalf("fixture row %d missing from store.Jobs", id)
		}
	}
}

// controlByIDEscaped posts a control action for a selection list, escaping
// the path segment.
func (f *handlerFixture) controlByIDEscaped(jobids, action string) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.controlByID(url.PathEscape(jobids), action)
}

// TestJobHandler_ControlTwoSelectionsDispatchOnce (bug-1899): a request with
// two selections dispatches each selected element exactly once and lists
// each once; the same holds for two ordinary jobs.
func TestJobHandler_ControlTwoSelectionsDispatchOnce(t *testing.T) {
	f := newHandlerFixture(t)
	X := handlerTestArrayBase + 30<<15
	Y := handlerTestArrayBase + 31<<15
	f.seedArrayElements(X, "foo", "RUN", 1, 2)
	f.seedArrayElements(Y, "bar", "RUN", 1, 2)
	A := handlerTestOrdinaryBase + 1
	B := handlerTestOrdinaryBase + 2
	f.seedOrdinaryJob(A, "a", "RUN")
	f.seedOrdinaryJob(B, "b", "RUN")

	x1, y2 := backend.MakeElementID(X, 1), backend.MakeElementID(Y, 2)
	r := decodeControlReply(t, f.controlByIDEscaped(jobIDString(X)+"[1],"+jobIDString(Y)+"[2]", "kill"))
	assertExactIDs(t, "X[1],Y[2] dispatched", f.dispatchedIDs(), []uint64{x1, y2})
	assertExactIDs(t, "X[1],Y[2] accepted", itemIDs(t, r.Accepted), []uint64{x1, y2})
	if len(r.Failed) != 0 {
		t.Errorf("X[1],Y[2] failed items: %+v", r.Failed)
	}

	f.dispatched = nil
	r = decodeControlReply(t, f.controlByIDEscaped(jobIDString(A)+","+jobIDString(B), "kill"))
	assertExactIDs(t, "A,B dispatched", f.dispatchedIDs(), []uint64{A, B})
	assertExactIDs(t, "A,B accepted", itemIDs(t, r.Accepted), []uint64{A, B})

	// a job selected by two selections is acted on once (de-duplicated by
	// unique_job_id)
	f.dispatched = nil
	x2 := backend.MakeElementID(X, 2)
	r = decodeControlReply(t, f.controlByIDEscaped(jobIDString(X)+","+jobIDString(X)+"[2]", "kill"))
	assertExactIDs(t, "X,X[2] dispatched", f.dispatchedIDs(), []uint64{x1, x2})
	assertExactIDs(t, "X,X[2] accepted", itemIDs(t, r.Accepted), []uint64{x1, x2})
}

// TestJobHandler_ControlRangeReportsMissing (5.3): X[1-3] with element 2
// missing dispatches elements 1 and 3 and reports 2 as not found, carrying
// its would-be element ID.
func TestJobHandler_ControlRangeReportsMissing(t *testing.T) {
	f := newHandlerFixture(t)
	X := handlerTestArrayBase + 32<<15
	f.seedArrayElements(X, "foo", "RUN", 1, 3, 4)
	x1, x2, x3 := backend.MakeElementID(X, 1), backend.MakeElementID(X, 2), backend.MakeElementID(X, 3)

	r := decodeControlReply(t, f.controlByIDEscaped(jobIDString(X)+"[1-3]", "kill"))
	assertExactIDs(t, "dispatched", f.dispatchedIDs(), []uint64{x1, x3})
	assertExactIDs(t, "accepted", itemIDs(t, r.Accepted), []uint64{x1, x3})
	assertExactIDs(t, "failed", itemIDs(t, r.Failed), []uint64{x2})
	if len(r.Failed) == 1 {
		it := r.Failed[0]
		if it.Msg != "job not found" || it.JobID != X || it.ArrayIdx != 2 {
			t.Errorf("not-found item = %+v, want job_id %d, array_idx 2, msg \"job not found\"", it, X)
		}
	}
	if r.Status != "partially_accepted" {
		t.Errorf("status = %q, want partially_accepted", r.Status)
	}

	// a range on an identity no array has: every number not found, no job
	f.dispatched = nil
	Z := handlerTestArrayBase + 33<<15
	r = decodeControlReply(t, f.controlByIDEscaped(jobIDString(Z)+"[1-2]", "kill"))
	if got := f.dispatchedIDs(); len(got) != 0 {
		t.Errorf("Z[1-2] dispatched %v, want none", got)
	}
	assertExactIDs(t, "Z[1-2] failed", itemIDs(t, r.Failed),
		[]uint64{backend.MakeElementID(Z, 1), backend.MakeElementID(Z, 2)})
}

// TestJobHandler_ControlElementZeroMissing (5.4): X[0] with no element 0
// acts on no job -- not on the array, not on an ordinary job with identity
// X -- and reports element 0 as not found.
func TestJobHandler_ControlElementZeroMissing(t *testing.T) {
	f := newHandlerFixture(t)
	X := handlerTestArrayBase + 34<<15
	elems := f.seedArrayElements(X, "foo", "RUN", 1, 2)
	// an ordinary job sharing the number X (array_idx 0)
	f.seedOrdinaryJob(X, "ordinary", "RUN")
	assertSeeded(t, uint64(elems[0].Unique_JobID), uint64(elems[1].Unique_JobID), X)

	for _, sel := range []string{"[0]", "[0-0]"} {
		f.dispatched = nil
		r := decodeControlReply(t, f.controlByIDEscaped(jobIDString(X)+sel, "kill"))
		if got := f.dispatchedIDs(); len(got) != 0 {
			t.Errorf("X%s dispatched %v, want none", sel, got)
		}
		if len(r.Accepted) != 0 {
			t.Errorf("X%s accepted %+v, want none", sel, r.Accepted)
		}
		assertExactIDs(t, "X"+sel+" failed", itemIDs(t, r.Failed), []uint64{backend.MakeElementID(X, 0)})
		if len(r.Failed) == 1 && r.Failed[0].Msg != "job not found" {
			t.Errorf("X%s failed msg = %q, want \"job not found\"", sel, r.Failed[0].Msg)
		}
	}
}

// TestJobHandler_ControlPreUpgradeElementZero (8.2, 8.3): a pre-upgrade
// element 0 is dispatched alone, also when the same request selects another
// element of that array, and is reported with its own unique_job_id.
func TestJobHandler_ControlPreUpgradeElementZero(t *testing.T) {
	f := newHandlerFixture(t)
	X := handlerTestArrayBase + 35<<15
	f.seedArrayElements(X, "old", "RUN", 0, 1, 2)
	x0, x2 := backend.MakeElementID(X, 0), backend.MakeElementID(X, 2)

	r := decodeControlReply(t, f.controlByIDEscaped(jobIDString(X)+"[0]", "kill"))
	assertExactIDs(t, "X[0] dispatched", f.dispatchedIDs(), []uint64{x0})
	assertExactIDs(t, "X[0] accepted", itemIDs(t, r.Accepted), []uint64{x0})

	f.dispatched = nil
	r = decodeControlReply(t, f.controlByIDEscaped(jobIDString(X)+"[0],"+jobIDString(X)+"[2]", "kill"))
	assertExactIDs(t, "X[0],X[2] dispatched", f.dispatchedIDs(), []uint64{x0, x2})
	assertExactIDs(t, "X[0],X[2] accepted", itemIDs(t, r.Accepted), []uint64{x0, x2})
}

// TestJobHandler_ControlValidThenInvalid (Terms): a valid selection followed
// by an invalid one rejects the whole command with 400 and dispatches
// nothing, without "not found" items.
func TestJobHandler_ControlValidThenInvalid(t *testing.T) {
	f := newHandlerFixture(t)
	X := handlerTestArrayBase + 36<<15
	elems := f.seedArrayElements(X, "foo", "RUN", 1, 2)
	assertSeeded(t, uint64(elems[0].Unique_JobID))

	cases := []struct{ sel, want string }{
		{jobIDString(X) + "[1]," + jobIDString(X) + "[40000]", "element numbers must be at most"},
		{jobIDString(X) + "[1]," + jobIDString(X) + "[3-1]", "reversed"},
		{jobIDString(X) + "[1]," + jobIDString(X) + "[a]", "invalid job identity or element selection"},
		{jobIDString(X) + "[1]," + jobIDString(uint64(elems[1].Unique_JobID)), "must be named as"},
	}
	for _, c := range cases {
		f.dispatched = nil
		assert400(t, c.sel, f.controlByIDEscaped(c.sel, "kill"), c.want)
		if got := f.dispatchedIDs(); len(got) != 0 {
			t.Errorf("%s: dispatched %v, want none", c.sel, got)
		}
	}
}

// ========== Control by filter with element selections (task 7.5) ==========

// controlFilterKill posts a kill through the filter-based ControlJob with
// the job-name filter name (the -J value as the client sends it).
func (f *handlerFixture) controlFilterKill(name string) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.controlByFilter(models.JobControlRequest{
		Filter: models.JobControlFilter{JobName: name},
		Action: "kill",
	})
}

// TestJobHandler_ControlFilterRejectsBadSelection (5.7, 5.16, 5.5): a
// bracketed -J value the name parser rejects returns 400 with its specific
// message before any job is fetched, and dispatches nothing.
func TestJobHandler_ControlFilterRejectsBadSelection(t *testing.T) {
	f := newHandlerFixture(t)
	X := handlerTestArrayBase + 40<<15
	elems := f.seedArrayElements(X, "foo", "RUN", 1, 2)
	assertSeeded(t, uint64(elems[0].Unique_JobID))

	cases := []struct{ name, want string }{
		{"foo[40000]", "element numbers must be at most"},
		{"foo[3-1]", "reversed"},
		{"foo[a]", "invalid"},
		{"foo [1]", "invalid"},
		{"foo[1", "invalid"},
	}
	for _, c := range cases {
		f.dispatched = nil
		assert400(t, "-J "+c.name, f.controlFilterKill(c.name), c.want)
		if got := f.dispatchedIDs(); len(got) != 0 {
			t.Errorf("-J %s: dispatched %v, want none", c.name, got)
		}
	}
}

// TestJobHandler_ControlFilterEmptyNameSelectsNothing (design ApiMatch):
// -J '[1]' (control sends eq) matches no row, not the unnamed array.
func TestJobHandler_ControlFilterEmptyNameSelectsNothing(t *testing.T) {
	f := newHandlerFixture(t)
	Z := handlerTestArrayBase + 41<<15
	elems := f.seedArrayElements(Z, "", "RUN", 1)
	assertSeeded(t, uint64(elems[0].Unique_JobID))

	w := f.controlFilterKill("[1]")
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	if got := f.dispatchedIDs(); len(got) != 0 {
		t.Errorf("-J [1] dispatched %v, want none", got)
	}
}

// TestJobHandler_ControlFilterMostRecentOneElementArray (5.10): with no
// element number in the filter, when the newest matching job is the element
// of a one-element array, only that element is acted on.
func TestJobHandler_ControlFilterMostRecentOneElementArray(t *testing.T) {
	f := newHandlerFixture(t)
	O := handlerTestOrdinaryBase + 50
	f.seedOrdinaryJob(O, "foo", "RUN")
	A := handlerTestArrayBase + 42<<15
	a1 := backend.MakeElementID(A, 1)
	f.seedArrayElements(A, "foo", "RUN", 1)
	assertSeeded(t, O, a1)

	r := decodeControlReply(t, f.controlFilterKill("foo"))
	assertExactIDs(t, "-J foo dispatched", f.dispatchedIDs(), []uint64{a1})
	assertExactIDs(t, "-J foo accepted", itemIDs(t, r.Accepted), []uint64{a1})

	// all-jobs: the rule does not apply, every matching job is acted on
	f.dispatched = nil
	w := f.controlByFilter(models.JobControlRequest{
		Filter: models.JobControlFilter{JobName: "foo", AllJobs: true},
		Action: "kill",
	})
	decodeControlReply(t, w)
	assertExactIDs(t, "-J foo -a dispatched", f.dispatchedIDs(), []uint64{O, a1})
}

// TestJobHandler_ControlFilterElementSelection (5.2): -J 'foo[1]' acts on
// exactly element 1 of every array named foo -- both one-element arrays and
// element 1 of a larger one -- never on an ordinary job named foo, another
// element, or another name, whatever the newest row is.
func TestJobHandler_ControlFilterElementSelection(t *testing.T) {
	f := newHandlerFixture(t)
	O := handlerTestOrdinaryBase + 51
	f.seedOrdinaryJob(O, "foo", "RUN")
	A := handlerTestArrayBase + 43<<15
	B := handlerTestArrayBase + 44<<15
	C := handlerTestArrayBase + 45<<15
	D := handlerTestArrayBase + 46<<15
	f.seedArrayElements(A, "foo", "RUN", 1)
	f.seedArrayElements(B, "foo", "RUN", 1)
	f.seedArrayElements(C, "foo", "RUN", 1, 2)
	f.seedArrayElements(D, "bar", "RUN", 1)
	a1, b1, c1 := backend.MakeElementID(A, 1), backend.MakeElementID(B, 1), backend.MakeElementID(C, 1)
	assertSeeded(t, O, a1, b1, c1, backend.MakeElementID(C, 2), backend.MakeElementID(D, 1))

	r := decodeControlReply(t, f.controlFilterKill("foo[1]"))
	assertExactIDs(t, "-J foo[1] dispatched", f.dispatchedIDs(), []uint64{a1, b1, c1})
	assertExactIDs(t, "-J foo[1] accepted", itemIDs(t, r.Accepted), []uint64{a1, b1, c1})

	// the range form 1-1 selects the same element 1 of every array named foo
	f.dispatched = nil
	r = decodeControlReply(t, f.controlFilterKill("foo[1-1]"))
	assertExactIDs(t, "-J foo[1-1] dispatched", f.dispatchedIDs(), []uint64{a1, b1, c1})
}

// TestJobHandler_ControlFilterElementSelectionNewestSingleElement (5.2,
// 5.10): when the most recent job named foo is the element of a one-element
// array, -J 'foo[1]' still acts on element 1 of every array named foo (the
// most-recent rule applies only to filters without an element number),
// while -J 'foo' acts on that most recent element alone.
func TestJobHandler_ControlFilterElementSelectionNewestSingleElement(t *testing.T) {
	f := newHandlerFixture(t)
	O := handlerTestOrdinaryBase + 53
	f.seedOrdinaryJob(O, "foo", "RUN")
	A := handlerTestArrayBase + 47<<15
	B := handlerTestArrayBase + 48<<15 // highest ID: the newest row
	f.seedArrayElements(A, "foo", "RUN", 1)
	f.seedArrayElements(B, "foo", "RUN", 1)
	a1, b1 := backend.MakeElementID(A, 1), backend.MakeElementID(B, 1)
	assertSeeded(t, O, a1, b1)

	r := decodeControlReply(t, f.controlFilterKill("foo[1]"))
	assertExactIDs(t, "-J foo[1] dispatched", f.dispatchedIDs(), []uint64{a1, b1})
	assertExactIDs(t, "-J foo[1] accepted", itemIDs(t, r.Accepted), []uint64{a1, b1})

	f.dispatched = nil
	r = decodeControlReply(t, f.controlFilterKill("foo"))
	assertExactIDs(t, "-J foo dispatched", f.dispatchedIDs(), []uint64{b1})
	assertExactIDs(t, "-J foo accepted", itemIDs(t, r.Accepted), []uint64{b1})
}

// TestJobHandler_ControlFilterPlainNameWithSpaces: an unnamed job carries
// its command as its name; kill -J 'sleep 100' keeps working (no brackets,
// no selection parsing).
func TestJobHandler_ControlFilterPlainNameWithSpaces(t *testing.T) {
	f := newHandlerFixture(t)
	O := handlerTestOrdinaryBase + 52
	f.seedOrdinaryJob(O, "sleep 100", "RUN")
	f.seedOrdinaryJob(O+1, "sleep 200", "RUN")

	r := decodeControlReply(t, f.controlFilterKill("sleep 100"))
	assertExactIDs(t, "-J 'sleep 100' dispatched", f.dispatchedIDs(), []uint64{O})
	assertExactIDs(t, "-J 'sleep 100' accepted", itemIDs(t, r.Accepted), []uint64{O})
}

// bug-1903: following GetJobsInfo's page_token yields the next page, every
// row exactly once, and the last page carries no token.
func TestJobHandler_QueryPagination(t *testing.T) {
	f := newHandlerFixture(t)
	s := seedQueryJobs(f)
	filter := url.QueryEscape("{submit_user:eq:" + handlerTestUser + "}")

	var got []uint64
	token := ""
	for page := 0; ; page++ {
		if page > 10 {
			t.Fatalf("paging did not terminate; rows so far %v", got)
		}
		path := "/xce/v1/jobs?limit=3&filter=" + filter
		if token != "" {
			path += "&page_token=" + url.QueryEscape(token)
		}
		w := f.do(http.MethodGet, path, nil)
		got = append(got, rowIDs(t, w)...)

		var reply struct {
			Data struct {
				PageToken string `json:"page_token"`
			} `json:"data"`
		}
		if err := json.Unmarshal(w.Body.Bytes(), &reply); err != nil {
			t.Fatalf("decode page_token: %v", err)
		}
		if reply.Data.PageToken == "" {
			break
		}
		token = reply.Data.PageToken
	}

	seen := map[uint64]int{}
	for _, id := range got {
		seen[id]++
	}
	for id, n := range seen {
		if n != 1 {
			t.Errorf("row %d returned %d times", id, n)
		}
	}
	assertIDs(t, "all pages", sortedIDs(got...), s.all())
}

// ========== Restart by identity (bug-1898) ==========

// seedRestartJob seeds a DONE job with a distinctive command.
func (f *handlerFixture) seedRestartJob(unique uint64, command string) {
	f.t.Helper()
	f.seedJob(models.JobInfo{
		Unique_JobID: int64(unique),
		JobID:        int64(backend.ExtractArrayBaseID(unique)),
		ArrayIdx:     backend.ExtractArrayIdx(unique),
		JobName:      "r",
		Command:      command,
		JobStatus:    models.JOBSTAT["DONE"],
	})
}

func (f *handlerFixture) restart(jobids string) *httptest.ResponseRecorder {
	f.t.Helper()
	return f.do(http.MethodPost, "/xce/v1/jobs/"+url.PathEscape(jobids)+"/restart", map[string]interface{}{})
}

// bug-1898: RestartJobByID copied jqueryReply[0] of an unevaluated job_id
// CompCondition -- any job, not the one named. It must restart exactly the
// named job (ordinary job or <array-id>[k] element), report an unknown one as
// not found, and refuse a bare array identity (several jobs).
func TestJobHandler_RestartNamedJob(t *testing.T) {
	f := newHandlerFixture(t)
	arr := handlerTestArrayBase + 30<<15
	ord := handlerTestOrdinaryBase + 31<<15
	// decoys first, so "the first row" is never the target by accident
	f.seedRestartJob(handlerTestOrdinaryBase+32<<15, "decoy-a")
	f.seedRestartJob(backend.MakeElementID(arr, 1), "elem-1")
	f.seedRestartJob(backend.MakeElementID(arr, 2), "elem-2")
	f.seedRestartJob(ord, "ordinary")

	cases := []struct {
		name, jobids, wantCommand string
	}{
		{"ordinary job by decimal id", strconv.FormatUint(ord, 10), "ordinary"},
		{"array element by <array-id>[k]", strconv.FormatUint(arr, 10) + "[2]", "elem-2"},
		{"array element by written <array-id>[k]", backend.FormatJobID(int64(arr)) + "[2]", "elem-2"},
		{"ordinary job by written id", backend.FormatJobID(int64(ord)), "ordinary"},
	}
	for _, c := range cases {
		before := len(f.submitted)
		w := f.restart(c.jobids)
		if w.Code != http.StatusOK && w.Code != http.StatusCreated {
			t.Errorf("%s: status %d; body %s", c.name, w.Code, w.Body.String())
			continue
		}
		if len(f.submitted) != before+1 {
			t.Errorf("%s: %d submits, want 1", c.name, len(f.submitted)-before)
			continue
		}
		if got := f.submitted[before].Job.Command; got != c.wantCommand {
			t.Errorf("%s: restarted a copy of %q, want %q", c.name, got, c.wantCommand)
		}
	}

	// unknown job and missing element: not found, nothing submitted
	for _, jobids := range []string{
		strconv.FormatUint(handlerTestOrdinaryBase+40<<15, 10),
		strconv.FormatUint(arr, 10) + "[9]",
	} {
		before := len(f.submitted)
		w := f.restart(jobids)
		if w.Code == http.StatusOK || w.Code == http.StatusCreated || len(f.submitted) != before {
			t.Errorf("%s: status %d, %d submits; want not found and none", jobids, w.Code, len(f.submitted)-before)
		}
	}

	// a bare array identity names several jobs: refused, nothing submitted
	before := len(f.submitted)
	w := f.restart(strconv.FormatUint(arr, 10))
	if w.Code != http.StatusBadRequest || len(f.submitted) != before {
		t.Errorf("bare array identity: status %d, %d submits; want 400 and none; body %s",
			w.Code, len(f.submitted)-before, w.Body.String())
	}
}

// bug-1968: restart, like submit (bug-1936), refuses an option the restarted
// job would never get: asked hosts (-m) have no field in the controller's
// submit data. Nothing is submitted.
func TestJobHandler_RestartRejectsAskedHosts(t *testing.T) {
	f := newHandlerFixture(t)
	ord := handlerTestOrdinaryBase + 50<<15
	f.seedRestartJob(ord, "sleep 1")

	before := len(f.submitted)
	w := f.do(http.MethodPost, "/xce/v1/jobs/"+strconv.FormatUint(ord, 10)+"/restart",
		map[string]interface{}{"asked_hosts": "hostA"})
	if w.Code != http.StatusBadRequest {
		t.Errorf("status = %d, want 400; body: %s", w.Code, w.Body.String())
	}
	if len(f.submitted) != before {
		t.Errorf("%d submits, want none", len(f.submitted)-before)
	}
}

// bug-1968: the restarted job is rebuilt from the old job's row. Options no
// job can get (asked hosts, begin/term time, I/O paths, out_dir; refused at
// submit since bug-1936) must not be carried over, while the options the job
// does get -- extended data (env, limits), cwd and exec home -- must be,
// whether or not the old job was checkpointed.
func TestJobHandler_RestartCarriesOnlyHonouredOptions(t *testing.T) {
	f := newHandlerFixture(t)
	ord := handlerTestOrdinaryBase + 51<<15
	exdata := `{"sub_env_vars":{"A":"1"},"limits":{"run_limit":60}}`
	f.seedJob(models.JobInfo{
		Unique_JobID: int64(ord),
		JobID:        int64(backend.ExtractArrayBaseID(ord)),
		JobName:      "r",
		Command:      "sleep 1",
		JobStatus:    models.JOBSTAT["DONE"],
		CWD:          "/data/run",
		ExecHome:     "/home/u",
		ResReq:       "rusage[mem=100]",
		ExtendData:   models.Jsonb{String: exdata},
		AskedHosts:   "hostA",
		BeginTime:    models.CustomTime{Time: time.Unix(1700000000, 0)},
		TermTime:     models.CustomTime{Time: time.Unix(1800000000, 0)},
		InFile:       "/in",
		OutFile:      "/out",
		ErrFile:      "/err",
		OutDir:       "/outdir",
	})

	before := len(f.submitted)
	w := f.restart(strconv.FormatUint(ord, 10))
	if w.Code != http.StatusOK && w.Code != http.StatusCreated {
		t.Fatalf("status = %d; body: %s", w.Code, w.Body.String())
	}
	if len(f.submitted) != before+1 {
		t.Fatalf("%d submits, want 1", len(f.submitted)-before)
	}
	job := f.submitted[before].Job

	dropped := map[string]bool{
		"asked_hosts": job.AskedHosts != "",
		"begin_time":  !job.BeginTime.Time.IsZero(),
		"term_time":   !job.TermTime.Time.IsZero(),
		"in_file":     job.InFile != "",
		"out_file":    job.OutFile != "",
		"err_file":    job.ErrFile != "",
		"out_dir":     job.OutDir != "",
	}
	for field, kept := range dropped {
		if kept {
			t.Errorf("restarted job kept %s, which no job gets", field)
		}
	}
	if job.ExtendData.String != exdata {
		t.Errorf("ExtendData = %q, want %q", job.ExtendData.String, exdata)
	}
	if job.CWD != "/data/run" {
		t.Errorf("CWD = %q, want /data/run", job.CWD)
	}
	if job.ExecHome != "/home/u" {
		t.Errorf("ExecHome = %q, want /home/u", job.ExecHome)
	}
	if job.ResReq != "rusage[mem=100]" {
		t.Errorf("ResReq = %q, want rusage[mem=100]", job.ResReq)
	}
}

// bug-1968: a checkpointed job restarts from its checkpoint: the restart
// command replaces the command and the checkpointed pid reaches the job as
// XCE_JOB_CHECKPOINT_PID next to the old job's environment. The stored
// chkpnt_info/extend_data are models.Jsonb, which the restart never read.
func TestJobHandler_RestartFromCheckpoint(t *testing.T) {
	f := newHandlerFixture(t)
	ord := handlerTestOrdinaryBase + 52<<15
	f.seedJob(models.JobInfo{
		Unique_JobID: int64(ord),
		JobID:        int64(backend.ExtractArrayBaseID(ord)),
		JobName:      "r",
		Command:      "sleep 1",
		JobStatus:    models.JOBSTAT["EXIT"],
		ChkpntInfo:   models.Jsonb{String: `{"restart_command":"criu restore","ckpt_pid":4321}`},
		ExtendData:   models.Jsonb{String: `{"sub_env_vars":{"A":"1"}}`},
	})

	before := len(f.submitted)
	w := f.restart(strconv.FormatUint(ord, 10))
	if w.Code != http.StatusOK && w.Code != http.StatusCreated {
		t.Fatalf("status = %d; body: %s", w.Code, w.Body.String())
	}
	if len(f.submitted) != before+1 {
		t.Fatalf("%d submits, want 1", len(f.submitted)-before)
	}
	job := f.submitted[before].Job
	if job.Command != "criu restore" {
		t.Errorf("Command = %q, want the checkpoint's restart command", job.Command)
	}
	var ex struct {
		Env map[string]string `json:"sub_env_vars"`
	}
	if err := json.Unmarshal([]byte(job.ExtendData.String), &ex); err != nil {
		t.Fatalf("ExtendData %q: %v", job.ExtendData.String, err)
	}
	if ex.Env["A"] != "1" || ex.Env["XCE_JOB_CHECKPOINT_PID"] != "4321" {
		t.Errorf("sub_env_vars = %v, want A=1 and XCE_JOB_CHECKPOINT_PID=4321", ex.Env)
	}
}

// bug-1936: the job handed to the backend carries the cwd and the extended
// data under the key names and units the worker's parse_exdata()/
// parse_limits() read (run_limit in seconds, memory limits in MB,
// sub_env_vars as a string map); modelJobToSubmit copies them into the
// start info.
func TestJobHandler_SubmitCarriesCwdEnvLimitsInExdata(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	w := f.submit(map[string]interface{}{
		"command": "sleep 120", "queue_name": "normal", "cwd": "/tmp/work",
		"env": []string{"A=1"}, "run_limit": "1", "mem_limit": "1 GB",
	})
	if w.Code != http.StatusOK {
		t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
	}
	subs := f.submissions()
	if len(subs) != 1 {
		t.Fatalf("submitJob calls = %d, want 1", len(subs))
	}
	job := subs[0].Job
	if job.CWD != "/tmp/work" {
		t.Errorf("CWD = %q, want /tmp/work", job.CWD)
	}
	var ex struct {
		Env    map[string]string  `json:"sub_env_vars"`
		Limits map[string]float64 `json:"limits"`
	}
	if err := json.Unmarshal([]byte(job.ExtendData.String), &ex); err != nil {
		t.Fatalf("ExtendData %q is not a JSON object: %v", job.ExtendData.String, err)
	}
	if ex.Env["A"] != "1" {
		t.Errorf("sub_env_vars = %v, want A=1", ex.Env)
	}
	if ex.Limits["run_limit"] != 60 || ex.Limits["rss_limit"] != 1024 {
		t.Errorf("limits = %v, want run_limit 60 (s) and rss_limit 1024 (MB)", ex.Limits)
	}
}

// bug-1936: extend_data the worker cannot parse (not a JSON object) would
// fail the job at launch; the submission is rejected instead.
func TestJobHandler_SubmitRejectsNonObjectExtendData(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	for _, ex := range []string{"abc", `[1,2]`} {
		before := len(f.submissions())
		w := f.submit(map[string]interface{}{
			"command": "sleep 1", "queue_name": "normal", "extend_data": ex,
		})
		if w.Code == http.StatusOK || len(f.submissions()) != before {
			t.Errorf("extend_data %q: status %d, %d submits; want an error and none; body %s",
				ex, w.Code, len(f.submissions())-before, w.Body.String())
		}
	}
}

// bug-1960: a submission the controller rejects for its queue (closed after
// the apiserver's own check, or unknown) is a client error naming the queue,
// not a 500 with a raw error number.
func TestJobHandler_SubmitQueueRejectedByController(t *testing.T) {
	for _, tc := range []struct {
		err  error
		want string
	}{
		{backend.ErrQueueClosed, "queue is closed"},
		{backend.ErrQueueInvalid, "invalid queue"},
	} {
		f := newHandlerFixture(t)
		f.seedQueue("normal")
		f.submitErr = tc.err
		w := f.submit(map[string]interface{}{"command": "sleep 1", "queue_name": "normal"})
		if w.Code != http.StatusBadRequest {
			t.Errorf("%v: status = %d, want 400; body: %s", tc.err, w.Code, w.Body.String())
			continue
		}
		msg, _ := decodeHandlerResponse(t, w).Msg.(string)
		if !strings.Contains(msg, tc.want) || !strings.Contains(msg, "normal") {
			t.Errorf("%v: msg = %q, want it to name the queue and say %q", tc.err, msg, tc.want)
		}
	}
}

// bug-1936: a submit option the job never gets must be refused with a 400
// naming it, not accepted and dropped. Neither the controller's submit data
// nor the worker carries stdin/stdout/stderr paths, an output directory,
// asked hosts or a begin/termination time, and the worker never stages
// transferred files in or out.
func TestJobHandler_SubmitRejectsUnsupportedOptions(t *testing.T) {
	for _, tc := range []struct {
		field string
		value interface{}
	}{
		{"in_file", "/tmp/in.txt"},
		{"out_file", "/tmp/out.%J"},
		{"err_file", "/tmp/err.%J"},
		{"out_dir", "/tmp/outdir"},
		{"asked_hosts", "host1"},
		{"begin_time", "2030-01-02 03:04"},
		{"term_time", "2030-01-02 03:04"},
		{"transfer_files", []string{"/tmp/a.txt > a.txt"}},
	} {
		f := newHandlerFixture(t)
		f.seedQueue("normal")
		w := f.submit(map[string]interface{}{
			"command": "sleep 1", "queue_name": "normal", tc.field: tc.value,
		})
		assert400(t, tc.field, w, tc.field)
		assert400(t, tc.field, w, "not supported")
		f.assertNothingCreated()
	}
}

// bug-1936: the job runs in cwd and with HOME set to exec_home on the
// worker, so both must be absolute paths that fit the worker's
// job_assignment_t buffers (PATH_LENGTH, 1024 bytes with the NUL).
func TestJobHandler_SubmitRejectsBadCwdAndExecHome(t *testing.T) {
	long := "/" + strings.Repeat("d", 1023)
	for _, tc := range []struct {
		field, value, want string
	}{
		{"cwd", "relative/dir", "absolute"},
		{"cwd", long, "too long"},
		{"exec_home", "home/u", "absolute"},
		{"exec_home", long, "too long"},
	} {
		f := newHandlerFixture(t)
		f.seedQueue("normal")
		w := f.submit(map[string]interface{}{
			"command": "sleep 1", "queue_name": "normal", tc.field: tc.value,
		})
		assert400(t, tc.field+"="+tc.value[:6], w, tc.want)
		f.assertNothingCreated()
	}
}

// bug-1936: exec_home is a submit field and reaches the submitted job (and
// from there submitStartInfo's exechome), next to cwd.
func TestJobHandler_SubmitCarriesCwdAndExecHome(t *testing.T) {
	f := newHandlerFixture(t)
	f.seedQueue("normal")

	submitData(t, f.submit(map[string]interface{}{
		"command": "sleep 1", "queue_name": "normal",
		"cwd": "/data/run/%J", "exec_home": "/home/alice",
	}))
	subs := f.submissions()
	if len(subs) != 1 {
		t.Fatalf("submitJob calls = %d, want 1", len(subs))
	}
	wantCwd := "/data/run/" + backend.FormatJobID(int64(handlerTestOrdinaryBase))
	if subs[0].Job.CWD != wantCwd || subs[0].Job.ExecHome != "/home/alice" {
		t.Errorf("cwd, exec_home = %q, %q; want %q, %q",
			subs[0].Job.CWD, subs[0].Job.ExecHome, wantCwd, "/home/alice")
	}
}

// bug-1936: num_files (and shell_limit.num_files) is the job's open-file
// limit; it lands in the exdata limits as nofile_limit, the key the
// worker's parse_limits() reads.
func TestJobHandler_SubmitNofileLimitInExdata(t *testing.T) {
	for _, body := range []map[string]interface{}{
		{"command": "sleep 1", "queue_name": "normal", "num_files": 2048},
		{"command": "sleep 1", "queue_name": "normal", "shell_limit": map[string]interface{}{"num_files": 2048}},
	} {
		f := newHandlerFixture(t)
		f.seedQueue("normal")
		submitData(t, f.submit(body))
		subs := f.submissions()
		if len(subs) != 1 {
			t.Fatalf("submitJob calls = %d, want 1", len(subs))
		}
		var ex struct {
			Limits map[string]float64 `json:"limits"`
		}
		if err := json.Unmarshal([]byte(subs[0].Job.ExtendData.String), &ex); err != nil {
			t.Fatalf("exdata %q: %v", subs[0].Job.ExtendData.String, err)
		}
		if ex.Limits["nofile_limit"] != 2048 {
			t.Errorf("exdata %q: nofile_limit = %v, want 2048", subs[0].Job.ExtendData.String, ex.Limits["nofile_limit"])
		}
	}
}