fix(apiserver): validate, default and document job.terminate_interval (bug-1957)

The loader now stores the effective interval (10 s when unset or 0, LSF's
JOB_TERMINATE_INTERVAL default) and refuses a negative one; app_schema.json
describes the setting; src/docs/xce.md documents kill/stop behaviour and
the interval.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/docs/xce.md

@@ -54,6 +54,23 @@ cluster. Common operations:
 Built as a standalone Go binary; communicates with the API server over
 its REST interface.
 
+## Killing and stopping jobs
+
+Job control follows LSF's TERMINATE and SUSPEND actions:
+
+- **Kill** (`xcecli jobs kill <id>`, no signal given): SIGINT, then SIGTERM,
+  then SIGKILL, one interval apart, each only while the job is still
+  running, so the job can catch SIGINT/SIGTERM and clean up.
+- **Stop** (`xcecli jobs stop <id>`): SIGTSTP first, so the job can catch
+  it, prepare and stop itself; SIGSTOP follows after the interval if it has
+  not stopped. **Resume** sends SIGCONT and cancels a stop still in progress.
+- **Explicit signal** (kill with a signal): that signal only, once.
+- **Force kill**: SIGKILL at once.
+
+The interval is the API server's `job.terminate_interval` setting
+(`config/app.yaml`, seconds; default 10 when unset or 0, as LSF's
+`JOB_TERMINATE_INTERVAL`; negative refuses startup).
+
 ## Job arrays
 
 A job array is a set of identical jobs (elements) submitted at once. XCE

src/frontend/apiserver/config/schema/app_schema.json

@@ -112,6 +112,11 @@
           "minimum": 1,
           "maximum": 2147483646,
           "description": "Default max array size for job submission"
+        },
+        "terminate_interval": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Seconds between the signals of a plain job kill (SIGINT, SIGTERM, SIGKILL) or stop (SIGTSTP, SIGSTOP); 0 or unset means 10 (LSF JOB_TERMINATE_INTERVAL)"
         }
       }
     },

src/frontend/apiserver/utils/config.go

@@ -247,6 +247,9 @@ func InitConf() error {
 	if terminateInterval < 0 {
 		return fmt.Errorf("job.terminate_interval must not be negative: %d", terminateInterval)
 	}
+	if terminateInterval == 0 {
+		terminateInterval = defaultTerminateInterval
+	}
 	JobYamlConfig = &JobConfig{
 		MaxArraySize:      maxArraySize,
 		TerminateInterval: terminateInterval,
@@ -316,6 +319,9 @@ func ParseMaxSize(maxSize string) (int64, error) {
 }
 
 const (
+	// defaultTerminateInterval is used when job.terminate_interval is unset
+	// or 0: LSF's JOB_TERMINATE_INTERVAL default (bug-1957).
+	defaultTerminateInterval = 10
 	// defaultMaxArraySize is used when job.max_array_size is unset or 0.
 	defaultMaxArraySize = 1000
 	// supportedMaxArraySize is the largest array the API server creates

src/frontend/apiserver/utils/config_test.go

@@ -299,3 +299,34 @@ func TestInitConf_MaxArraySizeNegativeRefused(t *testing.T) {
 		t.Fatalf("InitConf with -1: expected job.max_array_size error, got %v", err)
 	}
 }
+
+// bug-1957: job.terminate_interval, the seconds between the signals of a
+// plain kill or stop (LSF JOB_TERMINATE_INTERVAL)
+func TestInitConf_TerminateInterval(t *testing.T) {
+	setupInitConf(t, "job:\n  terminate_interval: 30")
+	if err := InitConf(); err != nil {
+		t.Fatalf("InitConf: %v", err)
+	}
+	if JobYamlConfig.TerminateInterval != 30 {
+		t.Errorf("TerminateInterval = %d, want 30", JobYamlConfig.TerminateInterval)
+	}
+}
+
+func TestInitConf_TerminateIntervalUnsetDefaults(t *testing.T) {
+	for _, section := range []string{"", "job:\n  terminate_interval: 0"} {
+		setupInitConf(t, section)
+		if err := InitConf(); err != nil {
+			t.Fatalf("%q: InitConf: %v", section, err)
+		}
+		if JobYamlConfig.TerminateInterval != 10 {
+			t.Errorf("%q: TerminateInterval = %d, want the default 10", section, JobYamlConfig.TerminateInterval)
+		}
+	}
+}
+
+func TestInitConf_TerminateIntervalNegativeRefused(t *testing.T) {
+	setupInitConf(t, "job:\n  terminate_interval: -1")
+	if err := InitConf(); err == nil || !strings.Contains(err.Error(), "terminate_interval") {
+		t.Fatalf("InitConf with a negative interval: err = %v, want an error naming terminate_interval", err)
+	}
+}