fix(worker): graceful job kill and catchable stop, as LSF (bug-1957)

A plain kill sent one SIGTERM and never escalated, and a plain stop sent
one uncatchable SIGSTOP. Following LSF's TERMINATE and SUSPEND actions, a
plain kill now sends SIGINT, then SIGTERM, then SIGKILL, and a plain stop
sends SIGTSTP (the job can catch it, prepare and stop itself), then SIGSTOP
if it has not stopped; each next signal follows after the grace period and
only while the job is alive (or not yet stopped). A resume cancels a
pending stop; the job's exit ends any sequence. An explicit signal is
delivered once; a force kill is SIGKILL at once.

- JobSignal gains grace (capnp @2, xce_signal_t.grace): > 0 asks for the
  sequence, in seconds between signals.
- apiserver: job.terminate_interval (default 10 s, LSF's
  JOB_TERMINATE_INTERVAL); jobCtrlParams sends kill = SIGKILL and stop =
  SIGSTOP with that grace when no signal is given; the handler no longer
  pre-fills a default signal.
- worker: grace travels through the job-control task, the kept-signal table
  (pending_signal_*_ex) and the IPC message; the job process runs the
  sequence (job_signal_seq_*, uv timer) and checks stop state in /proc.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/frontend/apiserver/backend/clib.go

@@ -703,6 +703,7 @@ func ControlJob(jobid uint64, action int, content string) error {
 		var sig C.xce_signal_t
 		sig.signo = C.int(args.Signo)
 		sig.force = C.int(args.Force)
+		sig.grace = C.int(args.Grace)
 		C.jobctrl_set_siginfo(&ctrl, &sig)
 	case models.JOB_CHKPNT:
 		var ckpt C.xce_ckpt_options_t

src/frontend/apiserver/backend/convert_pure.go

@@ -2,6 +2,7 @@ package backend
 
 import (
 	"apiserver/models"
+	"apiserver/utils"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -222,6 +223,7 @@ func jobCtrlUnionDiscriminant(action int) int {
 type jobCtrlArgs struct {
 	Signo    int
 	Force    int
+	Grace    int // seconds; > 0: graceful kill/stop sequence (bug-1957)
 	CkptKill int
 	Method   string
 }
@@ -266,6 +268,11 @@ func jobCtrlParams(action int, content string) (jobCtrlArgs, error) {
 	case models.JOB_STOP, models.JOB_RESUME, models.JOB_KILL:
 		if signo == 0 {
 			signo = defaultActionSignal(action)
+			// bug-1957: a plain kill or stop is graceful, as LSF's
+			// TERMINATE and SUSPEND actions; a force kill is immediate
+			if action == models.JOB_STOP || (action == models.JOB_KILL && !forceKill) {
+				args.Grace = jobTerminateInterval()
+			}
 		}
 		args.Signo = signo
 		if action == models.JOB_KILL && forceKill {
@@ -281,16 +288,30 @@ func jobCtrlParams(action int, content string) (jobCtrlArgs, error) {
 
 // defaultActionSignal is the signal a stop/resume/kill sends when the caller
 // gave none.
+//
+// bug-1957: kill and stop name the sequence's last signal (SIGKILL,
+// SIGSTOP); with a grace period the worker sends SIGINT/SIGTERM or SIGTSTP
+// first, so the job can clean up or prepare to stop.
 func defaultActionSignal(action int) int {
 	switch action {
 	case models.JOB_STOP:
-		// bug-1954: the worker delivers the requested signal, so a plain
-		// stop asks for the uncatchable SIGSTOP
 		return models.JobSignalMapTable["STOP"] // 19
 	case models.JOB_RESUME:
 		return models.JobSignalMapTable["CONT"] // 18
 	}
-	return models.JobSignalMapTable["TERM"] // 15
+	return models.JobSignalMapTable["KILL"] // 9
+}
+
+// defaultJobTerminateInterval is LSF's JOB_TERMINATE_INTERVAL default.
+const defaultJobTerminateInterval = 10
+
+// jobTerminateInterval is job.terminate_interval, or the default when it is
+// unset or 0.
+func jobTerminateInterval() int {
+	if utils.JobYamlConfig != nil && utils.JobYamlConfig.TerminateInterval > 0 {
+		return utils.JobYamlConfig.TerminateInterval
+	}
+	return defaultJobTerminateInterval
 }
 
 // isKnownSignal reports whether n is one of models.JobSignalMapTable's values.

src/frontend/apiserver/backend/jobctrl_params_test.go

@@ -2,6 +2,7 @@ package backend
 
 import (
 	"apiserver/models"
+	"apiserver/utils"
 	"encoding/json"
 	"testing"
 )
@@ -50,13 +51,17 @@ func TestJobCtrlParams_KillWithoutForce(t *testing.T) {
 
 // An event without a signal keeps the per-action defaults.
 func TestJobCtrlParams_DefaultSignalPerAction(t *testing.T) {
+	// bug-1957: a plain kill or stop is graceful: the worker runs LSF's
+	// TERMINATE (INT, TERM, KILL) or SUSPEND (TSTP, STOP) sequence with the
+	// configured interval; resume is a single SIGCONT
 	cases := []struct {
 		action int
 		want   int
+		grace  int
 	}{
-		{models.JOB_KILL, 15},
-		{models.JOB_STOP, 19}, // bug-1954: SIGSTOP
-		{models.JOB_RESUME, 18},
+		{models.JOB_KILL, 9, defaultJobTerminateInterval},
+		{models.JOB_STOP, 19, defaultJobTerminateInterval},
+		{models.JOB_RESUME, 18, 0},
 	}
 	for _, tc := range cases {
 		content := jobCtrlEventContent(t, models.JobControlEvent{UniqueJobID: 42})
@@ -64,8 +69,9 @@ func TestJobCtrlParams_DefaultSignalPerAction(t *testing.T) {
 		if err != nil {
 			t.Fatalf("action %d: unexpected error %v", tc.action, err)
 		}
-		if got.Signo != tc.want {
-			t.Errorf("action %d: got signo %d, want %d", tc.action, got.Signo, tc.want)
+		if got.Signo != tc.want || got.Grace != tc.grace {
+			t.Errorf("action %d: got signo %d grace %d, want %d grace %d",
+				tc.action, got.Signo, got.Grace, tc.want, tc.grace)
 		}
 	}
 }
@@ -74,7 +80,7 @@ func TestJobCtrlParams_DefaultSignalPerAction(t *testing.T) {
 // accepted (case-insensitive, optional SIG prefix).
 func TestJobCtrlParams_BareSignalNameOrNumber(t *testing.T) {
 	for content, want := range map[string]int{
-		"USR1": 10, "usr2": 12, "SIGHUP": 1, "9": 9, "": 15,
+		"USR1": 10, "usr2": 12, "SIGHUP": 1, "9": 9, "": 9, // "": graceful kill (bug-1957)
 	} {
 		got, err := jobCtrlParams(models.JOB_KILL, content)
 		if err != nil {
@@ -127,7 +133,30 @@ func TestJobCtrlParams_PlainStopSendsSigstop(t *testing.T) {
 	if args.Signo != models.JobSignalMapTable["STOP"] {
 		t.Errorf("plain stop: signo %d, want SIGSTOP (%d)", args.Signo, models.JobSignalMapTable["STOP"])
 	}
-	if models.DefaultActionSignal["stop"] != "STOP" {
-		t.Errorf("DefaultActionSignal[stop] = %q, want STOP", models.DefaultActionSignal["stop"])
+}
+
+// bug-1957: an explicit signal or a force kill is a single signal (no grace)
+func TestJobCtrlParams_ExplicitOrForcedKillIsNotGraceful(t *testing.T) {
+	explicit := jobCtrlEventContent(t, models.JobControlEvent{UniqueJobID: 42, Signal: 15})
+	got, err := jobCtrlParams(models.JOB_KILL, explicit)
+	if err != nil || got.Signo != 15 || got.Grace != 0 {
+		t.Errorf("explicit TERM: got %+v, %v; want signo 15 grace 0", got, err)
+	}
+	forced := jobCtrlEventContent(t, models.JobControlEvent{UniqueJobID: 42, ForceKill: true})
+	got, err = jobCtrlParams(models.JOB_KILL, forced)
+	if err != nil || got.Signo != 9 || got.Force != 1 || got.Grace != 0 {
+		t.Errorf("force kill: got %+v, %v; want signo 9 force 1 grace 0", got, err)
+	}
+}
+
+// bug-1957: the interval comes from job.terminate_interval
+func TestJobCtrlParams_GraceFromConfig(t *testing.T) {
+	old := utils.JobYamlConfig
+	t.Cleanup(func() { utils.JobYamlConfig = old })
+	utils.JobYamlConfig = &utils.JobConfig{MaxArraySize: 1000, TerminateInterval: 30}
+
+	got, err := jobCtrlParams(models.JOB_KILL, jobCtrlEventContent(t, models.JobControlEvent{UniqueJobID: 42}))
+	if err != nil || got.Grace != 30 {
+		t.Errorf("got %+v, %v; want grace 30", got, err)
 	}
 }

src/frontend/apiserver/config/app.yaml

@@ -78,6 +78,7 @@ file_upload:
     - "application/javascript"   # .js file
 
 job:
+  terminate_interval: 10 # seconds between the signals of a plain kill (INT, TERM, KILL) or stop (TSTP, STOP); default 10 if unset or 0
   max_array_size: 1000 # max elements per job array (default 1000 if unset or 0, at most 32767; larger values are clamped to 32767 with a startup warning; negative refuses startup)
 
 cache:

src/frontend/apiserver/controllers/jobs.go

@@ -2579,13 +2579,8 @@ func preCheckJobControlRequest(req *models.JobControlRequest) error {
 		return errors.New("signal must be empty for stop or resume actions")
 	}
 
-	// set default signal value if not provided
-	if req.Signal == "" {
-		if defaultSignal, ok := models.DefaultActionSignal[strings.ToLower(req.Action)]; ok {
-			req.Signal = defaultSignal
-		}
-
-	}
+	// bug-1957: no signal means the action's default, chosen by the backend
+	// (jobCtrlParams): a graceful kill or stop, a plain resume
 
 	// transform signal to numeric value if it is a string
 	if req.Signal != "" && !utils.IsNumeric(req.Signal) {

src/frontend/apiserver/models/jobs.go

@@ -125,9 +125,12 @@ var JobKillSignalControlMap = map[int][]int{
 	JOB_KILL: {JOBSTAT["RUN"], JOBSTAT["PEND"], JOBSTAT["SSUSP"], JOBSTAT["USUSP"], JOBSTAT["PSUSP"]},
 }
 
+// DefaultActionSignal names each action's default signal. bug-1957: kill and
+// stop name the last signal of their graceful sequences (backend
+// jobCtrlParams sends them with the configured grace period).
 var DefaultActionSignal = map[string]string{
-	"kill":   "TERM",
-	"stop":   "STOP", // bug-1954: the worker delivers exactly this signal
+	"kill":   "KILL",
+	"stop":   "STOP",
 	"resume": "CONT",
 }
 

src/frontend/apiserver/utils/config.go

@@ -94,6 +94,10 @@ type FileUploadConfig struct {
 
 type JobConfig struct {
 	MaxArraySize int `yaml:"max_array_size"`
+	// TerminateInterval is the seconds between the signals of a plain kill
+	// (SIGINT, SIGTERM, SIGKILL) and a plain stop (SIGTSTP, SIGSTOP), as
+	// LSF's JOB_TERMINATE_INTERVAL (bug-1957)
+	TerminateInterval int `yaml:"terminate_interval"`
 }
 
 type CORSConfig struct {
@@ -239,8 +243,13 @@ func InitConf() error {
 			zap.Int("configured", configuredArraySize),
 			zap.Int("effective", maxArraySize))
 	}
+	terminateInterval := viper.GetInt("job.terminate_interval")
+	if terminateInterval < 0 {
+		return fmt.Errorf("job.terminate_interval must not be negative: %d", terminateInterval)
+	}
 	JobYamlConfig = &JobConfig{
-		MaxArraySize: maxArraySize,
+		MaxArraySize:      maxArraySize,
+		TerminateInterval: terminateInterval,
 	}
 
 	// parse cache config section