fix(worker): graceful job kill and catchable stop, as LSF (bug-1957)
A plain kill sent one SIGTERM and never escalated, and a plain stop sent one uncatchable SIGSTOP. Following LSF's TERMINATE and SUSPEND actions, a plain kill now sends SIGINT, then SIGTERM, then SIGKILL, and a plain stop sends SIGTSTP (the job can catch it, prepare and stop itself), then SIGSTOP if it has not stopped; each next signal follows after the grace period and only while the job is alive (or not yet stopped). A resume cancels a pending stop; the job's exit ends any sequence. An explicit signal is delivered once; a force kill is SIGKILL at once. - JobSignal gains grace (capnp @2, xce_signal_t.grace): > 0 asks for the sequence, in seconds between signals. - apiserver: job.terminate_interval (default 10 s, LSF's JOB_TERMINATE_INTERVAL); jobCtrlParams sends kill = SIGKILL and stop = SIGSTOP with that grace when no signal is given; the handler no longer pre-fills a default signal. - worker: grace travels through the job-control task, the kept-signal table (pending_signal_*_ex) and the IPC message; the job process runs the sequence (job_signal_seq_*, uv timer) and checks stop state in /proc. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e5a0e42b66
commit
367af11af1
24 changed files with 580 additions and 39 deletions
|
|
@ -94,6 +94,10 @@ type FileUploadConfig struct {
|
|||
|
||||
type JobConfig struct {
|
||||
MaxArraySize int `yaml:"max_array_size"`
|
||||
// TerminateInterval is the seconds between the signals of a plain kill
|
||||
// (SIGINT, SIGTERM, SIGKILL) and a plain stop (SIGTSTP, SIGSTOP), as
|
||||
// LSF's JOB_TERMINATE_INTERVAL (bug-1957)
|
||||
TerminateInterval int `yaml:"terminate_interval"`
|
||||
}
|
||||
|
||||
type CORSConfig struct {
|
||||
|
|
@ -239,8 +243,13 @@ func InitConf() error {
|
|||
zap.Int("configured", configuredArraySize),
|
||||
zap.Int("effective", maxArraySize))
|
||||
}
|
||||
terminateInterval := viper.GetInt("job.terminate_interval")
|
||||
if terminateInterval < 0 {
|
||||
return fmt.Errorf("job.terminate_interval must not be negative: %d", terminateInterval)
|
||||
}
|
||||
JobYamlConfig = &JobConfig{
|
||||
MaxArraySize: maxArraySize,
|
||||
MaxArraySize: maxArraySize,
|
||||
TerminateInterval: terminateInterval,
|
||||
}
|
||||
|
||||
// parse cache config section
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue