diff --git a/src/docs/xce.md b/src/docs/xce.md index 79a08dcc..7408ccc1 100644 --- a/src/docs/xce.md +++ b/src/docs/xce.md @@ -54,6 +54,23 @@ cluster. Common operations: Built as a standalone Go binary; communicates with the API server over its REST interface. +## Killing and stopping jobs + +Job control follows LSF's TERMINATE and SUSPEND actions: + +- **Kill** (`xcecli jobs kill `, no signal given): SIGINT, then SIGTERM, + then SIGKILL, one interval apart, each only while the job is still + running, so the job can catch SIGINT/SIGTERM and clean up. +- **Stop** (`xcecli jobs stop `): SIGTSTP first, so the job can catch + it, prepare and stop itself; SIGSTOP follows after the interval if it has + not stopped. **Resume** sends SIGCONT and cancels a stop still in progress. +- **Explicit signal** (kill with a signal): that signal only, once. +- **Force kill**: SIGKILL at once. + +The interval is the API server's `job.terminate_interval` setting +(`config/app.yaml`, seconds; default 10 when unset or 0, as LSF's +`JOB_TERMINATE_INTERVAL`; negative refuses startup). + ## Job arrays A job array is a set of identical jobs (elements) submitted at once. XCE diff --git a/src/frontend/apiserver/config/schema/app_schema.json b/src/frontend/apiserver/config/schema/app_schema.json index ab399043..2eefd0f5 100644 --- a/src/frontend/apiserver/config/schema/app_schema.json +++ b/src/frontend/apiserver/config/schema/app_schema.json @@ -112,6 +112,11 @@ "minimum": 1, "maximum": 2147483646, "description": "Default max array size for job submission" + }, + "terminate_interval": { + "type": "integer", + "minimum": 0, + "description": "Seconds between the signals of a plain job kill (SIGINT, SIGTERM, SIGKILL) or stop (SIGTSTP, SIGSTOP); 0 or unset means 10 (LSF JOB_TERMINATE_INTERVAL)" } } }, diff --git a/src/frontend/apiserver/utils/config.go b/src/frontend/apiserver/utils/config.go index 4d4ecf80..b06cc4a4 100644 --- a/src/frontend/apiserver/utils/config.go +++ b/src/frontend/apiserver/utils/config.go @@ -247,6 +247,9 @@ func InitConf() error { if terminateInterval < 0 { return fmt.Errorf("job.terminate_interval must not be negative: %d", terminateInterval) } + if terminateInterval == 0 { + terminateInterval = defaultTerminateInterval + } JobYamlConfig = &JobConfig{ MaxArraySize: maxArraySize, TerminateInterval: terminateInterval, @@ -316,6 +319,9 @@ func ParseMaxSize(maxSize string) (int64, error) { } const ( + // defaultTerminateInterval is used when job.terminate_interval is unset + // or 0: LSF's JOB_TERMINATE_INTERVAL default (bug-1957). + defaultTerminateInterval = 10 // defaultMaxArraySize is used when job.max_array_size is unset or 0. defaultMaxArraySize = 1000 // supportedMaxArraySize is the largest array the API server creates diff --git a/src/frontend/apiserver/utils/config_test.go b/src/frontend/apiserver/utils/config_test.go index ee00fcba..e4997fea 100644 --- a/src/frontend/apiserver/utils/config_test.go +++ b/src/frontend/apiserver/utils/config_test.go @@ -299,3 +299,34 @@ func TestInitConf_MaxArraySizeNegativeRefused(t *testing.T) { t.Fatalf("InitConf with -1: expected job.max_array_size error, got %v", err) } } + +// bug-1957: job.terminate_interval, the seconds between the signals of a +// plain kill or stop (LSF JOB_TERMINATE_INTERVAL) +func TestInitConf_TerminateInterval(t *testing.T) { + setupInitConf(t, "job:\n terminate_interval: 30") + if err := InitConf(); err != nil { + t.Fatalf("InitConf: %v", err) + } + if JobYamlConfig.TerminateInterval != 30 { + t.Errorf("TerminateInterval = %d, want 30", JobYamlConfig.TerminateInterval) + } +} + +func TestInitConf_TerminateIntervalUnsetDefaults(t *testing.T) { + for _, section := range []string{"", "job:\n terminate_interval: 0"} { + setupInitConf(t, section) + if err := InitConf(); err != nil { + t.Fatalf("%q: InitConf: %v", section, err) + } + if JobYamlConfig.TerminateInterval != 10 { + t.Errorf("%q: TerminateInterval = %d, want the default 10", section, JobYamlConfig.TerminateInterval) + } + } +} + +func TestInitConf_TerminateIntervalNegativeRefused(t *testing.T) { + setupInitConf(t, "job:\n terminate_interval: -1") + if err := InitConf(); err == nil || !strings.Contains(err.Error(), "terminate_interval") { + t.Fatalf("InitConf with a negative interval: err = %v, want an error naming terminate_interval", err) + } +}