worker: reap unattached TTY jobs via ttyd watchdog with real-pid resolution
An attach job (xcecli -I without --browse) that no client ever connects to sat in RUN forever: ttyd only starts the job command on the first websocket client, so with no client the command never runs, ttyd never exits, and the job holds its ttyd, port and per-job worker indefinitely. New exdata key "attach":1, set by the CLI for TTY jobs that are not --browse and carried through the apiserver's extendDataMap to the worker's transfer_info->attach_mode. For such jobs the worker starts a 1 s watchdog when the job process starts: - no client attached within 60 s -> reap ttyd (SIGKILL to the monitored tree), job EXITs with status message "No client attached within 60 s" - client came and the command ran and exited -> give ttyd 5 s to stop on its own (-Q), then reap: "Command exited; ttyd did not stop on its own" The reap lands in the regular process_exit_callback path (JOB_EXITED, status reason, post-exec, worker exit), so kill and cleanup behave like any other job termination. Field-critical detail: uv_proc_get_pid returns the "/bin/sh -c ttyd ..." wrapper pid, and sh does not reliably exec -- on the field ttyd runs as a child of the wrapper. Judging the wrapper's children would see ttyd itself as "the command has started" and never reap. The watchdog resolves the real ttyd pid first (/proc/<pid>/comm walk down through the sh wrapper, capped depth) and judges client activity by the real ttyd's children; killing still goes through the monitored wrapper tree so both topologies (exec'd or forked) are covered. Web-viewed sessions (--browse / DISPLAY) are untouched: no attach exdata, no watchdog -- ttyd stays until a browser connects or the job is killed, as before. Attach failure on the CLI side prints that the job will be cleaned up by the worker if no client attaches within 60 s. Tests: xcecli (attach flag matrix), apiserver controllers (attach exdata end-to-end in ExtendData), worker native suites; live t1 cluster: normal attach completes DONE with no residue, unattached job EXITs at ~60 s with the reason in the worker log and port released, Ctrl-C-then-wait completes when the command ends, --browse job stays RUN past 60 s and kills cleanly. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
parent
6ce7bdbe11
commit
4f188310ba
9 changed files with 421 additions and 12 deletions
|
|
@ -2048,6 +2048,14 @@ func processJobRequest(ctx *gin.Context, req *models.JobSubmitRequest, defaultVa
|
|||
extendDataMap["interactive"] = req.Interactive
|
||||
}
|
||||
|
||||
// attach=1: a TTY job whose session ends with its command (xcecli -I
|
||||
// without --browse). The worker uses it to reap ttyd when no client ever
|
||||
// connects -- without it an unattached job would sit in RUN forever,
|
||||
// holding its ttyd, port and per-job worker.
|
||||
if req.Attach != 0 {
|
||||
extendDataMap["attach"] = req.Attach
|
||||
}
|
||||
|
||||
if len(extendDataMap) > 0 {
|
||||
updatedExtendDataBytes, err := json.Marshal(extendDataMap)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -428,6 +428,56 @@ func TestJobHandler_SmokeSubmitErrorIs500(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestJobHandler_SubmitInteractiveAttachExdata checks the submit handler
|
||||
// packs interactive/attach into the job's ExtendData: attach=1 only for the
|
||||
// terminal-attach TTY kind (the worker reaps ttyd when no client ever
|
||||
// connects), never for --browse or other interactive modes.
|
||||
func TestJobHandler_SubmitInteractiveAttachExdata(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
req map[string]interface{}
|
||||
wantAttach bool
|
||||
}{
|
||||
{"tty attach", map[string]interface{}{
|
||||
"job_name": "a", "command": "sleep 1", "interactive": 1, "attach": 1,
|
||||
}, true},
|
||||
{"tty web view (no attach)", map[string]interface{}{
|
||||
"job_name": "a", "command": "sleep 1", "interactive": 1,
|
||||
}, false},
|
||||
{"xjob with attach ignored worker-side but passed", map[string]interface{}{
|
||||
"job_name": "a", "command": "sleep 1", "interactive": 2, "attach": 1,
|
||||
}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newHandlerFixture(t)
|
||||
f.seedQueue("normal")
|
||||
|
||||
w := f.submit(tc.req)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String())
|
||||
}
|
||||
subs := f.submissions()
|
||||
if len(subs) != 1 {
|
||||
t.Fatalf("submitJob calls = %d, want 1", len(subs))
|
||||
}
|
||||
var ex map[string]interface{}
|
||||
if raw := subs[0].Job.ExtendData.String; raw != "" {
|
||||
if err := json.Unmarshal([]byte(raw), &ex); err != nil {
|
||||
t.Fatalf("ExtendData %q is not JSON: %v", raw, err)
|
||||
}
|
||||
}
|
||||
if got, ok := ex["interactive"]; !ok || int(got.(float64)) != tc.req["interactive"].(int) {
|
||||
t.Errorf("ExtendData interactive = %v (%v), want %d", got, ok, tc.req["interactive"])
|
||||
}
|
||||
_, hasAttach := ex["attach"]
|
||||
if hasAttach != tc.wantAttach {
|
||||
t.Errorf("ExtendData attach present = %v, want %v (exdata: %s)", hasAttach, tc.wantAttach, subs[0].Job.ExtendData.String)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestJobHandler_SmokeControlByID controls one seeded ordinary job by ID
|
||||
// and checks the dispatch is recorded once with its unique_job_id.
|
||||
func TestJobHandler_SmokeControlByID(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -669,6 +669,11 @@ type JobSubmitRequest struct {
|
|||
Exclusive bool `json:"exclusive,omitempty"`
|
||||
Rerunable bool `json:"rerunable,omitempty"`
|
||||
Interactive int `json:"interactive,omitempty"`
|
||||
// Attach marks an interactive TTY job as the terminal-attach kind
|
||||
// (xcecli -I without --browse): the session ends with the command, so
|
||||
// the worker may reap the ttyd when no client ever connects. 0 keeps
|
||||
// the web-viewed semantics.
|
||||
Attach int `json:"attach,omitempty"`
|
||||
WarningTimePeriod int `json:"warning_time_period,omitempty"`
|
||||
WarningAction string `json:"warning_action,omitempty"`
|
||||
DataSpecFile string `json:"data_spec_file,omitempty"`
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue