attach jobs end with their command: relay watcher, once-only start banner, worker-side ttyd cleanup

Three attach-mode (-I) fixes, verified end-to-end on the t1 cluster:

1. Session hang on job finish (job_attach.go): the relay's client->worker
   pump blocked forever in ReadMessage after the worker hop closed, so the
   old wg.Wait()-before-Close teardown never ran and the client hung, then
   died with an abnormal 1006. A broker watcher now subscribes to
   EventJobFinished and on job end sends the 4001 policy close and closes
   both hops, unblocking the pumps; the teardown follows normally. The
   client (attach.go) treats 4001/1000/1001 -- and 1006-after-output as a
   final safety net -- as a clean session end.

2. "<<Starting on t1 >>" printed repeatedly (submit.go): the status poll
   printed the banner on every transition; now gated on a reportedStart
   flag. The URL is also no longer printed in attach mode -- it only makes
   sense for browse (-I --tty --browse) and DISPLAY jobs.

3. ttyd left behind after an attach job exits (xcecd_worker.c): a new
   tty_stop_attach_session(), called from process_exit_callback for TTY
   jobs, kills the per-job ttyd right away. This is insurance -- the sh -c
   exec means uv_proc monitors ttyd's own pid, so ttyd is normally gone
   before the callback fires -- but it guarantees no orphan when the
   timing lines up badly. Web-viewed sessions are untouched: they keep
   ttyd and output alive for later browsing, per design.

Co-Authored-By: Claude Code <noreply@anthropic.com>

src/frontend/apiserver/controllers/job_attach.go

@@ -15,7 +15,6 @@ import (
 
 	logger "apiserver/utils"
 
-
 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"go.uber.org/zap"
@@ -166,7 +165,7 @@ func (j JobController) AttachJob(ctx *gin.Context) {
 		zap.Int64("jobid", job.Unique_JobID), zap.String("user", logonUser),
 		zap.String("worker_url", workerURL))
 
-	attachRelay(clientConn, workerConn)
+	attachRelay(uint64(job.Unique_JobID), clientConn, workerConn)
 	logger.Info("attach: relay finished", zap.Int64("jobid", job.Unique_JobID))
 }
 
@@ -196,13 +195,17 @@ func attachTTYDURL(raw string) (string, error) {
 	return u.String(), nil
 }
 
-// attachClose sends one close frame (best-effort) and stops reading.
+// attachClose sends one close frame (best-effort) and waits for the peer's
+// closing handshake before the caller tears the TCP connection down. Without
+// the drain the deferred Close() can race the frame out the socket and the
+// client sees 1006 (abnormal closure) instead of our policy code.
 func attachClose(conn *websocket.Conn, code int, text string) {
 	deadline := time.Now().Add(attachWriteTimeout)
 	conn.WriteControl(websocket.CloseMessage,
 		websocket.FormatCloseMessage(code, text), deadline)
-	// Drain so the close frame is actually processed by gorilla's internal
-	// control handler before the connection is torn down.
+	// Drain until the peer answers the handshake (or the deadline): each
+	// NextReader pumps gorilla's internal control-frame handler, which
+	// completes the close exchange on this side.
 	conn.SetReadDeadline(deadline)
 	for {
 		if _, _, err := conn.NextReader(); err != nil {
@@ -215,12 +218,13 @@ func attachClose(conn *websocket.Conn, code int, text string) {
 // goes away. Each direction runs in its own goroutine; the first direction
 // to finish tears the whole relay down (and closes both connections, which
 // unblocks the other direction's in-flight Read).
-func attachRelay(client, worker *websocket.Conn) {
-	// Oversized pipes: terminal output can burst; the pipes absorb jitter
-	// between the two hops. On ping/pong: both hops are terminal sessions,
-	// so application-level traffic flows constantly when attached; the
-	// deadlines below exist to catch a dead peer at all, not to time out
-	// interactive silence mid-command.
+//
+// The relay also watches the job's terminal state: an attach session is over
+// the moment its job finishes, even if the worker-side websocket lingers
+// (ttyd's --exit-with-program does not always fire promptly with a client
+// still attached). The watch fires a policy close to the client in that
+// case; the pumps' teardown below runs as usual.
+func attachRelay(jobID uint64, client, worker *websocket.Conn) {
 	var wg sync.WaitGroup
 	wg.Add(2)
 	done := make(chan struct{})
@@ -228,16 +232,61 @@ func attachRelay(client, worker *websocket.Conn) {
 	go func() {
 		defer wg.Done()
 		if !attachPump(worker, client, done) {
-			// worker side ended on its own: the session (or its job) ended
+			// worker side ended on its own: the session (or its job) ended.
+			// Send the policy close and complete the handshake BEFORE the
+			// teardown goroutine closes the socket -- attachClose blocks
+			// until the client answers or the deadline passes, so the
+			// client reliably reads 4001 instead of a bare 1006.
 			attachClose(client, attachCloseJobEnded, "interactive session ended")
 		}
 	}()
 	go func() {
 		defer wg.Done()
+		// client-initiated end: forward nothing, teardown below suffices.
 		attachPump(client, worker, done)
 	}()
 
-	// Unblock both read pumps when either direction finishes.
+	// The job ending is a session end regardless of the worker hop: send
+	// the policy close and tear the relay down. WriteControl (not the full
+	// attachClose handshake) here: the watcher may race a pump already
+	// closing this connection, and the teardown below follows immediately,
+	// so a best-effort frame is enough -- the client tolerates both a 4001
+	// and an abrupt end after output.
+	go func() {
+		ch := backend.DefaultBroker.Subscribe(4)
+		defer backend.DefaultBroker.Unsubscribe(ch)
+		for {
+			select {
+			case <-done:
+				return
+			case ev, ok := <-ch:
+				if !ok {
+					return
+				}
+				if ev.JobID == jobID && ev.Type == backend.EventJobFinished {
+					select {
+					case <-done:
+					default:
+						logger.Info("attach: job finished, ending relayed session",
+							zap.Uint64("jobid", jobID))
+						_ = client.WriteControl(websocket.CloseMessage,
+							websocket.FormatCloseMessage(attachCloseJobEnded,
+								"interactive session ended (job finished)"),
+							time.Now().Add(attachWriteTimeout))
+					}
+					// unblock the pumps; teardown closes both hops
+					go func() {
+						client.Close()
+						worker.Close()
+					}()
+					return
+				}
+			}
+		}
+	}()
+
+	// Unblock both read pumps when either direction finishes (including the
+	// attachClose handshake window above).
 	go func() {
 		wg.Wait()
 		close(done)
@@ -251,8 +300,9 @@ func attachRelay(client, worker *websocket.Conn) {
 // attachPump copies one direction: reads frames from src, writes them to
 // dst. Binary and text frames pass through with their type; close/other
 // control frames end the pump (gorilla surfaces close via the read error).
-// Returns false when src ended first (its peer outlived it), true when the
-// dst side ended.
+// Returns false when src ended on its own (its peer outlived it, and the
+// caller may want to send it a policy close); true when the end was caused
+// by dst or by our own teardown (nothing left to tell).
 func attachPump(src, dst *websocket.Conn, done <-chan struct{}) bool {
 	for {
 		msgType, data, err := src.ReadMessage()
@@ -268,6 +318,8 @@ func attachPump(src, dst *websocket.Conn, done <-chan struct{}) bool {
 		}
 		_ = dst.SetWriteDeadline(time.Now().Add(attachWriteTimeout))
 		if err := dst.WriteMessage(msgType, data); err != nil {
+			// dst went away first; unblock src's reads via teardown and
+			// report "not src-ended" so no policy close is sent to dst.
 			return true
 		}
 	}