feat(apiserver): SSE job-event stream + EventBroker (Phase 2, ADR-004)

Add a Server-Sent Events endpoint /xce/v1/events streaming job lifecycle
events (job.created/updated/deleted/finished) to subscribers. The
DefaultHandler now publishes every job event to a process-wide EventBroker;
the SSE handler subscribes and streams JSON frames with a keep-alive comment
during idle.

- backend/broker.go: thread-safe fan-out Broker (drops for slow consumers so
  the notification path never blocks), Event type with job_status string.
- backend/handler.go: OnJob* publish to DefaultBroker.
- controllers/events.go: SSE handler (gin, text/event-stream, keep-alive).
- router + whitelist + rbac: /xce/v1/events route, event:subscribe
  permission granted to mcp_readonly/mcp_operator/cluster_admin.
- Tests: broker fan-out/slow-consumer/unsubscribe, handler publishes, SSE
  end-to-end via httptest.

Verified live on t1: SSE stream delivered job.updated PEND→RUN events for a
submitted job.

Co-Authored-By: Claude <noreply@anthropic.com>

src/frontend/apiserver/backend/broker.go

@@ -0,0 +1,83 @@
+package backend
+
+import (
+	"sync"
+
+	"apiserver/models"
+)
+
+// Event types broadcast to SSE subscribers (scheduler-agent Phase 2, ADR-004).
+const (
+	EventJobCreated  = "job.created"
+	EventJobUpdated  = "job.updated"
+	EventJobDeleted  = "job.deleted"
+	EventJobFinished = "job.finished"
+)
+
+// Event is a single lifecycle event. Exactly one of Job / (JobID, ExitCode)
+// is populated depending on Type.
+type Event struct {
+	Type     string          `json:"type"`
+	JobID    uint64          `json:"job_id,omitempty"`
+	ExitCode int             `json:"exit_code,omitempty"`
+	Status   string          `json:"job_status,omitempty"` // JobStateStr of the job (PEND/RUN/DONE/EXIT/...)
+	Job      *models.JobInfo `json:"job,omitempty"`
+}
+
+// Broker is a thread-safe fan-out of lifecycle events to subscribed channels.
+// Subscribers register a buffered channel; a slow subscriber that cannot
+// drain before the next event has the event dropped (never blocking the
+// publisher / notification handler).
+type Broker struct {
+	mu   sync.RWMutex
+	subs map[chan Event]struct{}
+}
+
+// DefaultBroker is the process-wide event bus the DefaultHandler publishes to
+// and the SSE endpoint subscribes to.
+var DefaultBroker = NewBroker()
+
+// NewBroker returns an empty broker.
+func NewBroker() *Broker {
+	return &Broker{subs: make(map[chan Event]struct{})}
+}
+
+// Subscribe registers a new subscriber with the given buffer depth and
+// returns its channel. Callers must Unsubscribe when done.
+func (b *Broker) Subscribe(buffer int) chan Event {
+	ch := make(chan Event, buffer)
+	b.mu.Lock()
+	b.subs[ch] = struct{}{}
+	b.mu.Unlock()
+	return ch
+}
+
+// Unsubscribe removes a subscriber. Safe to call after the channel has been
+// drained; the channel itself is closed by the subscriber (see the SSE
+// handler).
+func (b *Broker) Unsubscribe(ch chan Event) {
+	b.mu.Lock()
+	delete(b.subs, ch)
+	b.mu.Unlock()
+}
+
+// Publish fans an event out to every subscriber, dropping it for slow
+// consumers. Never blocks the caller.
+func (b *Broker) Publish(ev Event) {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	for ch := range b.subs {
+		select {
+		case ch <- ev:
+		default:
+			// slow consumer — drop rather than block the notification path
+		}
+	}
+}
+
+// SubscriberCount reports the number of registered subscribers (tests / metrics).
+func (b *Broker) SubscriberCount() int {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	return len(b.subs)
+}

src/frontend/apiserver/backend/broker_test.go

@@ -0,0 +1,96 @@
+package backend
+
+import (
+	"testing"
+
+	"apiserver/models"
+)
+
+func TestBrokerSubscribePublishUnsubscribe(t *testing.T) {
+	b := NewBroker()
+	ch := b.Subscribe(8)
+	defer b.Unsubscribe(ch)
+
+	b.Publish(Event{Type: EventJobCreated, JobID: 42})
+
+	select {
+	case ev := <-ch:
+		if ev.Type != EventJobCreated || ev.JobID != 42 {
+			t.Fatalf("unexpected event: %+v", ev)
+		}
+	default:
+		t.Fatal("expected published event on subscriber channel")
+	}
+}
+
+func TestBrokerEventCarriesJobPayload(t *testing.T) {
+	b := NewBroker()
+	ch := b.Subscribe(8)
+	defer b.Unsubscribe(ch)
+
+	job := &models.JobInfo{Unique_JobID: 7, JobName: "x", Command: "sleep 1"}
+	b.Publish(Event{Type: EventJobUpdated, JobID: 7, Job: job})
+
+	ev := <-ch
+	if ev.Job == nil || ev.Job.Unique_JobID != 7 || ev.Job.Command != "sleep 1" {
+		t.Fatalf("event job payload not carried: %+v", ev)
+	}
+}
+
+func TestBrokerDoesNotBlockSlowConsumer(t *testing.T) {
+	// A full subscriber channel must not block Publish (the notification
+	// handler runs on the hot path). The event is dropped for the slow
+	// consumer; the publisher returns immediately.
+	b := NewBroker()
+	ch := b.Subscribe(1)
+	defer b.Unsubscribe(ch)
+
+	// Fill the single-slot channel.
+	b.Publish(Event{Type: EventJobCreated, JobID: 1})
+	if len(ch) != 1 {
+		t.Fatalf("expected full channel, got len %d", len(ch))
+	}
+
+	// This would deadlock if Publish blocked; a nil-drain on a goroutine
+	// would hang the test via the timeout. Just call it — if it blocks, the
+	// test framework's timeout catches it.
+	b.Publish(Event{Type: EventJobCreated, JobID: 2})
+
+	if len(ch) != 1 {
+		t.Fatalf("slow consumer should have the first event dropped, len=%d", len(ch))
+	}
+	if ev := <-ch; ev.JobID != 1 {
+		t.Fatalf("expected first event retained, got %+v", ev)
+	}
+}
+
+func TestBrokerUnsubscribeStopsDelivery(t *testing.T) {
+	b := NewBroker()
+	ch := b.Subscribe(8)
+	b.Unsubscribe(ch)
+
+	b.Publish(Event{Type: EventJobCreated, JobID: 9})
+
+	select {
+	case ev := <-ch:
+		t.Fatalf("event delivered after unsubscribe: %+v", ev)
+	default:
+		// expected — no delivery
+	}
+}
+
+func TestBrokerSubscriberCount(t *testing.T) {
+	b := NewBroker()
+	ch1 := b.Subscribe(8)
+	ch2 := b.Subscribe(8)
+	defer b.Unsubscribe(ch1)
+	defer b.Unsubscribe(ch2)
+
+	if n := b.SubscriberCount(); n != 2 {
+		t.Fatalf("expected 2 subscribers, got %d", n)
+	}
+	b.Unsubscribe(ch1)
+	if n := b.SubscriberCount(); n != 1 {
+		t.Fatalf("expected 1 subscriber after unsubscribe, got %d", n)
+	}
+}

src/frontend/apiserver/backend/handler.go

@@ -13,14 +13,17 @@ type DefaultHandler struct{}
 
 func (h *DefaultHandler) OnJobCreated(job *models.JobInfo) {
 	store.Jobs.Put(uint64(job.Unique_JobID), *job)
+	DefaultBroker.Publish(Event{Type: EventJobCreated, JobID: uint64(job.Unique_JobID), Status: models.JobStateStr(job.JobStatus), Job: job})
 }
 
 func (h *DefaultHandler) OnJobUpdated(job *models.JobInfo) {
 	store.Jobs.Put(uint64(job.Unique_JobID), *job)
+	DefaultBroker.Publish(Event{Type: EventJobUpdated, JobID: uint64(job.Unique_JobID), Status: models.JobStateStr(job.JobStatus), Job: job})
 }
 
 func (h *DefaultHandler) OnJobDeleted(jobid uint64) {
 	store.Jobs.Delete(jobid)
+	DefaultBroker.Publish(Event{Type: EventJobDeleted, JobID: jobid})
 }
 
 func (h *DefaultHandler) OnJobFinished(jobid uint64, exitCode int) {
@@ -34,6 +37,7 @@ func (h *DefaultHandler) OnJobFinished(jobid uint64, exitCode int) {
 		existing.JobStatus = models.JOBSTAT["EXIT"]
 	}
 	store.Jobs.Put(jobid, existing)
+	DefaultBroker.Publish(Event{Type: EventJobFinished, JobID: jobid, ExitCode: exitCode})
 }
 
 func (h *DefaultHandler) OnQueueUpdated(queue *models.QueueInfo) {

src/frontend/apiserver/backend/handler_test.go

@@ -16,6 +16,48 @@ func resetStores() {
 	store.InitStores()
 }
 
+// freshBroker swaps DefaultBroker for an isolated instance, restoring the
+// previous one on cleanup, so publish-behavior tests don't pollute other
+// tests that subscribe to the process-wide broker.
+func freshBroker(t *testing.T) *Broker {
+	t.Helper()
+	prev := DefaultBroker
+	b := NewBroker()
+	DefaultBroker = b
+	t.Cleanup(func() { DefaultBroker = prev })
+	return b
+}
+
+func TestHandlerPublishesJobEventsToBroker(t *testing.T) {
+	resetStores()
+	b := freshBroker(t)
+	ch := b.Subscribe(16)
+	defer b.Unsubscribe(ch)
+
+	h := DefaultHandler{}
+	h.OnJobCreated(&models.JobInfo{Unique_JobID: 1, JobName: "created"})
+	h.OnJobUpdated(&models.JobInfo{Unique_JobID: 2, JobName: "updated"})
+	// OnJobFinished only publishes when the job already exists in the store
+	// (it updates the stored record), so create job 3 before finishing it.
+	h.OnJobUpdated(&models.JobInfo{Unique_JobID: 3, JobName: "tofinish"})
+	h.OnJobFinished(3, 0)
+
+	var got []Event
+	for i := 0; i < 4; i++ {
+		got = append(got, <-ch)
+	}
+
+	if got[0].Type != EventJobCreated || got[0].JobID != 1 {
+		t.Fatalf("event[0] = %+v, want job.created 1", got[0])
+	}
+	if got[1].Type != EventJobUpdated || got[1].JobID != 2 {
+		t.Fatalf("event[1] = %+v, want job.updated 2", got[1])
+	}
+	if got[3].Type != EventJobFinished || got[3].JobID != 3 || got[3].ExitCode != 0 {
+		t.Fatalf("event[3] = %+v, want job.finished 3", got[3])
+	}
+}
+
 // captureLog swaps utils.Log for an observer core so a test can assert on
 // exactly which log lines fired, restoring the previous logger on cleanup.
 func captureLog(t *testing.T) *observer.ObservedLogs {