advisor: add compare subcommand for echo-vs-LLM measurement

One snapshot through BOTH providers per fired scenario, side by side:
CompareRun rows (scenario, trigger, per-side model/latency/tokens/
findings/unverified). The LLM side errors are reported in-band
(llm_error) rather than degrading to echo, so a flaky router shows up
as data instead of a silent echo-only row. The compare path is pure
measurement: nothing logged, nothing pushed — the run stays the only
writer of the decision log.

Live data (pend_stall, cmp job on t1-t3 cluster):
  echo   0ms      1 finding, unverified (reason "unknown" opaque)
  ds-v4  ~1.6s    cross-field catch: job PEND 876-902s while all
                  nodes idle + no dispatch -> severe, actionable

Echo answers in 0ms with a coarse degraded verdict; the LLM leg adds
cross-field correlation (pend vs host state vs dispatch) that the
static rules structurally cannot express. This is the quantified
incremental-value page for the advisor pitch.

Co-Authored-By: Claude Code <noreply@anthropic.com>

src/frontend/advisor/cmd/main.go

@@ -7,8 +7,9 @@
 //
 // Usage:
 //
-//	advisor run    # long-running observe loop
-//	advisor once   # one scan + evaluate round, report to stdout (no log)
+//	advisor run      # long-running observe loop
+//	advisor once     # one scan + evaluate round, report to stdout (no log)
+//	advisor compare  # one snapshot through echo AND the LLM, side by side
 package main
 
 import (
@@ -41,6 +42,8 @@ func main() {
 		cmdRun()
 	case "once":
 		cmdOnce()
+	case "compare":
+		cmdCompare()
 	case "-h", "--help", "help":
 		usage()
 	default:
@@ -54,8 +57,9 @@ func usage() {
 	fmt.Fprint(os.Stderr, `advisor — XCE scheduler advisor (observe mode)
 
 usage:
-  advisor run     start the observe loop (periodic scan + SSE events)
-  advisor once    run one scan/evaluate round and print the report
+  advisor run      start the observe loop (periodic scan + SSE events)
+  advisor once     run one scan/evaluate round and print the report
+  advisor compare  one snapshot through echo AND the LLM, side by side
 
 configuration (env):
   XCE_API_KEY          required — mcp_readonly service-account key
@@ -164,3 +168,26 @@ func discardLog() *declog.Log {
 	}
 	return l
 }
+
+// cmdCompare answers "what does the LLM add over echo?" with data: one
+// snapshot, every fired scenario evaluated by both providers, results
+// side-by-side on stdout. Nothing is logged or pushed — `run` stays the
+// only writer of the decision log.
+func cmdCompare() {
+	cfg := mustConfig()
+	if !cfg.LLMEnabled() {
+		log.Fatalf("advisor: compare needs ADVISOR_LLM_BASE_URL (echo vs echo is a tautology)")
+	}
+	rb := loadRunbook(cfg)
+
+	a := agent.New(mustClient(cfg), detect.NewDetector(detect.DefaultLimits()),
+		pickProvider(cfg), discardLog(), cfg.Interval)
+	a.RunbookText = rb.Render()
+
+	runs, err := a.Compare(context.Background())
+	if err != nil {
+		log.Fatalf("advisor: compare: %v", err)
+	}
+	out, _ := json.MarshalIndent(runs, "", "  ")
+	fmt.Println(string(out))
+}

src/frontend/advisor/internal/agent/agent.go

@@ -10,6 +10,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"log"
 	"strconv"
 	"time"
@@ -200,6 +201,90 @@ func (a *Agent) Once(ctx context.Context) (*OnceReport, error) {
 	return rep, nil
 }
 
+// CompareRun is one scenario evaluated by both providers on the same
+// snapshot — the echo-vs-LLM comparison row.
+type CompareRun struct {
+	Scenario string `json:"scenario"`
+	// Trigger is the detector fingerprint that fired this scenario.
+	Trigger string `json:"trigger"`
+	Echo    *CompareSide `json:"echo"`
+	LLM     *CompareSide `json:"llm"`
+	// LLMError is set when the LLM call failed (the row then shows the
+	// degradation the observe loop would have recorded).
+	LLMError string `json:"llm_error,omitempty"`
+}
+
+// CompareSide is one provider's answer on the shared snapshot.
+type CompareSide struct {
+	Model        string             `json:"model"`
+	LatencyMS    int64              `json:"latency_ms"`
+	InputTokens  int                `json:"input_tokens,omitempty"`
+	OutputTokens int                `json:"output_tokens,omitempty"`
+	Findings     []provider.Finding `json:"findings"`
+	// UnverifiedCount summarizes the hallucination firewall's verdicts.
+	UnverifiedCount int `json:"unverified_count"`
+}
+
+// Compare runs one snapshot through BOTH providers per fired scenario and
+// returns side-by-side results. Nothing is logged or pushed — this is the
+// measurement path for "what does the LLM add over echo" (decision-log
+// records from observe rounds are the other half of that evidence).
+func (a *Agent) Compare(ctx context.Context) ([]CompareRun, error) {
+	st, err := a.builder.Build(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	stateJSON, err := json.Marshal(st)
+	if err != nil {
+		return nil, err
+	}
+
+	runs := make([]CompareRun, 0, 4)
+	for _, tr := range a.detector.Detect(st) {
+		req := provider.EvaluateRequest{Scenario: tr.Scenario, State: stateJSON, Runbook: a.RunbookText}
+		run := CompareRun{Scenario: tr.Scenario, Trigger: tr.Fingerprint}
+
+		echoResp, err := a.evaluateWith(ctx, req, a.echo)
+		if err != nil {
+			// Echo never fails on valid state; a failure is a wiring bug.
+			return nil, fmt.Errorf("echo on %s: %w", tr.Scenario, err)
+		}
+		provider.VerifyEvidence(stateJSON, echoResp.Findings)
+		run.Echo = sideOf(echoResp)
+
+		llmResp, llmErr := a.evaluateWith(ctx, req, a.primary)
+		if llmErr != nil {
+			run.LLMError = llmErr.Error()
+		} else if a.primary != nil {
+			provider.VerifyEvidence(stateJSON, llmResp.Findings)
+			run.LLM = sideOf(llmResp)
+		}
+		runs = append(runs, run)
+	}
+	return runs, nil
+}
+
+func sideOf(resp *provider.EvaluateResponse) *CompareSide {
+	return &CompareSide{
+		Model:           resp.Model,
+		LatencyMS:       resp.Latency.Milliseconds(),
+		InputTokens:     resp.InputTokens,
+		OutputTokens:    resp.OutputTokens,
+		Findings:        resp.Findings,
+		UnverifiedCount: countUnverified(resp.Findings),
+	}
+}
+
+func countUnverified(fs []provider.Finding) int {
+	n := 0
+	for _, f := range fs {
+		if f.Unverified {
+			n++
+		}
+	}
+	return n
+}
+
 // noteEvent consumes one SSE event: finished jobs feed exit_burst on the
 // next scan (the event's exit code mirrors the apiserver's: 0 → DONE).
 func (a *Agent) noteEvent(ev sclient.JobEvent) {

src/frontend/advisor/internal/agent/agent_test.go

@@ -272,3 +272,84 @@ func indexByte(b []byte, c byte) int {
 	}
 	return -1
 }
+
+// TestCompare_BothProvidersOnSharedSnapshot: Compare runs echo AND the
+// primary on one snapshot, per fired scenario, and reports both sides with
+// latency/tokens/unverified — without writing a decision-log record.
+func TestCompare_BothProvidersOnSharedSnapshot(t *testing.T) {
+	fc := stalledFixture()
+	fp := &fakeProvider{name: "fake"}
+	a, _, path := newTestAgent(t, fc, fp)
+
+	runs, err := a.Compare(context.Background())
+	if err != nil {
+		t.Fatalf("compare: %v", err)
+	}
+	if len(runs) != 1 || runs[0].Scenario != "pend_stall" {
+		t.Fatalf("want one pend_stall run, got %+v", runs)
+	}
+	run := runs[0]
+	if run.Echo == nil || run.LLM == nil {
+		t.Fatalf("both sides required: %+v", run)
+	}
+	if run.Echo.Model != "echo-rules" {
+		t.Errorf("echo side model = %q", run.Echo.Model)
+	}
+	if run.LLM.Model != "fake-model" {
+		t.Errorf("llm side model = %q", run.LLM.Model)
+	}
+	if len(run.LLM.Findings) != 1 || run.LLM.Findings[0].Summary != "fake says stalled" {
+		t.Errorf("llm findings wrong: %+v", run.LLM.Findings)
+	}
+	if run.Echo.LatencyMS != 0 {
+		t.Errorf("echo latency should be 0, got %d", run.Echo.LatencyMS)
+	}
+
+	// Measurement path: the decision log must stay empty.
+	data, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read log: %v", err)
+	}
+	if len(data) != 0 {
+		t.Fatalf("compare must not write decision records, got %q", string(data))
+	}
+}
+
+// errPrimary fails on the LLM leg; Compare reports the error without
+// degrading — the echo column still carries the answer.
+func TestCompare_LLMErrorsAreReportedNotDegraded(t *testing.T) {
+	fc := stalledFixture()
+	fp := &fakeProvider{name: "fake", err: context.DeadlineExceeded}
+	a, _, _ := newTestAgent(t, fc, fp)
+
+	runs, err := a.Compare(context.Background())
+	if err != nil {
+		t.Fatalf("compare: %v", err)
+	}
+	if len(runs) != 1 {
+		t.Fatalf("want one run, got %+v", runs)
+	}
+	if runs[0].LLMError == "" {
+		t.Fatal("want LLMError recorded")
+	}
+	if runs[0].Echo == nil || len(runs[0].Echo.Findings) == 0 {
+		t.Fatal("echo side must still answer")
+	}
+}
+
+// Clean cluster: nothing fires, Compare returns no rows.
+func TestCompare_CleanClusterNoRows(t *testing.T) {
+	fc := &fakeClient{
+		nodes: []*sclient.URMNode{{Name: "t1", Status: "busy", CoresTotal: 8, CoresUsed: 4}},
+	}
+	fp := &fakeProvider{name: "fake"}
+	a, _, _ := newTestAgent(t, fc, fp)
+
+	runs, err := a.Compare(context.Background())
+	if err != nil {
+		t.Fatalf("compare: %v", err)
+	}
+	if len(runs) != 0 {
+		t.Fatalf("clean cluster must produce no rows: %+v", runs)
+	}
+}