diff --git a/src/frontend/advisor/cmd/main.go b/src/frontend/advisor/cmd/main.go index b10be69c..a6d918ac 100644 --- a/src/frontend/advisor/cmd/main.go +++ b/src/frontend/advisor/cmd/main.go @@ -7,8 +7,9 @@ // // Usage: // -// advisor run # long-running observe loop -// advisor once # one scan + evaluate round, report to stdout (no log) +// advisor run # long-running observe loop +// advisor once # one scan + evaluate round, report to stdout (no log) +// advisor compare # one snapshot through echo AND the LLM, side by side package main import ( @@ -41,6 +42,8 @@ func main() { cmdRun() case "once": cmdOnce() + case "compare": + cmdCompare() case "-h", "--help", "help": usage() default: @@ -54,8 +57,9 @@ func usage() { fmt.Fprint(os.Stderr, `advisor — XCE scheduler advisor (observe mode) usage: - advisor run start the observe loop (periodic scan + SSE events) - advisor once run one scan/evaluate round and print the report + advisor run start the observe loop (periodic scan + SSE events) + advisor once run one scan/evaluate round and print the report + advisor compare one snapshot through echo AND the LLM, side by side configuration (env): XCE_API_KEY required — mcp_readonly service-account key @@ -164,3 +168,26 @@ func discardLog() *declog.Log { } return l } + +// cmdCompare answers "what does the LLM add over echo?" with data: one +// snapshot, every fired scenario evaluated by both providers, results +// side-by-side on stdout. Nothing is logged or pushed — `run` stays the +// only writer of the decision log. +func cmdCompare() { + cfg := mustConfig() + if !cfg.LLMEnabled() { + log.Fatalf("advisor: compare needs ADVISOR_LLM_BASE_URL (echo vs echo is a tautology)") + } + rb := loadRunbook(cfg) + + a := agent.New(mustClient(cfg), detect.NewDetector(detect.DefaultLimits()), + pickProvider(cfg), discardLog(), cfg.Interval) + a.RunbookText = rb.Render() + + runs, err := a.Compare(context.Background()) + if err != nil { + log.Fatalf("advisor: compare: %v", err) + } + out, _ := json.MarshalIndent(runs, "", " ") + fmt.Println(string(out)) +} diff --git a/src/frontend/advisor/internal/agent/agent.go b/src/frontend/advisor/internal/agent/agent.go index 561c1b46..542b182a 100644 --- a/src/frontend/advisor/internal/agent/agent.go +++ b/src/frontend/advisor/internal/agent/agent.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "fmt" "log" "strconv" "time" @@ -200,6 +201,90 @@ func (a *Agent) Once(ctx context.Context) (*OnceReport, error) { return rep, nil } +// CompareRun is one scenario evaluated by both providers on the same +// snapshot — the echo-vs-LLM comparison row. +type CompareRun struct { + Scenario string `json:"scenario"` + // Trigger is the detector fingerprint that fired this scenario. + Trigger string `json:"trigger"` + Echo *CompareSide `json:"echo"` + LLM *CompareSide `json:"llm"` + // LLMError is set when the LLM call failed (the row then shows the + // degradation the observe loop would have recorded). + LLMError string `json:"llm_error,omitempty"` +} + +// CompareSide is one provider's answer on the shared snapshot. +type CompareSide struct { + Model string `json:"model"` + LatencyMS int64 `json:"latency_ms"` + InputTokens int `json:"input_tokens,omitempty"` + OutputTokens int `json:"output_tokens,omitempty"` + Findings []provider.Finding `json:"findings"` + // UnverifiedCount summarizes the hallucination firewall's verdicts. + UnverifiedCount int `json:"unverified_count"` +} + +// Compare runs one snapshot through BOTH providers per fired scenario and +// returns side-by-side results. Nothing is logged or pushed — this is the +// measurement path for "what does the LLM add over echo" (decision-log +// records from observe rounds are the other half of that evidence). +func (a *Agent) Compare(ctx context.Context) ([]CompareRun, error) { + st, err := a.builder.Build(ctx, nil) + if err != nil { + return nil, err + } + stateJSON, err := json.Marshal(st) + if err != nil { + return nil, err + } + + runs := make([]CompareRun, 0, 4) + for _, tr := range a.detector.Detect(st) { + req := provider.EvaluateRequest{Scenario: tr.Scenario, State: stateJSON, Runbook: a.RunbookText} + run := CompareRun{Scenario: tr.Scenario, Trigger: tr.Fingerprint} + + echoResp, err := a.evaluateWith(ctx, req, a.echo) + if err != nil { + // Echo never fails on valid state; a failure is a wiring bug. + return nil, fmt.Errorf("echo on %s: %w", tr.Scenario, err) + } + provider.VerifyEvidence(stateJSON, echoResp.Findings) + run.Echo = sideOf(echoResp) + + llmResp, llmErr := a.evaluateWith(ctx, req, a.primary) + if llmErr != nil { + run.LLMError = llmErr.Error() + } else if a.primary != nil { + provider.VerifyEvidence(stateJSON, llmResp.Findings) + run.LLM = sideOf(llmResp) + } + runs = append(runs, run) + } + return runs, nil +} + +func sideOf(resp *provider.EvaluateResponse) *CompareSide { + return &CompareSide{ + Model: resp.Model, + LatencyMS: resp.Latency.Milliseconds(), + InputTokens: resp.InputTokens, + OutputTokens: resp.OutputTokens, + Findings: resp.Findings, + UnverifiedCount: countUnverified(resp.Findings), + } +} + +func countUnverified(fs []provider.Finding) int { + n := 0 + for _, f := range fs { + if f.Unverified { + n++ + } + } + return n +} + // noteEvent consumes one SSE event: finished jobs feed exit_burst on the // next scan (the event's exit code mirrors the apiserver's: 0 → DONE). func (a *Agent) noteEvent(ev sclient.JobEvent) { diff --git a/src/frontend/advisor/internal/agent/agent_test.go b/src/frontend/advisor/internal/agent/agent_test.go index 40ecc4b3..4761fdf9 100644 --- a/src/frontend/advisor/internal/agent/agent_test.go +++ b/src/frontend/advisor/internal/agent/agent_test.go @@ -272,3 +272,84 @@ func indexByte(b []byte, c byte) int { } return -1 } + +// TestCompare_BothProvidersOnSharedSnapshot: Compare runs echo AND the +// primary on one snapshot, per fired scenario, and reports both sides with +// latency/tokens/unverified — without writing a decision-log record. +func TestCompare_BothProvidersOnSharedSnapshot(t *testing.T) { + fc := stalledFixture() + fp := &fakeProvider{name: "fake"} + a, _, path := newTestAgent(t, fc, fp) + + runs, err := a.Compare(context.Background()) + if err != nil { + t.Fatalf("compare: %v", err) + } + if len(runs) != 1 || runs[0].Scenario != "pend_stall" { + t.Fatalf("want one pend_stall run, got %+v", runs) + } + run := runs[0] + if run.Echo == nil || run.LLM == nil { + t.Fatalf("both sides required: %+v", run) + } + if run.Echo.Model != "echo-rules" { + t.Errorf("echo side model = %q", run.Echo.Model) + } + if run.LLM.Model != "fake-model" { + t.Errorf("llm side model = %q", run.LLM.Model) + } + if len(run.LLM.Findings) != 1 || run.LLM.Findings[0].Summary != "fake says stalled" { + t.Errorf("llm findings wrong: %+v", run.LLM.Findings) + } + if run.Echo.LatencyMS != 0 { + t.Errorf("echo latency should be 0, got %d", run.Echo.LatencyMS) + } + + // Measurement path: the decision log must stay empty. + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read log: %v", err) + } + if len(data) != 0 { + t.Fatalf("compare must not write decision records, got %q", string(data)) + } +} + +// errPrimary fails on the LLM leg; Compare reports the error without +// degrading — the echo column still carries the answer. +func TestCompare_LLMErrorsAreReportedNotDegraded(t *testing.T) { + fc := stalledFixture() + fp := &fakeProvider{name: "fake", err: context.DeadlineExceeded} + a, _, _ := newTestAgent(t, fc, fp) + + runs, err := a.Compare(context.Background()) + if err != nil { + t.Fatalf("compare: %v", err) + } + if len(runs) != 1 { + t.Fatalf("want one run, got %+v", runs) + } + if runs[0].LLMError == "" { + t.Fatal("want LLMError recorded") + } + if runs[0].Echo == nil || len(runs[0].Echo.Findings) == 0 { + t.Fatal("echo side must still answer") + } +} + +// Clean cluster: nothing fires, Compare returns no rows. +func TestCompare_CleanClusterNoRows(t *testing.T) { + fc := &fakeClient{ + nodes: []*sclient.URMNode{{Name: "t1", Status: "busy", CoresTotal: 8, CoresUsed: 4}}, + } + fp := &fakeProvider{name: "fake"} + a, _, _ := newTestAgent(t, fc, fp) + + runs, err := a.Compare(context.Background()) + if err != nil { + t.Fatalf("compare: %v", err) + } + if len(runs) != 0 { + t.Fatalf("clean cluster must produce no rows: %+v", runs) + } +}