fix(advisor): cluster-wide job view + change-triggered detection

snapshot: list jobs with AllUsers + submit_time:desc + limit 200 — the
advisor must see every user's PEND jobs newest-first, or pend_stall sees
nothing on a shared cluster (the default list is the caller's own jobs,
oldest first).

detect: drop the per-scenario cooldown from the fire path. The original
cooldown-first order suppressed a CHANGED situation for up to 10 minutes —
on the live cluster a new job joining the stalled set went unreported
until the cooldown expired. Semantics now: an unchanged situation (same
dedup key) never re-evaluates; any changed situation fires immediately.
A thrashing situation (A→B→A) firing per change is honest reporting — a
flapping node or oscillating stalled set is real signal.

Verified on the t1/t2/t3 cluster: two forced-PEND jobs (unsatisfiable
select; 64-slot limit rejection) both diagnosed in the first scan, a
kill changing the stalled set re-triggered within one 30s round, and
identical situations stayed deduped.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-10-19 07:06:53 +00:00
parent 46bb23ea91
commit 4db1aa8aef
3 changed files with 54 additions and 46 deletions

View file

@ -1,8 +1,8 @@
// Package detect holds the advisor's deterministic triggers — cheap, no
// LLM. A detector looks at one snapshot and decides whether a scenario is
// worth a provider round: threshold, then cooldown (don't re-evaluate the
// same scenario too often), then dedup (don't re-evaluate an identical
// situation at all).
// worth a provider round: threshold, then dedup (never re-evaluate an
// identical situation), then cooldown (dampen A→B→A alternations that
// last-key dedup cannot catch).
package detect
import (
@ -25,9 +25,8 @@ const (
// Trigger is one fired detection.
type Trigger struct {
Scenario string
// DedupKey hashes the situation (job set + reason set / node set) so an
// unchanged situation is not re-evaluated on every scan. Two rounds with
// the same key produce the second finding only after the cooldown.
// DedupKey hashes the situation (job set + reason set / node set). The
// same key never fires twice in a row; any change fires immediately.
DedupKey string
// Fingerprint is a human-readable summary of what matched (logged, and
// what the echo provider re-derives from the state).
@ -42,8 +41,6 @@ type Limits struct {
PendSpikeMin int
// ExitBurstMin: non-zero exits within the window that fire.
ExitBurstMin int
// Cooldowns per scenario; a scenario on cooldown is skipped entirely.
Cooldowns map[string]time.Duration
}
// DefaultLimits returns the P0 defaults.
@ -52,32 +49,24 @@ func DefaultLimits() Limits {
PendStallAfter: 5 * time.Minute,
PendSpikeMin: 5,
ExitBurstMin: 3,
Cooldowns: map[string]time.Duration{
ScenarioPendStall: 10 * time.Minute,
ScenarioWorkerHealth: 30 * time.Minute,
ScenarioUtilSkew: 30 * time.Minute,
ScenarioExitBurst: 30 * time.Minute,
},
}
}
// Detector evaluates snapshots against the limits. Stateful: tracks
// per-scenario cooldown and the last dedup key seen.
// Detector evaluates snapshots against the limits. Stateful: tracks the
// last dedup key per scenario.
type Detector struct {
lim Limits
now func() time.Time // injectable clock for tests
lastFired map[string]time.Time
lastKey map[string]string
lastKey map[string]string
}
// NewDetector returns a Detector with the given limits.
func NewDetector(lim Limits) *Detector {
return &Detector{
lim: lim,
now: time.Now,
lastFired: map[string]time.Time{},
lastKey: map[string]string{},
lim: lim,
now: time.Now,
lastKey: map[string]string{},
}
}
@ -100,19 +89,20 @@ func (d *Detector) Detect(st *snapshot.State) []Trigger {
return out
}
// fire applies cooldown + dedup; ok=true means the scenario should be
// evaluated this round and the state recorded.
// fire applies dedup; ok=true means the scenario should be evaluated this
// round and the state recorded.
//
// Semantics (P0 verification finding): an unchanged situation never
// re-evaluates (the dedup key); a CHANGED situation always fires immediately.
// The original cooldown-first order hid a genuinely new stall for up to 10
// minutes on the live cluster — unacceptable for "things are getting worse".
// Situations that thrash (A→B→A) therefore also fire per change, which is
// honest: a flapping node or an oscillating stalled set is real signal.
func (d *Detector) fire(scenario, key, fingerprint string) (Trigger, bool) {
if cd, ok := d.lim.Cooldowns[scenario]; ok {
if last, fired := d.lastFired[scenario]; fired && d.now().Sub(last) < cd {
return Trigger{}, false
}
}
// Identical situation as last fired round: skip regardless of cooldown.
// Identical situation as last fired round: skip.
if d.lastKey[scenario] == key {
return Trigger{}, false
}
d.lastFired[scenario] = d.now()
d.lastKey[scenario] = key
return Trigger{Scenario: scenario, DedupKey: key, Fingerprint: fingerprint}, true
}