fix(advisor): cluster-wide job view + change-triggered detection
snapshot: list jobs with AllUsers + submit_time:desc + limit 200 — the advisor must see every user's PEND jobs newest-first, or pend_stall sees nothing on a shared cluster (the default list is the caller's own jobs, oldest first). detect: drop the per-scenario cooldown from the fire path. The original cooldown-first order suppressed a CHANGED situation for up to 10 minutes — on the live cluster a new job joining the stalled set went unreported until the cooldown expired. Semantics now: an unchanged situation (same dedup key) never re-evaluates; any changed situation fires immediately. A thrashing situation (A→B→A) firing per change is honest reporting — a flapping node or oscillating stalled set is real signal. Verified on the t1/t2/t3 cluster: two forced-PEND jobs (unsatisfiable select; 64-slot limit rejection) both diagnosed in the first scan, a kill changing the stalled set re-triggered within one 30s round, and identical situations stayed deduped. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
parent
46bb23ea91
commit
4db1aa8aef
3 changed files with 54 additions and 46 deletions
|
|
@ -1,8 +1,8 @@
|
|||
// Package detect holds the advisor's deterministic triggers — cheap, no
|
||||
// LLM. A detector looks at one snapshot and decides whether a scenario is
|
||||
// worth a provider round: threshold, then cooldown (don't re-evaluate the
|
||||
// same scenario too often), then dedup (don't re-evaluate an identical
|
||||
// situation at all).
|
||||
// worth a provider round: threshold, then dedup (never re-evaluate an
|
||||
// identical situation), then cooldown (dampen A→B→A alternations that
|
||||
// last-key dedup cannot catch).
|
||||
package detect
|
||||
|
||||
import (
|
||||
|
|
@ -25,9 +25,8 @@ const (
|
|||
// Trigger is one fired detection.
|
||||
type Trigger struct {
|
||||
Scenario string
|
||||
// DedupKey hashes the situation (job set + reason set / node set) so an
|
||||
// unchanged situation is not re-evaluated on every scan. Two rounds with
|
||||
// the same key produce the second finding only after the cooldown.
|
||||
// DedupKey hashes the situation (job set + reason set / node set). The
|
||||
// same key never fires twice in a row; any change fires immediately.
|
||||
DedupKey string
|
||||
// Fingerprint is a human-readable summary of what matched (logged, and
|
||||
// what the echo provider re-derives from the state).
|
||||
|
|
@ -42,8 +41,6 @@ type Limits struct {
|
|||
PendSpikeMin int
|
||||
// ExitBurstMin: non-zero exits within the window that fire.
|
||||
ExitBurstMin int
|
||||
// Cooldowns per scenario; a scenario on cooldown is skipped entirely.
|
||||
Cooldowns map[string]time.Duration
|
||||
}
|
||||
|
||||
// DefaultLimits returns the P0 defaults.
|
||||
|
|
@ -52,32 +49,24 @@ func DefaultLimits() Limits {
|
|||
PendStallAfter: 5 * time.Minute,
|
||||
PendSpikeMin: 5,
|
||||
ExitBurstMin: 3,
|
||||
Cooldowns: map[string]time.Duration{
|
||||
ScenarioPendStall: 10 * time.Minute,
|
||||
ScenarioWorkerHealth: 30 * time.Minute,
|
||||
ScenarioUtilSkew: 30 * time.Minute,
|
||||
ScenarioExitBurst: 30 * time.Minute,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Detector evaluates snapshots against the limits. Stateful: tracks
|
||||
// per-scenario cooldown and the last dedup key seen.
|
||||
// Detector evaluates snapshots against the limits. Stateful: tracks the
|
||||
// last dedup key per scenario.
|
||||
type Detector struct {
|
||||
lim Limits
|
||||
now func() time.Time // injectable clock for tests
|
||||
|
||||
lastFired map[string]time.Time
|
||||
lastKey map[string]string
|
||||
lastKey map[string]string
|
||||
}
|
||||
|
||||
// NewDetector returns a Detector with the given limits.
|
||||
func NewDetector(lim Limits) *Detector {
|
||||
return &Detector{
|
||||
lim: lim,
|
||||
now: time.Now,
|
||||
lastFired: map[string]time.Time{},
|
||||
lastKey: map[string]string{},
|
||||
lim: lim,
|
||||
now: time.Now,
|
||||
lastKey: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -100,19 +89,20 @@ func (d *Detector) Detect(st *snapshot.State) []Trigger {
|
|||
return out
|
||||
}
|
||||
|
||||
// fire applies cooldown + dedup; ok=true means the scenario should be
|
||||
// evaluated this round and the state recorded.
|
||||
// fire applies dedup; ok=true means the scenario should be evaluated this
|
||||
// round and the state recorded.
|
||||
//
|
||||
// Semantics (P0 verification finding): an unchanged situation never
|
||||
// re-evaluates (the dedup key); a CHANGED situation always fires immediately.
|
||||
// The original cooldown-first order hid a genuinely new stall for up to 10
|
||||
// minutes on the live cluster — unacceptable for "things are getting worse".
|
||||
// Situations that thrash (A→B→A) therefore also fire per change, which is
|
||||
// honest: a flapping node or an oscillating stalled set is real signal.
|
||||
func (d *Detector) fire(scenario, key, fingerprint string) (Trigger, bool) {
|
||||
if cd, ok := d.lim.Cooldowns[scenario]; ok {
|
||||
if last, fired := d.lastFired[scenario]; fired && d.now().Sub(last) < cd {
|
||||
return Trigger{}, false
|
||||
}
|
||||
}
|
||||
// Identical situation as last fired round: skip regardless of cooldown.
|
||||
// Identical situation as last fired round: skip.
|
||||
if d.lastKey[scenario] == key {
|
||||
return Trigger{}, false
|
||||
}
|
||||
d.lastFired[scenario] = d.now()
|
||||
d.lastKey[scenario] = key
|
||||
return Trigger{Scenario: scenario, DedupKey: key, Fingerprint: fingerprint}, true
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue