fix(advisor): disarm dedup key on full resolution so recurrences report

Live-cluster gap found 2026-10-20 (t1/t2/t3): t3 went down, recovered,
went down again — and the second outage never reported. The dedup key
survived the healthy rounds, so the recurring situation looked
"unchanged" forever. Same hazard for pend_stall: a stall that empties
and later re-forms identically stays silent.

Fix: fire/clear pairing. When a detector's condition is fully absent
(all nodes healthy, no stalled jobs, no skew), it clears its scenario's
dedup key; the next occurrence is a fresh situation and fires. Unit
tests pin the outage→recovery→identical-outage cycle for worker_health
and the stall→resolve→re-form cycle for pend_stall.

exit_burst deliberately does NOT clear: RecentExits is a per-scan
window (consumed once), so an empty window is the steady state, not a
resolution — clearing there would re-fire bursts whose situation never
changed.

Cluster-verified cycle: t3 close → fired (glm, 11.1s); open → silent;
close again → fired again (22.1s, 2 findings). All detect tests green;
advisor restarted on the fix.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-10-20 02:21:42 +00:00
parent 321544f7be
commit e97f6b1687
2 changed files with 84 additions and 5 deletions

View file

@ -1,8 +1,9 @@
// Package detect holds the advisor's deterministic triggers — cheap, no // Package detect holds the advisor's deterministic triggers — cheap, no
// LLM. A detector looks at one snapshot and decides whether a scenario is // LLM. A detector looks at one snapshot and decides whether a scenario is
// worth a provider round: threshold, then dedup (never re-evaluate an // worth a provider round: threshold, then dedup. The dedup key hashes the
// identical situation), then cooldown (dampen A→B→A alternations that // situation: an unchanged situation never re-fires, any change fires
// last-key dedup cannot catch). // immediately, and a situation that resolves fully disarms its key so a
// later recurrence reports again (fire/clear pair).
package detect package detect
import ( import (
@ -26,7 +27,8 @@ const (
type Trigger struct { type Trigger struct {
Scenario string Scenario string
// DedupKey hashes the situation (job set + reason set / node set). The // DedupKey hashes the situation (job set + reason set / node set). The
// same key never fires twice in a row; any change fires immediately. // same key never fires twice in a row; any change fires immediately;
// a full resolution disarms the key (clear) so a recurrence fires again.
DedupKey string DedupKey string
// Fingerprint is a human-readable summary of what matched (logged, and // Fingerprint is a human-readable summary of what matched (logged, and
// what the echo provider re-derives from the state). // what the echo provider re-derives from the state).
@ -71,7 +73,9 @@ func NewDetector(lim Limits) *Detector {
} }
// Detect runs all detectors over the snapshot and returns the scenarios to // Detect runs all detectors over the snapshot and returns the scenarios to
// evaluate this round, in stable order. // evaluate this round, in stable order. Each detector arms its dedup key
// when it fires and disarms it (clear) when its condition is absent, so a
// resolved-then-recurred situation reports again.
func (d *Detector) Detect(st *snapshot.State) []Trigger { func (d *Detector) Detect(st *snapshot.State) []Trigger {
var out []Trigger var out []Trigger
if t, ok := d.pendStall(st); ok { if t, ok := d.pendStall(st); ok {
@ -107,8 +111,20 @@ func (d *Detector) fire(scenario, key, fingerprint string) (Trigger, bool) {
return Trigger{Scenario: scenario, DedupKey: key, Fingerprint: fingerprint}, true return Trigger{Scenario: scenario, DedupKey: key, Fingerprint: fingerprint}, true
} }
// clear forgets the scenario's dedup key when its condition is absent this
// round. Without this, a situation that resolves and then recurs identically
// (same node goes down twice, the same stall re-forms) is swallowed as
// "unchanged" forever — observed live (2026-10-20): t3 went down, recovered,
// went down again, and the second outage never reported. The key is armed by
// fire() and disarmed by clear(); each detector's healthy/empty early-return
// is the disarm point.
func (d *Detector) clear(scenario string) {
delete(d.lastKey, scenario)
}
func (d *Detector) pendStall(st *snapshot.State) (Trigger, bool) { func (d *Detector) pendStall(st *snapshot.State) (Trigger, bool) {
if len(st.PendTop) == 0 && st.Cluster.PendingJobs < d.lim.PendSpikeMin { if len(st.PendTop) == 0 && st.Cluster.PendingJobs < d.lim.PendSpikeMin {
d.clear(ScenarioPendStall)
return Trigger{}, false return Trigger{}, false
} }
// Key: the stalled job set and their reasons. // Key: the stalled job set and their reasons.
@ -126,6 +142,7 @@ func (d *Detector) workerHealth(st *snapshot.State) (Trigger, bool) {
} }
} }
if len(bad) == 0 { if len(bad) == 0 {
d.clear(ScenarioWorkerHealth)
return Trigger{}, false return Trigger{}, false
} }
key := hashOf(join(bad, ",")) key := hashOf(join(bad, ","))
@ -146,6 +163,7 @@ func (d *Detector) utilSkew(st *snapshot.State) (Trigger, bool) {
} }
} }
if len(hot) == 0 || len(idle) == 0 { if len(hot) == 0 || len(idle) == 0 {
d.clear(ScenarioUtilSkew)
return Trigger{}, false return Trigger{}, false
} }
key := hashOf("hot=" + join(hot, ",") + ";idle=" + join(idle, ",")) key := hashOf("hot=" + join(hot, ",") + ";idle=" + join(idle, ","))
@ -161,6 +179,10 @@ func (d *Detector) exitBurst(st *snapshot.State) (Trigger, bool) {
} }
} }
if len(ids) < d.lim.ExitBurstMin { if len(ids) < d.lim.ExitBurstMin {
// Note: NOT cleared. RecentExits is a per-scan window (exits are
// consumed once, see snapshot Build); an empty window is the steady
// state, not "the burst resolved" — arming/disarming here would
// re-fire a burst whose situation never actually changed.
return Trigger{}, false return Trigger{}, false
} }
key := hashOf(join(ids, ",")) key := hashOf(join(ids, ","))

View file

@ -98,6 +98,63 @@ func TestDetect_WorkerHealth(t *testing.T) {
} }
} }
// TestDetect_RecoveredThenRecurred fires again covers the live-cluster gap
// found 2026-10-20 (t1/t2/t3): t3 went down, recovered, went down again —
// and the second outage was silently deduped because the key survived the
// healthy rounds. Resolution must disarm the scenario's key.
func TestDetect_RecoveredThenRecurredFiresAgain(t *testing.T) {
st := healthyState()
st.Nodes[1].Status = "down"
d := NewDetector(DefaultLimits())
if trigs := d.Detect(st); len(trigs) != 1 {
t.Fatalf("first outage must fire: %+v", trigs)
}
// Recovery: all nodes healthy → no trigger, key disarmed.
st.Nodes[1].Status = "OK"
if got := d.Detect(st); len(got) != 0 {
t.Fatalf("recovery must not fire: %+v", got)
}
// The identical outage recurs → must fire again (was silently deduped).
st.Nodes[1].Status = "down"
trigs := d.Detect(st)
if len(trigs) != 1 || trigs[0].Scenario != ScenarioWorkerHealth {
t.Fatalf("recurred identical outage must fire again: %+v", trigs)
}
// And the fire/clear cycle is repeatable.
st.Nodes[1].Status = "OK"
if got := d.Detect(st); len(got) != 0 {
t.Fatalf("second recovery must not fire: %+v", got)
}
st.Nodes[1].Status = "down"
if trigs := d.Detect(st); len(trigs) != 1 {
t.Fatalf("third outage must fire: %+v", trigs)
}
}
// Same contract for pend_stall: the stalled set emptying disarms the key, so
// a stall re-forming from scratch reports again.
func TestDetect_PendStall_ResolvedThenReformed(t *testing.T) {
st := stalledState()
d := NewDetector(DefaultLimits())
if trigs := d.Detect(st); len(trigs) != 1 || trigs[0].Scenario != ScenarioPendStall {
t.Fatalf("stall must fire: %+v", trigs)
}
// Stall resolves (no pending at all) → silent, key disarmed.
empty := healthyState()
if got := d.Detect(empty); len(got) != 0 {
t.Fatalf("resolution must not fire: %+v", got)
}
// The same stall re-forms → fires again.
if trigs := d.Detect(st); len(trigs) != 1 || trigs[0].Scenario != ScenarioPendStall {
t.Fatalf("re-formed stall must fire again: %+v", trigs)
}
}
func TestDetect_UtilSkew(t *testing.T) { func TestDetect_UtilSkew(t *testing.T) {
st := healthyState() st := healthyState()
st.Nodes[0].Utilization = 0.9 st.Nodes[0].Utilization = 0.9