feat(advisor): snapshot builder + deterministic detectors
- internal/snapshot: compact cluster state via schedulerclient (nodes, PEND top-N most-starved with parsed pending_reason, cluster sums, queue garnish, recent exits). Field whitelists + stall-threshold filter keep the state a few KiB. Module-local PEND-reason table mirrors mcp-server/internal/pendreason.go (lift into schedulerclient is a P1 cleanup). - internal/detect: pend_stall / worker_health / util_skew / exit_burst triggers — threshold, then per-scenario cooldown, then dedup on a hash of the situation (job set + reasons / node set) so an unchanged cluster is never re-evaluated. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
parent
1ab96521b8
commit
fbc81d61d1
7 changed files with 850 additions and 5 deletions
95
src/frontend/advisor/internal/snapshot/pendreason.go
Normal file
95
src/frontend/advisor/internal/snapshot/pendreason.go
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
package snapshot
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// pendingReason mirrors the apiserver rawPendReason (convert_pure.go),
|
||||
// surfaced by schedulerclient as the JSON-encoded string
|
||||
// Extensions["pending_reason"].
|
||||
type pendingReason struct {
|
||||
MainReasonName string `json:"main_reason_name"`
|
||||
}
|
||||
|
||||
// reasonScenario maps a PREASON_* name to the human-readable scenario.
|
||||
// This mirrors mcp-server/internal/pendreason.go (module-local copy: the
|
||||
// advisor and mcp-server are separate modules; lifting the table into
|
||||
// schedulerclient is a deliberate P1 cleanup, not forgotten).
|
||||
func reasonScenario(name string) string {
|
||||
switch name {
|
||||
case "bad_resreq":
|
||||
return "the resource request is invalid or cannot be satisfied"
|
||||
case "no_resource":
|
||||
return "insufficient resource capacity"
|
||||
case "match_fail":
|
||||
return "no host matches the job's host-selection condition"
|
||||
case "limit_global":
|
||||
return "a global quota is exhausted"
|
||||
case "limit_host":
|
||||
return "a host-level quota is exhausted"
|
||||
case "queue_closed":
|
||||
return "the queue is closed (not accepting new jobs)"
|
||||
case "queue_draining":
|
||||
return "the queue is draining (no new jobs until it empties)"
|
||||
case "time_window":
|
||||
return "outside the queue's dispatch window"
|
||||
case "host_unavail":
|
||||
return "the host is unavailable"
|
||||
default:
|
||||
return fmt.Sprintf("unknown reason (%s)", name)
|
||||
}
|
||||
}
|
||||
|
||||
// parsePendingReason extracts the main reason name + scenario from a job's
|
||||
// Extensions. Returns nil when absent or unparseable — a missing reason is
|
||||
// reported as such, never guessed.
|
||||
func parsePendingReason(ext map[string]any) *JobReason {
|
||||
raw, ok := ext["pending_reason"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var pr pendingReason
|
||||
switch v := raw.(type) {
|
||||
case string:
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal([]byte(v), &pr); err != nil {
|
||||
return nil
|
||||
}
|
||||
case map[string]any:
|
||||
b, _ := json.Marshal(v)
|
||||
if err := json.Unmarshal(b, &pr); err != nil {
|
||||
return nil
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
if pr.MainReasonName == "" {
|
||||
return nil
|
||||
}
|
||||
return &JobReason{
|
||||
MainReasonName: pr.MainReasonName,
|
||||
Scenario: reasonScenario(pr.MainReasonName),
|
||||
}
|
||||
}
|
||||
|
||||
// round2 keeps snapshot numbers at two decimals so the JSON stays small.
|
||||
func round2(f float64) float64 {
|
||||
return float64(int(f*100+0.5*sign(f))) / 100
|
||||
}
|
||||
|
||||
func sign(f float64) float64 {
|
||||
if f < 0 {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
func frac(a, b int) float64 {
|
||||
if b == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(a) / float64(b)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue