fbc81d61d1
- internal/snapshot: compact cluster state via schedulerclient (nodes, PEND top-N most-starved with parsed pending_reason, cluster sums, queue garnish, recent exits). Field whitelists + stall-threshold filter keep the state a few KiB. Module-local PEND-reason table mirrors mcp-server/internal/pendreason.go (lift into schedulerclient is a P1 cleanup). - internal/detect: pend_stall / worker_health / util_skew / exit_burst triggers — threshold, then per-scenario cooldown, then dedup on a hash of the situation (job set + reasons / node set) so an unchanged cluster is never re-evaluated. Co-Authored-By: Claude Code <noreply@anthropic.com>
95 lines
2.4 KiB
Go
95 lines
2.4 KiB
Go
package snapshot
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
)
|
|
|
|
// pendingReason mirrors the apiserver rawPendReason (convert_pure.go),
|
|
// surfaced by schedulerclient as the JSON-encoded string
|
|
// Extensions["pending_reason"].
|
|
type pendingReason struct {
|
|
MainReasonName string `json:"main_reason_name"`
|
|
}
|
|
|
|
// reasonScenario maps a PREASON_* name to the human-readable scenario.
|
|
// This mirrors mcp-server/internal/pendreason.go (module-local copy: the
|
|
// advisor and mcp-server are separate modules; lifting the table into
|
|
// schedulerclient is a deliberate P1 cleanup, not forgotten).
|
|
func reasonScenario(name string) string {
|
|
switch name {
|
|
case "bad_resreq":
|
|
return "the resource request is invalid or cannot be satisfied"
|
|
case "no_resource":
|
|
return "insufficient resource capacity"
|
|
case "match_fail":
|
|
return "no host matches the job's host-selection condition"
|
|
case "limit_global":
|
|
return "a global quota is exhausted"
|
|
case "limit_host":
|
|
return "a host-level quota is exhausted"
|
|
case "queue_closed":
|
|
return "the queue is closed (not accepting new jobs)"
|
|
case "queue_draining":
|
|
return "the queue is draining (no new jobs until it empties)"
|
|
case "time_window":
|
|
return "outside the queue's dispatch window"
|
|
case "host_unavail":
|
|
return "the host is unavailable"
|
|
default:
|
|
return fmt.Sprintf("unknown reason (%s)", name)
|
|
}
|
|
}
|
|
|
|
// parsePendingReason extracts the main reason name + scenario from a job's
|
|
// Extensions. Returns nil when absent or unparseable — a missing reason is
|
|
// reported as such, never guessed.
|
|
func parsePendingReason(ext map[string]any) *JobReason {
|
|
raw, ok := ext["pending_reason"]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
var pr pendingReason
|
|
switch v := raw.(type) {
|
|
case string:
|
|
if v == "" {
|
|
return nil
|
|
}
|
|
if err := json.Unmarshal([]byte(v), &pr); err != nil {
|
|
return nil
|
|
}
|
|
case map[string]any:
|
|
b, _ := json.Marshal(v)
|
|
if err := json.Unmarshal(b, &pr); err != nil {
|
|
return nil
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
if pr.MainReasonName == "" {
|
|
return nil
|
|
}
|
|
return &JobReason{
|
|
MainReasonName: pr.MainReasonName,
|
|
Scenario: reasonScenario(pr.MainReasonName),
|
|
}
|
|
}
|
|
|
|
// round2 keeps snapshot numbers at two decimals so the JSON stays small.
|
|
func round2(f float64) float64 {
|
|
return float64(int(f*100+0.5*sign(f))) / 100
|
|
}
|
|
|
|
func sign(f float64) float64 {
|
|
if f < 0 {
|
|
return -1
|
|
}
|
|
return 1
|
|
}
|
|
|
|
func frac(a, b int) float64 {
|
|
if b == 0 {
|
|
return 0
|
|
}
|
|
return float64(a) / float64(b)
|
|
}
|