fix(frontend): count host job-state columns and EXEC_HOST by real slots
NJOBS/RUN/SSUSP/USUSP (xcecli hosts list -l) and EXEC_HOST (xcecli jobs list) always showed a hardcoded 1 slot per host regardless of a job's actual per-host allocation -- rawJobToModel()'s ExecHosts JSONB builder never read the real per-host "slots" resource from the C job allocation, so the already-correct downstream slot-counting logic (computeHostSlotUsage, bug-1743) and EXEC_HOST's "N*host" formatter never had real data to work with. convert.go now extracts each host's real slots allocation via a new CGO indexing helper, mirroring the scheduler's own get_nslots_in_alloc() convention. Also wires hosts list -l's "CURRENT LOAD USED FOR SCHEDULING" table (tmp/ swap/mem/slots and friends) to the host's real live resource values -- previously entirely hardcoded placeholder strings, never connected to any backend data.
This commit is contained in:
parent
f55b49f1a7
commit
e395eaa3ad
5 changed files with 223 additions and 28 deletions
|
|
@ -9,6 +9,10 @@ package backend
|
|||
|
||||
// Helper to index into xce_host_alloc_t** (CGO cannot dereference C pointer-to-pointer arrays).
|
||||
static xce_host_alloc_t *job_alloc_host(xce_job_alloc_t *a, int i) { return a->allocs[i]; }
|
||||
|
||||
// Helper to index into xce_alloc_t** (same CGO limitation as above) -- one
|
||||
// host's per-resource allocation list, scanned for the "slots" entry.
|
||||
static xce_alloc_t *host_alloc_res(xce_host_alloc_t *h, int i) { return h->allocs[i]; }
|
||||
*/
|
||||
import "C"
|
||||
|
||||
|
|
@ -114,9 +118,24 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
|
|||
n := int(job.allocs.nhosts)
|
||||
for i := 0; i < n; i++ {
|
||||
h := C.job_alloc_host(job.allocs, C.int(i))
|
||||
if h != nil && h.host != nil {
|
||||
r.execHosts = append(r.execHosts, C.GoString(h.host))
|
||||
if h == nil || h.host == nil {
|
||||
continue
|
||||
}
|
||||
// bug-1743-followup: this host's own "slots" resource
|
||||
// allocation (xce_host_alloc_t.allocs[]), mirroring the C
|
||||
// scheduler's own get_nslots_in_alloc()/repl_worker_alloc_
|
||||
// nslots() resname scan -- defaults to 1 when absent, same as
|
||||
// those.
|
||||
nslots := 1
|
||||
nallocs := int(h.nallocs)
|
||||
for j := 0; j < nallocs; j++ {
|
||||
res := C.host_alloc_res(h, C.int(j))
|
||||
if res != nil && res.resname != nil && C.GoString(res.resname) == "slots" {
|
||||
nslots = int(res.value)
|
||||
break
|
||||
}
|
||||
}
|
||||
r.execHosts = append(r.execHosts, rawHostSlot{host: C.GoString(h.host), slots: nslots})
|
||||
}
|
||||
}
|
||||
return rawJobToModel(r)
|
||||
|
|
|
|||
|
|
@ -130,6 +130,16 @@ type rawSubmitData struct {
|
|||
fromHost string
|
||||
}
|
||||
|
||||
// rawHostSlot mirrors one xce_host_alloc_t's host name plus its "slots"
|
||||
// resource allocation count (xce_host_alloc_t.allocs[], scanned for
|
||||
// resname=="slots" -- mirrors the C scheduler's own get_nslots_in_alloc()/
|
||||
// repl_worker_alloc_nslots() convention, defaulting to 1 when no explicit
|
||||
// slots entry is present).
|
||||
type rawHostSlot struct {
|
||||
host string
|
||||
slots int
|
||||
}
|
||||
|
||||
// rawJobData mirrors the C xce_job_t fields used by the Go model.
|
||||
type rawJobData struct {
|
||||
jobID int64
|
||||
|
|
@ -140,7 +150,7 @@ type rawJobData struct {
|
|||
startTime int64
|
||||
finishTime int64
|
||||
submit *rawSubmitData
|
||||
execHosts []string
|
||||
execHosts []rawHostSlot
|
||||
}
|
||||
|
||||
// rawWorkerData mirrors the C xce_worker_t fields used by the Go model.
|
||||
|
|
@ -225,7 +235,7 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
|
|||
}
|
||||
slots := make([]hostSlot, len(r.execHosts))
|
||||
for i, h := range r.execHosts {
|
||||
slots[i] = hostSlot{HostName: h, NumSlots: 1}
|
||||
slots[i] = hostSlot{HostName: h.host, NumSlots: h.slots}
|
||||
}
|
||||
data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)})
|
||||
info.ExecHosts = models.Jsonb{String: string(data)}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package backend
|
|||
|
||||
import (
|
||||
"apiserver/models"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -60,6 +61,80 @@ func TestRawJobToModel_BasicFields(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// bug-1743-followup: rawJobToModel must render each host's REAL per-host
|
||||
// slot count into ExecHosts' JSONB "num_slots" field (consumed by xcecli's
|
||||
// EXEC_HOST column, "N*host", and by controllers/hosts.go's
|
||||
// computeHostSlotUsage() for the NJOBS/RUN/SSUSP/USUSP columns) -- not a
|
||||
// hardcoded 1 regardless of how many slots the job actually consumed on
|
||||
// that host.
|
||||
func TestRawJobToModel_ExecHostsUsesRealPerHostSlotCount(t *testing.T) {
|
||||
r := rawJobData{
|
||||
jobID: 1,
|
||||
state: 2, // JOB_STAT_RUN
|
||||
execHosts: []rawHostSlot{
|
||||
{host: "pc13v", slots: 4},
|
||||
},
|
||||
}
|
||||
|
||||
info := rawJobToModel(r)
|
||||
|
||||
var parsed struct {
|
||||
HostSlots []struct {
|
||||
HostName string `json:"host_name"`
|
||||
NumSlots int `json:"num_slots"`
|
||||
} `json:"host_slots"`
|
||||
NumHosts int `json:"num_hosts"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(info.ExecHosts.String), &parsed); err != nil {
|
||||
t.Fatalf("failed to unmarshal ExecHosts JSONB: %v", err)
|
||||
}
|
||||
if len(parsed.HostSlots) != 1 {
|
||||
t.Fatalf("HostSlots: got %d entries, want 1", len(parsed.HostSlots))
|
||||
}
|
||||
if parsed.HostSlots[0].HostName != "pc13v" {
|
||||
t.Errorf("HostName: got %q, want %q", parsed.HostSlots[0].HostName, "pc13v")
|
||||
}
|
||||
if parsed.HostSlots[0].NumSlots != 4 {
|
||||
t.Errorf("NumSlots: got %d, want 4 (must reflect the job's real per-host allocation, not a hardcoded 1)",
|
||||
parsed.HostSlots[0].NumSlots)
|
||||
}
|
||||
}
|
||||
|
||||
// A job spanning multiple hosts with DIFFERENT slot counts (e.g. an
|
||||
// asymmetric multi-host allocation) must preserve each host's own count
|
||||
// independently, not collapse them to a single shared value.
|
||||
func TestRawJobToModel_ExecHostsPreservesPerHostSlotCountsAcrossHosts(t *testing.T) {
|
||||
r := rawJobData{
|
||||
jobID: 2,
|
||||
state: 2,
|
||||
execHosts: []rawHostSlot{
|
||||
{host: "hostA", slots: 2},
|
||||
{host: "hostB", slots: 6},
|
||||
},
|
||||
}
|
||||
|
||||
info := rawJobToModel(r)
|
||||
|
||||
var parsed struct {
|
||||
HostSlots []struct {
|
||||
HostName string `json:"host_name"`
|
||||
NumSlots int `json:"num_slots"`
|
||||
} `json:"host_slots"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(info.ExecHosts.String), &parsed); err != nil {
|
||||
t.Fatalf("failed to unmarshal ExecHosts JSONB: %v", err)
|
||||
}
|
||||
if len(parsed.HostSlots) != 2 {
|
||||
t.Fatalf("HostSlots: got %d entries, want 2", len(parsed.HostSlots))
|
||||
}
|
||||
want := map[string]int{"hostA": 2, "hostB": 6}
|
||||
for _, hs := range parsed.HostSlots {
|
||||
if want[hs.HostName] != hs.NumSlots {
|
||||
t.Errorf("host %q: got %d slots, want %d", hs.HostName, hs.NumSlots, want[hs.HostName])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1676: FromHost must round-trip through rawSubmitData into JobInfo like
|
||||
// every other submit-time field, and must be empty-safe (no submit data, or
|
||||
// submit data with no from_host, must not panic and must leave FromHost "").
|
||||
|
|
|
|||
|
|
@ -2,10 +2,66 @@ package output
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"xcecli/models"
|
||||
)
|
||||
|
||||
// formatHostResourceBytesHuman renders a raw byte count the same way the C
|
||||
// scheduler's own repl_format_bytes_human()/xce_test's format_bytes_human()
|
||||
// do -- successive /1024 until under 1024, B/KB/MB/GB/TB/PB.
|
||||
func formatHostResourceBytesHuman(bytesVal float64) string {
|
||||
units := []string{"B", "KB", "MB", "GB", "TB", "PB"}
|
||||
val := bytesVal
|
||||
i := 0
|
||||
for val >= 1024.0 && i < len(units)-1 {
|
||||
val /= 1024.0
|
||||
i++
|
||||
}
|
||||
return fmt.Sprintf("%.2f %s", val, units[i])
|
||||
}
|
||||
|
||||
// hostResourceValue looks up one named resource's raw numeric wire value
|
||||
// from the host's live ResourceValues (populated from the worker's own
|
||||
// resval pushes -- see apiserver/backend/convert.go's cNameValToWorkerResval).
|
||||
func hostResourceValue(host *models.XCEHost, name string) (float64, bool) {
|
||||
for _, rv := range host.ResourceValues {
|
||||
if rv.ResourceName == name {
|
||||
return rv.NumericValue, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// formatHostLoadValue formats one "CURRENT LOAD USED FOR SCHEDULING" column
|
||||
// value per this codebase's established per-resource unit conventions
|
||||
// (mirroring the scheduler REPL's repl_is_bytes_valued_resource()/is_swp
|
||||
// special case, repl_fmt.c/repl_info.c): mem/tmp arrive already in bytes;
|
||||
// swp arrives in MB (node_information.c's node_info_get_swap()) and needs
|
||||
// the same *1024*1024 conversion the REPL applies before byte-formatting;
|
||||
// slots is a plain integer; every other (load-average/utilization-style)
|
||||
// metric is printed as a bare decimal, matching the REPL's own Load Metrics
|
||||
// section (no unit suffix, no percent sign).
|
||||
func formatHostLoadValue(host *models.XCEHost, name string) string {
|
||||
val, ok := hostResourceValue(host, name)
|
||||
if !ok {
|
||||
return "-"
|
||||
}
|
||||
switch name {
|
||||
case models.HEADER_HOST_MEM, models.HEADER_HOST_TMP:
|
||||
return formatHostResourceBytesHuman(val)
|
||||
case models.HEADER_HOST_SWP:
|
||||
return formatHostResourceBytesHuman(val * 1024.0 * 1024.0)
|
||||
case models.HEADER_HOST_SLOTS:
|
||||
return fmt.Sprintf("%.0f", val)
|
||||
default:
|
||||
if val == math.Trunc(val) {
|
||||
return fmt.Sprintf("%.0f", val)
|
||||
}
|
||||
return fmt.Sprintf("%.2f", val)
|
||||
}
|
||||
}
|
||||
|
||||
// HostsPagePrinter
|
||||
type HostsPagePrinter struct {
|
||||
*BasePagePrinter
|
||||
|
|
@ -284,13 +340,30 @@ func printHostsVerboseOutput(hosts *models.HostsResponse) {
|
|||
|
||||
// table2
|
||||
headers2 := append([]string{""}, models.HostLoadSchedulingHeaders...)
|
||||
var totalVals, reserveVals []string
|
||||
if isUnavail {
|
||||
totalVals = []string{"Total", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"}
|
||||
reserveVals = []string{"Reserved", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"}
|
||||
} else {
|
||||
totalVals = []string{"Total", "0.0", "0.0", "0.0", "1%", "0.0", "33", "1", "1137", "51G", "3.9G", "6.8G", "8", "0.0"}
|
||||
reserveVals = []string{"Reserved", "0.0", "0.0", "0.0", "0%", "0.0", "0", "0", "0", "0M", "0M", "0M", "-", "-"}
|
||||
totalVals := make([]string, 0, len(headers2))
|
||||
reserveVals := make([]string, 0, len(headers2))
|
||||
totalVals = append(totalVals, "Total")
|
||||
reserveVals = append(reserveVals, "Reserved")
|
||||
for _, h := range models.HostLoadSchedulingHeaders {
|
||||
if isUnavail {
|
||||
totalVals = append(totalVals, "-")
|
||||
reserveVals = append(reserveVals, "-")
|
||||
continue
|
||||
}
|
||||
totalVals = append(totalVals, formatHostLoadValue(&host, h))
|
||||
// Reserved/in-use is only a meaningful concept for a CONSUMABLE
|
||||
// resource (mirrors the scheduler REPL's own Consumable/
|
||||
// Capacity-vs-Load-Metrics split, repl_info.c) -- slots is the
|
||||
// only one the Go apiserver currently tracks in-use counts for
|
||||
// (computeHostSlotUsage(), same live counts the RUN/SSUSP/USUSP
|
||||
// columns above show); mem/tmp/swp in-use and every load metric
|
||||
// are honestly "-" rather than a fabricated number.
|
||||
if h == models.HEADER_HOST_SLOTS {
|
||||
reserveVals = append(reserveVals,
|
||||
fmt.Sprintf("%d", host.NumRun+host.NumSSusp+host.NumUSusp))
|
||||
} else {
|
||||
reserveVals = append(reserveVals, "-")
|
||||
}
|
||||
}
|
||||
|
||||
widths2 := make([]int, len(headers2))
|
||||
|
|
|
|||
|
|
@ -98,6 +98,24 @@ type XCEHost struct {
|
|||
GPUTopo interface{} `json:"gpu_topo"` // JSONB
|
||||
SchedStop SchedStop `json:"sched_stop"` // JSONB
|
||||
RealLoad interface{} `json:"real_load"`
|
||||
|
||||
// ResourceValues carries this host's live per-resource wire values
|
||||
// (r15s/r1m/r15m/ut/pg/io/ls/it/tmp/swp/mem/slots, among others), the
|
||||
// same set the C scheduler's own "info worker" REPL command shows --
|
||||
// populated by the apiserver from the worker's own resval pushes.
|
||||
ResourceValues []ResourceDistributionInfo `json:"resource_values"`
|
||||
}
|
||||
|
||||
// ResourceDistributionInfo mirrors apiserver/models.ResourceDistributionInfo's
|
||||
// JSON shape -- one resource's current value for one host/worker.
|
||||
type ResourceDistributionInfo struct {
|
||||
ResourceName string `json:"res_name"`
|
||||
IsDynamic bool `json:"is_dynamic"`
|
||||
ValueType int `json:"value_type"`
|
||||
BooleanValue bool `json:"boolean_value"`
|
||||
StringValue string `json:"string_value"`
|
||||
NumericValue float64 `json:"numeric_value"`
|
||||
Location string `json:"location"`
|
||||
}
|
||||
|
||||
var HostFieldsMap = map[string]string{
|
||||
|
|
@ -135,20 +153,20 @@ type HostControlResponseItem struct {
|
|||
HostName string `json:"host_name"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
|
||||
func (h *XCEHost) UnmarshalJSON(data []byte) error {
|
||||
type Alias XCEHost
|
||||
var a Alias
|
||||
if err := UnmarshalOrderMapRow(data, &a); err != nil {
|
||||
return err
|
||||
}
|
||||
*h = XCEHost(a)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ResReqSelectRequest struct {
|
||||
Select string `json:"select"`
|
||||
Order string `json:"order,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
PageToken string `json:"page_token,omitempty"`
|
||||
}
|
||||
|
||||
func (h *XCEHost) UnmarshalJSON(data []byte) error {
|
||||
type Alias XCEHost
|
||||
var a Alias
|
||||
if err := UnmarshalOrderMapRow(data, &a); err != nil {
|
||||
return err
|
||||
}
|
||||
*h = XCEHost(a)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ResReqSelectRequest struct {
|
||||
Select string `json:"select"`
|
||||
Order string `json:"order,omitempty"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
PageToken string `json:"page_token,omitempty"`
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue