diff --git a/.wolf/buglog.json b/.wolf/buglog.json index c7561007..82f018ca 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -20190,6 +20190,41 @@ ], "occurrences": 1, "last_seen": "2026-09-01T04:40:00Z" + }, + { + "id": "bug-1743", + "timestamp": "2026-09-01T04:10:00Z", + "gitea_issue": 46, + "related_bugs": [ + "bug-1673", + "bug-1556" + ], + "error_message": "Gitea issue #46 (\u8fd8\u9700\u8981\u4e00\u4e2aslots\u7684\u6982\u5ff5): host test01 shows NCPUS=16, MAX_SLOT=16 via `xcecli hosts list`, but of 9 submitted single-slot `sleep` jobs on queue `normal`, only 8 reach RUN and the 9th is stuck PEND -- dispatch stops at 8/16 host capacity. Secondary ask: `hosts list` should show slots used/free, not just NCPUS/MAX_SLOT.", + "file": "src/scheduler/src/resource.c (sched_check_host_satisfy_:328-377), src/scheduler/src/plugins/limit.c (build_vres_name/per-user-queue-project virtual resource accounting:183-227,312-376), src/configs/resources.yaml (slots resource:4, consumable:true), src/configs/limits.yaml (ug1_normal_slots example, value:8, lines 41-49), src/frontend/apiserver/models/hosts.go (HostInfo.NumRun:87)", + "root_cause": "Two independent findings. (1) Dispatch-stops-at-8 is most likely NOT a slot-counting bug in the scheduler -- MAX_SLOT genuinely gates per-host admission in sched_check_host_satisfy_() the same way `mem` does (per .kiro/steering/resource-accounting.md, reservation==usage for virtual resources like slots), but there is a SEPARATE, independent admission gate: cluster-wide usergroup+queue slot limits enforced by the scheduler's limit plugin (limit.c), configured in limits.yaml. This repo's example limits.yaml defines `ug1_normal_slots`: users in group ug1 on queue `normal` share a pool capped at exactly 8 (limits.yaml:41-49) -- an exact match to the reporter's queue (`normal`) and the observed 8/16 cutoff. This exact scenario (a queue/usergroup slot limit as an alternate explanation to a host-capacity bug) was explicitly predicted-but-unconfirmed in bug-1673 (filed against the related earlier report Gitea #40) and appears to be that prediction now reproduced. CAVEAT: limits.yaml/users.yaml in this repo look like sample/demo config (users.yaml's stock ug1 group only contains user1/user2, not `admin`) -- confirming this requires checking the reporter's actual deployed limits.yaml/users.yaml or their job's pend-reason output, neither of which was available this session (static analysis only). (2) The feature-request half (show slot usage in hosts list) is NOT starting from scratch: models.HostInfo.NumRun (hosts.go:87) already exists in the schema and is already an output column in `xcecli hosts list` (matching the reporter's own output), but `grep -rn \"\\.NumRun\\s*=\" apiserver --include=*.go` returns zero hits anywhere in the apiserver -- the field is never assigned and is permanently stuck at Go's zero value, which is exactly why the reporter's own output shows NUM_RUN=0 while 8 jobs are actually running on that host. Same for NumSSusp/NumUSusp.", + "fix": "PARTIALLY FIXED (2026-09-01). Part 2 (hosts-list slot visibility) is FIXED: added computeHostSlotUsage() (controllers/hosts.go) which scans store.Jobs' live ExecHosts JSONB (the same {\"host_slots\":[{\"host_name\",\"num_slots\"}],...} shape backend/convert_pure.go populates) and JobStatus bits (JOBSTAT[\"RUN\"]/[\"SSUSP\"]/[\"USUSP\"]) to count occupied slots per host, per sub-state -- counting SLOTS (not just job count) so the numbers are directly comparable to max_slot, matching the reporter's actual ask ('host\u4e0a\u9700\u8981\u4f53\u73b0\u7528\u4e86\u591a\u5c11slots'). processHostData() now overwrites row.Data[\"num_run\"/\"num_ssusp\"/\"num_ususp\"] with these live counts instead of leaving HostInfo's always-zero struct fields untouched. TDD: added TestProcessHostData_ComputesSlotUsageFromJobs (host_control_test.go) covering plain RUN, USUSP, a 2-slot SSUSP job (verifies slot-counting, not job-counting), a PEND job with no exec host (must not count), and a job on a different host (must not bleed across hosts). Confirmed red (num_run=0, want 6) before the fix, green after. Full apiserver suite passes with zero regressions.\n\nPart 1 (dispatch-stops-at-8 root cause) remains UNRESOLVED/NOT ACTIONABLE this session: confirming whether a real usergroup/queue limits.yaml entry is gating the reporter's deployment requires their actual live config or pend-reason output, neither available statically. No scheduler/limit.c code change was made pending that confirmation -- see root_cause for the open question to ask the reporter.", + "tags": [ + "scheduler", + "slots", + "admission-control", + "limits.yaml", + "usergroup-limit", + "queue-limit", + "hosts-list", + "NumRun", + "dead-field", + "visibility-gap", + "feature-request", + "new", + "found-static-analysis", + "needs-live-repro", + "related-to-bug-1673", + "related-to-bug-1556", + "partially-fixed", + "needs-reporter-confirmation" + ], + "occurrences": 1, + "last_seen": "2026-09-01T05:10:00Z" } ] } diff --git a/src/frontend/apiserver/controllers/host_control_test.go b/src/frontend/apiserver/controllers/host_control_test.go index 55393fe6..39fce0ed 100644 --- a/src/frontend/apiserver/controllers/host_control_test.go +++ b/src/frontend/apiserver/controllers/host_control_test.go @@ -137,3 +137,63 @@ func TestPreCheckHostControlRequest_CommentTooLong(t *testing.T) { t.Error("expected error for comment exceeding 512 chars, got nil") } } + +// TestProcessHostData_ComputesSlotUsageFromJobs is a regression guard for +// bug-1743 (gitea #46): models.HostInfo.NumRun/.NumSSusp/.NumUSusp exist in +// the schema and are already shown by `xcecli hosts list`, but nothing in +// the apiserver ever assigned them -- they were permanently stuck at Go's +// zero value, so `hosts list` always showed NUM_RUN=0 regardless of how many +// jobs were actually running on a host. processHostData must now derive +// these counts from store.Jobs' live exec_hosts/job_status data (the same +// JSONB shape backend/convert_pure.go's job-conversion path populates: +// {"host_slots":[{"host_name":..,"num_slots":N}],"num_hosts":N}), counting +// slots (not just job count) so the numbers are comparable to max_slot. +func TestProcessHostData_ComputesSlotUsageFromJobs(t *testing.T) { + resetHostControlState() + + store.Hosts.Put("test01", models.HostInfo{ + HostName: "test01", + NumCPUs: 16, + MaxSlot: 16, + }) + + putJob := func(id int64, execHosts string, status int) { + store.Jobs.Put(uint64(id), models.JobInfo{ + Unique_JobID: id, + JobID: id, + JobStatus: status, + ExecHosts: models.Jsonb{String: execHosts}, + }) + } + + // 6 plain RUN jobs, 1 slot each on test01. + for i := int64(1); i <= 6; i++ { + putJob(i, `{"host_slots":[{"host_name":"test01","num_slots":1}],"num_hosts":1}`, models.JOBSTAT["RUN"]) + } + // 1 user-suspended job (RUN|USUSP) -- must count as USUSP, not RUN. + putJob(7, `{"host_slots":[{"host_name":"test01","num_slots":1}],"num_hosts":1}`, models.JOBSTAT["RUN"]|models.JOBSTAT["USUSP"]) + // 1 system-suspended job (RUN|SSUSP) using 2 slots on test01. + putJob(8, `{"host_slots":[{"host_name":"test01","num_slots":2}],"num_hosts":1}`, models.JOBSTAT["RUN"]|models.JOBSTAT["SSUSP"]) + // 1 PEND job with no exec host yet -- must not be counted anywhere. + putJob(9, "", models.JOBSTAT["PEND"]) + // 1 job on a different host -- must not count toward test01. + putJob(10, `{"host_slots":[{"host_name":"other-host","num_slots":1}],"num_hosts":1}`, models.JOBSTAT["RUN"]) + + rows := store.StoreToOrderMaps(store.Hosts) + result := processHostData(rows, "") + + if result.Count != 1 { + t.Fatalf("expected 1 host row, got %d", result.Count) + } + row := result.Rows[0] + + if got := row.Data["num_run"]; got != 6 { + t.Errorf("num_run = %v, want 6", got) + } + if got := row.Data["num_ssusp"]; got != 2 { + t.Errorf("num_ssusp = %v, want 2 (2-slot SSUSP job)", got) + } + if got := row.Data["num_ususp"]; got != 1 { + t.Errorf("num_ususp = %v, want 1", got) + } +} diff --git a/src/frontend/apiserver/controllers/hosts.go b/src/frontend/apiserver/controllers/hosts.go index 92c4e9de..152014f2 100644 --- a/src/frontend/apiserver/controllers/hosts.go +++ b/src/frontend/apiserver/controllers/hosts.go @@ -8,6 +8,7 @@ import ( "apiserver/store" "apiserver/utils" logger "apiserver/utils" + "encoding/json" "errors" "reflect" "strings" @@ -565,7 +566,56 @@ func preCheckHostControlRequest(req *models.HostControlRequest) (bool, error) { return isAll, nil } +// hostSlotUsage holds counts of slots currently occupied on one host, +// broken down by run/suspend sub-state, mirroring HostInfo.NumRun/ +// NumSSusp/NumUSusp. +type hostSlotUsage struct { + Run, SSusp, USusp int +} + +// execHostsJSON mirrors the JSONB shape backend/convert_pure.go's +// buildJobInfoFromReplayJob() writes into JobInfo.ExecHosts. +type execHostsJSON struct { + HostSlots []struct { + HostName string `json:"host_name"` + NumSlots int `json:"num_slots"` + } `json:"host_slots"` +} + +// computeHostSlotUsage derives per-host slot usage from the live job store, +// since store.Hosts' own NumRun/NumSSusp/NumUSusp fields are never written +// (bug-1743): a host's occupied-slot count can only be known by scanning +// which non-terminal jobs are currently placed on it. +func computeHostSlotUsage() map[string]hostSlotUsage { + usage := make(map[string]hostSlotUsage) + for _, job := range store.Jobs.List() { + if job.ExecHosts.String == "" { + continue + } + var parsed execHostsJSON + if err := json.Unmarshal([]byte(job.ExecHosts.String), &parsed); err != nil { + continue + } + for _, hs := range parsed.HostSlots { + u := usage[hs.HostName] + switch { + case job.JobStatus&models.JOBSTAT["USUSP"] != 0: + u.USusp += hs.NumSlots + case job.JobStatus&models.JOBSTAT["SSUSP"] != 0: + u.SSusp += hs.NumSlots + case job.JobStatus&models.JOBSTAT["RUN"] != 0: + u.Run += hs.NumSlots + default: + continue + } + usage[hs.HostName] = u + } + } + return usage +} + func processHostData(rows []utils.OrderMap, page_token string) utils.QueryResult { + slotUsage := computeHostSlotUsage() for _, row := range rows { if v, exist := row.Data["host_status"]; exist { if v != nil { @@ -574,6 +624,16 @@ func processHostData(rows []utils.OrderMap, page_token string) utils.QueryResult } } + // bug-1743: HostInfo.NumRun/.NumSSusp/.NumUSusp are never assigned + // anywhere else -- derive them here from the live job store so + // `hosts list` reflects actual slot usage instead of always 0. + if hostName, ok := row.Data["host_name"].(string); ok { + usage := slotUsage[hostName] + row.Data["num_run"] = usage.Run + row.Data["num_ssusp"] = usage.SSusp + row.Data["num_ususp"] = usage.USusp + } + // Process JSONB fields based on predefined list for _, jsonbField := range models.JSONBHFields { if v, exist := row.Data[jsonbField]; exist {