package backend import ( "apiserver/models" "apiserver/store" "apiserver/utils" "testing" "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest/observer" ) // resetStores reinitializes all global stores for test isolation. func resetStores() { store.InitStores() } // captureLog swaps utils.Log for an observer core so a test can assert on // exactly which log lines fired, restoring the previous logger on cleanup. func captureLog(t *testing.T) *observer.ObservedLogs { t.Helper() core, logs := observer.New(zapcore.InfoLevel) prev := utils.Log utils.Log = zap.New(core) t.Cleanup(func() { utils.Log = prev }) return logs } func TestOnHostConfigUpdated_UnregisteredHostIgnored(t *testing.T) { // Config-only entries (e.g. "default", hosts whose daemons haven't started) // must not be inserted into the Hosts store. Only OnHostUpdated (worker push) creates entries. resetStores() h := DefaultHandler{} h.OnHostConfigUpdated(&models.HostInfo{ HostName: "node01", HostModel: "x86_64", HostType: "LINUX64", MaxSlot: 8, }) _, ok := store.Hosts.Get("node01") if ok { t.Error("OnHostConfigUpdated must not create a new host entry; only workers can register hosts") } } func TestOnHostConfigUpdated_DefaultPseudoHostIgnored(t *testing.T) { // The "default" host is a cluster-wide config sentinel, never a real worker. resetStores() h := DefaultHandler{} h.OnHostConfigUpdated(&models.HostInfo{ HostName: "default", HostType: "x86_64-linux-gnu", MaxSlot: -1, }) _, ok := store.Hosts.Get("default") if ok { t.Error("OnHostConfigUpdated must not create an entry for the 'default' pseudo-host") } } func TestOnHostConfigUpdated_PartialMerge(t *testing.T) { resetStores() h := DefaultHandler{} // Simulate a prior worker push that set worker-sourced fields. store.Hosts.Put("node01", models.HostInfo{ HostName: "node01", HostStatus: 0x00010000, // UNAVAIL CPUFactor: 2.5, NumCPUs: 32, MaxMem: 131072, NumRun: 5, }) // Config push arrives — should merge config fields, preserve worker fields. h.OnHostConfigUpdated(&models.HostInfo{ HostName: "node01", HostModel: "modelA", HostType: "CENTOS7", MaxSlot: 16, }) got, ok := store.Hosts.Get("node01") if !ok { t.Fatal("host node01 should exist after merge") } // Config-sourced fields taken from the push. if got.HostModel != "modelA" { t.Errorf("HostModel: got %q, want %q", got.HostModel, "modelA") } if got.HostType != "CENTOS7" { t.Errorf("HostType: got %q, want %q", got.HostType, "CENTOS7") } if got.MaxSlot != 16 { t.Errorf("MaxSlot: got %d, want 16", got.MaxSlot) } // Worker-sourced fields preserved from the prior entry. if got.HostStatus != 0x00010000 { t.Errorf("HostStatus: got 0x%08X, want 0x00010000 (should be preserved)", got.HostStatus) } if got.CPUFactor != 2.5 { t.Errorf("CPUFactor: got %f, want 2.5 (should be preserved)", got.CPUFactor) } if got.NumCPUs != 32 { t.Errorf("NumCPUs: got %d, want 32 (should be preserved)", got.NumCPUs) } if got.MaxMem != 131072 { t.Errorf("MaxMem: got %d, want 131072 (should be preserved)", got.MaxMem) } if got.NumRun != 5 { t.Errorf("NumRun: got %d, want 5 (should be preserved)", got.NumRun) } } func TestOnClusterUpdated_NewCluster(t *testing.T) { resetStores() h := DefaultHandler{} cluster := &models.ClusterInfo{ ClusterName: "mycluster", HeartbeatInterval: 30, Vcpus: 16, InactiveHeartbeatCount: 3, } h.OnClusterUpdated(cluster) got, ok := store.Clusters.Get("mycluster") if !ok { t.Fatal("cluster mycluster should exist in store after OnClusterUpdated") } if got.HeartbeatInterval != 30 { t.Errorf("HeartbeatInterval: got %d, want 30", got.HeartbeatInterval) } if got.Vcpus != 16 { t.Errorf("Vcpus: got %d, want 16", got.Vcpus) } if got.InactiveHeartbeatCount != 3 { t.Errorf("InactiveHeartbeatCount: got %d, want 3", got.InactiveHeartbeatCount) } } func TestOnClusterUpdated_PartialMerge(t *testing.T) { resetStores() h := DefaultHandler{} // Simulate built-in data loaded at startup. store.Clusters.Put("mycluster", models.ClusterInfo{ ClusterName: "mycluster", Version: "10.1.0", Status: 0x01, // CONNECTED ControllerName: "ctrl01", NumServers: 3, }) // Config push arrives — should merge config fields, preserve built-in fields. h.OnClusterUpdated(&models.ClusterInfo{ ClusterName: "mycluster", HeartbeatInterval: 45, Vcpus: 64, InactiveHeartbeatCount: 5, }) got, ok := store.Clusters.Get("mycluster") if !ok { t.Fatal("cluster mycluster should exist after merge") } // Config-sourced fields from the push. if got.HeartbeatInterval != 45 { t.Errorf("HeartbeatInterval: got %d, want 45", got.HeartbeatInterval) } if got.Vcpus != 64 { t.Errorf("Vcpus: got %d, want 64", got.Vcpus) } if got.InactiveHeartbeatCount != 5 { t.Errorf("InactiveHeartbeatCount: got %d, want 5", got.InactiveHeartbeatCount) } // Built-in fields preserved. if got.Version != "10.1.0" { t.Errorf("Version: got %q, want %q (should be preserved)", got.Version, "10.1.0") } if got.Status != 0x01 { t.Errorf("Status: got 0x%02X, want 0x01 (should be preserved)", got.Status) } if got.ControllerName != "ctrl01" { t.Errorf("ControllerName: got %q, want %q (should be preserved)", got.ControllerName, "ctrl01") } if got.NumServers != 3 { t.Errorf("NumServers: got %d, want 3 (should be preserved)", got.NumServers) } } func TestOnHostUpdated_PartialMerge(t *testing.T) { resetStores() h := DefaultHandler{} // Simulate a prior config push that set config-sourced fields. store.Hosts.Put("node01", models.HostInfo{ HostName: "node01", HostModel: "x86_64", HostType: "LINUX64", MaxSlot: 8, DisptWindow: "always", }) // Worker push arrives — should merge worker fields, preserve config fields. h.OnHostUpdated(&models.HostInfo{ HostName: "node01", HostStatus: 3, CPUFactor: 1.8, NumCPUs: 16, MaxMem: 65536, NumRun: 2, }) got, ok := store.Hosts.Get("node01") if !ok { t.Fatal("host node01 should exist after merge") } // Worker-sourced fields from the push. if got.HostStatus != 3 { t.Errorf("HostStatus: got %d, want 3", got.HostStatus) } if got.CPUFactor != 1.8 { t.Errorf("CPUFactor: got %f, want 1.8", got.CPUFactor) } if got.NumCPUs != 16 { t.Errorf("NumCPUs: got %d, want 16", got.NumCPUs) } if got.MaxMem != 65536 { t.Errorf("MaxMem: got %d, want 65536", got.MaxMem) } if got.NumRun != 2 { t.Errorf("NumRun: got %d, want 2", got.NumRun) } // Config-sourced fields preserved. if got.HostModel != "x86_64" { t.Errorf("HostModel: got %q, want %q (should be preserved)", got.HostModel, "x86_64") } if got.HostType != "LINUX64" { t.Errorf("HostType: got %q, want %q (should be preserved)", got.HostType, "LINUX64") } if got.MaxSlot != 8 { t.Errorf("MaxSlot: got %d, want 8 (should be preserved)", got.MaxSlot) } if got.DisptWindow != "always" { t.Errorf("DisptWindow: got %q, want %q (should be preserved)", got.DisptWindow, "always") } } func TestOnHostUpdated_NewHost(t *testing.T) { resetStores() h := DefaultHandler{} // No prior entry in store — first push is from worker. h.OnHostUpdated(&models.HostInfo{ HostName: "node05", HostStatus: 1, CPUFactor: 3.0, }) got, ok := store.Hosts.Get("node05") if !ok { t.Fatal("host node05 should exist after first push") } if got.HostStatus != 1 { t.Errorf("HostStatus: got %d, want 1", got.HostStatus) } } func TestDispatchConfigPush_MergesConfigOntoRegisteredHosts(t *testing.T) { // Config push should merge onto hosts already registered by workers, and // silently skip any host not yet in the store (e.g. "default", offline workers). resetStores() h := &DefaultHandler{} // Simulate worker registration arriving before config push. store.Hosts.Put("node01", models.HostInfo{HostName: "node01", NumCPUs: 32}) store.Hosts.Put("node02", models.HostInfo{HostName: "node02", NumCPUs: 64}) hosts := []models.HostInfo{ {HostName: "node01", HostModel: "x86_64", HostType: "LINUX64", MaxSlot: 4}, {HostName: "node02", HostModel: "arm64", HostType: "LINUX64", MaxSlot: 8}, {HostName: "default", HostModel: "x86_64", HostType: "LINUX64", MaxSlot: -1}, } dispatchConfigPush(h, nil, nil, hosts, nil) got1, ok1 := store.Hosts.Get("node01") if !ok1 { t.Fatal("node01 should remain in store after dispatchConfigPush") } if got1.HostModel != "x86_64" { t.Errorf("node01 HostModel: got %q, want x86_64", got1.HostModel) } if got1.MaxSlot != 4 { t.Errorf("node01 MaxSlot: got %d, want 4", got1.MaxSlot) } if got1.NumCPUs != 32 { t.Errorf("node01 NumCPUs: got %d, want 32 (worker field must be preserved)", got1.NumCPUs) } got2, ok2 := store.Hosts.Get("node02") if !ok2 { t.Fatal("node02 should remain in store after dispatchConfigPush") } if got2.HostModel != "arm64" { t.Errorf("node02 HostModel: got %q, want arm64", got2.HostModel) } if got2.MaxSlot != 8 { t.Errorf("node02 MaxSlot: got %d, want 8", got2.MaxSlot) } if _, ok := store.Hosts.Get("default"); ok { t.Error("'default' pseudo-host must not be inserted by config push") } if n := store.Hosts.Count(); n != 2 { t.Errorf("Hosts store should have exactly 2 entries, got %d", n) } } func TestDispatchConfigPush_CallsOnClusterUpdated(t *testing.T) { resetStores() h := &DefaultHandler{} cluster := &models.ClusterInfo{ ClusterName: "mycluster", HeartbeatInterval: 30, Vcpus: 64, InactiveHeartbeatCount: 5, } dispatchConfigPush(h, nil, nil, nil, cluster) got, ok := store.Clusters.Get("mycluster") if !ok { t.Fatal("mycluster should be in store after dispatchConfigPush") } if got.HeartbeatInterval != 30 { t.Errorf("HeartbeatInterval: got %d, want 30", got.HeartbeatInterval) } if got.Vcpus != 64 { t.Errorf("Vcpus: got %d, want 64", got.Vcpus) } if got.InactiveHeartbeatCount != 5 { t.Errorf("InactiveHeartbeatCount: got %d, want 5", got.InactiveHeartbeatCount) } } func TestDispatchConfigPush_NilClusterIsSkipped(t *testing.T) { resetStores() h := &DefaultHandler{} dispatchConfigPush(h, nil, nil, nil, nil) if n := store.Clusters.Count(); n != 0 { t.Errorf("Clusters store should be empty when cluster is nil, got %d entries", n) } } // --- job-array-single-wal-record task 5.4: dispatchJobArray --- // // dispatchJobArray is the sibling of the single-job XceData_data_jobs push // path (clib.go handleDataset, which calls gHandler.OnJobUpdated per job // with no separate dedup step because store.Jobs is a map keyed by jobid). // For the array path, dedup happens BEFORE calling the handler so a // redelivered push can never clobber a jobid whose state has since moved // on, mirroring the scheduler's sched_on_job_array() (handler.c) skip-if- // exists behavior. Requirement 6.4/6.5/8.4: exactly one confirmation log // line per successful (>=1 new element) expansion, zero on a fully- // duplicate redelivery. func TestDispatchJobArray_NewElementsStoredAndLogged(t *testing.T) { resetStores() logs := captureLog(t) h := &DefaultHandler{} elements := []*models.JobInfo{ {Unique_JobID: 100, JobID: 100, ArrayIdx: 0, QueueName: "normal"}, {Unique_JobID: 101, JobID: 100, ArrayIdx: 1, QueueName: "normal"}, {Unique_JobID: 102, JobID: 100, ArrayIdx: 2, QueueName: "normal"}, } nNew := dispatchJobArray(h, elements, 100, 3) if nNew != 3 { t.Errorf("nNew: got %d, want 3", nNew) } for _, jobid := range []uint64{100, 101, 102} { got, ok := store.Jobs.Get(jobid) if !ok { t.Errorf("jobid %d should exist in store.Jobs after expansion", jobid) continue } if got.QueueName != "normal" { t.Errorf("jobid %d: QueueName got %q, want normal", jobid, got.QueueName) } } if got := logs.Len(); got != 1 { t.Errorf("expected exactly 1 confirmation log line, got %d", got) } } func TestDispatchJobArray_FullRedeliveryIsNoOpAndDoesNotLog(t *testing.T) { resetStores() logs := captureLog(t) h := &DefaultHandler{} elements := []*models.JobInfo{ {Unique_JobID: 200, JobID: 200, ArrayIdx: 0}, {Unique_JobID: 201, JobID: 200, ArrayIdx: 1}, } // First delivery: both elements are new. if nNew := dispatchJobArray(h, elements, 200, 2); nNew != 2 { t.Fatalf("first delivery nNew: got %d, want 2", nNew) } logs.TakeAll() // discard the first delivery's own confirmation line // Redelivery of the identical push (e.g. after a reconnect): every // index already exists, so this must be a pure no-op. nNew := dispatchJobArray(h, elements, 200, 2) if nNew != 0 { t.Errorf("redelivery nNew: got %d, want 0", nNew) } if got := logs.Len(); got != 0 { t.Errorf("expected zero confirmation log lines on full redelivery, got %d", got) } if n := store.Jobs.Count(); n != 2 { t.Errorf("store.Jobs should still have exactly 2 entries after redelivery, got %d", n) } } func TestDispatchJobArray_PartialRedeliverySkipsExistingOnly(t *testing.T) { resetStores() h := &DefaultHandler{} // Pre-seed jobid 300 with state that a stale redelivered array push // must NOT clobber (dedup-by-identity is a correctness guard, not // just a log-suppression detail). store.Jobs.Put(300, models.JobInfo{Unique_JobID: 300, JobStatus: models.JOBSTAT["RUN"]}) logs := captureLog(t) elements := []*models.JobInfo{ {Unique_JobID: 300, JobID: 300, ArrayIdx: 0, JobStatus: models.JOBSTAT["PEND"]}, {Unique_JobID: 301, JobID: 300, ArrayIdx: 1, JobStatus: models.JOBSTAT["PEND"]}, } nNew := dispatchJobArray(h, elements, 300, 2) if nNew != 1 { t.Errorf("nNew: got %d, want 1 (only jobid 301 is new)", nNew) } got300, _ := store.Jobs.Get(300) if got300.JobStatus != models.JOBSTAT["RUN"] { t.Errorf("jobid 300 must not be clobbered by a stale redelivered array push: got status %d, want %d", got300.JobStatus, models.JOBSTAT["RUN"]) } if _, ok := store.Jobs.Get(301); !ok { t.Error("jobid 301 should have been created") } if got := logs.Len(); got != 1 { t.Errorf("expected exactly 1 confirmation log line for a partial (1 new) expansion, got %d", got) } } func TestDispatchJobArray_NilHandlerIsNoOp(t *testing.T) { resetStores() elements := []*models.JobInfo{{Unique_JobID: 400}} nNew := dispatchJobArray(nil, elements, 400, 1) if nNew != 0 { t.Errorf("nNew: got %d, want 0 for nil handler", nNew) } } func TestDispatchConfigPush_EmptySlicesNoOp(t *testing.T) { resetStores() h := &DefaultHandler{} dispatchConfigPush(h, []models.QueueInfo{}, []models.ResourceDefinitionInfo{}, []models.HostInfo{}, nil, ) if n := store.Hosts.Count(); n != 0 { t.Errorf("Hosts store should be empty, got %d entries", n) } if n := store.Queues.Count(); n != 0 { t.Errorf("Queues store should be empty, got %d entries", n) } if n := store.Clusters.Count(); n != 0 { t.Errorf("Clusters store should be empty, got %d entries", n) } }