myxce/src/frontend/apiserver/backend/handler_test.go

package backend

import (
	"apiserver/models"
	"apiserver/store"
	"apiserver/utils"
	"testing"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
)

// resetStores reinitializes all global stores for test isolation.
func resetStores() {
	store.InitStores()
}

// captureLog swaps utils.Log for an observer core so a test can assert on
// exactly which log lines fired, restoring the previous logger on cleanup.
func captureLog(t *testing.T) *observer.ObservedLogs {
	t.Helper()
	core, logs := observer.New(zapcore.InfoLevel)
	prev := utils.Log
	utils.Log = zap.New(core)
	t.Cleanup(func() { utils.Log = prev })
	return logs
}

func TestOnHostConfigUpdated_UnregisteredHostIgnored(t *testing.T) {
	// Config-only entries (e.g. "default", hosts whose daemons haven't started)
	// must not be inserted into the Hosts store. Only OnHostUpdated (worker push) creates entries.
	resetStores()
	h := DefaultHandler{}

	h.OnHostConfigUpdated(&models.HostInfo{
		HostName:  "node01",
		HostModel: "x86_64",
		HostType:  "LINUX64",
		MaxSlot:   8,
	})

	_, ok := store.Hosts.Get("node01")
	if ok {
		t.Error("OnHostConfigUpdated must not create a new host entry; only workers can register hosts")
	}
}

func TestOnHostConfigUpdated_DefaultPseudoHostIgnored(t *testing.T) {
	// The "default" host is a cluster-wide config sentinel, never a real worker.
	resetStores()
	h := DefaultHandler{}

	h.OnHostConfigUpdated(&models.HostInfo{
		HostName: "default",
		HostType: "x86_64-linux-gnu",
		MaxSlot:  -1,
	})

	_, ok := store.Hosts.Get("default")
	if ok {
		t.Error("OnHostConfigUpdated must not create an entry for the 'default' pseudo-host")
	}
}

func TestOnHostConfigUpdated_PartialMerge(t *testing.T) {
	resetStores()
	h := DefaultHandler{}

	// Simulate a prior worker push that set worker-sourced fields.
	store.Hosts.Put("node01", models.HostInfo{
		HostName:   "node01",
		HostStatus: 0x00010000, // UNAVAIL
		CPUFactor:  2.5,
		NumCPUs:    32,
		MaxMem:     131072,
		NumRun:     5,
	})

	// Config push arrives — should merge config fields, preserve worker fields.
	h.OnHostConfigUpdated(&models.HostInfo{
		HostName:  "node01",
		HostModel: "modelA",
		HostType:  "CENTOS7",
		MaxSlot:   16,
	})

	got, ok := store.Hosts.Get("node01")
	if !ok {
		t.Fatal("host node01 should exist after merge")
	}
	// Config-sourced fields taken from the push.
	if got.HostModel != "modelA" {
		t.Errorf("HostModel: got %q, want %q", got.HostModel, "modelA")
	}
	if got.HostType != "CENTOS7" {
		t.Errorf("HostType: got %q, want %q", got.HostType, "CENTOS7")
	}
	if got.MaxSlot != 16 {
		t.Errorf("MaxSlot: got %d, want 16", got.MaxSlot)
	}
	// Worker-sourced fields preserved from the prior entry.
	if got.HostStatus != 0x00010000 {
		t.Errorf("HostStatus: got 0x%08X, want 0x00010000 (should be preserved)", got.HostStatus)
	}
	if got.CPUFactor != 2.5 {
		t.Errorf("CPUFactor: got %f, want 2.5 (should be preserved)", got.CPUFactor)
	}
	if got.NumCPUs != 32 {
		t.Errorf("NumCPUs: got %d, want 32 (should be preserved)", got.NumCPUs)
	}
	if got.MaxMem != 131072 {
		t.Errorf("MaxMem: got %d, want 131072 (should be preserved)", got.MaxMem)
	}
	if got.NumRun != 5 {
		t.Errorf("NumRun: got %d, want 5 (should be preserved)", got.NumRun)
	}
}

func TestOnClusterUpdated_NewCluster(t *testing.T) {
	resetStores()
	h := DefaultHandler{}

	cluster := &models.ClusterInfo{
		ClusterName:            "mycluster",
		HeartbeatInterval:      30,
		Vcpus:                  16,
		InactiveHeartbeatCount: 3,
	}

	h.OnClusterUpdated(cluster)

	got, ok := store.Clusters.Get("mycluster")
	if !ok {
		t.Fatal("cluster mycluster should exist in store after OnClusterUpdated")
	}
	if got.HeartbeatInterval != 30 {
		t.Errorf("HeartbeatInterval: got %d, want 30", got.HeartbeatInterval)
	}
	if got.Vcpus != 16 {
		t.Errorf("Vcpus: got %d, want 16", got.Vcpus)
	}
	if got.InactiveHeartbeatCount != 3 {
		t.Errorf("InactiveHeartbeatCount: got %d, want 3", got.InactiveHeartbeatCount)
	}
}

func TestOnClusterUpdated_PartialMerge(t *testing.T) {
	resetStores()
	h := DefaultHandler{}

	// Simulate built-in data loaded at startup.
	store.Clusters.Put("mycluster", models.ClusterInfo{
		ClusterName:    "mycluster",
		Version:        "10.1.0",
		Status:         0x01, // CONNECTED
		ControllerName: "ctrl01",
		NumServers:     3,
	})

	// Config push arrives — should merge config fields, preserve built-in fields.
	h.OnClusterUpdated(&models.ClusterInfo{
		ClusterName:            "mycluster",
		HeartbeatInterval:      45,
		Vcpus:                  64,
		InactiveHeartbeatCount: 5,
	})

	got, ok := store.Clusters.Get("mycluster")
	if !ok {
		t.Fatal("cluster mycluster should exist after merge")
	}
	// Config-sourced fields from the push.
	if got.HeartbeatInterval != 45 {
		t.Errorf("HeartbeatInterval: got %d, want 45", got.HeartbeatInterval)
	}
	if got.Vcpus != 64 {
		t.Errorf("Vcpus: got %d, want 64", got.Vcpus)
	}
	if got.InactiveHeartbeatCount != 5 {
		t.Errorf("InactiveHeartbeatCount: got %d, want 5", got.InactiveHeartbeatCount)
	}
	// Built-in fields preserved.
	if got.Version != "10.1.0" {
		t.Errorf("Version: got %q, want %q (should be preserved)", got.Version, "10.1.0")
	}
	if got.Status != 0x01 {
		t.Errorf("Status: got 0x%02X, want 0x01 (should be preserved)", got.Status)
	}
	if got.ControllerName != "ctrl01" {
		t.Errorf("ControllerName: got %q, want %q (should be preserved)", got.ControllerName, "ctrl01")
	}
	if got.NumServers != 3 {
		t.Errorf("NumServers: got %d, want 3 (should be preserved)", got.NumServers)
	}
}

func TestOnHostUpdated_PartialMerge(t *testing.T) {
	resetStores()
	h := DefaultHandler{}

	// Simulate a prior config push that set config-sourced fields.
	store.Hosts.Put("node01", models.HostInfo{
		HostName:    "node01",
		HostModel:   "x86_64",
		HostType:    "LINUX64",
		MaxSlot:     8,
		DisptWindow: "always",
	})

	// Worker push arrives — should merge worker fields, preserve config fields.
	h.OnHostUpdated(&models.HostInfo{
		HostName:   "node01",
		HostStatus: 3,
		CPUFactor:  1.8,
		NumCPUs:    16,
		MaxMem:     65536,
		NumRun:     2,
	})

	got, ok := store.Hosts.Get("node01")
	if !ok {
		t.Fatal("host node01 should exist after merge")
	}
	// Worker-sourced fields from the push.
	if got.HostStatus != 3 {
		t.Errorf("HostStatus: got %d, want 3", got.HostStatus)
	}
	if got.CPUFactor != 1.8 {
		t.Errorf("CPUFactor: got %f, want 1.8", got.CPUFactor)
	}
	if got.NumCPUs != 16 {
		t.Errorf("NumCPUs: got %d, want 16", got.NumCPUs)
	}
	if got.MaxMem != 65536 {
		t.Errorf("MaxMem: got %d, want 65536", got.MaxMem)
	}
	if got.NumRun != 2 {
		t.Errorf("NumRun: got %d, want 2", got.NumRun)
	}
	// Config-sourced fields preserved.
	if got.HostModel != "x86_64" {
		t.Errorf("HostModel: got %q, want %q (should be preserved)", got.HostModel, "x86_64")
	}
	if got.HostType != "LINUX64" {
		t.Errorf("HostType: got %q, want %q (should be preserved)", got.HostType, "LINUX64")
	}
	if got.MaxSlot != 8 {
		t.Errorf("MaxSlot: got %d, want 8 (should be preserved)", got.MaxSlot)
	}
	if got.DisptWindow != "always" {
		t.Errorf("DisptWindow: got %q, want %q (should be preserved)", got.DisptWindow, "always")
	}
}

func TestOnHostUpdated_NewHost(t *testing.T) {
	resetStores()
	h := DefaultHandler{}

	// No prior entry in store — first push is from worker.
	h.OnHostUpdated(&models.HostInfo{
		HostName:   "node05",
		HostStatus: 1,
		CPUFactor:  3.0,
	})

	got, ok := store.Hosts.Get("node05")
	if !ok {
		t.Fatal("host node05 should exist after first push")
	}
	if got.HostStatus != 1 {
		t.Errorf("HostStatus: got %d, want 1", got.HostStatus)
	}
}

func TestDispatchConfigPush_MergesConfigOntoRegisteredHosts(t *testing.T) {
	// Config push should merge onto hosts already registered by workers, and
	// silently skip any host not yet in the store (e.g. "default", offline workers).
	resetStores()
	h := &DefaultHandler{}

	// Simulate worker registration arriving before config push.
	store.Hosts.Put("node01", models.HostInfo{HostName: "node01", NumCPUs: 32})
	store.Hosts.Put("node02", models.HostInfo{HostName: "node02", NumCPUs: 64})

	hosts := []models.HostInfo{
		{HostName: "node01", HostModel: "x86_64", HostType: "LINUX64", MaxSlot: 4},
		{HostName: "node02", HostModel: "arm64", HostType: "LINUX64", MaxSlot: 8},
		{HostName: "default", HostModel: "x86_64", HostType: "LINUX64", MaxSlot: -1},
	}

	dispatchConfigPush(h, nil, nil, hosts, nil)

	got1, ok1 := store.Hosts.Get("node01")
	if !ok1 {
		t.Fatal("node01 should remain in store after dispatchConfigPush")
	}
	if got1.HostModel != "x86_64" {
		t.Errorf("node01 HostModel: got %q, want x86_64", got1.HostModel)
	}
	if got1.MaxSlot != 4 {
		t.Errorf("node01 MaxSlot: got %d, want 4", got1.MaxSlot)
	}
	if got1.NumCPUs != 32 {
		t.Errorf("node01 NumCPUs: got %d, want 32 (worker field must be preserved)", got1.NumCPUs)
	}

	got2, ok2 := store.Hosts.Get("node02")
	if !ok2 {
		t.Fatal("node02 should remain in store after dispatchConfigPush")
	}
	if got2.HostModel != "arm64" {
		t.Errorf("node02 HostModel: got %q, want arm64", got2.HostModel)
	}
	if got2.MaxSlot != 8 {
		t.Errorf("node02 MaxSlot: got %d, want 8", got2.MaxSlot)
	}

	if _, ok := store.Hosts.Get("default"); ok {
		t.Error("'default' pseudo-host must not be inserted by config push")
	}
	if n := store.Hosts.Count(); n != 2 {
		t.Errorf("Hosts store should have exactly 2 entries, got %d", n)
	}
}

func TestDispatchConfigPush_CallsOnClusterUpdated(t *testing.T) {
	resetStores()
	h := &DefaultHandler{}

	cluster := &models.ClusterInfo{
		ClusterName:            "mycluster",
		HeartbeatInterval:      30,
		Vcpus:                  64,
		InactiveHeartbeatCount: 5,
	}

	dispatchConfigPush(h, nil, nil, nil, cluster)

	got, ok := store.Clusters.Get("mycluster")
	if !ok {
		t.Fatal("mycluster should be in store after dispatchConfigPush")
	}
	if got.HeartbeatInterval != 30 {
		t.Errorf("HeartbeatInterval: got %d, want 30", got.HeartbeatInterval)
	}
	if got.Vcpus != 64 {
		t.Errorf("Vcpus: got %d, want 64", got.Vcpus)
	}
	if got.InactiveHeartbeatCount != 5 {
		t.Errorf("InactiveHeartbeatCount: got %d, want 5", got.InactiveHeartbeatCount)
	}
}

func TestDispatchConfigPush_NilClusterIsSkipped(t *testing.T) {
	resetStores()
	h := &DefaultHandler{}

	dispatchConfigPush(h, nil, nil, nil, nil)

	if n := store.Clusters.Count(); n != 0 {
		t.Errorf("Clusters store should be empty when cluster is nil, got %d entries", n)
	}
}

// --- job-array-single-wal-record task 5.4: dispatchJobArray ---
//
// dispatchJobArray is the sibling of the single-job XceData_data_jobs push
// path (clib.go handleDataset, which calls gHandler.OnJobUpdated per job
// with no separate dedup step because store.Jobs is a map keyed by jobid).
// For the array path, dedup happens BEFORE calling the handler so a
// redelivered push can never clobber a jobid whose state has since moved
// on, mirroring the scheduler's sched_on_job_array() (handler.c) skip-if-
// exists behavior. Requirement 6.4/6.5/8.4: exactly one confirmation log
// line per successful (>=1 new element) expansion, zero on a fully-
// duplicate redelivery.

func TestDispatchJobArray_NewElementsStoredAndLogged(t *testing.T) {
	resetStores()
	logs := captureLog(t)
	h := &DefaultHandler{}

	elements := []*models.JobInfo{
		{Unique_JobID: 100, JobID: 100, ArrayIdx: 0, QueueName: "normal"},
		{Unique_JobID: 101, JobID: 100, ArrayIdx: 1, QueueName: "normal"},
		{Unique_JobID: 102, JobID: 100, ArrayIdx: 2, QueueName: "normal"},
	}

	nNew := dispatchJobArray(h, elements, 100, 3)

	if nNew != 3 {
		t.Errorf("nNew: got %d, want 3", nNew)
	}
	for _, jobid := range []uint64{100, 101, 102} {
		got, ok := store.Jobs.Get(jobid)
		if !ok {
			t.Errorf("jobid %d should exist in store.Jobs after expansion", jobid)
			continue
		}
		if got.QueueName != "normal" {
			t.Errorf("jobid %d: QueueName got %q, want normal", jobid, got.QueueName)
		}
	}
	if got := logs.Len(); got != 1 {
		t.Errorf("expected exactly 1 confirmation log line, got %d", got)
	}
}

func TestDispatchJobArray_FullRedeliveryIsNoOpAndDoesNotLog(t *testing.T) {
	resetStores()
	logs := captureLog(t)
	h := &DefaultHandler{}

	elements := []*models.JobInfo{
		{Unique_JobID: 200, JobID: 200, ArrayIdx: 0},
		{Unique_JobID: 201, JobID: 200, ArrayIdx: 1},
	}

	// First delivery: both elements are new.
	if nNew := dispatchJobArray(h, elements, 200, 2); nNew != 2 {
		t.Fatalf("first delivery nNew: got %d, want 2", nNew)
	}
	logs.TakeAll() // discard the first delivery's own confirmation line

	// Redelivery of the identical push (e.g. after a reconnect): every
	// index already exists, so this must be a pure no-op.
	nNew := dispatchJobArray(h, elements, 200, 2)

	if nNew != 0 {
		t.Errorf("redelivery nNew: got %d, want 0", nNew)
	}
	if got := logs.Len(); got != 0 {
		t.Errorf("expected zero confirmation log lines on full redelivery, got %d", got)
	}
	if n := store.Jobs.Count(); n != 2 {
		t.Errorf("store.Jobs should still have exactly 2 entries after redelivery, got %d", n)
	}
}

func TestDispatchJobArray_PartialRedeliverySkipsExistingOnly(t *testing.T) {
	resetStores()
	h := &DefaultHandler{}

	// Pre-seed jobid 300 with state that a stale redelivered array push
	// must NOT clobber (dedup-by-identity is a correctness guard, not
	// just a log-suppression detail).
	store.Jobs.Put(300, models.JobInfo{Unique_JobID: 300, JobStatus: models.JOBSTAT["RUN"]})

	logs := captureLog(t)
	elements := []*models.JobInfo{
		{Unique_JobID: 300, JobID: 300, ArrayIdx: 0, JobStatus: models.JOBSTAT["PEND"]},
		{Unique_JobID: 301, JobID: 300, ArrayIdx: 1, JobStatus: models.JOBSTAT["PEND"]},
	}

	nNew := dispatchJobArray(h, elements, 300, 2)

	if nNew != 1 {
		t.Errorf("nNew: got %d, want 1 (only jobid 301 is new)", nNew)
	}
	got300, _ := store.Jobs.Get(300)
	if got300.JobStatus != models.JOBSTAT["RUN"] {
		t.Errorf("jobid 300 must not be clobbered by a stale redelivered array push: got status %d, want %d",
			got300.JobStatus, models.JOBSTAT["RUN"])
	}
	if _, ok := store.Jobs.Get(301); !ok {
		t.Error("jobid 301 should have been created")
	}
	if got := logs.Len(); got != 1 {
		t.Errorf("expected exactly 1 confirmation log line for a partial (1 new) expansion, got %d", got)
	}
}

func TestDispatchJobArray_NilHandlerIsNoOp(t *testing.T) {
	resetStores()
	elements := []*models.JobInfo{{Unique_JobID: 400}}
	nNew := dispatchJobArray(nil, elements, 400, 1)
	if nNew != 0 {
		t.Errorf("nNew: got %d, want 0 for nil handler", nNew)
	}
}

func TestDispatchConfigPush_EmptySlicesNoOp(t *testing.T) {
	resetStores()
	h := &DefaultHandler{}

	dispatchConfigPush(h,
		[]models.QueueInfo{},
		[]models.ResourceDefinitionInfo{},
		[]models.HostInfo{},
		nil,
	)

	if n := store.Hosts.Count(); n != 0 {
		t.Errorf("Hosts store should be empty, got %d entries", n)
	}
	if n := store.Queues.Count(); n != 0 {
		t.Errorf("Queues store should be empty, got %d entries", n)
	}
	if n := store.Clusters.Count(); n != 0 {
		t.Errorf("Clusters store should be empty, got %d entries", n)
	}
}