Merge Go frontend (apiserver + xcecli) into meson build

src/frontend/apiserver/utils/cgo_stub.go

@@ -0,0 +1,5 @@
+package utils
+
+// This file is reserved for future CGo bindings to the backend C library.
+// The CGo infrastructure (compiler flags, linker flags) is kept in meson.build
+// and will be used when the backend C library integration is implemented.

src/frontend/apiserver/utils/config.go

@@ -0,0 +1,293 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/viper"
+)
+
+type BackendConfig struct {
+	Address string `yaml:"address"`
+	Port    int    `yaml:"port"`
+	From    string `yaml:"from"`
+	Key     string `yaml:"key"`
+}
+
+type AppServerConfig struct {
+	AppConfig      AppConfig       `yaml:"app"`
+	Security       Security        `yaml:"security"`
+	RateLimit      RateLimitConfig `yaml:"rate_limit"`
+	Authentication Authentication  `yaml:"authentication"`
+	JobConfig      JobConfig       `yaml:"job"`
+	CacheConfig    CacheConfig     `yaml:"cache"`
+	Backend        BackendConfig   `yaml:"backend"`
+}
+
+type AppConfig struct {
+	AppName    string `yaml:"name"`
+	AppMode    string `yaml:"mode"`
+	HTTPPort   string `yaml:"http_port"`
+	AppVersion string `yaml:"version"`
+}
+
+// TokenConfig is the configuration for the token
+type TokenConfig struct {
+	Type            string `yaml:"type"`
+	SecretKey       string `yaml:"secret_key"`
+	TokenExpireTime int    `yaml:"token_expire_time"`
+}
+
+// Authentication is the configuration for the authentication
+type Authentication struct {
+	Token TokenConfig
+}
+
+type Security struct {
+	HTTPS HTTPSConfig `yaml:"https"`
+}
+type HTTPSConfig struct {
+	Enabled  bool   `yaml:"enabled"`
+	Port     string `yaml:"port"`
+	CertFile string `yaml:"cert_file"`
+	KeyFile  string `yaml:"key_file"`
+}
+
+type RateLimitConfig struct {
+	Global    EndpointLimit            `yaml:"global"`
+	Endpoints map[string]EndpointLimit `yaml:"endpoints"`
+}
+
+type EndpointLimit struct {
+	Rate   int    `yaml:"rate"`   // requests per second
+	Burst  int    `yaml:"burst"`  // requests per minute
+	Window string `yaml:"window"` // Time window (second, minute)
+}
+
+type WhitelistConfig struct {
+	Whitelist []string `yaml:"whitelist"`
+}
+
+type FileUploadConfig struct {
+	MaxSize         string   `yaml:"max_size"`         // maximum file size in MB
+	AllowedTypes    []string `yaml:"allowed_types"`    // allowed file types
+	DisallowedTypes []string `yaml:"disallowed_types"` // disallowed file types
+}
+
+type JobConfig struct {
+	MaxArraySize int `yaml:"max_array_size"`
+}
+
+type CORSConfig struct {
+	Enabled        bool     `yaml:"enabled"`
+	AllowedOrigins []string `yaml:"allowed_origins"`
+}
+
+type CacheConfig struct {
+	RefreshInterval int `yaml:"refresh_interval"`
+}
+
+var DirPerm os.FileMode
+
+// Add global variables to store configs
+var (
+	AppYamlConfig            *AppConfig
+	AuthenticationYamlConfig *Authentication
+	HttpsYamlConfig          *HTTPSConfig
+	RateLimitYamlConfig      *RateLimitConfig
+	WhitelistYamlConfig      *WhitelistConfig
+	FileUploadYamlConfig     *FileUploadConfig
+	JobYamlConfig            *JobConfig
+	CacheYamlConfig          *CacheConfig
+	CORSYamlConfig           *CORSConfig
+	BackendYamlConfig        *BackendConfig
+)
+
+func init() {
+	if runtime.GOOS == "windows" {
+		DirPerm = 0666
+	} else {
+		DirPerm = 0700
+	}
+}
+
+func InitConf() error {
+	Debug("Enter InitConf() function")
+
+	viper.SetConfigType("yaml")
+	viper.AddConfigPath(Global_TopDir + "/config")
+	viper.SetConfigName("app")
+	// validate the app.yaml file whether is validate
+	data, err := ValidateYamlFormat(Global_TopDir + "/config/app.yaml")
+	if err != nil {
+		Error("Fail to call DiagnoseLogCreationError with error")
+		return err
+	}
+	if data == nil {
+		Error("Fail to call ValidateYamlFormat and data is nil")
+		return err
+	}
+
+	err1 := viper.ReadInConfig()
+	if err1 != nil {
+		_, ok := err1.(viper.ConfigFileNotFoundError)
+		if ok {
+			Error("Can't find the configuration files of app.yaml")
+			return err1
+		} else {
+			log.Println("The configuration file of app.yaml with some error")
+			return err1
+		}
+	}
+	// parse app section
+	AppYamlConfig = &AppConfig{
+		AppName:    viper.GetString("app.name"),
+		AppMode:    viper.GetString("app.runmode"),
+		HTTPPort:   viper.GetString("app.http_port"),
+		AppVersion: viper.GetString("app.version"),
+	}
+	// parse authentication section
+	AuthenticationYamlConfig = &Authentication{
+		Token: TokenConfig{
+			Type:            viper.GetString("authentication.token.type"),
+			SecretKey:       resolveJWTSecret(viper.GetString("authentication.token.secretKey")),
+			TokenExpireTime: int(viper.GetFloat64("authentication.token.token_expire_time")),
+		},
+	}
+	// parse security section
+	HttpsYamlConfig = &HTTPSConfig{
+		Enabled:  viper.GetBool("security.https.enabled"),
+		Port:     viper.GetString("security.https.port"),
+		CertFile: viper.GetString("security.https.cert_file"),
+		KeyFile:  viper.GetString("security.https.key_file"),
+	}
+
+	// parse rate limit section
+	RateLimitYamlConfig = &RateLimitConfig{
+		Global: EndpointLimit{
+			Rate:   viper.GetInt("rate_limit.global.rate"),
+			Burst:  viper.GetInt("rate_limit.global.burst"),
+			Window: viper.GetString("rate_limit.global.window"),
+		},
+		Endpoints: make(map[string]EndpointLimit),
+	}
+	// parse endpoints
+	endpoints := viper.GetStringMap("rate_limit.endpoints")
+	for path := range endpoints {
+		RateLimitYamlConfig.Endpoints[path] = EndpointLimit{
+			Rate:   viper.GetInt("rate_limit.endpoints." + path + ".rate"),
+			Burst:  viper.GetInt("rate_limit.endpoints." + path + ".burst"),
+			Window: viper.GetString("rate_limit.endpoints." + path + ".window"),
+		}
+	}
+
+	WhitelistYamlConfig = &WhitelistConfig{
+		Whitelist: viper.GetStringSlice("whitelist"),
+	}
+	// parse file upload section
+	rawMaxSize := viper.GetString("file_upload.max_size")
+	fileMaxSizeMB, err := ParseMaxSize(rawMaxSize)
+	if err != nil {
+		return fmt.Errorf("failed to parse max_size: %v", err)
+	}
+	FileUploadYamlConfig = &FileUploadConfig{
+		MaxSize:         strconv.FormatInt(fileMaxSizeMB, 10) + " MB",
+		AllowedTypes:    viper.GetStringSlice("file_upload.allowed_types"),
+		DisallowedTypes: viper.GetStringSlice("file_upload.disallowed_types"),
+	}
+
+	// parse job config section
+	// parse job config section
+	if !viper.IsSet("job") {
+		// if job section is not set, use default values
+		JobYamlConfig = &JobConfig{
+			MaxArraySize: 1000,
+		}
+	} else {
+		maxArraySize := viper.GetInt("job.max_array_size")
+		if maxArraySize == 0 {
+			maxArraySize = 1000
+		}
+		if maxArraySize < 1 || maxArraySize > 2147483646 {
+			return fmt.Errorf("invalid job.max_array_size value: %d, must be between 1 and 2147483646", maxArraySize)
+		}
+		JobYamlConfig = &JobConfig{
+			MaxArraySize: maxArraySize,
+		}
+	}
+
+	// parse cache config section
+	if !viper.IsSet("cache") {
+		// if cache section is not set, use default values
+		CacheYamlConfig = &CacheConfig{
+			RefreshInterval: 300,
+		}
+	} else {
+		refreshInterval := viper.GetInt("cache.refresh_interval")
+		if refreshInterval == 0 {
+			refreshInterval = 300
+		}
+		if refreshInterval < 1 || refreshInterval > 300 {
+			return fmt.Errorf("invalid cache.refresh_interval value: %d, must be between 1 and 300", refreshInterval)
+		}
+		CacheYamlConfig = &CacheConfig{
+			RefreshInterval: refreshInterval,
+		}
+	}
+
+	CORSYamlConfig = &CORSConfig{
+		Enabled:        viper.GetBool("cors.enabled"),
+		AllowedOrigins: viper.GetStringSlice("cors.allowed_origins"),
+	}
+
+	BackendYamlConfig = &BackendConfig{
+		Address: viper.GetString("backend.address"),
+		Port:    viper.GetInt("backend.port"),
+		From:    viper.GetString("backend.from"),
+		Key:     viper.GetString("backend.key"),
+	}
+
+	return nil
+}
+
+func ParseMaxSize(maxSize string) (int64, error) {
+	parts := strings.Fields(maxSize)
+	if len(parts) != 2 {
+		return 0, errors.New("invalid max_size format, expected '<number> <unit>'")
+	}
+
+	size, err := strconv.ParseInt(parts[0], 10, 64)
+	if err != nil {
+		return 0, errors.New("invalid number in max_size")
+	}
+
+	unit := strings.ToUpper(parts[1])
+	switch unit {
+	case "MB":
+		size *= 1 // MB remains as is
+	case "GB":
+		size *= 1024 // convert GB to MB
+	default:
+		return 0, errors.New("unsupported unit in max_size, use MB or GB")
+	}
+
+	if size > 10240 {
+		return 0, errors.New("max_size cannot exceed 10GB (10240 MB)")
+	}
+
+	return size, nil
+}
+
+// resolveJWTSecret returns the JWT secret from XCE_JWT_SECRET env var if set and non-empty,
+// otherwise falls back to the value from config file.
+func resolveJWTSecret(configValue string) string {
+	if env := os.Getenv("XCE_JWT_SECRET"); env != "" {
+		return env
+	}
+	return configValue
+}

src/frontend/apiserver/utils/config_test.go

@@ -0,0 +1,35 @@
+package utils
+
+import (
+	"os"
+	"testing"
+)
+
+func TestResolveJWTSecret_EnvOverridesConfig(t *testing.T) {
+	os.Setenv("XCE_JWT_SECRET", "env-secret")
+	defer os.Unsetenv("XCE_JWT_SECRET")
+
+	got := resolveJWTSecret("config-secret")
+	if got != "env-secret" {
+		t.Errorf("expected env-secret, got %q", got)
+	}
+}
+
+func TestResolveJWTSecret_FallsBackToConfig(t *testing.T) {
+	os.Unsetenv("XCE_JWT_SECRET")
+
+	got := resolveJWTSecret("config-secret")
+	if got != "config-secret" {
+		t.Errorf("expected config-secret, got %q", got)
+	}
+}
+
+func TestResolveJWTSecret_EmptyEnvUsesConfig(t *testing.T) {
+	os.Setenv("XCE_JWT_SECRET", "")
+	defer os.Unsetenv("XCE_JWT_SECRET")
+
+	got := resolveJWTSecret("config-secret")
+	if got != "config-secret" {
+		t.Errorf("expected config-secret, got %q", got)
+	}
+}

src/frontend/apiserver/utils/crypto.go

@@ -0,0 +1,63 @@
+package utils
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"io"
+	"os"
+)
+
+var EncryptionKey = ")ZLxR6X@h5tYGhkwin<j"
+
+// use sha256 to get 32 bytes key
+func getAESKey(key string) []byte {
+	selfSecretKey := os.Getenv("ENCRYPTION_KEY")
+	if selfSecretKey != "" {
+		key = selfSecretKey
+	} else {
+		key = EncryptionKey
+	}
+	hash := sha256.Sum256([]byte(key))
+	return hash[:]
+}
+
+func Encrypt(plaintext, secret string) (string, error) {
+	key := getAESKey(secret)
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+	ciphertext := make([]byte, aes.BlockSize+len(plaintext))
+	iv := ciphertext[:aes.BlockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return "", err
+	}
+
+	stream := cipher.NewCFBEncrypter(block, iv)
+	stream.XORKeyStream(ciphertext[aes.BlockSize:], []byte(plaintext))
+	return base64.StdEncoding.EncodeToString(ciphertext), nil
+}
+
+func Decrypt(encryptedText, secret string) (string, error) {
+	key := getAESKey(secret)
+	ciphertext, err := base64.StdEncoding.DecodeString(encryptedText)
+	if err != nil {
+		return "", err
+	}
+	if len(ciphertext) < aes.BlockSize {
+		return "", errors.New("the ciphertext is too short")
+	}
+	iv := ciphertext[:aes.BlockSize]
+	ciphertext = ciphertext[aes.BlockSize:]
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+	stream := cipher.NewCFBDecrypter(block, iv)
+	stream.XORKeyStream(ciphertext, ciphertext)
+	return string(ciphertext), nil
+}

src/frontend/apiserver/utils/jobidgen.go

@@ -0,0 +1,80 @@
+package utils
+
+import (
+	"errors"
+	"sync"
+	"time"
+)
+
+/**
+ * 31 bit timestamp(s)  |  | 5 bit clusterID | 10 bit sequenceBits
+ */
+const (
+	clusterIDBits = uint64(5) // 5 bits clusterID
+	sequenceBits  = uint64(10)
+
+	maxClusterID = int64(-1) ^ (int64(-1) << clusterIDBits)
+	maxSequence  = int64(-1) ^ (int64(-1) << sequenceBits)
+
+	timeLeft    = uint8(15) // timeLeft = clusterIDBits + sequenceBits //
+	clusterLeft = uint8(10)
+
+	twepoch = int64(1735228800) // Dec 27 00:00:00 2024
+)
+
+type Worker struct {
+	mu        sync.Mutex
+	LastStamp int64 // last timestamp
+	//WorkerID int64 //
+	ClusterID int64 // cluster id
+	Sequence  int64
+}
+
+// under distributed env, we need assign uniform id to each cluster: max 32 cluster.
+func NewWorker(clusterID int) *Worker {
+	return &Worker{
+		//WorkerID: workerID,
+		LastStamp: 0,
+		Sequence:  0,
+		ClusterID: int64(clusterID),
+	}
+}
+
+func (w *Worker) getSeconds() int64 {
+	//return time.Now().UnixNano() / 1e9 // get timestamp in second.
+	return time.Now().Unix() // / 1e9 // get timestamp in second.
+}
+
+func (w *Worker) NextID() (int64, error) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	return w.nextID()
+}
+
+func (w *Worker) nextID() (int64, error) {
+	timeStamp := w.getSeconds()
+	if timeStamp < w.LastStamp {
+		return 0, errors.New("time is moving backwards,waiting until")
+	}
+
+	if w.LastStamp == timeStamp {
+		w.Sequence = (w.Sequence + 1) & maxSequence
+
+		if w.Sequence == 0 {
+			for timeStamp <= w.LastStamp {
+				timeStamp = w.getSeconds()
+			}
+		}
+	} else {
+		w.Sequence = 0
+	}
+
+	w.LastStamp = timeStamp
+
+	id := ((timeStamp - twepoch) << timeLeft) |
+		(w.ClusterID << clusterLeft) |
+		w.Sequence
+
+	return int64(id), nil
+}

src/frontend/apiserver/utils/logger.go

@@ -0,0 +1,188 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	Log    *zap.Logger
+	once   sync.Once
+	config Config
+)
+
+type Config struct {
+	Logging struct {
+		Level      string `yaml:"level"`
+		Encoding   string `yaml:"encoding"`
+		OutputPath string `yaml:"outputPath"`
+		File       struct {
+			MaxSize    int  `yaml:"maxSize"`
+			MaxBackups int  `yaml:"maxBackups"`
+			MaxAge     int  `yaml:"maxAge"`
+			Compress   bool `yaml:"compress"`
+		} `yaml:"file"`
+		EncoderConfig struct {
+			MessageKey    string `yaml:"messageKey"`
+			LevelKey      string `yaml:"levelKey"`
+			TimeKey       string `yaml:"timeKey"`
+			NameKey       string `yaml:"nameKey"`
+			CallerKey     string `yaml:"callerKey"`
+			FunctionKey   string `yaml:"functionKey"`
+			StacktraceKey string `yaml:"stacktraceKey"`
+			EncodeTime    string `yaml:"timeEncoder"`
+			EncodeLevel   string `yaml:"levelEncoder"`
+		} `yaml:"encoderConfig"`
+	} `yaml:"logging"`
+}
+
+func InitLogger() error {
+	var err error
+	once.Do(func() {
+		err = loadLoggerConfig()
+	})
+	return err
+}
+
+func loadLoggerConfig() error {
+	// read logger configuration file
+	configPath := filepath.Join(Global_TopDir, "config", "logger.yaml")
+
+	// validate the yaml file format
+	data, err := ValidateYamlFormat(configPath)
+	if err != nil {
+		DiagnoseLogCreationError("Fail to call DiagnoseLogCreationError with error", err)
+		return err
+	}
+	if data == nil {
+		err = errors.New("invalid logger.yaml file")
+		DiagnoseLogCreationError("Fail to call ValidateYamlFormat and data is nil", err)
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		DiagnoseLogCreationError("Failed to parse logger configuration file", err)
+		return err
+	}
+
+	// Handle both formats for OutputPath
+	if !filepath.IsAbs(config.Logging.OutputPath) {
+		// If OutputPath is relative, prepend the logs directory
+		config.Logging.OutputPath = filepath.Join(Global_TopDir, "logs", config.Logging.OutputPath)
+	}
+
+	// Get the directory from the absolute path
+	logFilePath := config.Logging.OutputPath // it store the absolute path
+	logDir := filepath.Dir(logFilePath)      // Extract the directory
+	// Create the directory if it does not exist
+	if err := os.MkdirAll(logDir, 0755); err != nil {
+		DiagnoseLogCreationError("Failed to create logs directory", err)
+		return err
+	}
+
+	Log, err = createLogger(config)
+	if err != nil {
+		DiagnoseLogCreationError("Failed to create logger", err)
+		return err
+	}
+	return nil
+}
+
+func createLogger(config Config) (*zap.Logger, error) {
+	configLevel, err := zapcore.ParseLevel(config.Logging.Level)
+	if err != nil {
+		return nil, err
+	}
+
+	// create log file and set the permission
+	file, err := os.OpenFile(config.Logging.OutputPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
+	if err != nil {
+		return nil, err
+	}
+	file.Close()
+
+	// configure lumberjack for log rotation
+	logWriter := &lumberjack.Logger{
+		Filename:   config.Logging.OutputPath,
+		MaxSize:    config.Logging.File.MaxSize,
+		MaxBackups: config.Logging.File.MaxBackups,
+		MaxAge:     config.Logging.File.MaxAge,
+		Compress:   config.Logging.File.Compress,
+	}
+
+	// Create cores for different log levels
+	core := zapcore.NewCore(
+		zapcore.NewConsoleEncoder(zapcore.EncoderConfig{
+			MessageKey:       config.Logging.EncoderConfig.MessageKey,
+			LevelKey:         config.Logging.EncoderConfig.LevelKey,
+			TimeKey:          config.Logging.EncoderConfig.TimeKey,
+			NameKey:          config.Logging.EncoderConfig.NameKey,
+			CallerKey:        config.Logging.EncoderConfig.CallerKey,
+			StacktraceKey:    "", // disable stacktrace
+			EncodeTime:       zapcore.ISO8601TimeEncoder,
+			EncodeLevel:      zapcore.CapitalLevelEncoder,
+			EncodeCaller:     zapcore.ShortCallerEncoder,
+			ConsoleSeparator: " ",
+		}),
+		zapcore.AddSync(logWriter),
+		configLevel,
+	)
+	logger := zap.New(core,
+		zap.AddCaller(),                   // caller info
+		zap.AddStacktrace(zap.ErrorLevel), // show error stacktrace
+		zap.AddCallerSkip(1),              // skip one layer of caller
+	)
+	return logger, nil
+
+}
+
+func Info(msg string, fields ...zap.Field) {
+	Log.WithOptions(zap.WithCaller(false)).Info(msg, fields...)
+}
+func Error(msg string, fields ...zap.Field) {
+	Log.WithOptions(zap.WithCaller(false)).Error(msg, fields...)
+}
+
+func Warn(msg string, fields ...zap.Field) {
+	Log.WithOptions(zap.WithCaller(false)).Warn(msg, fields...)
+}
+
+func Debug(msg string, fields ...zap.Field) {
+	Log.WithOptions(zap.WithCaller(true)).Debug(msg, fields...)
+}
+
+func Fatal(msg string, fields ...zap.Field) {
+	Log.WithOptions(zap.WithCaller(false)).Fatal(msg, fields...)
+}
+
+// DiagnoseLogCreationError is a helper function to diagnose the error when creating log file
+func DiagnoseLogCreationError(context string, err error) {
+	// construct the error message
+	logMsg := fmt.Sprintf("[%s] %s: %v\n", time.Now().Format("2006-01-02 15:04:05"), context, err)
+	// define logger_error.log file path
+	logFilePath := filepath.Join(Global_TopDir, "logs", "logger_error.log")
+	// open log file
+	f, fileErr := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if fileErr != nil {
+		fmt.Printf("Failed to open logger_error.log: %v\n", fileErr)
+		// if failed to open log file, print the error message to console
+		fmt.Print(logMsg)
+		return
+	}
+	defer f.Close()
+	// write the error message
+	if _, writeErr := f.WriteString(logMsg); writeErr != nil {
+		fmt.Printf("Failed to write to logger_error.log: %v\n", writeErr)
+		// if failed to write to log file, print the error message to console
+		fmt.Print(logMsg)
+	}
+}

src/frontend/apiserver/utils/types.go

@@ -0,0 +1,532 @@
+package utils
+
+import (
+	"apiserver/models"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+// ValueType represents the type of value stored in a field.
+type ValueType int
+
+const (
+	TypeNull      ValueType = 0
+	TypeInt       ValueType = 1
+	TypeDouble    ValueType = 2
+	TypeString    ValueType = 3
+	TypeBool      ValueType = 4
+	TypeJSON      ValueType = 5
+	TypeTimestamp ValueType = 6
+	TypeArray     ValueType = 7
+)
+
+// OperatorType represents a filter comparison operator.
+type OperatorType int
+
+const (
+	OpEQ         OperatorType = 0
+	OpNE         OperatorType = 1
+	OpLT         OperatorType = 2
+	OpLTE        OperatorType = 3
+	OpGT         OperatorType = 4
+	OpGTE        OperatorType = 5
+	OpIN         OperatorType = 6
+	OpNOTIN      OperatorType = 7
+	OpLIKE       OperatorType = 8
+	OpNOTLIKE    OperatorType = 9
+	OpBETWEEN    OperatorType = 10
+	OpNOTBETWEEN OperatorType = 11
+	OpISNULL     OperatorType = 12
+	OpNOTNULL    OperatorType = 13
+	OpREGEX      OperatorType = 14
+	OpNOTREGEX   OperatorType = 15
+	OpAND        OperatorType = 16
+	OpOR         OperatorType = 17
+	OpNOT        OperatorType = 18
+)
+
+// Value represents a typed value.
+type Value struct {
+	Type  ValueType
+	Value interface{}
+}
+
+// Field represents a named value.
+type Field struct {
+	Name  string
+	Value Value
+}
+
+// Entity represents a collection of fields.
+type Entity struct {
+	Fields []Field
+}
+
+// Condition represents a filter condition on a field.
+type Condition struct {
+	Column   string
+	Operator OperatorType
+	Value    Value
+}
+
+// ConditionArray represents a compound condition with OR sub-conditions.
+type ConditionArray struct {
+	MainCondition   Condition
+	SubORConditions []Condition
+}
+
+// CompCondition maps field names to arrays of compound conditions.
+type CompCondition map[string][]ConditionArray
+
+// SortOrder represents a sort specification.
+type SortOrder struct {
+	Column    string
+	Ascending bool
+}
+
+// QueryOptions holds filter, sort, pagination parameters (pure Go, no CGo).
+type QueryOptions struct {
+	Conditions     []Condition
+	CompConditions CompCondition
+	Orders         []SortOrder
+	Limit          int
+	Bookmark       string
+}
+
+// OrderMap represents a row of data with ordered field names.
+type OrderMap struct {
+	Fields []string
+	Data   map[string]interface{}
+}
+
+// QueryResult is the API response for list queries.
+type QueryResult struct {
+	Count     int        `json:"count"`
+	Rows      []OrderMap `json:"rows"`
+	Pagetoken string     `json:"page_token"`
+}
+
+// Operator mapping tables (indexed by filter operator string).
+var (
+	IntOperatorMappingTable = map[string]OperatorType{
+		"eq":      OpEQ,
+		"ne":      OpNE,
+		"lt":      OpLT,
+		"lte":     OpLTE,
+		"gt":      OpGT,
+		"gte":     OpGTE,
+		"in":      OpIN,
+		"notin":   OpNOTIN,
+		"between": OpBETWEEN,
+		"null":    OpISNULL,
+		"notnull": OpNOTNULL,
+	}
+
+	TimeOperatorMappingTable = map[string]OperatorType{
+		"eq":      OpEQ,
+		"ne":      OpNE,
+		"lt":      OpLT,
+		"lte":     OpLTE,
+		"gt":      OpGT,
+		"gte":     OpGTE,
+		"in":      OpIN,
+		"notin":   OpNOTIN,
+		"between": OpBETWEEN,
+	}
+
+	StringOperatorMappingTable = map[string]OperatorType{
+		"contains":    OpLIKE,
+		"notcontains": OpNOTLIKE,
+		"startwith":   OpLIKE,
+		"endwith":     OpLIKE,
+		"eq":          OpEQ,
+		"ne":          OpNE,
+		"in":          OpIN,
+		"notin":       OpNOTIN,
+		"null":        OpISNULL,
+		"notnull":     OpNOTNULL,
+	}
+
+	BoolOperatorMappingTable = map[string]OperatorType{
+		"eq": OpEQ,
+		"ne": OpNE,
+	}
+)
+
+// getOperatorMappingTable returns the operator mapping table for a given reflect kind.
+func getOperatorMappingTable(kind reflect.Kind, typ reflect.Type) (map[string]OperatorType, error) {
+	switch kind {
+	case reflect.Int, reflect.Int64:
+		return IntOperatorMappingTable, nil
+	case reflect.Float64:
+		return IntOperatorMappingTable, nil
+	case reflect.String:
+		return StringOperatorMappingTable, nil
+	case reflect.Bool:
+		return BoolOperatorMappingTable, nil
+	case reflect.Struct:
+		if typ == reflect.TypeOf(time.Time{}) || typ == reflect.TypeOf(models.CustomTime{}) {
+			return TimeOperatorMappingTable, nil
+		}
+		return nil, fmt.Errorf("unsupported struct type for operator mapping: %v", typ)
+	default:
+		return nil, fmt.Errorf("unsupported type for operator mapping: %v", kind)
+	}
+}
+
+// BuildFilterConditions converts parsed filter conditions into typed Condition values.
+func BuildFilterConditions(filters []FilterCondition, datatype interface{}) ([]Condition, error) {
+	conditions := make([]Condition, 0)
+	for _, filter := range filters {
+		var cond Condition
+
+		val := reflect.ValueOf(datatype)
+		if val.Kind() == reflect.Ptr {
+			val = val.Elem()
+		} else {
+			return nil, fmt.Errorf("datatype must be a pointer to a struct")
+		}
+
+		typ := val.Type()
+
+		for i := 0; i < val.NumField(); i++ {
+			field := val.Field(i)
+			fieldType := typ.Field(i)
+
+			if filter.Field == strings.Split(fieldType.Tag.Get("json"), ",")[0] {
+				filterOperator := strings.ToLower(strings.TrimSpace(filter.Operator))
+
+				operatorMappingTable, err := getOperatorMappingTable(field.Kind(), field.Type())
+				if err != nil {
+					return nil, err
+				}
+
+				operator, ok := operatorMappingTable[filterOperator]
+				if !ok {
+					return nil, fmt.Errorf("unsupported operator for %s type: %s", field.Kind(), filter.Operator)
+				}
+
+				switch field.Kind() {
+				case reflect.Int64, reflect.Int:
+					cond, err = buildIntCondition(filter, filterOperator, operator)
+					if err != nil {
+						return nil, err
+					}
+
+				case reflect.String:
+					cond, err = buildStringCondition(filter, filterOperator, operator)
+					if err != nil {
+						return nil, err
+					}
+
+				case reflect.Float64:
+					cond, err = buildFloatCondition(filter, filterOperator, operator)
+					if err != nil {
+						return nil, err
+					}
+
+				case reflect.Bool:
+					boolVal, err := strconv.ParseBool(filter.Value)
+					if err != nil {
+						return nil, fmt.Errorf("failed to parse bool: %v", err)
+					}
+					cond = Condition{
+						Column:   filter.Field,
+						Operator: operator,
+						Value:    Value{Type: TypeBool, Value: boolVal},
+					}
+
+				case reflect.Struct:
+					if field.Type() == reflect.TypeOf(time.Time{}) || field.Type() == reflect.TypeOf(models.CustomTime{}) {
+						cond, err = buildTimeCondition(filter, filterOperator, operator)
+						if err != nil {
+							return nil, err
+						}
+					}
+				}
+
+				conditions = append(conditions, cond)
+				break
+			}
+		}
+	}
+	return conditions, nil
+}
+
+func buildIntCondition(filter FilterCondition, filterOperator string, operator OperatorType) (Condition, error) {
+	if filterOperator == "in" || filterOperator == "notin" || filterOperator == "between" {
+		fvs := strings.SplitN(strings.Trim(filter.Value, "()"), ",", -1)
+		if filterOperator == "between" && len(fvs) != 2 {
+			return Condition{}, fmt.Errorf("failed to parse between: %v", fvs)
+		}
+
+		// Special bitmask handling for job_status and host_status
+		if filter.Field == "job_status" || filter.Field == "host_status" {
+			var bitmask int64
+			for _, fv := range fvs {
+				fv = strings.ToUpper(strings.TrimSpace(fv))
+				var statMap map[string]int
+				if filter.Field == "job_status" {
+					statMap = models.JOBSTAT
+				} else {
+					statMap = models.HOSTSTAT
+				}
+				if targV, ok := statMap[fv]; ok {
+					bitmask |= int64(targV)
+				} else {
+					return Condition{}, fmt.Errorf("invalid %s: %s", filter.Field, fv)
+				}
+			}
+			op := OpNE
+			if filterOperator == "notin" {
+				op = OpEQ
+			}
+			return Condition{
+				Column:   filter.Field,
+				Operator: op,
+				Value:    Value{Type: TypeInt, Value: bitmask},
+			}, nil
+		}
+
+		// Normal int array
+		intValues := make([]Value, 0, len(fvs))
+		for _, fv := range fvs {
+			fv = strings.TrimSpace(fv)
+			v, err := strconv.ParseInt(fv, 10, 64)
+			if err != nil {
+				return Condition{}, fmt.Errorf("failed to parse int: %v", err)
+			}
+			intValues = append(intValues, Value{Type: TypeInt, Value: v})
+		}
+		return Condition{
+			Column:   filter.Field,
+			Operator: operator,
+			Value:    Value{Type: TypeArray, Value: intValues},
+		}, nil
+
+	}
+
+	// Scalar: eq, ne, lt, lte, gt, gte, null, notnull
+	if filterOperator == "null" || filterOperator == "notnull" {
+		return Condition{
+			Column:   filter.Field,
+			Operator: operator,
+			Value:    Value{Type: TypeInt, Value: int64(0)},
+		}, nil
+	}
+
+	// Bitmask fields for scalar operators
+	if filter.Field == "job_status" || filter.Field == "host_status" {
+		fv := strings.ToUpper(filter.Value)
+		var statMap map[string]int
+		if filter.Field == "job_status" {
+			statMap = models.JOBSTAT
+		} else {
+			statMap = models.HOSTSTAT
+		}
+		targV, ok := statMap[fv]
+		if !ok {
+			return Condition{}, fmt.Errorf("invalid %s: %s", filter.Field, filter.Value)
+		}
+		op := OpNE // "eq" for bitmask means (val & mask) != 0
+		if filterOperator == "eq" {
+			op = OpNE
+		} else {
+			op = OpEQ
+		}
+		return Condition{
+			Column:   filter.Field,
+			Operator: op,
+			Value:    Value{Type: TypeInt, Value: int64(targV)},
+		}, nil
+	}
+
+	intV, err := strconv.ParseInt(filter.Value, 10, 64)
+	if err != nil {
+		return Condition{}, fmt.Errorf("failed to parse int: %v", err)
+	}
+	return Condition{
+		Column:   filter.Field,
+		Operator: operator,
+		Value:    Value{Type: TypeInt, Value: intV},
+	}, nil
+}
+
+func buildStringCondition(filter FilterCondition, filterOperator string, operator OperatorType) (Condition, error) {
+	if filterOperator == "in" || filterOperator == "notin" || filterOperator == "between" {
+		fvs := strings.SplitN(strings.Trim(filter.Value, "()"), ",", -1)
+		if filterOperator == "between" && len(fvs) != 2 {
+			return Condition{}, fmt.Errorf("failed to parse between: %v", fvs)
+		}
+		strValues := make([]Value, 0, len(fvs))
+		for _, fv := range fvs {
+			fv = strings.TrimSpace(fv)
+			strValues = append(strValues, Value{Type: TypeString, Value: fv})
+		}
+		if len(strValues) == 0 {
+			return Condition{}, fmt.Errorf("%s has no operands specified", filter.Operator)
+		}
+		return Condition{
+			Column:   filter.Field,
+			Operator: operator,
+			Value:    Value{Type: TypeArray, Value: strValues},
+		}, nil
+	}
+
+	if filterOperator == "null" || filterOperator == "notnull" {
+		return Condition{
+			Column:   filter.Field,
+			Operator: operator,
+			Value:    Value{Type: TypeString, Value: ""},
+		}, nil
+	}
+
+	// Store the original operator string for LIKE pattern matching
+	return Condition{
+		Column:   filter.Field,
+		Operator: operator,
+		Value:    Value{Type: TypeString, Value: filter.Value},
+	}, nil
+}
+
+func buildFloatCondition(filter FilterCondition, filterOperator string, operator OperatorType) (Condition, error) {
+	if filterOperator == "null" || filterOperator == "notnull" {
+		return Condition{
+			Column:   filter.Field,
+			Operator: operator,
+			Value:    Value{Type: TypeDouble, Value: float64(0)},
+		}, nil
+	}
+	floatV, err := strconv.ParseFloat(filter.Value, 64)
+	if err != nil {
+		return Condition{}, fmt.Errorf("failed to parse float: %v", err)
+	}
+	return Condition{
+		Column:   filter.Field,
+		Operator: operator,
+		Value:    Value{Type: TypeDouble, Value: floatV},
+	}, nil
+}
+
+func buildTimeCondition(filter FilterCondition, filterOperator string, operator OperatorType) (Condition, error) {
+	if filterOperator == "between" {
+		fvs := strings.SplitN(strings.Trim(filter.Value, "()"), ",", -1)
+		if len(fvs) != 2 {
+			return Condition{}, fmt.Errorf("failed to parse between: %v", fvs)
+		}
+		timeValues := make([]Value, 0, 2)
+		for _, fv := range fvs {
+			fv = strings.TrimSpace(fv)
+			t, err := time.Parse("2006-01-02 15:04:05", fv)
+			if err != nil {
+				return Condition{}, fmt.Errorf("failed to parse time: %v", err)
+			}
+			timeValues = append(timeValues, Value{Type: TypeTimestamp, Value: t.Unix()})
+		}
+		return Condition{
+			Column:   filter.Field,
+			Operator: operator,
+			Value:    Value{Type: TypeArray, Value: timeValues},
+		}, nil
+	}
+
+	t, err := time.Parse("2006-01-02 15:04:05", filter.Value)
+	if err != nil {
+		return Condition{}, fmt.Errorf("failed to parse time: %v", err)
+	}
+	return Condition{
+		Column:   filter.Field,
+		Operator: operator,
+		Value:    Value{Type: TypeTimestamp, Value: t.Unix()},
+	}, nil
+}
+
+// BuildQueryOptions creates a QueryOptions from parsed filter, sort, and pagination parameters.
+func BuildQueryOptions(filtercds []Condition, compfiltercd CompCondition, orders []SortOrder, limit int, bookmark string) (*QueryOptions, error) {
+	return &QueryOptions{
+		Conditions:     filtercds,
+		CompConditions: compfiltercd,
+		Orders:         orders,
+		Limit:          limit,
+		Bookmark:       bookmark,
+	}, nil
+}
+
+// ParseJSONB parses a JSONB string into a Go interface.
+func ParseJSONB(jsonStr string) (interface{}, error) {
+	if jsonStr == "" {
+		return nil, nil
+	}
+	var result interface{}
+	err := json.Unmarshal([]byte(jsonStr), &result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+// FormatTimestamp converts a unix timestamp to local time string.
+func FormatTimestamp(ts int64) string {
+	if ts == 0 {
+		return ""
+	}
+	t := time.Unix(ts, 0)
+	return t.Local().Format("2006-01-02 15:04:05")
+}
+
+
+// Supported query parameter lists for validation.
+var (
+	SupportedQueryParams         = []string{"filter", "sort", "limit", "page_token", "fields"}
+	SpecificSupportedQueryParams = []string{"filter", "sort", "limit", "page_token", "fields", "ids"}
+	TypeQueryParams              = []string{"type"}
+	UniqueQueryParams            = []string{"role_name", "user_name"}
+)
+
+// ConvertToNewFilterArray converts compound filter conditions into typed CompCondition.
+func ConvertToNewFilterArray(compfilters CompFilterCondition, datatype interface{}) (CompCondition, error) {
+	result := make(CompCondition)
+	for key, fcArrays := range compfilters {
+		var condArrays []ConditionArray
+		for _, fca := range fcArrays {
+			// Build main condition
+			mainConds, err := BuildFilterConditions([]FilterCondition{fca.MainFiltCond}, datatype)
+			if err != nil {
+				return nil, err
+			}
+			var mainCond Condition
+			if len(mainConds) > 0 {
+				mainCond = mainConds[0]
+			}
+
+			// Build sub OR conditions
+			subConds, err := BuildFilterConditions(fca.SubFiltConds, datatype)
+			if err != nil {
+				return nil, err
+			}
+
+			condArrays = append(condArrays, ConditionArray{
+				MainCondition:   mainCond,
+				SubORConditions: subConds,
+			})
+		}
+		result[key] = condArrays
+	}
+	return result, nil
+}
+
+// ConvertToNewFilter is an alias for BuildFilterConditions.
+func ConvertToNewFilter(filters []FilterCondition, datatype interface{}) ([]Condition, error) {
+	return BuildFilterConditions(filters, datatype)
+}
+
+// init suppresses unused import warnings
+func init() {
+	_ = zap.String
+}

src/frontend/apiserver/utils/utils.go

@@ -0,0 +1,646 @@
+package utils
+
+import (
+	"apiserver/models"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v3"
+)
+
+var (
+	Global_TopDir      string
+	PER_PAGE_LIMIT     = 50
+	PER_PAGE_LIMIT_MAX = 1000
+	FileChunkSize      = 1024 * 1024 // 1 MB
+)
+
+type FilterCondition struct {
+	Field    string
+	Operator string
+	Value    string
+}
+
+// to store filter condition for array job. 101[1-3, 6-8]
+// mainFiltCond: job_id=101
+// SubFiltConds: [ array_idx:btw:(1,3), array_idx:btw:(6,8) ]
+type FilterConditionArray struct {
+	MainFiltCond FilterCondition
+	SubFiltConds []FilterCondition
+}
+
+type CompFilterCondition map[string][]FilterConditionArray
+
+// check the starting top directory whether correctly
+func PromptForTopDir(topDir *string) string {
+	if *topDir == "" {
+		fmt.Fprintln(os.Stderr, "Error: -d parameter is required")
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// convert to absolute path if the directory is relative
+	absPath, err := filepath.Abs(*topDir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: failed to convert to absolute path: %v\n", err)
+		os.Exit(1)
+	}
+	*topDir = absPath
+
+	// check if the directory exists
+	if _, err := os.Stat(*topDir); os.IsNotExist(err) {
+		fmt.Fprintf(os.Stderr, "Error: directory %s does not exist, fails to run the command\n", *topDir)
+		os.Exit(1)
+	}
+
+	// check if the config directory exists
+	configDir := filepath.Join(*topDir, "config")
+	if _, err := os.Stat(configDir); os.IsNotExist(err) {
+		fmt.Fprintf(os.Stderr, "Error: config directory %s does not exist, fails to run the command\n", configDir)
+		os.Exit(1)
+	}
+
+	return *topDir
+}
+
+// bindJSONStrict strict check the json format, if the json format is not correct, return error.
+func BindJSONStrict(ctx *gin.Context, obj interface{}) error {
+	// First read the raw body to check for duplicate fields
+	body, err := io.ReadAll(ctx.Request.Body)
+	if err != nil {
+		return fmt.Errorf("error reading request body: %v", err)
+	}
+
+	// Reset the request body so it can be read again
+	ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
+
+	// Check for duplicate fields using json.Decoder
+	dec := json.NewDecoder(bytes.NewBuffer(body))
+	seenFields := make(map[string]bool)
+
+	// Read the opening brace
+	t, err := dec.Token()
+	if err != nil {
+		return fmt.Errorf("error reading JSON: %v", err)
+	}
+	if delim, ok := t.(json.Delim); !ok || delim != '{' {
+		return fmt.Errorf("expected JSON object")
+	}
+
+	// Read key-value pairs
+	for dec.More() {
+		// Read key
+		t, err := dec.Token()
+		if err != nil {
+			return fmt.Errorf("error reading JSON key: %v", err)
+		}
+		key, ok := t.(string)
+		if !ok {
+			return fmt.Errorf("expected string key")
+		}
+
+		// Check for duplicate key
+		if seenFields[key] {
+			return fmt.Errorf("duplicate field found in JSON: %s", key)
+		}
+		seenFields[key] = true
+
+		// Skip the value
+		var value interface{}
+		if err := dec.Decode(&value); err != nil {
+			return fmt.Errorf("error reading JSON value: %v", err)
+		}
+	}
+
+	// Read the closing brace
+	t, err = dec.Token()
+	if err != nil {
+		return fmt.Errorf("error reading JSON: %v", err)
+	}
+	if delim, ok := t.(json.Delim); !ok || delim != '}' {
+		return fmt.Errorf("expected closing brace")
+	}
+
+	// Now decode into the target object
+	dec = json.NewDecoder(bytes.NewBuffer(body))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(obj); err != nil {
+		var syntaxError *json.SyntaxError
+		var unmarshalTypeError *json.UnmarshalTypeError
+		var invalidUnmarshalError *json.InvalidUnmarshalError
+
+		switch {
+		case errors.As(err, &syntaxError):
+			return fmt.Errorf("request body contains badly-formed JSON (at position %d)", syntaxError.Offset)
+
+		case errors.Is(err, io.ErrUnexpectedEOF):
+			return errors.New("request body contains badly-formed JSON")
+
+		case errors.As(err, &unmarshalTypeError):
+			if unmarshalTypeError.Field != "" {
+				return fmt.Errorf("request body contains incorrect JSON type for field %q", unmarshalTypeError.Field)
+			}
+			return fmt.Errorf("request body contains incorrect JSON type (at position %d)", unmarshalTypeError.Offset)
+
+		case errors.Is(err, io.EOF):
+			return errors.New("request body must not be empty")
+
+		case strings.HasPrefix(err.Error(), "json: unknown field "):
+			fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ")
+			return fmt.Errorf("request body contains unknown field %s", fieldName)
+
+		case errors.As(err, &invalidUnmarshalError):
+			return fmt.Errorf("error unmarshalling json: %v", err)
+
+		default:
+			return errors.New(err.Error())
+		}
+	}
+
+	val := reflect.ValueOf(obj)
+	if val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+
+	for i := 0; i < val.NumField(); i++ {
+		field := val.Type().Field(i)
+		if field.Tag.Get("binding") == "required" {
+			if val.Field(i).Kind() == reflect.String {
+				trimmedValue := strings.TrimSpace(val.Field(i).String())
+				val.Field(i).SetString(trimmedValue)
+			}
+			if isZero(val.Field(i)) {
+				// get json tag name
+				jsonTag := field.Tag.Get("json")
+				if jsonTag == "" {
+					jsonTag = field.Name // if no tag name, using field name
+				}
+				return fmt.Errorf("json key of %s is required", jsonTag)
+			}
+		}
+	}
+	return nil
+}
+
+func isZero(v reflect.Value) bool {
+	return v.IsZero()
+}
+
+// ParseFilters parses filter string into FilterConditions
+func ParseFilters(filterStr string) []FilterCondition {
+	Debug("Enter ParseFilters function", zap.String("filterStr", filterStr))
+	// Remove brackets
+	//filterStr = strings.TrimPrefix(filterStr, "[")
+	//filterStr = strings.TrimSuffix(filterStr, "]") // since job_name will add index
+
+	// remove double quotes
+	filterStr = strings.ReplaceAll(filterStr, "\"", "")
+	filterStr = strings.ReplaceAll(filterStr, "}, {", "},{")
+
+	if filterStr == "" {
+		Debug("filterStr is empty, return nil")
+		return nil
+	}
+
+	conditions := strings.Split(filterStr, "},{")
+	filters := make([]FilterCondition, 0)
+
+	for _, condition := range conditions {
+		condition = strings.TrimPrefix(condition, "{")
+		condition = strings.TrimSuffix(condition, "}")
+
+		// need handle submit_time:eq:2025-03-31 14:34:07}
+		parts := strings.SplitN(condition, ":", 3)
+		if len(parts) != 3 {
+			continue
+		}
+
+		// check if there are same filter key and operator
+		exists := false
+		for _, filter := range filters {
+			if filter.Field == parts[0] && filter.Operator == parts[1] {
+				exists = true
+				break
+			}
+		}
+		if exists {
+			Error("There are duplicated filter key and operator, please check the input.")
+			return nil // return error for filter
+		}
+
+		filters = append(filters, FilterCondition{
+			Field:    parts[0],
+			Operator: parts[1],
+			Value:    strings.TrimSpace(parts[2]),
+		})
+		Debug("In function of ParseFilters, Field, Operator, Value are:", zap.Any("Field", parts[0]),
+			zap.Any("Op", parts[1]), zap.Any("Value", strings.TrimSpace(parts[2])))
+	}
+	if len(filters) > 0 {
+		Debug("Finish calling ParseFilters function, filter results", zap.Any("filters", filters))
+		return filters
+	} else {
+		Debug("The format of filter is not correct!")
+		return nil
+	}
+}
+
+// parseFields will parse the fields string
+// eg: field=col1,col2,col3
+func ParseFields(fieldsStr string) []string {
+	// Parse fields
+	var fields []string
+
+	if fieldsStr == "" {
+		return nil
+	}
+	fields = strings.Split(fieldsStr, ",")
+	// Trim spaces
+	result := make([]string, 0, len(fields))
+	for _, field := range fields {
+		field = strings.TrimSpace(field)
+		if field != "" {
+			result = append(result, field)
+		}
+	}
+	return result
+}
+
+func ParsePageToken(pagingStr string) (string, error) {
+	if pagingStr != "" {
+		pagingStr = strings.TrimPrefix(pagingStr, "\"")
+		pagingStr = strings.TrimSuffix(pagingStr, "\"")
+		pagingStr = strings.TrimSpace(pagingStr)
+		return pagingStr, nil
+	}
+	return "", nil
+}
+
+// Parse sort string in query parameters
+// sort=job_id:asc,submit_time:desc
+func ParseSortParam(s string, supportSortList []string) ([]SortOrder, error) {
+	if s == "" {
+		return nil, fmt.Errorf("the sort params is empty")
+	}
+
+	sortStr := strings.TrimSpace(s)
+	sorts := strings.SplitN(sortStr, ",", -1)
+	sortList := make([]SortOrder, 0)
+
+	// Convert supportSortList to map for quick lookup
+	sortMap := make(map[string]bool)
+	for _, item := range supportSortList {
+		sortMap[item] = true
+	}
+
+	if len(sorts) > 0 {
+		for _, sort := range sorts {
+			var col, order string
+			if strings.Contains(sort, ":") {
+				col = strings.SplitN(sort, ":", -1)[0]
+				order = strings.SplitN(sort, ":", -1)[1]
+				if order != "asc" && order != "desc" {
+					return nil, fmt.Errorf("the order string %s not in \"asc\" or \"desc\" ", order)
+				}
+			} else {
+				col = sort
+				order = "asc" // by default
+			}
+			if _, ok := sortMap[col]; !ok {
+				return nil, fmt.Errorf("unsupported sort column: %s", col)
+			}
+			sortList = append(sortList, SortOrder{
+				Column:    col,
+				Ascending: order == "asc",
+			})
+		}
+	} else {
+		return nil, fmt.Errorf("the sort params is empty")
+	}
+	return sortList, nil
+}
+
+func CheckAllMapKeyInList(sorts []SortOrder, fields []string) error {
+	// check the sort column is among the query fields
+	for _, col := range sorts {
+		found := ContainsField(fields, col.Column)
+		if !found {
+			err := fmt.Errorf("key \"%s\" is not in fields:%v ", col.Column, fields)
+			return err
+		}
+	}
+	return nil
+}
+
+// judge if a field exist in supported fields list
+func ContainsField(fields []string, field string) bool {
+	for _, v := range fields {
+		if v == field {
+			return true
+		}
+	}
+	return false
+}
+
+// judge if a tagName exist in structType json tag ;
+func JsonTagExists(structType interface{}, tagName string) bool {
+	val := reflect.ValueOf(structType)
+	for i := 0; i < val.NumField(); i++ {
+		field := val.Type().Field(i)
+		if field.Tag.Get("json") == tagName {
+			return true
+		}
+	}
+	return false
+}
+
+// in a map, get value's corresponding key
+func GetKeyFromValue(m map[string]int, value int) (string, error) {
+	for k, v := range m {
+		if v == value {
+			return k, nil
+		}
+	}
+	return "", errors.New("value is not recognized.")
+}
+
+// GetMappedField returns the mapped field if it exists, otherwise returns the original field.
+func GetMappedField(jsonTag string) string {
+	if mappedField, exists := models.JobFieldMappings[jsonTag]; exists {
+		return mappedField
+	}
+	return jsonTag
+}
+
+// GetSupportedFields returns the supported fields for a given model, applying any field mappings.
+func GetSupportedFields(model interface{}) []string {
+	var fields []string
+	t := reflect.TypeOf(model)
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+		jsonTag := field.Tag.Get("json")
+		if jsonTag != "" {
+			// Only take the first part before any comma
+			jsonTag = strings.Split(jsonTag, ",")[0]
+			// Apply field mapping
+			fields = append(fields, jsonTag)
+		}
+	}
+	return fields
+}
+
+// IntOrInt64 converts an interface{} value that is either int or int64 to int.
+// Returns (0, false) for any other type.
+func IntOrInt64(v interface{}) (int, bool) {
+	switch t := v.(type) {
+	case int:
+		return t, true
+	case int64:
+		return int(t), true
+	default:
+		return 0, false
+	}
+}
+
+// ValidateYamlFormat reads and validates a YAML file, returning its raw bytes.
+func ValidateYamlFormat(filePath string) ([]byte, error) {
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read YAML file %s: %v", filePath, err)
+	}
+	var out interface{}
+	if err := yaml.Unmarshal(data, &out); err != nil {
+		return nil, fmt.Errorf("invalid YAML format in %s: %v", filePath, err)
+	}
+	return data, nil
+}
+
+func ParseLimit(limitStr string) (int, error) {
+	if limitStr == "" {
+		return PER_PAGE_LIMIT, nil
+	}
+	limit, err := strconv.Atoi(limitStr)
+	if err != nil {
+		return 0, err
+	}
+	if limit > PER_PAGE_LIMIT_MAX {
+		return 0, fmt.Errorf("limit=%d exceed MAX=%d", limit, PER_PAGE_LIMIT_MAX)
+	}
+	return limit, nil
+}
+
+func ParseType(typeStr string, availableTypes []string) (string, error) {
+	if typeStr == "" {
+		return "", nil
+	}
+	typeStr = strings.TrimSpace(typeStr)
+	for _, t := range availableTypes {
+		if typeStr == t {
+			return typeStr, nil
+		}
+	}
+	return "", fmt.Errorf("type=%s is not supported, available types: %v", typeStr, availableTypes)
+}
+
+func ParseUnique(uniqueStr string) bool {
+	if uniqueStr == "" {
+		return false
+	}
+
+	// Convert to lowercase and check if it's exactly "true"
+	return strings.ToLower(strings.TrimSpace(uniqueStr)) == "true"
+}
+
+func ConvertStringToJSON(str []string) (string, error) {
+	// eg:"VAR1=value1","VAR2=value2"
+	jsonMap := make(map[string]string)
+	for _, e := range str {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) == 2 {
+			jsonMap[parts[0]] = parts[1]
+		}
+	}
+	strIntoJson, err := json.Marshal(jsonMap)
+	if err != nil {
+		return "", err
+	}
+	return string(strIntoJson), nil
+}
+
+func ConvertStringToJsonb(str []string) (models.Jsonb, error) {
+	// eg:"VAR1=value1","VAR2=value2"
+	jsonMap := make(map[string]string)
+	for _, e := range str {
+		parts := strings.SplitN(e, "=", 2)
+		if len(parts) == 2 {
+			jsonMap[parts[0]] = parts[1]
+		}
+	}
+	strIntoJson, err := json.Marshal(jsonMap)
+	if err != nil {
+		return models.Jsonb{}, err
+	}
+	jsonb := models.Jsonb{
+		String: string(strIntoJson),
+	}
+	return jsonb, nil
+}
+
+func AddFieldsToSupportedList(target *[]string, fields interface{}) {
+	t := reflect.TypeOf(fields)
+	for i := 0; i < t.NumField(); i++ {
+		jsonTag := t.Field(i).Tag.Get("json")
+		if jsonTag != "" && jsonTag != "-" {
+			*target = append(*target, jsonTag)
+		}
+	}
+}
+
+func GetServerHostPort(ctx *gin.Context) string {
+	scheme := "http"
+	if ctx.Request.TLS != nil {
+		scheme = "https"
+	}
+	return fmt.Sprintf("%s://%s", scheme, ctx.Request.Host)
+}
+
+// convert timestamp(s) to local time string with TZ.
+func convertTimestampToLocalTime(timestamp int64) string {
+	utcTime := time.Unix(timestamp, 0).UTC()
+	localTime := utcTime.Local()
+	return localTime.Format("2006-01-02 15:04:05 MST")
+}
+
+// convert time string with TZ to timestamp
+func convertTimestrToTimestamp(timeStr string) (int64, error) {
+	// Define the layout for parsing the time string.
+	// "2006-01-02 15:04:05 MST" is a reference time in Go.
+	layout := "2006-01-02 15:04:05 MST"
+
+	// Parse the time string into a time.Time object.
+	parsedTime, err := time.Parse(layout, timeStr)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse time: %w", err)
+	}
+	// Get the timestamp in seconds.
+	timestamp := parsedTime.Unix()
+
+	return timestamp, nil
+}
+
+func IsNumeric(s string) bool {
+	_, err := strconv.ParseFloat(s, 64)
+	return err == nil
+}
+
+func ValidateControlAction(controlAction string, supportedActions []string) error {
+	controlAction = strings.ToLower(strings.TrimSpace(controlAction))
+	for _, action := range supportedActions {
+		if controlAction == action {
+			return nil
+		}
+	}
+
+	return errors.New("unsupported control action: " + controlAction)
+}
+
+func IsControlActionAllowed(actionType int, status int64, actionControlMap map[int][]int) bool {
+	Debug("Enter IsActionAllowed function",
+		zap.Any("actionType", actionType),
+		zap.Any("status", status))
+
+	allowedStatuses, exists := actionControlMap[actionType]
+	if !exists {
+		Error("Unsupported action type", zap.Any("actionType", actionType))
+		return false
+	}
+
+	for _, allowedStatus := range allowedStatuses {
+		if allowedStatus == int(status) {
+			Debug("Action is allowed",
+				zap.Any("status", status),
+				zap.Any("allowedStatus", allowedStatus))
+			return true
+		}
+	}
+	Debug("Action is not allowed", zap.Any("status", status))
+	return false
+}
+
+// RemoveDuplicates removes duplicate elements from any slice.
+func RemoveDuplicates[T comparable](input []T) []T {
+	seen := make(map[T]bool)
+	result := []T{}
+
+	for _, value := range input {
+		if !seen[value] {
+			seen[value] = true
+			result = append(result, value)
+		}
+	}
+	return result
+}
+
+func TruncateString(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
+	}
+	runes := []rune(s)
+	if len(runes) > maxLen {
+		return string(runes[:maxLen])
+	}
+	return s
+}
+
+func IncludeBuiltinRoles(roleNames []string) (builtinIncluded []string) {
+	for _, roleName := range roleNames {
+		if ContainsField(models.BUILTIN_ROLE_NAMES, roleName) {
+			builtinIncluded = append(builtinIncluded, roleName)
+		}
+	}
+	return builtinIncluded
+}
+
+func IsValidUserName(userName string) error {
+	// Check if name is empty
+	if userName == "" {
+		return errors.New("username cannot be empty")
+	}
+
+	// Check maximum length (32 characters)
+	if len(userName) > 32 {
+		return errors.New("username cannot exceed 32 characters")
+	}
+
+	// Check if it begins with a letter
+	if !((userName[0] >= 'a' && userName[0] <= 'z') || (userName[0] >= 'A' && userName[0] <= 'Z')) {
+		return errors.New("username must begin with a letter")
+	}
+
+	// Check for valid characters (0-9, a-z, A-Z, -, _, space)
+	for i := 0; i < len(userName); i++ {
+		c := userName[i]
+		if !((c >= 'a' && c <= 'z') ||
+			(c >= 'A' && c <= 'Z') ||
+			(c >= '0' && c <= '9') ||
+			c == '-' ||
+			c == '_' ||
+			c == ' ') {
+			return errors.New("username can only contain letters, numbers, hyphens, underscores, or spaces")
+		}
+	}
+
+	return nil
+}

src/frontend/apiserver/utils/utils_test.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"testing"
+)
+
+func TestIntOrInt64(t *testing.T) {
+	tests := []struct {
+		name  string
+		input interface{}
+		want  int
+		ok    bool
+	}{
+		{"int zero", int(0), 0, true},
+		{"int positive", int(42), 42, true},
+		{"int negative", int(-1), -1, true},
+		{"int64 zero", int64(0), 0, true},
+		{"int64 positive", int64(42), 42, true},
+		{"int64 negative", int64(-1), -1, true},
+		{"int64 large", int64(9223372036854775807), int(9223372036854775807), true}, // int==int64 on 64-bit
+		{"string", "42", 0, false},
+		{"float64", 42.0, 0, false},
+		{"nil", nil, 0, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := IntOrInt64(tt.input)
+			if ok != tt.ok {
+				t.Errorf("IntOrInt64(%v) ok = %v, want %v", tt.input, ok, tt.ok)
+			}
+			if ok && got != tt.want {
+				t.Errorf("IntOrInt64(%v) = %d, want %d", tt.input, got, tt.want)
+			}
+		})
+	}
+}