package utils import ( "apiserver/models" "bytes" "encoding/json" "errors" "flag" "fmt" "io" "os" "path/filepath" "reflect" "strconv" "strings" "time" "github.com/gin-gonic/gin" "go.uber.org/zap" "gopkg.in/yaml.v3" ) var ( Global_TopDir string PER_PAGE_LIMIT = 50 PER_PAGE_LIMIT_MAX = 1000 FileChunkSize = 1024 * 1024 // 1 MB ) type FilterCondition struct { Field string Operator string Value string } // to store filter condition for array job. 101[1-3, 6-8] // mainFiltCond: job_id=101 // SubFiltConds: [ array_idx:btw:(1,3), array_idx:btw:(6,8) ] type FilterConditionArray struct { MainFiltCond FilterCondition SubFiltConds []FilterCondition } type CompFilterCondition map[string][]FilterConditionArray // check the starting top directory whether correctly func PromptForTopDir(topDir *string) string { if *topDir == "" { fmt.Fprintln(os.Stderr, "Error: -d parameter is required") flag.Usage() os.Exit(1) } // convert to absolute path if the directory is relative absPath, err := filepath.Abs(*topDir) if err != nil { fmt.Fprintf(os.Stderr, "Error: failed to convert to absolute path: %v\n", err) os.Exit(1) } *topDir = absPath // check if the directory exists if _, err := os.Stat(*topDir); os.IsNotExist(err) { fmt.Fprintf(os.Stderr, "Error: directory %s does not exist, fails to run the command\n", *topDir) os.Exit(1) } // check if the config directory exists configDir := filepath.Join(*topDir, "config") if _, err := os.Stat(configDir); os.IsNotExist(err) { fmt.Fprintf(os.Stderr, "Error: config directory %s does not exist, fails to run the command\n", configDir) os.Exit(1) } return *topDir } // bindJSONStrict strict check the json format, if the json format is not correct, return error. func BindJSONStrict(ctx *gin.Context, obj interface{}) error { // First read the raw body to check for duplicate fields body, err := io.ReadAll(ctx.Request.Body) if err != nil { return fmt.Errorf("error reading request body: %v", err) } // Reset the request body so it can be read again ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body)) // Check for duplicate fields using json.Decoder dec := json.NewDecoder(bytes.NewBuffer(body)) seenFields := make(map[string]bool) // Read the opening brace t, err := dec.Token() if err != nil { return fmt.Errorf("error reading JSON: %v", err) } if delim, ok := t.(json.Delim); !ok || delim != '{' { return fmt.Errorf("expected JSON object") } // Read key-value pairs for dec.More() { // Read key t, err := dec.Token() if err != nil { return fmt.Errorf("error reading JSON key: %v", err) } key, ok := t.(string) if !ok { return fmt.Errorf("expected string key") } // Check for duplicate key if seenFields[key] { return fmt.Errorf("duplicate field found in JSON: %s", key) } seenFields[key] = true // Skip the value var value interface{} if err := dec.Decode(&value); err != nil { return fmt.Errorf("error reading JSON value: %v", err) } } // Read the closing brace t, err = dec.Token() if err != nil { return fmt.Errorf("error reading JSON: %v", err) } if delim, ok := t.(json.Delim); !ok || delim != '}' { return fmt.Errorf("expected closing brace") } // Now decode into the target object dec = json.NewDecoder(bytes.NewBuffer(body)) dec.DisallowUnknownFields() if err := dec.Decode(obj); err != nil { var syntaxError *json.SyntaxError var unmarshalTypeError *json.UnmarshalTypeError var invalidUnmarshalError *json.InvalidUnmarshalError switch { case errors.As(err, &syntaxError): return fmt.Errorf("request body contains badly-formed JSON (at position %d)", syntaxError.Offset) case errors.Is(err, io.ErrUnexpectedEOF): return errors.New("request body contains badly-formed JSON") case errors.As(err, &unmarshalTypeError): if unmarshalTypeError.Field != "" { return fmt.Errorf("request body contains incorrect JSON type for field %q", unmarshalTypeError.Field) } return fmt.Errorf("request body contains incorrect JSON type (at position %d)", unmarshalTypeError.Offset) case errors.Is(err, io.EOF): return errors.New("request body must not be empty") case strings.HasPrefix(err.Error(), "json: unknown field "): fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") return fmt.Errorf("request body contains unknown field %s", fieldName) case errors.As(err, &invalidUnmarshalError): return fmt.Errorf("error unmarshalling json: %v", err) default: return errors.New(err.Error()) } } val := reflect.ValueOf(obj) if val.Kind() == reflect.Ptr { val = val.Elem() } for i := 0; i < val.NumField(); i++ { field := val.Type().Field(i) if field.Tag.Get("binding") == "required" { if val.Field(i).Kind() == reflect.String { trimmedValue := strings.TrimSpace(val.Field(i).String()) val.Field(i).SetString(trimmedValue) } if isZero(val.Field(i)) { // get json tag name jsonTag := field.Tag.Get("json") if jsonTag == "" { jsonTag = field.Name // if no tag name, using field name } return fmt.Errorf("json key of %s is required", jsonTag) } } } return nil } func isZero(v reflect.Value) bool { return v.IsZero() } // ParseFilters parses filter string into FilterConditions func ParseFilters(filterStr string) []FilterCondition { Debug("Enter ParseFilters function", zap.String("filterStr", filterStr)) // Remove brackets //filterStr = strings.TrimPrefix(filterStr, "[") //filterStr = strings.TrimSuffix(filterStr, "]") // since job_name will add index // remove double quotes filterStr = strings.ReplaceAll(filterStr, "\"", "") filterStr = strings.ReplaceAll(filterStr, "}, {", "},{") if filterStr == "" { Debug("filterStr is empty, return nil") return nil } conditions := strings.Split(filterStr, "},{") filters := make([]FilterCondition, 0) for _, condition := range conditions { condition = strings.TrimPrefix(condition, "{") condition = strings.TrimSuffix(condition, "}") // need handle submit_time:eq:2025-03-31 14:34:07} parts := strings.SplitN(condition, ":", 3) if len(parts) != 3 { continue } // check if there are same filter key and operator exists := false for _, filter := range filters { if filter.Field == parts[0] && filter.Operator == parts[1] { exists = true break } } if exists { Error("There are duplicated filter key and operator, please check the input.") return nil // return error for filter } filters = append(filters, FilterCondition{ Field: parts[0], Operator: parts[1], Value: strings.TrimSpace(parts[2]), }) Debug("In function of ParseFilters, Field, Operator, Value are:", zap.Any("Field", parts[0]), zap.Any("Op", parts[1]), zap.Any("Value", strings.TrimSpace(parts[2]))) } if len(filters) > 0 { Debug("Finish calling ParseFilters function, filter results", zap.Any("filters", filters)) return filters } else { Debug("The format of filter is not correct!") return nil } } // parseFields will parse the fields string // eg: field=col1,col2,col3 func ParseFields(fieldsStr string) []string { // Parse fields var fields []string if fieldsStr == "" { return nil } fields = strings.Split(fieldsStr, ",") // Trim spaces result := make([]string, 0, len(fields)) for _, field := range fields { field = strings.TrimSpace(field) if field != "" { result = append(result, field) } } return result } func ParsePageToken(pagingStr string) (string, error) { if pagingStr != "" { pagingStr = strings.TrimPrefix(pagingStr, "\"") pagingStr = strings.TrimSuffix(pagingStr, "\"") pagingStr = strings.TrimSpace(pagingStr) return pagingStr, nil } return "", nil } // Parse sort string in query parameters // sort=job_id:asc,submit_time:desc func ParseSortParam(s string, supportSortList []string) ([]SortOrder, error) { if s == "" { return nil, fmt.Errorf("the sort params is empty") } sortStr := strings.TrimSpace(s) sorts := strings.SplitN(sortStr, ",", -1) sortList := make([]SortOrder, 0) // Convert supportSortList to map for quick lookup sortMap := make(map[string]bool) for _, item := range supportSortList { sortMap[item] = true } if len(sorts) > 0 { for _, sort := range sorts { var col, order string if strings.Contains(sort, ":") { col = strings.SplitN(sort, ":", -1)[0] order = strings.SplitN(sort, ":", -1)[1] if order != "asc" && order != "desc" { return nil, fmt.Errorf("the order string %s not in \"asc\" or \"desc\" ", order) } } else { col = sort order = "asc" // by default } if _, ok := sortMap[col]; !ok { return nil, fmt.Errorf("unsupported sort column: %s", col) } sortList = append(sortList, SortOrder{ Column: col, Ascending: order == "asc", }) } } else { return nil, fmt.Errorf("the sort params is empty") } return sortList, nil } func CheckAllMapKeyInList(sorts []SortOrder, fields []string) error { // check the sort column is among the query fields for _, col := range sorts { found := ContainsField(fields, col.Column) if !found { err := fmt.Errorf("key \"%s\" is not in fields:%v ", col.Column, fields) return err } } return nil } // judge if a field exist in supported fields list func ContainsField(fields []string, field string) bool { for _, v := range fields { if v == field { return true } } return false } // judge if a tagName exist in structType json tag ; func JsonTagExists(structType interface{}, tagName string) bool { val := reflect.ValueOf(structType) for i := 0; i < val.NumField(); i++ { field := val.Type().Field(i) if field.Tag.Get("json") == tagName { return true } } return false } // in a map, get value's corresponding key func GetKeyFromValue(m map[string]int, value int) (string, error) { for k, v := range m { if v == value { return k, nil } } return "", errors.New("value is not recognized.") } // GetMappedField returns the mapped field if it exists, otherwise returns the original field. func GetMappedField(jsonTag string) string { if mappedField, exists := models.JobFieldMappings[jsonTag]; exists { return mappedField } return jsonTag } // GetSupportedFields returns the supported fields for a given model, applying any field mappings. func GetSupportedFields(model interface{}) []string { var fields []string t := reflect.TypeOf(model) for i := 0; i < t.NumField(); i++ { field := t.Field(i) jsonTag := field.Tag.Get("json") if jsonTag != "" { // Only take the first part before any comma jsonTag = strings.Split(jsonTag, ",")[0] // Apply field mapping fields = append(fields, jsonTag) } } return fields } // IntOrInt64 converts an interface{} value that is either int or int64 to int. // Returns (0, false) for any other type. func IntOrInt64(v interface{}) (int, bool) { switch t := v.(type) { case int: return t, true case int64: return int(t), true default: return 0, false } } // ValidateYamlFormat reads and validates a YAML file, returning its raw bytes. func ValidateYamlFormat(filePath string) ([]byte, error) { data, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("failed to read YAML file %s: %v", filePath, err) } var out interface{} if err := yaml.Unmarshal(data, &out); err != nil { return nil, fmt.Errorf("invalid YAML format in %s: %v", filePath, err) } return data, nil } func ParseLimit(limitStr string) (int, error) { if limitStr == "" { return PER_PAGE_LIMIT, nil } limit, err := strconv.Atoi(limitStr) if err != nil { return 0, err } if limit > PER_PAGE_LIMIT_MAX { return 0, fmt.Errorf("limit=%d exceed MAX=%d", limit, PER_PAGE_LIMIT_MAX) } return limit, nil } func ParseType(typeStr string, availableTypes []string) (string, error) { if typeStr == "" { return "", nil } typeStr = strings.TrimSpace(typeStr) for _, t := range availableTypes { if typeStr == t { return typeStr, nil } } return "", fmt.Errorf("type=%s is not supported, available types: %v", typeStr, availableTypes) } func ParseUnique(uniqueStr string) bool { if uniqueStr == "" { return false } // Convert to lowercase and check if it's exactly "true" return strings.ToLower(strings.TrimSpace(uniqueStr)) == "true" } func ConvertStringToJSON(str []string) (string, error) { // eg:"VAR1=value1","VAR2=value2" jsonMap := make(map[string]string) for _, e := range str { parts := strings.SplitN(e, "=", 2) if len(parts) == 2 { jsonMap[parts[0]] = parts[1] } } strIntoJson, err := json.Marshal(jsonMap) if err != nil { return "", err } return string(strIntoJson), nil } func ConvertStringToJsonb(str []string) (models.Jsonb, error) { // eg:"VAR1=value1","VAR2=value2" jsonMap := make(map[string]string) for _, e := range str { parts := strings.SplitN(e, "=", 2) if len(parts) == 2 { jsonMap[parts[0]] = parts[1] } } strIntoJson, err := json.Marshal(jsonMap) if err != nil { return models.Jsonb{}, err } jsonb := models.Jsonb{ String: string(strIntoJson), } return jsonb, nil } func AddFieldsToSupportedList(target *[]string, fields interface{}) { t := reflect.TypeOf(fields) for i := 0; i < t.NumField(); i++ { jsonTag := t.Field(i).Tag.Get("json") if jsonTag != "" && jsonTag != "-" { *target = append(*target, jsonTag) } } } func GetServerHostPort(ctx *gin.Context) string { scheme := "http" if ctx.Request.TLS != nil { scheme = "https" } return fmt.Sprintf("%s://%s", scheme, ctx.Request.Host) } // convert timestamp(s) to local time string with TZ. func convertTimestampToLocalTime(timestamp int64) string { utcTime := time.Unix(timestamp, 0).UTC() localTime := utcTime.Local() return localTime.Format("2006-01-02 15:04:05 MST") } // convert time string with TZ to timestamp func convertTimestrToTimestamp(timeStr string) (int64, error) { // Define the layout for parsing the time string. // "2006-01-02 15:04:05 MST" is a reference time in Go. layout := "2006-01-02 15:04:05 MST" // Parse the time string into a time.Time object. parsedTime, err := time.Parse(layout, timeStr) if err != nil { return 0, fmt.Errorf("failed to parse time: %w", err) } // Get the timestamp in seconds. timestamp := parsedTime.Unix() return timestamp, nil } func IsNumeric(s string) bool { _, err := strconv.ParseFloat(s, 64) return err == nil } func ValidateControlAction(controlAction string, supportedActions []string) error { controlAction = strings.ToLower(strings.TrimSpace(controlAction)) for _, action := range supportedActions { if controlAction == action { return nil } } return errors.New("unsupported control action: " + controlAction) } func IsControlActionAllowed(actionType int, status int64, actionControlMap map[int][]int) bool { Debug("Enter IsActionAllowed function", zap.Any("actionType", actionType), zap.Any("status", status)) allowedStatuses, exists := actionControlMap[actionType] if !exists { Error("Unsupported action type", zap.Any("actionType", actionType)) return false } for _, allowedStatus := range allowedStatuses { if allowedStatus == int(status) { Debug("Action is allowed", zap.Any("status", status), zap.Any("allowedStatus", allowedStatus)) return true } } Debug("Action is not allowed", zap.Any("status", status)) return false } // RemoveDuplicates removes duplicate elements from any slice. func RemoveDuplicates[T comparable](input []T) []T { seen := make(map[T]bool) result := []T{} for _, value := range input { if !seen[value] { seen[value] = true result = append(result, value) } } return result } func TruncateString(s string, maxLen int) string { if len(s) <= maxLen { return s } runes := []rune(s) if len(runes) > maxLen { return string(runes[:maxLen]) } return s } func IncludeBuiltinRoles(roleNames []string) (builtinIncluded []string) { for _, roleName := range roleNames { if ContainsField(models.BUILTIN_ROLE_NAMES, roleName) { builtinIncluded = append(builtinIncluded, roleName) } } return builtinIncluded } func IsValidUserName(userName string) error { // Check if name is empty if userName == "" { return errors.New("username cannot be empty") } // Check maximum length (32 characters) if len(userName) > 32 { return errors.New("username cannot exceed 32 characters") } // Check if it begins with a letter if !((userName[0] >= 'a' && userName[0] <= 'z') || (userName[0] >= 'A' && userName[0] <= 'Z')) { return errors.New("username must begin with a letter") } // Check for valid characters (0-9, a-z, A-Z, -, _, space) for i := 0; i < len(userName); i++ { c := userName[i] if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == ' ') { return errors.New("username can only contain letters, numbers, hyphens, underscores, or spaces") } } return nil }