myxce/src/frontend/apiserver/utils/config.go

package utils

import (
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"runtime"
	"strconv"
	"strings"

	"github.com/spf13/viper"
	"go.uber.org/zap"
)

type BackendConfig struct {
	Address string `yaml:"address"`
	Port    int    `yaml:"port"`
	From    string `yaml:"from"`
	Key     string `yaml:"key"`
	// Peers lists additional controller candidates for HA mode in "host:port"
	// format.  When non-empty, RegisterToBackend tries each candidate in order
	// and connects to whichever accepts the join as primary.  Address+Port
	// serve as a single-host fallback when Peers is empty.
	Peers []string `yaml:"peers"`
}

type AppServerConfig struct {
	AppConfig      AppConfig       `yaml:"app"`
	Security       Security        `yaml:"security"`
	RateLimit      RateLimitConfig `yaml:"rate_limit"`
	Authentication Authentication  `yaml:"authentication"`
	JobConfig      JobConfig       `yaml:"job"`
	CacheConfig    CacheConfig     `yaml:"cache"`
	Backend        BackendConfig   `yaml:"backend"`
}

type AppConfig struct {
	AppName    string `yaml:"name"`
	AppMode    string `yaml:"mode"`
	HTTPPort   string `yaml:"http_port"`
	AppVersion string `yaml:"version"`
}

// TokenConfig is the configuration for the token
type TokenConfig struct {
	Type            string `yaml:"type"`
	SecretKey       string `yaml:"secret_key"`
	TokenExpireTime int    `yaml:"token_expire_time"`
}

// Authentication is the configuration for the authentication
type Authentication struct {
	Token TokenConfig
}

type Security struct {
	HTTPS HTTPSConfig `yaml:"https"`
	// TrustedProxies lists the reverse proxies (IP or CIDR) whose
	// X-Forwarded-For / X-Real-IP headers set the client IP. Empty (the
	// default) trusts no proxy: the client IP is the TCP peer address.
	TrustedProxies []string `yaml:"trusted_proxies"`
}
type HTTPSConfig struct {
	Enabled  bool   `yaml:"enabled"`
	Port     string `yaml:"port"`
	CertFile string `yaml:"cert_file"`
	KeyFile  string `yaml:"key_file"`
}

type RateLimitConfig struct {
	Global    EndpointLimit            `yaml:"global"`
	Endpoints map[string]EndpointLimit `yaml:"endpoints"`
}

type EndpointLimit struct {
	Rate   int    `yaml:"rate"`   // requests per second
	Burst  int    `yaml:"burst"`  // requests per minute
	Window string `yaml:"window"` // Time window (second, minute)
}

type WhitelistConfig struct {
	Whitelist []string `yaml:"whitelist"`
}

type FileUploadConfig struct {
	MaxSize         string   `yaml:"max_size"`         // maximum file size in MB
	AllowedTypes    []string `yaml:"allowed_types"`    // allowed file types
	DisallowedTypes []string `yaml:"disallowed_types"` // disallowed file types
}

type JobConfig struct {
	MaxArraySize int `yaml:"max_array_size"`
	// TerminateInterval is the seconds between the signals of a plain kill
	// (SIGINT, SIGTERM, SIGKILL) and a plain stop (SIGTSTP, SIGSTOP), as
	// LSF's JOB_TERMINATE_INTERVAL (bug-1957)
	TerminateInterval int `yaml:"terminate_interval"`
}

type CORSConfig struct {
	Enabled        bool     `yaml:"enabled"`
	AllowedOrigins []string `yaml:"allowed_origins"`
}

type CacheConfig struct {
	RefreshInterval int `yaml:"refresh_interval"`
}

var DirPerm os.FileMode

// Add global variables to store configs
var (
	AppYamlConfig            *AppConfig
	AuthenticationYamlConfig *Authentication
	HttpsYamlConfig          *HTTPSConfig
	TrustedProxiesYamlConfig []string
	RateLimitYamlConfig      *RateLimitConfig
	WhitelistYamlConfig      *WhitelistConfig
	FileUploadYamlConfig     *FileUploadConfig
	JobYamlConfig            *JobConfig
	CacheYamlConfig          *CacheConfig
	CORSYamlConfig           *CORSConfig
	BackendYamlConfig        *BackendConfig
)

func init() {
	if runtime.GOOS == "windows" {
		DirPerm = 0666
	} else {
		DirPerm = 0700
	}
}

func InitConf() error {
	Debug("Enter InitConf() function")

	viper.SetConfigType("yaml")
	viper.AddConfigPath(Global_TopDir + "/config")
	viper.SetConfigName("app")
	// validate the app.yaml file whether is validate
	data, err := ValidateYamlFormat(Global_TopDir + "/config/app.yaml")
	if err != nil {
		Error("Fail to call DiagnoseLogCreationError with error")
		return err
	}
	if data == nil {
		Error("Fail to call ValidateYamlFormat and data is nil")
		return err
	}

	err1 := viper.ReadInConfig()
	if err1 != nil {
		_, ok := err1.(viper.ConfigFileNotFoundError)
		if ok {
			Error("Can't find the configuration files of app.yaml")
			return err1
		} else {
			log.Println("The configuration file of app.yaml with some error")
			return err1
		}
	}
	// parse app section
	AppYamlConfig = &AppConfig{
		AppName:    viper.GetString("app.name"),
		AppMode:    viper.GetString("app.runmode"),
		HTTPPort:   viper.GetString("app.http_port"),
		AppVersion: viper.GetString("app.version"),
	}
	// parse authentication section
	configuredSecret := viper.GetString("authentication.token.secretKey")
	jwtSecret := resolveJWTSecret(configuredSecret)
	if jwtSecret != configuredSecret && jwtSecret != os.Getenv("XCE_JWT_SECRET") {
		Warn("No JWT secret configured (XCE_JWT_SECRET or authentication.token.secretKey), " +
			"or it is the publicly known default: using a random per-process secret. " +
			"Tokens do not survive an apiserver restart, and apiservers sharing one endpoint " +
			"must all set the same XCE_JWT_SECRET")
	}
	AuthenticationYamlConfig = &Authentication{
		Token: TokenConfig{
			Type:            viper.GetString("authentication.token.type"),
			SecretKey:       jwtSecret,
			TokenExpireTime: int(viper.GetFloat64("authentication.token.token_expire_time")),
		},
	}
	// parse security section
	HttpsYamlConfig = &HTTPSConfig{
		Enabled:  viper.GetBool("security.https.enabled"),
		Port:     viper.GetString("security.https.port"),
		CertFile: viper.GetString("security.https.cert_file"),
		KeyFile:  viper.GetString("security.https.key_file"),
	}
	trustedProxies, err := resolveTrustedProxies(viper.GetStringSlice("security.trusted_proxies"))
	if err != nil {
		return err
	}
	TrustedProxiesYamlConfig = trustedProxies

	// parse rate limit section
	RateLimitYamlConfig = &RateLimitConfig{
		Global: EndpointLimit{
			Rate:   viper.GetInt("rate_limit.global.rate"),
			Burst:  viper.GetInt("rate_limit.global.burst"),
			Window: viper.GetString("rate_limit.global.window"),
		},
		Endpoints: make(map[string]EndpointLimit),
	}
	// parse endpoints
	endpoints := viper.GetStringMap("rate_limit.endpoints")
	for path := range endpoints {
		RateLimitYamlConfig.Endpoints[path] = EndpointLimit{
			Rate:   viper.GetInt("rate_limit.endpoints." + path + ".rate"),
			Burst:  viper.GetInt("rate_limit.endpoints." + path + ".burst"),
			Window: viper.GetString("rate_limit.endpoints." + path + ".window"),
		}
	}

	WhitelistYamlConfig = &WhitelistConfig{
		Whitelist: viper.GetStringSlice("whitelist"),
	}
	// parse file upload section
	rawMaxSize := viper.GetString("file_upload.max_size")
	fileMaxSizeMB, err := ParseMaxSize(rawMaxSize)
	if err != nil {
		return fmt.Errorf("failed to parse max_size: %v", err)
	}
	FileUploadYamlConfig = &FileUploadConfig{
		MaxSize:         strconv.FormatInt(fileMaxSizeMB, 10) + " MB",
		AllowedTypes:    viper.GetStringSlice("file_upload.allowed_types"),
		DisallowedTypes: viper.GetStringSlice("file_upload.disallowed_types"),
	}

	// parse job config section; an absent section or key reads as 0 (default)
	configuredArraySize := viper.GetInt("job.max_array_size")
	maxArraySize, clamped, err := resolveMaxArraySize(configuredArraySize)
	if err != nil {
		return err
	}
	if clamped {
		Warn("job.max_array_size above supported maximum",
			zap.Int("configured", configuredArraySize),
			zap.Int("effective", maxArraySize))
	}
	terminateInterval := viper.GetInt("job.terminate_interval")
	if terminateInterval < 0 {
		return fmt.Errorf("job.terminate_interval must not be negative: %d", terminateInterval)
	}
	if terminateInterval == 0 {
		terminateInterval = defaultTerminateInterval
	}
	JobYamlConfig = &JobConfig{
		MaxArraySize:      maxArraySize,
		TerminateInterval: terminateInterval,
	}

	// parse cache config section
	if !viper.IsSet("cache") {
		// if cache section is not set, use default values
		CacheYamlConfig = &CacheConfig{
			RefreshInterval: 300,
		}
	} else {
		refreshInterval := viper.GetInt("cache.refresh_interval")
		if refreshInterval == 0 {
			refreshInterval = 300
		}
		if refreshInterval < 1 || refreshInterval > 300 {
			return fmt.Errorf("invalid cache.refresh_interval value: %d, must be between 1 and 300", refreshInterval)
		}
		CacheYamlConfig = &CacheConfig{
			RefreshInterval: refreshInterval,
		}
	}

	CORSYamlConfig = &CORSConfig{
		Enabled:        viper.GetBool("cors.enabled"),
		AllowedOrigins: viper.GetStringSlice("cors.allowed_origins"),
	}

	BackendYamlConfig = &BackendConfig{
		Address: viper.GetString("backend.address"),
		Port:    viper.GetInt("backend.port"),
		From:    viper.GetString("backend.from"),
		Key:     viper.GetString("backend.key"),
		Peers:   viper.GetStringSlice("backend.peers"),
	}

	return nil
}

func ParseMaxSize(maxSize string) (int64, error) {
	parts := strings.Fields(maxSize)
	if len(parts) != 2 {
		return 0, errors.New("invalid max_size format, expected '<number> <unit>'")
	}

	size, err := strconv.ParseInt(parts[0], 10, 64)
	if err != nil {
		return 0, errors.New("invalid number in max_size")
	}

	unit := strings.ToUpper(parts[1])
	switch unit {
	case "MB":
		size *= 1 // MB remains as is
	case "GB":
		size *= 1024 // convert GB to MB
	default:
		return 0, errors.New("unsupported unit in max_size, use MB or GB")
	}

	if size > 10240 {
		return 0, errors.New("max_size cannot exceed 10GB (10240 MB)")
	}

	return size, nil
}

const (
	// defaultTerminateInterval is used when job.terminate_interval is unset
	// or 0: LSF's JOB_TERMINATE_INTERVAL default (bug-1957).
	defaultTerminateInterval = 10
	// defaultMaxArraySize is used when job.max_array_size is unset or 0.
	defaultMaxArraySize = 1000
	// supportedMaxArraySize is the largest array the API server creates
	// (array element numbers are 1..32767).
	supportedMaxArraySize = 32767
)

// resolveMaxArraySize turns the configured job.max_array_size into the
// effective array size limit: 0 (unset) gives the default 1000, 1..32767 is
// used as is, anything above 32767 is clamped to 32767 (clamped = true, the
// caller logs a warning), and a negative value is an error.
func resolveMaxArraySize(v int) (effective int, clamped bool, err error) {
	switch {
	case v < 0:
		return 0, false, fmt.Errorf("invalid job.max_array_size value: %d, must not be negative", v)
	case v == 0:
		return defaultMaxArraySize, false, nil
	case v > supportedMaxArraySize:
		return supportedMaxArraySize, true, nil
	default:
		return v, false, nil
	}
}

// resolveTrustedProxies validates security.trusted_proxies: every entry must
// be an IP address or a CIDR. An empty list means no proxy is trusted.
func resolveTrustedProxies(entries []string) ([]string, error) {
	proxies := make([]string, 0, len(entries))
	for _, e := range entries {
		e = strings.TrimSpace(e)
		if e == "" {
			continue
		}
		if net.ParseIP(e) == nil {
			if _, _, err := net.ParseCIDR(e); err != nil {
				return nil, fmt.Errorf("invalid security.trusted_proxies entry %q: must be an IP address or CIDR", e)
			}
		}
		proxies = append(proxies, e)
	}
	return proxies, nil
}

// knownDefaultJWTSecret is the secret older app.yaml files shipped with; it is
// public, so it is treated as "no secret configured" (bug-1945).
const knownDefaultJWTSecret = "xce-api-key"

// usableJWTSecret reports whether s may sign tokens: non-empty and not the
// publicly known default.
func usableJWTSecret(s string) bool {
	return s != "" && s != knownDefaultJWTSecret
}

// resolveJWTSecret returns the JWT secret: XCE_JWT_SECRET if usable, else
// the config file value if usable, else a random per-process secret (the
// caller detects that case because the result differs from both inputs).
func resolveJWTSecret(configValue string) string {
	if env := os.Getenv("XCE_JWT_SECRET"); usableJWTSecret(env) {
		return env
	}
	if usableJWTSecret(configValue) {
		return configValue
	}
	buf := make([]byte, 32)
	if _, err := rand.Read(buf); err != nil {
		// crypto/rand never fails on supported platforms; refuse to fall
		// back to a predictable secret if it ever does.
		panic(fmt.Sprintf("cannot generate a JWT secret: %v", err))
	}
	return hex.EncodeToString(buf)
}