Merge Go frontend (apiserver + xcecli) into meson build

This commit is contained in:
Rongsong Shen 2026-04-14 11:20:17 +08:00
parent 43dc12a36c
commit c9dc8c4676
205 changed files with 30793 additions and 328 deletions

View file

@ -0,0 +1 @@
go test -v load_test.go

View file

@ -0,0 +1,58 @@
package middleware
import (
"net/http"
"strings"
"github.com/gin-gonic/gin"
)
// CORSMiddleware returns a middleware that sets CORS headers.
// allowedOrigins: list of allowed origins; pass nil to disable CORS entirely.
// Pass []string{"*"} to allow all origins.
func CORSMiddleware(allowedOrigins []string) gin.HandlerFunc {
return func(c *gin.Context) {
if len(allowedOrigins) == 0 {
c.Next()
return
}
origin := c.Request.Header.Get("Origin")
allowed := ""
for _, o := range allowedOrigins {
if o == "*" || o == origin {
allowed = o
break
}
}
if allowed != "" {
c.Header("Access-Control-Allow-Origin", allowed)
c.Header("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS")
c.Header("Access-Control-Allow-Headers", "Origin, Content-Type, Authorization")
c.Header("Access-Control-Max-Age", "86400")
}
if c.Request.Method == http.MethodOptions {
c.AbortWithStatus(http.StatusNoContent)
return
}
c.Next()
}
}
// CORSOriginsFromConfig builds the allowed origins list from a config string slice.
// Returns nil if CORS is disabled (empty slice).
func CORSOriginsFromConfig(enabled bool, origins []string) []string {
if !enabled || len(origins) == 0 {
return nil
}
result := make([]string, 0, len(origins))
for _, o := range origins {
if s := strings.TrimSpace(o); s != "" {
result = append(result, s)
}
}
return result
}

View file

@ -0,0 +1,60 @@
package middleware
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
)
func TestCORSMiddleware_SetsHeaders(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(CORSMiddleware([]string{"*"}))
r.GET("/ping", func(c *gin.Context) { c.Status(200) })
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/ping", nil)
req.Header.Set("Origin", "http://example.com")
r.ServeHTTP(w, req)
if w.Header().Get("Access-Control-Allow-Origin") == "" {
t.Error("expected Access-Control-Allow-Origin header")
}
}
func TestCORSMiddleware_Preflight(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(CORSMiddleware([]string{"http://example.com"}))
w := httptest.NewRecorder()
req, _ := http.NewRequest("OPTIONS", "/xce/v1/jobs", nil)
req.Header.Set("Origin", "http://example.com")
req.Header.Set("Access-Control-Request-Method", "POST")
r.ServeHTTP(w, req)
if w.Code != http.StatusNoContent {
t.Errorf("preflight: got %d want 204", w.Code)
}
if w.Header().Get("Access-Control-Allow-Methods") == "" {
t.Error("expected Access-Control-Allow-Methods header")
}
}
func TestCORSMiddleware_Disabled(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(CORSMiddleware(nil)) // nil = disabled
r.GET("/ping", func(c *gin.Context) { c.Status(200) })
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/ping", nil)
req.Header.Set("Origin", "http://example.com")
r.ServeHTTP(w, req)
if w.Header().Get("Access-Control-Allow-Origin") != "" {
t.Error("CORS headers must not be set when disabled")
}
}

View file

@ -0,0 +1,111 @@
package middleware
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"sync"
"testing"
"time"
)
type AuthResponse struct {
Code int `json:"code"`
Msg string `json:"msg"`
Data struct {
Token string `json:"token"`
Path string `json:"path"`
IP string `json:"ip"`
} `json:"data"`
}
// login
func login(username, password string) (string, error) {
const loginURL = "http://localhost:8080/xce/v1/auth/logon" //replace with your login url
// create login request json data
loginData := map[string]string{
"username": username,
"password": password,
}
jsonData, err := json.Marshal(loginData)
if err != nil {
return "", err
}
// send post request
resp, err := http.Post(loginURL, "application/json", bytes.NewBuffer(jsonData))
if err != nil {
return "", err
}
defer resp.Body.Close()
// check response status
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("login failed with status: %s", resp.Status)
}
// parse token from response
var authResponse AuthResponse
if err := json.NewDecoder(resp.Body).Decode(&authResponse); err != nil {
return "", err
}
// check if code is 200
if authResponse.Code != 200 {
return "", fmt.Errorf("login failed: %s", authResponse.Msg)
}
return authResponse.Data.Token, nil
}
// TestLoadJobs test function
func TestLoadJobs(t *testing.T) {
const numRequests = 100
const username = "user01" // replace with your username
const password = "Welcome@1212" // replace with your password
fmt.Println("before login")
// login to get token
token, err := login(username, password)
if err != nil {
t.Fatalf("Failed to log in: %v", err)
}
fmt.Println("Obtained Bearer Token:", token)
var wg sync.WaitGroup
wg.Add(numRequests)
startTime := time.Now()
for i := 0; i < numRequests; i++ {
go func() {
defer wg.Done()
req, err := http.NewRequest("GET", "http://localhost:8080/xce/v1/jobs", nil) // replace with your actual url
if err != nil {
t.Error("Error creating request:", err)
return
}
// set bearer token
req.Header.Set("Authorization", "Bearer "+token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Error("Error:", err)
return
}
defer resp.Body.Close()
// optional: check response status
if resp.StatusCode != http.StatusOK {
t.Errorf("Expected status OK, got %v", resp.Status)
}
}()
}
wg.Wait() // wait for all requests to complete
elapsedTime := time.Since(startTime)
fmt.Printf("Completed %d requests in %s\n", numRequests, elapsedTime)
}

View file

@ -0,0 +1,136 @@
package middleware
import (
"apiserver/common"
"apiserver/controllers"
"apiserver/utils"
logger "apiserver/utils"
"strings"
"sync"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
"golang.org/x/time/rate"
)
var (
limiters = make(map[string]*rate.Limiter)
limitersMux sync.RWMutex
)
// RateLimiter struct to hold the limiter
type RateLimiter struct {
limiters map[string]*utils.EndpointLimit
globalLimiter *utils.EndpointLimit
}
// Initialize RateLimiter with configuration
func NewRateLimiter(cfg *utils.RateLimitConfig) *RateLimiter {
logger.Debug("Enter NewRateLimiter function", zap.Any("cfg", cfg))
rl := &RateLimiter{
limiters: make(map[string]*utils.EndpointLimit),
globalLimiter: &cfg.Global,
}
for path, endpoint := range cfg.Endpoints {
rl.limiters[path] = &endpoint
}
return rl
}
// Middleware function to enforce rate limiting
func (r *RateLimiter) Limit() gin.HandlerFunc {
logger.Debug("Enter Limit() function")
return func(ctx *gin.Context) {
path := ctx.FullPath()
limiter := getRateLimiter(path, ctx)
if limiter == nil {
ctx.Next()
return
}
if !limiter.Allow() {
logger.Error("Rate limit exceeded", zap.String("path", path), zap.String("client_ip", ctx.ClientIP()), zap.Int("errorcode", common.ERROR_TOO_MANY_REQUESTS_CODE))
controllers.ReturnError(ctx, common.ERROR_TOO_MANY_REQUESTS_CODE, common.ERROR_TOO_MANY_REQUESTS_MSG, nil)
ctx.Abort()
return
}
ctx.Next()
}
}
// getRateLimiter returns a rate limiter for the given endpoint
func getRateLimiter(endpoint string, ctx *gin.Context) *rate.Limiter {
// get submit user from token
token := ctx.GetHeader("Authorization")
if token != "" {
submitUser, err := controllers.GetUsernameFromToken(strings.TrimPrefix(token, "Bearer "))
if err != nil {
logger.Error("Failed to get submit user from token", zap.Any("err", err))
}
logger.Debug("Enter getRateLimiter() function", zap.Any("endpoint", endpoint), zap.Any("client_ip", ctx.ClientIP()), zap.Any("submitUser", submitUser))
} else {
logger.Debug("Enter getRateLimiter() function", zap.Any("endpoint", endpoint), zap.Any("client_ip", ctx.ClientIP()))
}
limitersMux.RLock()
limiter, exists := limiters[endpoint]
limitersMux.RUnlock()
if exists {
return limiter
}
limitersMux.Lock()
defer limitersMux.Unlock()
// Double check after acquiring write lock
if limiter, exists = limiters[endpoint]; exists {
return limiter
}
// Get rate limit configuration
config := utils.RateLimitYamlConfig
if config == nil {
return nil
}
var limit utils.EndpointLimit
if endpointLimit, ok := config.Endpoints[endpoint]; ok {
limit = endpointLimit
} else {
// Check for wildcard matches
matchFlag := false
for path, endpointLimit := range config.Endpoints {
// Remove trailing slash for comparison
cleanedEndpoint := endpoint
cleanedPath := path
// Compare cleaned paths
if cleanedPath == "*" || (len(cleanedPath) > 1 && string(cleanedPath[len(cleanedPath)-1]) == "*" && cleanedEndpoint == string(cleanedPath[:len(cleanedPath)-2])) {
limit = endpointLimit
matchFlag = true
break
}
}
if !matchFlag {
limit = config.Global
}
}
// Calculate rate based on window
var r rate.Limit
switch limit.Window {
case "minute":
r = rate.Limit(float64(limit.Rate) / 60.0)
case "second":
r = rate.Limit(limit.Rate)
default:
r = rate.Limit(limit.Rate)
}
logger.Debug("double check rate limit setting for endpoint", zap.Any("endpoint", endpoint), zap.Any("r", r),
zap.Any("burst", limit.Burst))
limiter = rate.NewLimiter(r, limit.Burst)
limiters[endpoint] = limiter
return limiter
}

View file

@ -0,0 +1,121 @@
package middleware
import (
"apiserver/utils"
logger "apiserver/utils"
"fmt"
"net"
"os"
"os/user"
"path/filepath"
"runtime"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
// openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout server.key -out server.crt -config openssl.cnf -extensions v3_req
func StartServer(r *gin.Engine) error {
logger.Debug("Enter StartServer() function")
if runtime.GOOS != "windows" {
currentUser, err := user.Current()
if err != nil {
logger.Error("Failed to get current user:", zap.Error(err))
os.Exit(1)
}
if currentUser.Uid != "0" {
logger.Warn("Not running as root — PAM authentication will not work. OK for dev/testing.")
}
}
ips, err := GetIPAddresses()
if err != nil {
logger.Error("Failed to get server IP address:", zap.Error(err))
os.Exit(1)
}
ip := ips[0]
logger.Debug("API Server IP address:", zap.String("ip", ip))
httpsConfig := utils.HttpsYamlConfig //get only once
if httpsConfig.Enabled {
addr := ":" + httpsConfig.Port
logger.Info("Starting HTTPS server on " + addr)
fmt.Println("[GIN-debug] The API server will be run on https://" + ip + addr)
if err := r.RunTLS(
addr,
filepath.Join(utils.Global_TopDir, httpsConfig.CertFile),
filepath.Join(utils.Global_TopDir, httpsConfig.KeyFile),
); err != nil {
logger.Error("HTTPS Server fails to start:", zap.Error(err))
os.Exit(1)
}
} else {
addr := ":" + utils.AppYamlConfig.HTTPPort
logger.Info("Starting HTTP server on " + addr)
fmt.Println("[GIN-debug] The API server will be run on http://" + ip + addr)
if err := r.Run(addr); err != nil {
logger.Error("API Server fails to start:", zap.Error(err))
os.Exit(1)
}
}
return nil
}
func getServerIP() (string, error) {
// get the server IP address
addrs, err := net.InterfaceAddrs()
if err != nil {
logger.Error("Failed to get server IP address:", zap.Error(err))
return "", err
}
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() != nil {
return ipnet.IP.String(), nil // real ip address
}
}
return "", nil
}
func GetIPAddresses() ([]string, error) {
var ips []string
interfaces, err := net.Interfaces()
if err != nil {
return nil, err
}
for _, iface := range interfaces {
if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 {
// Skip down or loopback interfaces
continue
}
addrs, err := iface.Addrs()
if err != nil {
continue
}
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
// Skip IPv6 link-local or loopback addresses
if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
continue
}
ips = append(ips, ip.String())
}
}
return ips, nil
}

View file

@ -0,0 +1,12 @@
@echo off
echo Testing Rate Limit for /ping
echo ========================
for /l %%x in (1, 1, 15) do (
echo Request %%x:
curl -i http://localhost:8443/ping
echo.
echo ========================
)
pause

View file

@ -0,0 +1,143 @@
package middleware
import (
"apiserver/common"
"apiserver/controllers"
"apiserver/models"
"apiserver/utils"
logger "apiserver/utils"
"bytes"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"github.com/gin-gonic/gin"
"github.com/golang-jwt/jwt/v4"
"go.uber.org/zap"
)
func ValidateHandler() gin.HandlerFunc {
logger.Debug("Enter ValidateHandler() function")
return func(ctx *gin.Context) {
if isExcludedPath(ctx.Request.URL.Path) {
ctx.Next()
return
}
validateHeader()(ctx)
if ctx.IsAborted() {
return
}
flag := !strings.Contains("multipart/form-data", ctx.Request.Header.Get("Content-Type"))
if ctx.Request.Method == http.MethodPost && flag == false {
isValidBody, err := validateRequestBody(ctx)
if err != nil || !isValidBody {
controllers.ReturnError(ctx, common.INVALID_PARAMS_CODE,
common.ERROR_READ_REQUEST_BODY_MSG+": "+err.Error(), nil)
return
}
}
if ctx.IsAborted() {
return
}
ctx.Next()
}
}
func isExcludedPath(path string) bool {
allowed := false
// define some excluded paths
excludedPaths := []string{
"/ping",
"/xce/v1/auth/logon",
"/swagger/*any",
"/xce/v1/auth/logout",
}
for _, excludedPath := range excludedPaths {
if strings.HasPrefix(path, excludedPath[:len(excludedPath)-1]) { // 去掉最后的 *
allowed = true
break
}
}
return allowed
}
// Token validation middleware
func validateHeader() gin.HandlerFunc {
logger.Debug("Enter validateHeader() function")
return func(ctx *gin.Context) {
//continue to validate token
token := ctx.GetHeader("Authorization")
if token == "" {
logger.Error("Token is empty")
controllers.ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE,
common.ERROR_AUTH_CHECK_TOKEN_EMPTY_MSG, nil)
ctx.Abort()
return
}
// Validate token (this is a placeholder for your actual token validation logic)
isValid, err := isValidToken(strings.TrimPrefix(token, "Bearer "), ctx.ClientIP())
if err != nil {
logger.Error("Validate token error", zap.Error(err))
controllers.ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE,
common.ERROR_AUTH_MSG+": "+err.Error(), nil)
ctx.Abort()
return
}
if !isValid {
logger.Error("Token is invalid")
controllers.ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE,
common.ERROR_AUTH_MSG, nil)
ctx.Abort()
return
}
}
}
func isValidToken(tokenString string, IP string) (bool, error) {
token, err := jwt.ParseWithClaims(tokenString, &models.JWTClaims{}, func(token *jwt.Token) (interface{}, error) {
return []byte(utils.AuthenticationYamlConfig.Token.SecretKey), nil
})
if err != nil {
return false, err
}
if claims, ok := token.Claims.(*models.JWTClaims); ok && token.Valid {
if claims.IP == IP {
return true, nil
}
}
return false, nil
}
func validateRequestBody(ctx *gin.Context) (bool, error) {
logger.Debug("Enter validateRequestBody function")
// read request body and resolve EOF error
body, err := ctx.GetRawData()
if err != nil {
logger.Error("Failed to read request body", zap.Error(err))
ctx.Abort()
return false, err
}
if len(body) == 0 {
logger.Error("Request body is empty")
ctx.Abort()
return false, errors.New(common.ERROR_REQUEST_BODY_EMPTY_MSG)
}
// reset request body
ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
// check request body is valid
if !json.Valid(body) {
logger.Error("Request body is not valid", zap.Error(err))
ctx.Abort()
return false, errors.New(common.ERROR_REQUEST_BODY_NOT_JSON_MSG)
}
return true, nil
}

View file

@ -0,0 +1,162 @@
package middleware
import (
"apiserver/common"
"apiserver/controllers"
logger "apiserver/utils"
"net"
"strings"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
var WhitelistGETURL = []string{
"/ping", // ping test
"/swagger", // Swagger interface for document generation
"/xce/v1/cluster", // cluster (legacy)
"/xce/v1/clusters", // clusters
"/xce/v1/hosts", // hosts
"/xce/v1/hosts/*", // hosts
"/xce/v1/jobs", // jobs
"/xce/v1/jobs/*", // jobs
"/xce/v1/queues", // queues
"/xce/v1/queues/*", // queues
"/xce/v1/files", // get all files
"/xce/v1/files/*", // get download file
"/xce/v1/resources/definition", // resources definition
"/xce/v1/resources/distribution", // resources distribution
"/xce/v1/roles", // roles list
"/xce/v1/roles/*", // role detail
"/xce/v1/permissions", // permissions
"/xce/v1/users", // users list
"/xce/v1/users/*", // user detail / current
}
var WhitelistPOSTURL = []string{
"/xce/v1/auth/logon", // logon
"/xce/v1/auth/logout", // logout
"/xce/v1/hosts/control", // control hosts
"/xce/v1/hosts/select", // filter host with -R select string
"/xce/v1/queues/control", // control queues
"/xce/v1/jobs/create", // create job
"/xce/v1/jobs/control", // control job
"/xce/v1/jobs/*", // control specified job
"/xce/v1/files/upload", // upload file
"/xce/v1/roles/create", // create role
"/xce/v1/roles/*", // add permissions for role
"/xce/v1/users/*", // add roles for user
}
var WhitelistDELETEURL = []string{
"/xce/v1/files",
"/xce/v1/files/*",
"/xce/v1/users/*",
"/xce/v1/roles/*",
}
// pathMatchesPattern checks if path matches a whitelist pattern.
// Patterns ending in /* match any path with that prefix.
// Exact patterns match only that exact path.
func pathMatchesPattern(path, pattern string) bool {
if len(pattern) == 0 {
return false
}
if pattern[len(pattern)-1] == '*' {
return strings.HasPrefix(path, pattern[:len(pattern)-1])
}
return path == pattern
}
// WhitelistURLMiddleware returns a middleware that checks if the URL path is in the whitelist.
func WhitelistURLMiddleware() gin.HandlerFunc {
return func(ctx *gin.Context) {
path := ctx.Request.URL.Path
method := ctx.Request.Method
allowed := false
var whitelist []string
if method == "GET" {
whitelist = WhitelistGETURL
} else if method == "POST" {
whitelist = WhitelistPOSTURL
} else if method == "DELETE" {
whitelist = WhitelistDELETEURL
}
for _, pattern := range whitelist {
if pathMatchesPattern(path, pattern) {
allowed = true
break
}
}
if !allowed {
logger.Error("URL is not allowed", zap.String("path", path))
controllers.ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_REQUEST_URL_NOT_FOUND, nil)
ctx.Abort()
return
}
ctx.Next()
}
}
// WhitelistMiddleware returns a middleware that checks if the client IP is in the whitelist,
// supporting both single IP addresses and CIDR notation.
func WhitelistIPMiddleware(allowedIPs []string) gin.HandlerFunc {
// parse the allowed IPs
var allowedNetworks []*net.IPNet
var allowedIPList []net.IP
for _, s := range allowedIPs {
// parse CIDR format eg:"172.29.6.0/24"
if _, network, err := net.ParseCIDR(s); err == nil {
allowedNetworks = append(allowedNetworks, network)
logger.Debug("allowedNetworks", zap.String("allowedNetworks", network.String()))
} else {
// parse single IP format
ip := net.ParseIP(s)
if ip != nil {
allowedIPList = append(allowedIPList, ip)
logger.Debug("allowedIPList", zap.String("allowedIPList", ip.String()))
}
}
}
return func(ctx *gin.Context) {
clientIPStr := ctx.ClientIP()
clientIP := net.ParseIP(clientIPStr)
logger.Debug("clientIP", zap.String("clientIP", clientIPStr))
if clientIP == nil {
logger.Error("invalid client IP", zap.String("clientIP", clientIPStr))
controllers.ReturnError(ctx, common.INVALID_PARAMS_CODE, common.ERROR_INVALID_CLIENT_IP_MSG, nil)
return
}
allowed := false
// single IP
for _, ip := range allowedIPList {
if ip.Equal(clientIP) {
allowed = true
break
}
}
// ip array
if !allowed {
for _, network := range allowedNetworks {
if network.Contains(clientIP) {
allowed = true
break
}
}
}
if !allowed {
logger.Error("access forbidden", zap.String("clientIP", clientIPStr))
controllers.ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil)
ctx.Abort()
return
}
ctx.Next()
}
}

View file

@ -0,0 +1,33 @@
package middleware
import "testing"
func TestPathMatchesPattern(t *testing.T) {
cases := []struct {
pattern string
path string
want bool
}{
// wildcard patterns
{"/xce/v1/hosts/*", "/xce/v1/hosts/host1", true},
{"/xce/v1/jobs/*", "/xce/v1/jobs/123/control", true},
{"/xce/v1/roles/*", "/xce/v1/roles/cluster_admin", true},
{"/xce/v1/users/*", "/xce/v1/users/current", true},
{"/xce/v1/users/*", "/xce/v1/users/admin/basic_user", true},
// exact patterns
{"/xce/v1/hosts", "/xce/v1/hosts", true},
{"/xce/v1/hosts", "/xce/v1/host", false},
{"/xce/v1/roles", "/xce/v1/roles", true},
{"/xce/v1/roles", "/xce/v1/role", false},
{"/xce/v1/clusters", "/xce/v1/clusters", true},
{"/ping", "/ping", true},
{"/ping", "/pin", false},
}
for _, tc := range cases {
got := pathMatchesPattern(tc.path, tc.pattern)
if got != tc.want {
t.Errorf("pathMatchesPattern(%q, %q) = %v, want %v", tc.path, tc.pattern, got, tc.want)
}
}
}