diff --git a/.gitignore b/.gitignore index e4e5f6c8..790fc0ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,18 @@ -*~ \ No newline at end of file +*~ +*.AppImage +result +\#*\# + +# Editor swap/backup files +*.swp +*.swo +*~ +.*.swp + +# Build outputs +src/frontend/apiserver/apiserver +src/frontend/xcecli/xcecli + +# Logs and tokens (runtime state) +src/frontend/apiserver/logs/ +src/frontend/apiserver/tokens/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..807da69e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,68 @@ +# XCE Cluster Management System + +## Overview +XCE is an HPC cluster management system providing job scheduling, queue management, and cluster resource control. The Go-based API/CLI tier fronts C/C++ system daemons (controller, worker, scheduler). + +## Tech Stack +- **Go 1.23.4**: REST API server (`src/frontend/apiserver`), CLI tool (`src/frontend/xcecli`) +- **C/C++**: Core daemons — controller, worker, scheduler, shared lib +- **Build**: Meson + Nix flakes for C components; `make` for Go components +- **Go key deps**: Gin v1.10.0, JWT v4, Zap logging, Viper config, Cobra CLI + +## Development Commands +```bash +# From src/frontend/ +make build-server # Build apiserver binary → /tmp/apiserver +make build-cli # Build xcecli binary → /tmp/xce_static (CGO_ENABLED=0) +make test # Unit tests (no live server required) +make test-integration # Regression suite (requires XCE_BIN and SRV_URL env vars) +make test-all # Unit + integration tests + +# Full dev build — all components (from src/) +nix develop # Enter dev environment +meson setup build && ninja -C build # Build everything (all components) + +# Nix package builds (from src/) +nix build --option sandbox false .#xcecore # C daemons + frontend/apiserver +nix build --option sandbox false .#xcecore_debug # Same with debug/ASAN +nix build --option sandbox false .#xcecli # CLI tool only +``` + +## Test-Driven Development +Follow TDD strictly throughout the full development process: + +1. **Red** — Write a failing test that specifies the desired behaviour before writing any implementation code. +2. **Green** — Write the minimal implementation to make the test pass. Do not over-engineer. +3. **Refactor** — Clean up the implementation and tests while keeping all tests green. + +### Rules +- Never implement a feature without a corresponding test written first. +- Tests must be runnable without a live server or external services (use table-driven tests, stubs, and `t.TempDir()` for fixtures). +- A change is not complete until `make test` (and `make test-integration` where applicable) passes. +- For C components, add a test case in the relevant `testsuite/` before modifying the implementation. +- Tests live next to the code they cover (`_test.go` for Go, `testsuite/` for C). +- Keep test scope narrow — unit-test business logic; use integration tests for HTTP/API surface. + +## Architecture +``` +Request → Gin Router → Middleware (CORS/RateLimit/IPWhitelist/JWT) → RBAC Check → Controller → Services → In-Memory Store +``` +- **3 independent Go modules**: `apiserver`, `xcecli`, `bcctool` (no go.work) +- **Stateful API server**: in-memory stores seeded at startup via `store.LoadBuiltinData()` +- **Pluggable auth**: configfile / LDAP / OAuth2 / PAM (set in `config/auth.yaml`) +- **RBAC**: domain-based role-action-domain model loaded from `config/rbac.yaml` + +## Key Files +- `src/frontend/Makefile` — primary build/test orchestration +- `src/frontend/apiserver/cmd/main.go` — server startup, init sequence +- `src/frontend/apiserver/router/routers.go` — all routes and middleware chain +- `src/frontend/apiserver/controllers/rbac_check.go` — RBAC loading and enforcement +- `src/frontend/apiserver/config/rbac.yaml` — role/permission definitions +- `src/frontend/apiserver/config/auth.yaml` — auth provider selection + +## Subdirectory Guidelines +- `src/frontend/` — See `src/frontend/CLAUDE.md` +- `src/controller/` — See `src/controller/CLAUDE.md` +- `src/worker/` — See `src/worker/CLAUDE.md` +- `src/scheduler/` — See `src/scheduler/CLAUDE.md` +- `src/lib/` — See `src/lib/CLAUDE.md` diff --git a/src/apiserver/cmd/main.go b/src/apiserver/cmd/main.go deleted file mode 100644 index 19358971..00000000 --- a/src/apiserver/cmd/main.go +++ /dev/null @@ -1,211 +0,0 @@ -package main - -/* - #include - #include - #include - - #include "client.h" - #include "netio.h" - #include "controller.capnp.h" - #include "helper.h" - - typedef struct { - xcecli_ctx_t ctx; - uintptr_t go_handler; - void* go_apiserver; - } apiserver_ctx_t; - - extern int go_callback(uintptr_t, xce_request_t*, apiserver_ctx_t *); - - static int xce_apiserver_handler(xce_request_t *req, void *arg) { - int rc = -EINVAL; - apiserver_ctx_t *ctx = (apiserver_ctx_t *)arg; - - if (ctx == NULL) { - return -EINVAL; - } - - if (ctx->go_handler != 0) { - rc = go_callback(ctx->go_handler, req, arg); - } - - return rc; - } - - static int xce_apiserver_init(xce_client_config_t *config, apiserver_ctx_t *ctx) { - int rc; - - if ((config == NULL) || (ctx == NULL)) { - return -EINVAL; - } - - rc = xcecli_ctx_init(&(ctx->ctx), config); - if (rc < 0) { - return rc; - } - - xcecli_set_handler(ctx->ctx, (xce_request_handler_t)xce_apiserver_handler, ctx); - - rc = xcecli_start(ctx->ctx); - if (rc < 0) { - return rc; - } - - rc = xcecli_join(ctx->ctx, HostRole_apiserver, NULL); - - return rc; - } - - static xce_job_t **dataset_jobs(dataset_t *d) { - return (xce_job_t **)d->rows; - } - - static xce_worker_t **dataset_workers(dataset_t *d) { - return (xce_worker_t **)d->rows; - } -*/ -import "C" - -import ( - "fmt" - "unsafe" - "runtime/cgo" - "log" - "net/http" -) - -type ApiServer struct { - // add data here -} - -//export go_callback -func go_callback(h C.uintptr_t, req *C.xce_request_t, ctx *C.apiserver_ctx_t) C.int { - fn := cgo.Handle(h).Value().(func(*C.xce_request_t,*C.apiserver_ctx_t) C.int) - return fn(req, ctx); -} - -func (apiserver *ApiServer) handle_config(config *C.xce_configdata_t) C.int { - switch (config.data_which) { - case C.ConfigData_data_clusterconfig: - fmt.Printf("cluster configuration received\n") - // Extract configuration information from config - // - queues - // - resources information - // - and other information - default: - // - fmt.Printf("unhandled config request %d\n", config.data_which) - } - return 0 -} - -func print_job(job *C.xce_job_t) { - fmt.Printf("jobid `%v' user `%s' queue `%s'\n",uint64(job.jobid), C.GoString(job.submit.user), C.GoString(job.submit.queue)) -} - -func print_worker(w *C.xce_worker_t) { - fmt.Printf("worker `%s'\n", C.GoString(w.name)) -} - -func (apiserver *ApiServer) handle_job_push(jobs []*C.xce_job_t) C.int { - // add all jobs to apiserver - for i := range jobs { - print_job(jobs[i]) - } - - return 0 -} - -func (apiserver *ApiServer) handle_worker_push(workers []*C.xce_worker_t) C.int { - // add all workers to api server - for i := range workers { - print_worker(workers[i]) - } - return 0 -} - -func (apiserver *ApiServer)handle_dataset(dataset *C.dataset_t) C.int { - - fmt.Printf("handle dataset table `%s'\n", C.GoString(C.xce_dataset_table(dataset.table))) - switch(dataset.table) { - case C.Data_data_jobs: - fmt.Printf("%d jobs arrived\n", dataset.nrows) - var theJobs **C.xce_job_t = C.dataset_jobs(dataset) - - jobs := unsafe.Slice(theJobs, dataset.nrows) - - return apiserver.handle_job_push(jobs) - case C.Data_data_workers: - fmt.Printf("%d workers arrived\n", dataset.nrows) - var theWorkers **C.xce_worker_t = C.dataset_workers(dataset) - - workers := unsafe.Slice(theWorkers, dataset.nrows) - return apiserver.handle_worker_push(workers) - default: - // - } - return 0 -} -func go_req_handler(request *C.xce_request_t, ctx *C.apiserver_ctx_t) C.int { - var from = C.GoString(request.from.name) - var body_which = request.body_which - var apiserver = (*ApiServer)(unsafe.Pointer(ctx.go_apiserver)) - - fmt.Println("api server handler called") - fmt.Printf("op `%s' msg `%s' from %s\n", C.GoString(C.xce_request_op_name(request)), C.GoString(C.xce_request_msg_name(request)), from) - - switch (request.opcode) { - case C.OPCode_push: - switch (body_which) { - case C.Request_body_configpushmsg: - // handle push config - return apiserver.handle_config((*C.xce_configdata_t)(unsafe.Pointer(C.xce_request_body(request)))) - case C.Request_body_datamsg: - // handle pushed data msg - return apiserver.handle_dataset((*C.dataset_t)(unsafe.Pointer(C.xce_request_body(request)))) - default: - // something wrong - } - default: - } - - C.free_Request(request) - - return 0 -} - -func init_config(config *C.xce_client_config_t) C.int { - config.nmasters =1 - config.masters[0] = C.CString("localhost") - config.master_ports[0] = C.int(9990); - config.from = C.CString("xce_apiserver") - config.key = C.CString("keepsecret$") - - return 0 -} - -func (apiserver *ApiServer) handle_job_query(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, "URL.Path = %q\n", r.URL.Path) -} - -func main() { - var config C.xce_client_config_t - var ctx C.apiserver_ctx_t - var apiserver ApiServer - - init_config(&config) - - h := cgo.NewHandle(go_req_handler) - defer h.Delete() - - ctx.go_handler = C.uintptr_t(h) - ctx.go_apiserver = unsafe.Pointer(&apiserver) - - rc := C.xce_apiserver_init(&config, &ctx) - fmt.Printf("rc %d\n", rc); - - http.HandleFunc("/jobs", (&apiserver).handle_job_query) - - log.Fatal(http.ListenAndServe("localhost:8000", nil)) -} diff --git a/src/apiserver/meson.build b/src/apiserver/meson.build deleted file mode 100644 index 8c97ea40..00000000 --- a/src/apiserver/meson.build +++ /dev/null @@ -1,74 +0,0 @@ -cgo_cflags ='' -cgo_ldflags = '' - -if with_debug - cgo_cflags += '-fsanitize=address -g -DWITH_DEBUG=1 ' - cgo_ldflags += '-fsanitize=address -g ' -endif - -apiserver_ext_deps = [ - libxce_dep, - librtti_dep, - c_capnproto_dep, - ssl_dep, - tinylog_dep, - math_dep, - rt_dep, -] - -foreach e: apiserver_ext_deps - cflags_result = run_command(pkgconfig, '--cflags', e.name(), check: false) - - if cflags_result.returncode() == 0 - cgo_cflags += ' ' - cgo_cflags += cflags_result.stdout().strip() - endif - ldflags_result = run_command(pkgconfig, '--libs', e.name(), check: false) - - if ldflags_result.returncode() == 0 - cgo_ldflags += ' ' - cgo_ldflags += ldflags_result.stdout().strip() - endif -endforeach - -apiserver_deps = [ - apiserver_ext_deps, - libxce_dep, -] - -cgo_cflags += ' -I@CURRENT_SOURCE_DIR@/../include -I@CURRENT_SOURCE_DIR@/include -I@BUILD_ROOT@/lib ' -cgo_ldflags += ' -L@BUILD_ROOT@/lib -L@BUILD_ROOT@/apiserver -lxce' - -go_env = environment() -go_env.set('CGO_CFLAGS', cgo_cflags) -go_env.set('CGO_LDFLAGS', cgo_ldflags) -go_env.set('BUILDTYPE', get_option('buildtype')) -go_env.set('FIX_ENVARS','CGO_CFLAGS CGO_LDFLAGS') -go_env.set('DEPENDENCIES','thread math') -go_env.set('ENVARS', 'CURRENT_SOURCE_DIR BUILD_ROOT') - -#libapiserver_src = [ -# 'src/init.c', -#] - -#libapiserver_incdir = include_directories('../include', 'include', '../lib') - -#libapiserver_shared = shared_library('apiserver', -# libapiserver_src, -# include_directories: [libapiserver_incdir], -# dependencies: [libxce_dep], -# install: true) - -apiserver = custom_target('apiserver', - input: 'cmd/main.go', - output: 'xce_apiserver', - command: [env_program, - 'CURRENT_SOURCE_DIR="@CURRENT_SOURCE_DIR@"', - 'BUILD_ROOT="@BUILD_ROOT@"', - go_build_program, - '@INPUT@', '@OUTPUT@'], - env: go_env, - depends: [libxce], - install: true, - install_dir: get_option('prefix') / 'bin') - diff --git a/src/controller/CLAUDE.md b/src/controller/CLAUDE.md new file mode 100644 index 00000000..be22b508 --- /dev/null +++ b/src/controller/CLAUDE.md @@ -0,0 +1,34 @@ +# XCE Controller + +## Overview +C daemon managing cluster-level job and queue operations. Acts as the central coordinator between the scheduler, worker nodes, and the Go API server. + +## Build +```bash +# From workspace root +nix develop +meson build && ninja -C build +``` + +## Architecture +Core source files handle distinct domains: +- `job.c` — Job lifecycle management +- `queue.c` — Queue state and control +- `cluster.c` — Cluster topology +- `dataex.c` — Data exchange with other daemons + +Links against `libxce` from `src/lib/`. + +## Conventions +- C standard: C99 or later +- Format with `clang-format` +- Component-specific headers in `include/` +- Shared types defined in `src/include/xce/` (libxce) + +## Key Files +- `src/controller/` — Controller source files +- `include/` — Controller-specific headers + +## See Also +- Shared library: `src/lib/CLAUDE.md` +- Root guidelines: `/workspace/CLAUDE.md` diff --git a/src/frontend/CLAUDE.md b/src/frontend/CLAUDE.md new file mode 100644 index 00000000..1d6c0b64 --- /dev/null +++ b/src/frontend/CLAUDE.md @@ -0,0 +1,77 @@ +# XCE Frontend (API Server + CLI) + +## Overview +Go-based REST API server (Gin) and CLI tool (Cobra) that expose cluster management operations. Two independent Go modules sharing no code at the module level. + +## Modules +| Module | Path | Entry Point | +|--------|------|-------------| +| `apiserver` | `src/frontend/apiserver/` | `cmd/main.go` | +| `xcecli` | `src/frontend/xcecli/` | `cmd/main.go` | + +## Build & Test +```bash +make build-server # → /tmp/apiserver +make build-cli # → /tmp/xce_static (CGO_ENABLED=0, static binary) +make test # unit tests only (no live server required) +make test-all # unit + integration (needs XCE_BIN and SRV_URL env vars) +``` + +## API Server Architecture +``` +cmd/ Entry point — init logger, config, stores, RBAC, auth, then start Gin +router/ Route registration and middleware chain +controllers/ HTTP handlers: one file per resource (jobs, queues, hosts, clusters, users...) +services/ Business logic +store/ Thread-safe in-memory key-value stores (generics-based) +models/ Request/response DTOs and domain structs +middleware/ CORS, rate limiting, IP whitelist, JWT validation +backend/ C library integration stubs +backend/auth Pluggable auth providers (configfile, ldap, oauth2, pam) +utils/ Logger (zap), config parsing, crypto helpers +common/ Global error codes and messages +config/ YAML configs and JSON schemas +``` + +## Conventions + +### Controllers +- Struct: `type {Resource}Controller struct{}` +- Handler signature: `func (c {Resource}Controller) {Action}{Resource}(ctx *gin.Context)` +- Always return via helpers — never write response directly: + ```go + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, data) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + ``` + +### Error Handling +- Early return on every error — no deferred error accumulation +- Log with zap fields before returning: `logger.Error("msg", zap.Error(err))` +- Use `preCheck{Resource}Request()` helpers for input validation before business logic + +### RBAC +- Protect routes with `controllers.RequirePermission("action:domain")` middleware +- Permission strings follow `action:scope` pattern (e.g., `job:view:self`, `queue:submit:all`) + +### Logging +- Import as: `logger "apiserver/utils"` +- Always include context: `zap.String("key", val)`, `zap.Error(err)` + +### Models +- JSON tags use snake_case: `json:"field_name"` +- Required fields: `binding:"required"` +- YAML tags for config structs: `yaml:"field_name,omitempty"` + +### Tests +- Use `t.TempDir()` for file fixtures +- Reset global state with a helper (e.g., `resetRBACState()` pattern in rbac_check_test.go) +- Table-driven tests for filter/validation logic + +## Key Files +- `controllers/response.go` — ReturnSuccess / ReturnError helpers +- `controllers/rbac_check.go` — RBAC loading, RequirePermission middleware +- `middleware/validatehandler.go` — JWT token validation middleware +- `backend/auth/provider.go` — AuthProvider interface definition +- `store/store.go` — Generic in-memory store implementation +- `config/rbac.yaml` — Role and permission definitions +- `config/auth.yaml` — Auth provider configuration diff --git a/src/frontend/Makefile b/src/frontend/Makefile new file mode 100644 index 00000000..32c6913f --- /dev/null +++ b/src/frontend/Makefile @@ -0,0 +1,41 @@ +# XCE project — test targets +# Usage: +# make test # unit tests only (no server needed, CGO_ENABLED=0) +# make test-backend # backend CGO conversion tests (requires nix dev shell + build dir) +# make test-integration # regression suite against live server +# make test-all # unit + integration tests + +XCE_BIN ?= /tmp/xce_static +SRV_URL ?= http://localhost:8080 +GOFLAGS := CGO_ENABLED=0 + +.PHONY: build-cli build-server test test-backend test-integration test-all + +build-cli: + cd xcecli && $(GOFLAGS) go build -o $(XCE_BIN) ./cmd/main.go + +build-server: + cd apiserver && go build -o /tmp/apiserver ./cmd/main.go + +## Run all Go unit tests that do not require CGO or a live server. +test: + @echo "=== Unit tests: apiserver ===" + cd apiserver && $(GOFLAGS) go test ./store/... ./utils/... ./controllers/... ./backend/... -count=1 + @echo "" + @echo "=== Unit tests: xcecli ===" + cd xcecli && $(GOFLAGS) go test ./... -count=1 + +## Run backend CGO conversion tests. +## Requires CGO_ENABLED=1 and CGO_CFLAGS/CGO_LDFLAGS pointing at the build tree. +## Run from within the nix develop shell after a full meson build. +test-backend: + @echo "=== Backend CGO tests: apiserver/backend ===" + cd apiserver && CGO_ENABLED=1 go test ./backend/... -count=1 + +## Run regression suite against a live server (requires XCE_BIN and SRV_URL) +test-integration: build-cli + @echo "=== Integration regression: $(SRV_URL) ===" + XCE=$(XCE_BIN) URL=$(SRV_URL) bash tests/regression/run.sh + +## Run everything +test-all: test test-integration diff --git a/src/frontend/apiserver/README.md b/src/frontend/apiserver/README.md new file mode 100644 index 00000000..9d5fc712 --- /dev/null +++ b/src/frontend/apiserver/README.md @@ -0,0 +1,10 @@ +(1) run server command: + +cd cmd +go run main.go + +(2) run server with different log level: +cd cmd +go run main.go -level=debug + +level options: info/debug/warn/error/fatal diff --git a/src/frontend/apiserver/backend/auth/config_file.go b/src/frontend/apiserver/backend/auth/config_file.go new file mode 100644 index 00000000..7ce6c90c --- /dev/null +++ b/src/frontend/apiserver/backend/auth/config_file.go @@ -0,0 +1,84 @@ +package auth + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +// AuthFileConfig is the top-level structure of auth.yaml. +type AuthFileConfig struct { + Provider string `yaml:"provider"` + ConfigFile *ConfigFileAuth `yaml:"config_file"` +} + +// ConfigFileAuth holds the user entries for config_file provider. +type ConfigFileAuth struct { + Users []UserEntry `yaml:"users"` +} + +// UserEntry represents a single user in auth.yaml. +type UserEntry struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + Roles []string `yaml:"roles"` +} + +// ConfigFileProvider authenticates users against a YAML config file. +type ConfigFileProvider struct { + users map[string]UserEntry +} + +// NewConfigFileProvider creates a provider from raw YAML bytes. +func NewConfigFileProvider(data []byte) (*ConfigFileProvider, error) { + var cfg AuthFileConfig + if err := yaml.Unmarshal(data, &cfg); err != nil { + return nil, fmt.Errorf("failed to parse auth config: %w", err) + } + + if len(cfg.ConfigFile.Users) == 0 { + return nil, fmt.Errorf("no users defined in auth config") + } + + users := make(map[string]UserEntry, len(cfg.ConfigFile.Users)) + for _, u := range cfg.ConfigFile.Users { + if u.Username == "" { + return nil, fmt.Errorf("empty username in auth config") + } + users[u.Username] = u + } + + return &ConfigFileProvider{users: users}, nil +} + +// Authenticate checks username/password against the config file. +func (p *ConfigFileProvider) Authenticate(username, password string) error { + entry, ok := p.users[username] + if !ok { + return fmt.Errorf("invalid username or password") + } + if entry.Password != password { + return fmt.Errorf("invalid username or password") + } + return nil +} + +// GetUserRoles returns roles defined in the config for the given user. +func (p *ConfigFileProvider) GetUserRoles(username string) []string { + if entry, ok := p.users[username]; ok { + return entry.Roles + } + return nil +} + +// Name returns the provider identifier. +func (p *ConfigFileProvider) Name() string { return "config_file" } + +// AllUsers returns all usernames known to this provider (implements UserRoleSeeder). +func (p *ConfigFileProvider) AllUsers() []string { + users := make([]string, 0, len(p.users)) + for u := range p.users { + users = append(users, u) + } + return users +} diff --git a/src/frontend/apiserver/backend/auth/config_file_test.go b/src/frontend/apiserver/backend/auth/config_file_test.go new file mode 100644 index 00000000..99873327 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/config_file_test.go @@ -0,0 +1,133 @@ +package auth + +import ( + "testing" +) + +func TestConfigFileProvider_Authenticate(t *testing.T) { + yaml := ` +provider: config_file +config_file: + users: + - username: admin + password: secret + roles: [cluster_admin] + - username: viewer + password: pass + roles: [basic_user] +` + + provider, err := NewConfigFileProvider([]byte(yaml)) + if err != nil { + t.Fatalf("NewConfigFileProvider: %v", err) + } + + tests := []struct { + name string + user string + pass string + wantErr bool + }{ + {"valid admin", "admin", "secret", false}, + {"valid viewer", "viewer", "pass", false}, + {"wrong password", "admin", "wrong", true}, + {"unknown user", "nobody", "pass", true}, + {"empty user", "", "pass", true}, + {"empty password", "admin", "", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := provider.Authenticate(tt.user, tt.pass) + if (err != nil) != tt.wantErr { + t.Errorf("Authenticate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestConfigFileProvider_GetUserRoles(t *testing.T) { + yaml := ` +provider: config_file +config_file: + users: + - username: admin + password: admin + roles: [cluster_admin, operator] + - username: guest + password: guest + roles: [] +` + + provider, err := NewConfigFileProvider([]byte(yaml)) + if err != nil { + t.Fatalf("NewConfigFileProvider: %v", err) + } + + if roles := provider.GetUserRoles("admin"); len(roles) != 2 { + t.Errorf("admin roles = %v, want 2", roles) + } + if roles := provider.GetUserRoles("guest"); len(roles) != 0 { + t.Errorf("guest roles = %v, want empty", roles) + } + if roles := provider.GetUserRoles("unknown"); roles != nil { + t.Errorf("unknown roles = %v, want nil", roles) + } +} + +func TestConfigFileProvider_Name(t *testing.T) { + p := &ConfigFileProvider{} + if got := p.Name(); got != "config_file" { + t.Errorf("Name() = %q, want %q", got, "config_file") + } +} + +func TestNewConfigFileProvider_Errors(t *testing.T) { + t.Run("invalid yaml", func(t *testing.T) { + _, err := NewConfigFileProvider([]byte("{{invalid")) + if err == nil { + t.Error("expected error for invalid yaml") + } + }) + + t.Run("no users", func(t *testing.T) { + _, err := NewConfigFileProvider([]byte("provider: config_file\nconfig_file:\n users: []")) + if err == nil { + t.Error("expected error for empty users list") + } + }) + + t.Run("empty username", func(t *testing.T) { + _, err := NewConfigFileProvider([]byte("provider: config_file\nconfig_file:\n users:\n - username: ''\n password: x")) + if err == nil { + t.Error("expected error for empty username") + } + }) +} + +func TestPAMProvider_Name(t *testing.T) { + p := &PAMProvider{} + if got := p.Name(); got != "pam" { + t.Errorf("Name() = %q, want %q", got, "pam") + } +} + +func TestLDAPProvider_Name(t *testing.T) { + p := &LDAPProvider{} + if got := p.Name(); got != "ldap" { + t.Errorf("Name() = %q, want %q", got, "ldap") + } + if err := p.Authenticate("u", "p"); err == nil { + t.Error("expected not-implemented error") + } +} + +func TestOAuth2Provider_Name(t *testing.T) { + p := &OAuth2Provider{} + if got := p.Name(); got != "oauth2" { + t.Errorf("Name() = %q, want %q", got, "oauth2") + } + if err := p.Authenticate("u", "p"); err == nil { + t.Error("expected not-implemented error") + } +} diff --git a/src/frontend/apiserver/backend/auth/init.go b/src/frontend/apiserver/backend/auth/init.go new file mode 100644 index 00000000..96d2229c --- /dev/null +++ b/src/frontend/apiserver/backend/auth/init.go @@ -0,0 +1,44 @@ +package auth + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +var GlobalProvider AuthProvider + +// InitAuthProvider reads auth.yaml and creates the appropriate provider. +func InitAuthProvider(configPath string) error { + data, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("failed to read auth config %s: %w", configPath, err) + } + + var cfg struct { + Provider string `yaml:"provider"` + } + if err := yaml.Unmarshal(data, &cfg); err != nil { + return fmt.Errorf("failed to parse auth config: %w", err) + } + + switch cfg.Provider { + case "config_file": + provider, err := NewConfigFileProvider(data) + if err != nil { + return fmt.Errorf("failed to create config_file provider: %w", err) + } + GlobalProvider = provider + case "pam": + GlobalProvider = &PAMProvider{} + case "ldap": + GlobalProvider = &LDAPProvider{} + case "oauth2": + GlobalProvider = &OAuth2Provider{} + default: + return fmt.Errorf("unsupported auth provider: %s", cfg.Provider) + } + + return nil +} diff --git a/src/frontend/apiserver/backend/auth/ldap.go b/src/frontend/apiserver/backend/auth/ldap.go new file mode 100644 index 00000000..2f735d03 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/ldap.go @@ -0,0 +1,17 @@ +package auth + +import "fmt" + +// LDAPProvider is a stub for future LDAP authentication. +type LDAPProvider struct{} + +// Authenticate verifies credentials via LDAP (not yet implemented). +func (p *LDAPProvider) Authenticate(username, password string) error { + return fmt.Errorf("LDAP authentication not yet implemented") +} + +// GetUserRoles returns nil (not yet implemented). +func (p *LDAPProvider) GetUserRoles(username string) []string { return nil } + +// Name returns the provider identifier. +func (p *LDAPProvider) Name() string { return "ldap" } diff --git a/src/frontend/apiserver/backend/auth/oauth2.go b/src/frontend/apiserver/backend/auth/oauth2.go new file mode 100644 index 00000000..c5f514f6 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/oauth2.go @@ -0,0 +1,17 @@ +package auth + +import "fmt" + +// OAuth2Provider is a stub for future OAuth2/OIDC authentication. +type OAuth2Provider struct{} + +// Authenticate verifies credentials via OAuth2 (not yet implemented). +func (p *OAuth2Provider) Authenticate(username, password string) error { + return fmt.Errorf("OAuth2 authentication not yet implemented") +} + +// GetUserRoles returns nil (not yet implemented). +func (p *OAuth2Provider) GetUserRoles(username string) []string { return nil } + +// Name returns the provider identifier. +func (p *OAuth2Provider) Name() string { return "oauth2" } diff --git a/src/frontend/apiserver/backend/auth/pam.go b/src/frontend/apiserver/backend/auth/pam.go new file mode 100644 index 00000000..32badbf8 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/pam.go @@ -0,0 +1,11 @@ +package auth + +// PAMProvider authenticates users via the system PAM stack. +// Only available on Linux. +type PAMProvider struct{} + +// GetUserRoles returns nil — PAM users get roles from RBAC config. +func (p *PAMProvider) GetUserRoles(username string) []string { return nil } + +// Name returns the provider identifier. +func (p *PAMProvider) Name() string { return "pam" } diff --git a/src/frontend/apiserver/backend/auth/pam_linux.go b/src/frontend/apiserver/backend/auth/pam_linux.go new file mode 100644 index 00000000..825adbd3 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/pam_linux.go @@ -0,0 +1,112 @@ +//go:build linux && cgo + +package auth + +/* +#cgo LDFLAGS: -ldl +#include +#include +#include + +// PAM types and constants (from ) +#define PAM_SUCCESS 0 +#define PAM_PROMPT_ECHO_OFF 1 + +typedef struct pam_message { + int msg_style; + const char *msg; +} pam_message; + +typedef struct pam_response { + char *resp; + int resp_retcode; +} pam_response; + +typedef struct pam_conv { + int (*conv)(int, const struct pam_message **, struct pam_response **, void *); + void *appdata_ptr; +} pam_conv; + +typedef void* pam_handle_t; + +// Function pointer types +typedef int (*pam_start_fn)(const char*, const char*, const pam_conv*, pam_handle_t**); +typedef int (*pam_authenticate_fn)(pam_handle_t*, int); +typedef int (*pam_end_fn)(pam_handle_t*, int); + +static void *pam_handle = NULL; +static pam_start_fn fn_pam_start = NULL; +static pam_authenticate_fn fn_pam_authenticate = NULL; +static pam_end_fn fn_pam_end = NULL; + +// load_pam opens libpam.so dynamically. Returns 0 on success. +static int load_pam() { + if (pam_handle) return 0; + const char *libs[] = {"libpam.so.0", "libpam.so", NULL}; + for (int i = 0; libs[i]; i++) { + pam_handle = dlopen(libs[i], RTLD_LAZY); + if (pam_handle) break; + } + if (!pam_handle) return -1; + fn_pam_start = (pam_start_fn) dlsym(pam_handle, "pam_start"); + fn_pam_authenticate = (pam_authenticate_fn) dlsym(pam_handle, "pam_authenticate"); + fn_pam_end = (pam_end_fn) dlsym(pam_handle, "pam_end"); + if (!fn_pam_start || !fn_pam_authenticate || !fn_pam_end) { + dlclose(pam_handle); + pam_handle = NULL; + return -2; + } + return 0; +} + +// conv_fn is the PAM conversation callback — returns the password. +static int conv_fn(int num_msg, const pam_message **msg, + pam_response **resp, void *appdata) { + *resp = (pam_response*)calloc(num_msg, sizeof(pam_response)); + if (!*resp) return 4; // PAM_BUF_ERR + for (int i = 0; i < num_msg; i++) { + if (msg[i]->msg_style == PAM_PROMPT_ECHO_OFF) { + (*resp)[i].resp = strdup((const char*)appdata); + } + } + return PAM_SUCCESS; +} + +// pam_auth_dynamic authenticates username/password via dlopen'd libpam. +// Returns 0 on success, -1 if libpam not found, PAM error code otherwise. +static int pam_auth_dynamic(const char *username, const char *password) { + if (load_pam() != 0) return -1; + pam_conv conv = { conv_fn, (void*)password }; + pam_handle_t *pamh = NULL; + int ret = fn_pam_start("system-auth", username, &conv, &pamh); + if (ret != PAM_SUCCESS) return ret; + ret = fn_pam_authenticate(pamh, 0); + fn_pam_end(pamh, ret); + return ret; +} +*/ +import "C" +import ( + "fmt" + "unsafe" +) + +// Authenticate verifies credentials via dynamically loaded libpam.so. +// The binary does not link against libpam at build time — it is loaded +// at runtime only when this provider is active. +func (p *PAMProvider) Authenticate(username, password string) error { + cu := C.CString(username) + cp := C.CString(password) + defer C.free(unsafe.Pointer(cu)) + defer C.free(unsafe.Pointer(cp)) + + ret := C.pam_auth_dynamic(cu, cp) + switch ret { + case 0: + return nil + case -1: + return fmt.Errorf("PAM library (libpam.so) not found on this system") + default: + return fmt.Errorf("invalid username or password") + } +} diff --git a/src/frontend/apiserver/backend/auth/pam_stub.go b/src/frontend/apiserver/backend/auth/pam_stub.go new file mode 100644 index 00000000..bc863726 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/pam_stub.go @@ -0,0 +1,13 @@ +//go:build !linux || !cgo + +package auth + +import ( + "fmt" + "runtime" +) + +// Authenticate returns an error when PAM is not available (non-Linux or CGO disabled). +func (p *PAMProvider) Authenticate(username, password string) error { + return fmt.Errorf("PAM authentication is not supported on %s or without CGO; use config_file provider", runtime.GOOS) +} diff --git a/src/frontend/apiserver/backend/auth/provider.go b/src/frontend/apiserver/backend/auth/provider.go new file mode 100644 index 00000000..4a912823 --- /dev/null +++ b/src/frontend/apiserver/backend/auth/provider.go @@ -0,0 +1,14 @@ +package auth + +// UserRoleSeeder is an optional interface for providers that can enumerate all users. +// Used to pre-populate the role store at startup so permissions work after restart. +type UserRoleSeeder interface { + AllUsers() []string +} + +// AuthProvider is the interface for authentication strategies. +type AuthProvider interface { + Authenticate(username, password string) error + GetUserRoles(username string) []string + Name() string +} diff --git a/src/frontend/apiserver/backend/clib.go b/src/frontend/apiserver/backend/clib.go new file mode 100644 index 00000000..f5bcd505 --- /dev/null +++ b/src/frontend/apiserver/backend/clib.go @@ -0,0 +1,319 @@ +//go:build cgo + +package backend + +/* +#include +#include +#include +#include "client.h" +#include "netio.h" +#include "helper.h" +#include "controller.capnp.h" + +// backend_ctx_t wraps xcecli_ctx_t for use as the C handler's arg. +typedef struct { + xcecli_ctx_t ctx; +} backend_ctx_t; + +// go_push_handler is exported from Go and called from C for every incoming request. +extern int go_push_handler(xce_request_t*, void*); + +static int xce_backend_handler(xce_request_t *req, void *arg) { + return go_push_handler(req, arg); +} + +// xce_backend_init initialises the client context, sets the handler, +// starts the network loop, then joins as HostRole_apiserver. +static int xce_backend_init(xce_client_config_t *config, backend_ctx_t *ctx) { + int rc = xcecli_ctx_init(&(ctx->ctx), config); + if (rc < 0) return rc; + xcecli_set_handler(ctx->ctx, (xce_request_handler_t)xce_backend_handler, NULL); + rc = xcecli_start(ctx->ctx); + if (rc < 0) return rc; + return xcecli_join(ctx->ctx, HostRole_apiserver, NULL); +} + +// Helpers to extract typed row pointers from a dataset_t. +static xce_job_t **dataset_jobs(dataset_t *d) { return (xce_job_t **)d->rows; } +static xce_worker_t **dataset_workers(dataset_t *d) { return (xce_worker_t **)d->rows; } + +// CGO cannot access C union fields by name (they appear as [N]byte). +// These thin helpers set the correct union branch from C so Go callers +// don't need unsafe pointer arithmetic. +static void jobctrl_set_siginfo(xce_jobctrl_t *ctrl, xce_signal_t *sig) { + ctrl->u.siginfo = sig; +} +static void jobctrl_set_ckptoption(xce_jobctrl_t *ctrl, xce_ckpt_options_t *ckpt) { + ctrl->u.ckptoption = ckpt; +} +*/ +import "C" + +import ( + "apiserver/models" + logger "apiserver/utils" + "fmt" + "unsafe" + + "go.uber.org/zap" +) + +// gCtx holds the C client context for the lifetime of the process. +var gCtx C.backend_ctx_t + +// gHandler receives push notifications from the backend. +// Must be set via StartNotificationListener before RegisterToBackend. +var gHandler NotificationHandler + +// RegisterToBackend initialises the XCE client library and joins as an API +// server. StartNotificationListener should be called first so that push +// events have a handler ready before the first push can arrive. +func RegisterToBackend(config *BackendConfig) error { + from := config.From + if from == "" { + from = "xce_apiserver" + } + key := config.Key + if key == "" { + key = "keepsecret$" + } + + cAddr := C.CString(config.Address) + cFrom := C.CString(from) + cKey := C.CString(key) + defer func() { + C.free(unsafe.Pointer(cAddr)) + C.free(unsafe.Pointer(cFrom)) + C.free(unsafe.Pointer(cKey)) + }() + + var cConfig C.xce_client_config_t + cConfig.nmasters = 1 + cConfig.masters[0] = cAddr + cConfig.master_ports[0] = C.int(config.Port) + cConfig.from = cFrom + cConfig.key = cKey + + rc := C.xce_backend_init(&cConfig, &gCtx) + if rc < 0 { + return fmt.Errorf("xce_backend_init failed: %d", int(rc)) + } + + logger.Info("Backend connection established", + zap.String("address", config.Address), + zap.Int("port", config.Port)) + return nil +} + +// StartNotificationListener registers the handler that receives backend push +// events. Call this before RegisterToBackend. +func StartNotificationListener(handler NotificationHandler) error { + gHandler = handler + return nil +} + +// go_push_handler is called by the C library on every incoming request. +// It must be exported so the C trampoline can call it from a C thread. +// +//export go_push_handler +func go_push_handler(req *C.xce_request_t, _ unsafe.Pointer) C.int { + if req == nil { + return 0 + } + defer C.free_Request(req) + + if gHandler == nil { + return 0 + } + + switch req.opcode { + case C.OPCode_push: + return handlePush(req) + } + return 0 +} + +// handlePush dispatches an OPCode_push request to the appropriate handler. +func handlePush(req *C.xce_request_t) C.int { + switch req.body_which { + case C.Request_body_datamsg: + dataset := (*C.dataset_t)(unsafe.Pointer(C.xce_request_body(req))) + return handleDataset(dataset) + case C.Request_body_configpushmsg: + cfgdata := (*C.xce_configdata_t)(unsafe.Pointer(C.xce_request_body(req))) + return handleConfigPush(cfgdata) + } + return 0 +} + +// handleDataset processes a dataset push (jobs or workers). +func handleDataset(d *C.dataset_t) C.int { + switch d.table { + case C.Data_data_jobs: + n := int(d.nrows) + if n == 0 { + return 0 + } + jobs := unsafe.Slice(C.dataset_jobs(d), n) + for _, jp := range jobs { + if jp == nil { + continue + } + info := cJobToModel(jp) + gHandler.OnJobUpdated(info) + } + + case C.Data_data_workers: + n := int(d.nrows) + if n == 0 { + return 0 + } + workers := unsafe.Slice(C.dataset_workers(d), n) + for _, wp := range workers { + if wp == nil { + continue + } + info := cWorkerToModel(wp) + gHandler.OnHostUpdated(info) + } + } + return 0 +} + +// handleConfigPush processes a cluster configuration push. +func handleConfigPush(cfgdata *C.xce_configdata_t) C.int { + if cfgdata.data_which != C.ConfigData_data_clusterconfig { + return 0 + } + + conf := (*C.cluster_config_t)(C.xce_configdata_ptr(cfgdata)) + if conf == nil { + return 0 + } + + for _, q := range cClusterConfQueues(conf) { + qi := q + gHandler.OnQueueUpdated(&qi) + } + for _, r := range cClusterConfResources(conf) { + ri := r + gHandler.OnResourceUpdated(&ri) + } + return 0 +} + +// SubmitJob sends a job submission to the backend via xcecli_jobsubmit. +// The store is updated only when the backend pushes the resulting OnJobCreated +// notification — not immediately here. +func SubmitJob(job *models.JobInfo) error { + js, sd, si := modelJobToSubmit(job) + defer freeJobSubmit(js, sd, si) + + var outJobID C.uint64_t + rc := C.xcecli_jobsubmit(gCtx.ctx, js, &outJobID) + if rc < 0 { + return fmt.Errorf("xcecli_jobsubmit failed: %d", int(rc)) + } + if outJobID != 0 { + job.Unique_JobID = int64(outJobID) + job.JobID = int64(outJobID) + } + return nil +} + +// SubmitArrayJobs submits an array job via xcecli_jobsubmit with array_size > 1. +func SubmitArrayJobs(arrayjob *models.ArrayJobInfo, jobs []models.JobInfo) error { + if len(jobs) == 0 { + return fmt.Errorf("no jobs in array") + } + + // Use the first job as the template; set array_size to the total count. + js, sd, si := modelJobToSubmit(&jobs[0]) + defer freeJobSubmit(js, sd, si) + js.array_size = C.uint64_t(len(jobs)) + + var outJobID C.uint64_t + rc := C.xcecli_jobsubmit(gCtx.ctx, js, &outJobID) + if rc < 0 { + return fmt.Errorf("xcecli_jobsubmit (array) failed: %d", int(rc)) + } + if outJobID != 0 { + arrayjob.JobID = int64(outJobID) + } + return nil +} + +// ControlJob sends a job control action to the backend via xcecli_jobctrl. +// action must be one of the models.JOB_* constants; content is an optional +// signal name (e.g. "TERM") for kill operations. +func ControlJob(jobid uint64, action int, content string) error { + var ctrl C.xce_jobctrl_t + ctrl.jobid = C.uint64_t(jobid) + ctrl.action = C.int(action) + + // For signal-based actions set signo; for checkpoint set ckpt options. + // xce_jobctrl_t.u is a C union — we use a static C helper to set the + // right branch safely. + switch action { + case models.JOB_STOP, models.JOB_RESUME, models.JOB_KILL: + var sig C.xce_signal_t + sig.signo = C.int(signalNumber(content, action)) + sig.force = 0 + C.jobctrl_set_siginfo(&ctrl, &sig) + case models.JOB_CHKPNT: + var ckpt C.xce_ckpt_options_t + ckpt.kill = 0 + C.jobctrl_set_ckptoption(&ctrl, &ckpt) + } + + rc := C.xcecli_jobctrl(gCtx.ctx, &ctrl) + if rc < 0 { + return fmt.Errorf("xcecli_jobctrl failed: %d", int(rc)) + } + return nil +} + +// ControlQueue sends a queue open/close/activate/deactivate action to the +// backend via xcecli_queuectrl. +func ControlQueue(queuename string, action int) error { + cQueue := C.CString(queuename) + defer C.free(unsafe.Pointer(cQueue)) + + ctrl := C.xce_queuectrl_t{ + action: C.int(action), + queue: cQueue, + } + + rc := C.xcecli_queuectrl(gCtx.ctx, &ctrl) + if rc < 0 { + return fmt.Errorf("xcecli_queuectrl failed: %d", int(rc)) + } + return nil +} + +// ControlHost is a placeholder; host control via xcecli is not yet exposed +// in client.h. Use the REST API workaround until the C library adds +// xcecli_hostctrl. +func ControlHost(hostname string, action int) error { + return fmt.Errorf("host control not yet supported via C library (host: %s, action: %d)", hostname, action) +} + +// signalNumber maps a job action and optional signal name to a POSIX signal +// number, falling back to sensible defaults. +func signalNumber(signame string, action int) int { + if signame != "" { + if n, ok := models.JobSignalMapTable[signame]; ok { + return n + } + } + switch action { + case models.JOB_KILL: + return models.JobSignalMapTable["TERM"] // 15 + case models.JOB_STOP: + return models.JobSignalMapTable["TSTP"] // 20 + case models.JOB_RESUME: + return models.JobSignalMapTable["CONT"] // 18 + } + return 15 // SIGTERM as safe default +} diff --git a/src/frontend/apiserver/backend/config.go b/src/frontend/apiserver/backend/config.go new file mode 100644 index 00000000..e1245c3a --- /dev/null +++ b/src/frontend/apiserver/backend/config.go @@ -0,0 +1,10 @@ +package backend + +// BackendConfig holds connection parameters for the XCE controller. +// Used by RegisterToBackend to establish the C client connection. +type BackendConfig struct { + Address string // controller hostname (e.g. "localhost") + Port int // controller port (e.g. 9990) + From string // identity sent to controller (default: "xce_apiserver") + Key string // authentication key +} diff --git a/src/frontend/apiserver/backend/convert.go b/src/frontend/apiserver/backend/convert.go new file mode 100644 index 00000000..f544b948 --- /dev/null +++ b/src/frontend/apiserver/backend/convert.go @@ -0,0 +1,210 @@ +//go:build cgo + +package backend + +/* +#include +#include "client.h" +*/ +import "C" + +import ( + "apiserver/models" + "unsafe" +) + +// goString safely converts a C string to a Go string, handling nil. +func goString(s *C.char) string { + if s == nil { + return "" + } + return C.GoString(s) +} + +// cJobToModel extracts C xce_job_t fields into a rawJobData and maps it to JobInfo. +func cJobToModel(job *C.xce_job_t) *models.JobInfo { + r := rawJobData{ + jobID: int64(job.jobid), + state: int(job.state), + runTime: int(job.run_time), + subTime: int64(job.sub_time), + queueTime: int64(job.queue_time), + startTime: int64(job.start_time), + finishTime: int64(job.finish_time), + } + if job.submit != nil { + s := job.submit + rd := &rawSubmitData{ + queue: goString(s.queue), + user: goString(s.user), + jobName: goString(s.job_name), + projectName: goString(s.project_name), + userGroup: goString(s.user_group), + jobDesc: goString(s.job_description), + resReq: goString(s.resreq), + minTasks: int(s.mintasks), + maxTasks: int(s.maxtasks), + } + if s.start_info != nil { + si := s.start_info + rd.startInfo = &rawStartInfo{ + cmdline: goString(si.cmdline), + launchType: goString(si.launch_type), + preExec: goString(si.job_pre_exec), + postExec: goString(si.job_post_exec), + execHome: goString(si.exechome), + execCWD: goString(si.execcwd), + execUser: goString(si.execuser), + } + } + r.submit = rd + } + return rawJobToModel(r) +} + +// cWorkerToModel extracts C xce_worker_t fields into a rawWorkerData and maps it to HostInfo. +func cWorkerToModel(worker *C.xce_worker_t) *models.HostInfo { + r := rawWorkerData{ + name: goString(worker.name), + status: int(worker.status), + } + if worker.attrs != nil { + a := worker.attrs + r.hasAttrs = true + r.hostType = goString(a.hosttype) + r.hostModel = goString(a.hostmodel) + r.cpuFactor = float64(a.cpufactor) + r.maxCPUs = int(a.maxcpus) + r.maxMem = int(a.maxmem) + r.maxSwap = int(a.maxswap) + r.maxTmp = int(a.maxtmp) + r.nDisks = int(a.ndisks) + } + return rawWorkerToModel(r) +} + +// cQueueToModel extracts C xce_queue_t fields and maps them to QueueInfo. +func cQueueToModel(queue *C.xce_queue_t) *models.QueueInfo { + return rawQueueToModel(rawQueueData{ + name: goString(queue.queue), + status: int(queue.status), + priority: int(queue.priority), + }) +} + +// cResourceToModel extracts C xce_resource_t fields and maps them to ResourceDefinitionInfo. +func cResourceToModel(res *C.xce_resource_t) *models.ResourceDefinitionInfo { + return rawResourceToModel(rawResourceData{ + name: goString(res.name), + desc: goString(res.desc), + valType: int(res.valtype), + ordType: int(res.ordtype), + flags: int(res.flags), + interval: int(res.interval), + }) +} + +// cClusterConfQueues extracts queue models from a cluster_config_t. +func cClusterConfQueues(conf *C.cluster_config_t) []models.QueueInfo { + n := int(conf.nqueues) + if n == 0 { + return nil + } + qps := unsafe.Slice(conf.queues, n) + out := make([]models.QueueInfo, 0, n) + for _, qp := range qps { + if qp != nil { + out = append(out, *cQueueToModel(qp)) + } + } + return out +} + +// cClusterConfResources extracts resource definition models from a cluster_config_t. +func cClusterConfResources(conf *C.cluster_config_t) []models.ResourceDefinitionInfo { + n := int(conf.nres) + if n == 0 { + return nil + } + rps := unsafe.Slice(conf.resources, n) + out := make([]models.ResourceDefinitionInfo, 0, n) + for _, rp := range rps { + if rp != nil { + out = append(out, *cResourceToModel(rp)) + } + } + return out +} + +// modelJobToSubmit converts a Go JobInfo to C xce_job_submit_t heap structs. +// The caller must free all returned C memory by calling freeJobSubmit. +func modelJobToSubmit(job *models.JobInfo) (*C.xce_job_submit_t, *C.job_submit_data_t, *C.job_startinfo_t) { + si := (*C.job_startinfo_t)(C.calloc(1, C.sizeof_job_startinfo_t)) + si.cmdline = C.CString(job.Command) + if job.LaunchType != "" { + si.launch_type = C.CString(job.LaunchType) + } + if job.PreExecCmd != "" { + si.job_pre_exec = C.CString(job.PreExecCmd) + } + if job.PostExecCmd != "" { + si.job_post_exec = C.CString(job.PostExecCmd) + } + + sd := (*C.job_submit_data_t)(C.calloc(1, C.sizeof_job_submit_data_t)) + sd.queue = C.CString(job.QueueName) + sd.user = C.CString(job.SubmitUser) + sd.job_name = C.CString(job.JobName) + if job.ProjectName != "" { + sd.project_name = C.CString(job.ProjectName) + } + if job.UserGroup != "" { + sd.user_group = C.CString(job.UserGroup) + } + if job.JobDescription != "" { + sd.job_description = C.CString(job.JobDescription) + } + if job.ResReq != "" { + sd.resreq = C.CString(job.ResReq) + } + sd.mintasks = C.uint(job.NumProcessors) + if sd.mintasks == 0 { + sd.mintasks = 1 + } + sd.maxtasks = C.uint(job.MaxNumProcessors) + if sd.maxtasks < sd.mintasks { + sd.maxtasks = sd.mintasks + } + sd.start_info = si + + js := (*C.xce_job_submit_t)(C.calloc(1, C.sizeof_xce_job_submit_t)) + js.jobid = C.uint64_t(job.Unique_JobID) + js.array_size = 1 + js.submit = sd + + return js, sd, si +} + +// freeJobSubmit releases C memory allocated by modelJobToSubmit. +func freeJobSubmit(js *C.xce_job_submit_t, sd *C.job_submit_data_t, si *C.job_startinfo_t) { + if si != nil { + C.free(unsafe.Pointer(si.cmdline)) + C.free(unsafe.Pointer(si.launch_type)) + C.free(unsafe.Pointer(si.job_pre_exec)) + C.free(unsafe.Pointer(si.job_post_exec)) + C.free(unsafe.Pointer(si)) + } + if sd != nil { + C.free(unsafe.Pointer(sd.queue)) + C.free(unsafe.Pointer(sd.user)) + C.free(unsafe.Pointer(sd.job_name)) + C.free(unsafe.Pointer(sd.project_name)) + C.free(unsafe.Pointer(sd.user_group)) + C.free(unsafe.Pointer(sd.job_description)) + C.free(unsafe.Pointer(sd.resreq)) + C.free(unsafe.Pointer(sd)) + } + if js != nil { + C.free(unsafe.Pointer(js)) + } +} diff --git a/src/frontend/apiserver/backend/convert_pure.go b/src/frontend/apiserver/backend/convert_pure.go new file mode 100644 index 00000000..479561ab --- /dev/null +++ b/src/frontend/apiserver/backend/convert_pure.go @@ -0,0 +1,161 @@ +package backend + +import ( + "apiserver/models" + "time" +) + +// rawStartInfo mirrors the C job_startinfo_t fields used by the Go model. +type rawStartInfo struct { + cmdline string + launchType string + preExec string + postExec string + execHome string + execCWD string + execUser string +} + +// rawSubmitData mirrors the C job_submit_data_t fields used by the Go model. +type rawSubmitData struct { + queue string + user string + jobName string + projectName string + userGroup string + jobDesc string + resReq string + minTasks int + maxTasks int + startInfo *rawStartInfo +} + +// rawJobData mirrors the C xce_job_t fields used by the Go model. +type rawJobData struct { + jobID int64 + state int + runTime int + subTime int64 + queueTime int64 + startTime int64 + finishTime int64 + submit *rawSubmitData +} + +// rawWorkerData mirrors the C xce_worker_t fields used by the Go model. +type rawWorkerData struct { + name string + status int + hasAttrs bool + hostType string + hostModel string + cpuFactor float64 + maxCPUs int + maxMem int + maxSwap int + maxTmp int + nDisks int +} + +// rawQueueData mirrors the C xce_queue_t fields used by the Go model. +type rawQueueData struct { + name string + status int + priority int +} + +// rawResourceData mirrors the C xce_resource_t fields used by the Go model. +type rawResourceData struct { + name string + desc string + valType int + ordType int + flags int + interval int +} + +// rawJobToModel maps a rawJobData to a models.JobInfo. +// This pure-Go function contains the field-mapping logic and is tested without CGO. +func rawJobToModel(r rawJobData) *models.JobInfo { + info := &models.JobInfo{ + Unique_JobID: r.jobID, + JobID: r.jobID, + JobStatus: r.state, + RunTime: r.runTime, + } + if r.submit != nil { + s := r.submit + info.QueueName = s.queue + info.SubmitUser = s.user + info.JobName = s.jobName + info.ProjectName = s.projectName + info.UserGroup = s.userGroup + info.JobDescription = s.jobDesc + info.ResReq = s.resReq + info.NumProcessors = s.minTasks + info.MaxNumProcessors = s.maxTasks + if s.startInfo != nil { + si := s.startInfo + info.Command = si.cmdline + info.LaunchType = si.launchType + info.PreExecCmd = si.preExec + info.PostExecCmd = si.postExec + info.ExecHome = si.execHome + info.ExecCWD = si.execCWD + info.ExecUser = si.execUser + } + } + if r.subTime > 0 { + info.SubmitTime = time.Unix(r.subTime, 0) + } + if r.queueTime > 0 { + info.QueueTime = time.Unix(r.queueTime, 0) + } + if r.startTime > 0 { + info.StartTime = time.Unix(r.startTime, 0) + } + if r.finishTime > 0 { + info.EndTime = time.Unix(r.finishTime, 0) + } + return info +} + +// rawWorkerToModel maps a rawWorkerData to a models.HostInfo. +func rawWorkerToModel(r rawWorkerData) *models.HostInfo { + info := &models.HostInfo{ + HostName: r.name, + HostStatus: r.status, + } + if r.hasAttrs { + info.HostType = r.hostType + info.HostModel = r.hostModel + info.CPUFactor = r.cpuFactor + info.NumCPUs = r.maxCPUs + info.MaxMem = r.maxMem + info.MaxSwap = r.maxSwap + info.MaxTmp = r.maxTmp + info.NumDisk = r.nDisks + } + return info +} + +// rawQueueToModel maps a rawQueueData to a models.QueueInfo. +func rawQueueToModel(r rawQueueData) *models.QueueInfo { + return &models.QueueInfo{ + QueueName: r.name, + QueueStatus: r.status, + Priority: r.priority, + } +} + +// rawResourceToModel maps a rawResourceData to a models.ResourceDefinitionInfo. +func rawResourceToModel(r rawResourceData) *models.ResourceDefinitionInfo { + return &models.ResourceDefinitionInfo{ + ResourceName: r.name, + Description: r.desc, + ValueType: r.valType, + OrderType: r.ordType, + Flag: r.flags, + Interval: r.interval, + } +} diff --git a/src/frontend/apiserver/backend/convert_test.go b/src/frontend/apiserver/backend/convert_test.go new file mode 100644 index 00000000..10f51c1d --- /dev/null +++ b/src/frontend/apiserver/backend/convert_test.go @@ -0,0 +1,199 @@ +package backend + +import ( + "testing" + "time" +) + +func TestRawJobToModel_BasicFields(t *testing.T) { + r := rawJobData{ + jobID: 123, + state: 2, // JOB_STAT_RUN + runTime: 42, + submit: &rawSubmitData{ + queue: "normal", + user: "alice", + jobName: "myjob", + minTasks: 1, + maxTasks: 4, + startInfo: &rawStartInfo{ + cmdline: "sleep 60", + }, + }, + } + + info := rawJobToModel(r) + + if info.Unique_JobID != 123 { + t.Errorf("Unique_JobID: got %d, want 123", info.Unique_JobID) + } + if info.JobID != 123 { + t.Errorf("JobID: got %d, want 123", info.JobID) + } + if info.QueueName != "normal" { + t.Errorf("QueueName: got %q, want %q", info.QueueName, "normal") + } + if info.SubmitUser != "alice" { + t.Errorf("SubmitUser: got %q, want %q", info.SubmitUser, "alice") + } + if info.JobName != "myjob" { + t.Errorf("JobName: got %q, want %q", info.JobName, "myjob") + } + if info.Command != "sleep 60" { + t.Errorf("Command: got %q, want %q", info.Command, "sleep 60") + } + if info.JobStatus != 2 { + t.Errorf("JobStatus: got %d, want 2", info.JobStatus) + } + if info.NumProcessors != 1 { + t.Errorf("NumProcessors: got %d, want 1", info.NumProcessors) + } + if info.MaxNumProcessors != 4 { + t.Errorf("MaxNumProcessors: got %d, want 4", info.MaxNumProcessors) + } + if info.RunTime != 42 { + t.Errorf("RunTime: got %d, want 42", info.RunTime) + } +} + +func TestRawJobToModel_NilSubmit(t *testing.T) { + r := rawJobData{jobID: 99, state: 1 /* JOB_STAT_PEND */} + + info := rawJobToModel(r) + + if info.Unique_JobID != 99 { + t.Errorf("Unique_JobID: got %d, want 99", info.Unique_JobID) + } + if info.QueueName != "" { + t.Errorf("QueueName should be empty for nil submit, got %q", info.QueueName) + } +} + +func TestRawJobToModel_Timestamps(t *testing.T) { + r := rawJobData{ + jobID: 1, + subTime: 1700000000, + queueTime: 1700000010, + startTime: 1700000020, + finishTime: 1700000030, + } + + info := rawJobToModel(r) + + if info.SubmitTime != time.Unix(1700000000, 0) { + t.Errorf("SubmitTime mismatch") + } + if info.QueueTime != time.Unix(1700000010, 0) { + t.Errorf("QueueTime mismatch") + } + if info.StartTime != time.Unix(1700000020, 0) { + t.Errorf("StartTime mismatch") + } + if info.EndTime != time.Unix(1700000030, 0) { + t.Errorf("EndTime mismatch") + } +} + +func TestRawWorkerToModel_BasicFields(t *testing.T) { + r := rawWorkerData{ + name: "node01", + status: 3, // XCE_WORKER_UNAVAIL + hasAttrs: true, + hostType: "linux", + hostModel: "x86_64", + cpuFactor: 1.5, + maxCPUs: 16, + maxMem: 65536, + maxSwap: 32768, + maxTmp: 1024, + nDisks: 2, + } + + info := rawWorkerToModel(r) + + if info.HostName != "node01" { + t.Errorf("HostName: got %q, want %q", info.HostName, "node01") + } + if info.HostType != "linux" { + t.Errorf("HostType: got %q, want %q", info.HostType, "linux") + } + if info.HostModel != "x86_64" { + t.Errorf("HostModel: got %q, want %q", info.HostModel, "x86_64") + } + if info.NumCPUs != 16 { + t.Errorf("NumCPUs: got %d, want 16", info.NumCPUs) + } + if info.MaxMem != 65536 { + t.Errorf("MaxMem: got %d, want 65536", info.MaxMem) + } + if info.NumDisk != 2 { + t.Errorf("NumDisk: got %d, want 2", info.NumDisk) + } + if info.HostStatus != 3 { + t.Errorf("HostStatus: got %d, want 3", info.HostStatus) + } +} + +func TestRawWorkerToModel_NoAttrs(t *testing.T) { + r := rawWorkerData{name: "node02", status: 1, hasAttrs: false} + + info := rawWorkerToModel(r) + + if info.HostName != "node02" { + t.Errorf("HostName: got %q, want node02", info.HostName) + } + if info.HostType != "" { + t.Errorf("HostType should be empty when hasAttrs=false, got %q", info.HostType) + } + if info.NumCPUs != 0 { + t.Errorf("NumCPUs should be 0 when hasAttrs=false, got %d", info.NumCPUs) + } +} + +func TestRawQueueToModel_BasicFields(t *testing.T) { + r := rawQueueData{ + name: "normal", + status: 1, // QUEUE_STAT_OPEN + priority: 30, + } + + info := rawQueueToModel(r) + + if info.QueueName != "normal" { + t.Errorf("QueueName: got %q, want %q", info.QueueName, "normal") + } + if info.Priority != 30 { + t.Errorf("Priority: got %d, want 30", info.Priority) + } + if info.QueueStatus != 1 { + t.Errorf("QueueStatus: got %d, want 1", info.QueueStatus) + } +} + +func TestRawResourceToModel_BasicFields(t *testing.T) { + r := rawResourceData{ + name: "mem", + desc: "memory resource", + valType: 1, // XCE_VAL_NUMERIC + flags: 4, // XCE_RESF_CONSUMABLE + interval: 60, + } + + info := rawResourceToModel(r) + + if info.ResourceName != "mem" { + t.Errorf("ResourceName: got %q, want %q", info.ResourceName, "mem") + } + if info.Description != "memory resource" { + t.Errorf("Description: got %q, want %q", info.Description, "memory resource") + } + if info.ValueType != 1 { + t.Errorf("ValueType: got %d, want 1", info.ValueType) + } + if info.Flag != 4 { + t.Errorf("Flag: got %d, want 4", info.Flag) + } + if info.Interval != 60 { + t.Errorf("Interval: got %d, want 60", info.Interval) + } +} diff --git a/src/frontend/apiserver/backend/handler.go b/src/frontend/apiserver/backend/handler.go new file mode 100644 index 00000000..bcc82b8c --- /dev/null +++ b/src/frontend/apiserver/backend/handler.go @@ -0,0 +1,38 @@ +package backend + +import ( + "apiserver/models" + "apiserver/store" +) + +// DefaultHandler implements NotificationHandler by writing to stores. +type DefaultHandler struct{} + +func (h *DefaultHandler) OnJobCreated(job *models.JobInfo) { + store.Jobs.Put(uint64(job.Unique_JobID), *job) +} + +func (h *DefaultHandler) OnJobUpdated(job *models.JobInfo) { + store.Jobs.Put(uint64(job.Unique_JobID), *job) +} + +func (h *DefaultHandler) OnJobDeleted(jobid uint64) { + store.Jobs.Delete(jobid) +} + +func (h *DefaultHandler) OnQueueUpdated(queue *models.QueueInfo) { + store.Queues.Put(queue.QueueName, *queue) +} + +func (h *DefaultHandler) OnHostUpdated(host *models.HostInfo) { + store.Hosts.Put(host.HostName, *host) +} + +func (h *DefaultHandler) OnResourceUpdated(res *models.ResourceDefinitionInfo) { + store.Resources.Put(res.ResourceName, *res) +} + +func (h *DefaultHandler) OnResourceLocationUpdated(res *models.ResourceDistributionInfo) { + key := res.ResourceName + ":" + res.Location + store.ResourceLocations.Put(key, *res) +} diff --git a/src/frontend/apiserver/backend/notification.go b/src/frontend/apiserver/backend/notification.go new file mode 100644 index 00000000..eb3eea92 --- /dev/null +++ b/src/frontend/apiserver/backend/notification.go @@ -0,0 +1,16 @@ +package backend + +import "apiserver/models" + +// NotificationHandler defines callbacks for backend push events. +// When the backend creates/updates/deletes objects, these callbacks +// are invoked to update the API server's in-memory stores. +type NotificationHandler interface { + OnJobCreated(job *models.JobInfo) + OnJobUpdated(job *models.JobInfo) + OnJobDeleted(jobid uint64) + OnQueueUpdated(queue *models.QueueInfo) + OnHostUpdated(host *models.HostInfo) + OnResourceUpdated(res *models.ResourceDefinitionInfo) + OnResourceLocationUpdated(res *models.ResourceDistributionInfo) +} diff --git a/src/frontend/apiserver/backend/stub.go b/src/frontend/apiserver/backend/stub.go new file mode 100644 index 00000000..f38e5168 --- /dev/null +++ b/src/frontend/apiserver/backend/stub.go @@ -0,0 +1,31 @@ +//go:build !cgo + +package backend + +import ( + "apiserver/models" + "errors" +) + +var errNoCGO = errors.New("backend requires CGO: rebuild with CGO_ENABLED=1 and libxce") + +// RegisterToBackend stub — real implementation in clib.go (requires CGO). +func RegisterToBackend(config *BackendConfig) error { return errNoCGO } + +// StartNotificationListener stub — real implementation in clib.go (requires CGO). +func StartNotificationListener(handler NotificationHandler) error { return errNoCGO } + +// SubmitJob stub — real implementation in clib.go (requires CGO). +func SubmitJob(job *models.JobInfo) error { return errNoCGO } + +// SubmitArrayJobs stub — real implementation in clib.go (requires CGO). +func SubmitArrayJobs(arrayjob *models.ArrayJobInfo, jobs []models.JobInfo) error { return errNoCGO } + +// ControlJob stub — real implementation in clib.go (requires CGO). +func ControlJob(jobid uint64, action int, content string) error { return errNoCGO } + +// ControlQueue stub — real implementation in clib.go (requires CGO). +func ControlQueue(queuename string, action int) error { return errNoCGO } + +// ControlHost stub — real implementation in clib.go (requires CGO). +func ControlHost(hostname string, action int) error { return errNoCGO } diff --git a/src/frontend/apiserver/cmd/main.go b/src/frontend/apiserver/cmd/main.go new file mode 100644 index 00000000..ecb77e73 --- /dev/null +++ b/src/frontend/apiserver/cmd/main.go @@ -0,0 +1,117 @@ +package main + +import ( + _ "apiserver/docs" + "apiserver/backend" + "apiserver/backend/auth" + "apiserver/middleware" + "apiserver/router" + "apiserver/store" + "apiserver/utils" + logger "apiserver/utils" + "flag" + "fmt" + "io" + "os" + + "apiserver/controllers" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +func main() { + gin.SetMode(gin.ReleaseMode) + gin.DefaultWriter = io.Discard + gin.DefaultErrorWriter = io.Discard + + topDir := flag.String("d", "", "Top directory (default: current directory)") + flag.Usage = func() { + fmt.Fprintln(os.Stderr, ` + Usage: apiserver -d + Top directory (default: current directory) + `) + } + flag.Parse() + + utils.Global_TopDir = utils.PromptForTopDir(topDir) + + if err := utils.InitLogger(); err != nil { + os.Exit(1) + } + logger.Info("********** Initialize Logger done!") + + if err := utils.InitConf(); err != nil { + logger.Error("Failed to initialize configuration files", zap.Error(err)) + os.Exit(1) + } + logger.Info("********** Initialize configuration files done!") + + // Initialize in-memory stores + store.InitStores() + logger.Info("********** Initialize in-memory stores done!") + + // Load builtin test data and RBAC from config + if err := store.LoadBuiltinData(); err != nil { + logger.Error("Failed to load builtin data", zap.Error(err)) + os.Exit(1) + } + logger.Info("********** Load builtin data done!") + + // Connect to the XCE controller backend for live data. + // StartNotificationListener must be called before RegisterToBackend so the + // handler is ready before any push events can arrive. + if utils.BackendYamlConfig != nil && utils.BackendYamlConfig.Address != "" { + handler := &backend.DefaultHandler{} + if err := backend.StartNotificationListener(handler); err != nil { + logger.Error("Failed to start notification listener", zap.Error(err)) + os.Exit(1) + } + bcfg := &backend.BackendConfig{ + Address: utils.BackendYamlConfig.Address, + Port: utils.BackendYamlConfig.Port, + From: utils.BackendYamlConfig.From, + Key: utils.BackendYamlConfig.Key, + } + if err := backend.RegisterToBackend(bcfg); err != nil { + logger.Error("Failed to connect to backend", zap.Error(err)) + os.Exit(1) + } + logger.Info("********** Backend connection registered!") + } else { + logger.Info("********** Backend not configured — running in stub mode") + } + + // Load RBAC config + rbacPath := utils.Global_TopDir + "/config/rbac.yaml" + if err := controllers.LoadRBACConfig(rbacPath); err != nil { + logger.Error("Failed to load RBAC config", zap.Error(err)) + os.Exit(1) + } + logger.Info("********** Load RBAC config done!") + + // Initialize auth provider + authConfigPath := utils.Global_TopDir + "/config/auth.yaml" + if err := auth.InitAuthProvider(authConfigPath); err != nil { + logger.Error("Failed to initialize auth provider", zap.Error(err)) + os.Exit(1) + } + logger.Info("********** Initialize auth provider done! (" + auth.GlobalProvider.Name() + ")") + + // Pre-populate user-role store from auth provider so permissions work + // immediately after restart without requiring users to re-login. + if seeder, ok := auth.GlobalProvider.(auth.UserRoleSeeder); ok { + for _, username := range seeder.AllUsers() { + for _, role := range auth.GlobalProvider.GetUserRoles(username) { + store.AddRoleForUser(username, role) + } + } + logger.Info("********** Pre-populated user-role store from auth provider done!") + } + + r := router.Router() + if err := middleware.StartServer(r); err != nil { + logger.Error("Server failed to start:", zap.Error(err)) + os.Exit(1) + } +} diff --git a/src/frontend/apiserver/common/common.go b/src/frontend/apiserver/common/common.go new file mode 100644 index 00000000..9f14cd61 --- /dev/null +++ b/src/frontend/apiserver/common/common.go @@ -0,0 +1,61 @@ +package common + +const ( + SUCCESS_MSG = "Success" + ERROR_MSG = "Fail" + INVALID_PARAMS_MSG = "Invalid parameters" + ERROR_AUTH_CHECK_TOKEN_MSG = "invalid token" + ERROR_AUTH_CHECK_TOKEN_EMPTY_MSG = "Authorization token is required" + ERROR_AUTH_TOKEN_TIMEOUT_MSG = "Token is expired" + ERROR_AUTH_MSG = "Authentication failed" + ERROR_NOT_FOUND_MSG = "resource not found" + ERROR_HOST_NOT_FOUND_MSG = "host not found" + ERROR_JOB_NOT_FOUND_MSG = "job not found" + ERROR_QUEUE_NOT_FOUND_MSG = "queue not found" + ERROR_JWT_GENERATION_MSG = "Failed to generate token" + ERROR_JWT_VALIDATION_MSG = "Failed to validate token" + ERROR_READ_REQUEST_BODY_MSG = "Request body invalid" + ERROR_REQUEST_BODY_EMPTY_MSG = "request body is empty" + ERROR_REQUEST_BODY_NOT_JSON_MSG = "request body is not json format" + ERROR_REQUEST_URL_NOT_FOUND = "request url is not found" + ERROR_GENERATE_JOBID_MSG = "Failed to generate job ID" + PARTIAL_DELETE_FILES_MSG = "Partially deleting files" + ERROR_CONNECT_BACKEND_MSG = "Failed to connect to backend" + ERROR_WRITE_DATA_MSG = "Failed to write data" + ERROR_QUERY_JOBINFO_MSG = "Failed to query job info" + ERROR_QUEUR_ROLEINFO_MSG = "Failed to query role info" + ERROR_QUERY_HOSTINFO_MSG = "Failed to query host info" + ERROR_QUERY_QUEUEINFO_MSG = "Failed to query queue info" + ERROR_QUERY_RESINFO_MSG = "Failed to query resource info" + ERROR_QUERY_CLUSTERINFO_MSG = "Failed to query cluster info" + ERROR_QUERY_PERMISSIONS_MSG = "Failed to query permissions" + ERROR_PARSE_JSON_STRING_MSG = "Failed to parse json string" + ERROR_INVALID_CLIENT_IP_MSG = "Invalid client IP" + ERROR_ACCESS_FORBIDDEN_MSG = "Access forbidden" + + //read file error + ERROR_READ_FILE_MSG = "Failed to read file" + ERROR_PARSE_FILE_MSG = "Failed to parse file" + ERROR_FILTER_FORMAT_MSG = "Invalid filter format" + ERROR_TOO_MANY_REQUESTS_MSG = "Too many requests. Please try again later." + ERROR_GET_USERNAME_MSG = "Failed to get username from token" + ERROR_GET_USERPATH_MSG = "Failed to get user path" + ERROR_REMOVE_TOKEN_MSG = "Failed to remove token file" +) + +const ( + SUCCESS_CODE = 200 + CREATED = 201 + ACCEPTED = 202 + StatusPartialContent = 206 + ERROR_CODE = 500 + INVALID_PARAMS_CODE = 400 + ERROR_AUTH_CHECK_TOKEN_CODE = 401 + ERROR_AUTH_CODE = 403 + ERROR_NOT_FOUND_CODE = 404 + ERROR_TOO_MANY_REQUESTS_CODE = 429 + + //read file error + ERROR_READ_FILE_CODE = 501 + ERROR_PARSE_FILE_CODE = 502 +) diff --git a/src/frontend/apiserver/config/app.yaml b/src/frontend/apiserver/config/app.yaml new file mode 100644 index 00000000..5c847dba --- /dev/null +++ b/src/frontend/apiserver/config/app.yaml @@ -0,0 +1,85 @@ +app: + name: "xce_apiserver" + runmode: "dev" # debug, release, test + http_port: 8080 + version: "1.0.0" + +security: + https: + enabled: false + port: 8443 + cert_file: "config/ssl/server.crt" + key_file: "config/ssl/server.key" + +authentication: + token: + type: "jwt" + secretKey: "xce-api-key" + token_expire_time: 86400 # 1 day + +rate_limit: + global: + rate: 1000 # requests per second + burst: 2000 + window: second # second, minute + endpoints: + /xce/v1/auth/logon: + rate: 100 # requests per minute + burst: 200 + window: second + /xce/v1/cluster: + rate: 1000 # requests per second + burst: 2000 + window: second + /xce/v1/jobs/*: + rate: 1000 # requests per second + burst: 2000 + window: second + /xce/v1/files/*: + rate: 1000 # requests per second + burst: 2000 + window: second + +whitelist: + #- "127.0.0.1" + #- "::1" + #- "172.24.100.0/24" + #- "172.29.6.0/24" + #- "192.168.0.0/16" + #- "172.19.0.0/16" + - "0.0.0.0/0" + - "::/0" + +file_upload: + max_size: 100 MB # maximum file size, support unit MB or GB + allowed_types: + - "application/x-tar" + - "application/zip" + - "application/x-gzip" + - "application/x-bzip2" + - "application/x-xz" + - "application/gzip" + - "application/bzip2" + - "application/x-xz" + - "application/octet-stream" + - "application/x-zip-compressed" + disallowed_types: # disallowed file types + - "application/x-msdownload" # .exe file + - "application/javascript" # .js file + +job: + max_array_size: 1000 # default max array size for job submission + +cache: + refresh_interval: 30 # seconds, available values: 1-300 + +cors: + enabled: true + allowed_origins: + - "*" + +backend: + address: "localhost" + port: 9990 + from: "xce_apiserver" + key: "keepsecret$" diff --git a/src/frontend/apiserver/config/auth.yaml b/src/frontend/apiserver/config/auth.yaml new file mode 100644 index 00000000..04a2d73c --- /dev/null +++ b/src/frontend/apiserver/config/auth.yaml @@ -0,0 +1,19 @@ +# Authentication provider: config_file | pam | ldap | oauth2 +# - config_file: dev mode, users defined in this file +# - pam: Linux system PAM authentication (Linux only) +# - ldap: LDAP authentication (future) +# - oauth2: OAuth2/OIDC authentication (future) +provider: config_file + +# ConfigFile provider settings +config_file: + users: + - username: admin + password: admin + roles: [cluster_admin] + - username: user1 + password: user1 + roles: [basic_user] + - username: user2 + password: user2 + roles: [basic_user] diff --git a/src/frontend/apiserver/config/auth.yaml.example b/src/frontend/apiserver/config/auth.yaml.example new file mode 100644 index 00000000..04a2d73c --- /dev/null +++ b/src/frontend/apiserver/config/auth.yaml.example @@ -0,0 +1,19 @@ +# Authentication provider: config_file | pam | ldap | oauth2 +# - config_file: dev mode, users defined in this file +# - pam: Linux system PAM authentication (Linux only) +# - ldap: LDAP authentication (future) +# - oauth2: OAuth2/OIDC authentication (future) +provider: config_file + +# ConfigFile provider settings +config_file: + users: + - username: admin + password: admin + roles: [cluster_admin] + - username: user1 + password: user1 + roles: [basic_user] + - username: user2 + password: user2 + roles: [basic_user] diff --git a/src/frontend/apiserver/config/logger.yaml b/src/frontend/apiserver/config/logger.yaml new file mode 100644 index 00000000..8907fe38 --- /dev/null +++ b/src/frontend/apiserver/config/logger.yaml @@ -0,0 +1,27 @@ +logging: + # Log level: debug, info, warn, error, fatal + level: "debug" + # Log encoding: json or console + encoding: console + + # standard output + outputPath: "apiserver_out.log" + + # File rotation settings + file: + maxSize: 100 # megabytes before rotation + maxBackups: 10 # number of old files to keep + maxAge: 14 # days to keep old files + compress: true # compress rotated files + + # Encoder configuration + encoderConfig: + messageKey: "message" + levelKey: "level" + timeKey: "timestamp" + nameKey: "logger" + callerKey: "caller" + functionKey: "func" + stacktraceKey: "stacktrace" + timeEncoder: "ISO8601" + levelEncoder: "capital" diff --git a/src/frontend/apiserver/config/nginx.http.conf b/src/frontend/apiserver/config/nginx.http.conf new file mode 100644 index 00000000..0b90ad10 --- /dev/null +++ b/src/frontend/apiserver/config/nginx.http.conf @@ -0,0 +1,60 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + ' $status $body_bytes_sent "$http_referer" ' + ' $http_user_agent $http_x_forwarded_for $request_time; ' + ' $upstream_response_time $upstream_status $upstream_addr'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + # define upstream cluster, configure primary and backup instances + upstream apiserver_backend { + # valid values include IP, domain, and port + server 172.29.6.117:8080 max_fails=3 fail_timeout=30s; + server 172.29.6.101:8080 max_fails=3 fail_timeout=30s; + # server apiserver2.example.com:80 max_fails=3 fail_timeout=30s; + # backup server + #server backup.example.com:80 backup; + } + + server { + listen 8080; + server_name _; + + location / { + proxy_pass http://apiserver_backend; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_next_upstream error timeout http_500 http_502 http_503 http_504; + proxy_connect_timeout 3s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + } + + # optional configuration to detect and redirect to a maintenance page + # if all upstream servers are down + error_log /var/log/nginx/apiserver_error.log; + access_log /var/log/nginx/apiserver_access.log main; + } +} diff --git a/src/frontend/apiserver/config/nginx.https.conf b/src/frontend/apiserver/config/nginx.https.conf new file mode 100644 index 00000000..67a00715 --- /dev/null +++ b/src/frontend/apiserver/config/nginx.https.conf @@ -0,0 +1,65 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + ' $status $body_bytes_sent "$http_referer" ' + ' $http_user_agent $http_x_forwarded_for $request_time; ' + ' $upstream_response_time $upstream_status $upstream_addr'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; + + # define upstream cluster, configure primary and backup instances + upstream apiserver_backend { + # valid values include IP, domain, and port + server 172.29.6.117:8443 max_fails=3 fail_timeout=30s; + server 172.29.6.101:8443 max_fails=3 fail_timeout=30s; + # server apiserver2.example.com:80 max_fails=3 fail_timeout=30s; + # backup server + #server backup.example.com:80 backup; + } + + server { + listen 8443 ssl; + server_name *.akrostar-tech.com; # replace with your domain name + + ssl_certificate /etc/nginx/ssl/server.crt; # cetificate file path + ssl_certificate_key /etc/nginx/ssl/server.key; # key file path + ssl_session_cache shared:SSL:1m; + ssl_session_timeout 10m; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + location / { + proxy_pass https://apiserver_backend; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_next_upstream error timeout http_500 http_502 http_503 http_504; + proxy_connect_timeout 3s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + } + + access_log /var/log/nginx/apiserver_access.log main; + error_log /var/log/nginx/apiserver_error.log; + } +} diff --git a/src/frontend/apiserver/config/rbac.yaml b/src/frontend/apiserver/config/rbac.yaml new file mode 100644 index 00000000..7dcb92bf --- /dev/null +++ b/src/frontend/apiserver/config/rbac.yaml @@ -0,0 +1,151 @@ +permissions: + - queue:view # view queue info + - queue:submit # submit job + - queue:control # close, open, active, deactive + - job:view # view job info + - job:control # stop, resume, kill, requeue, peek??, modify + - host:view # view host info + - host:control # close, open + - cluster:view # view cluster info + - cluster:control # shutdown, restart, reconfigure + - user:view # view user info) + - rbac:control # create, update, delete roles + - rbac:view # view role info + - file:control # upload, download,delete + - file:view # view file info + - hostgroup:view # view hostgroup info + - hostgroup:control # add + - usergroup:view # view usergroup info + - resource:view # view resource info + - ar:view # view advanced reservation info + - ar:submit # submit advanced reservation + - ar:control # delete, modify, create + +roles: + basic_user: + permissions: + + - action: cluster:view + domain: + - type: all + - action: queue:view + domain: + - type: all + - action: queue:submit + domain: + - type: all + - action: job:view + domain: + - type: self + - type: self_ug + - action: host:view + domain: + - type: all + - action: rbac:view + domain: + - type: all + - action: user:view + domain: + - type: all + - action: file:view + domain: + - type: self + - action: job:control + domain: + - type: self + - action: hostgroup:view + domain: + - type: all + - action: usergroup:view + domain: + - type: all + - action: resource:view + domain: + - type: all + cluster_admin: + permissions: + - action: cluster:control + domain: + - type: self + - action: cluster:view + domain: + - type: all + - action: queue:submit + domain: + - type: all + - action: queue:control + domain: + - type: all + - action: queue:view + domain: + - type: all + - action: job:control + domain: + - type: all + - action: job:view + domain: + - type: all + - action: host:control + domain: + - type: all + - action: host:view + domain: + - type: all + - action: user:view + domain: + - type: all + - action: rbac:view + domain: + - type: all + - action: rbac:control + domain: + - type: all + - action: file:control + domain: + - type: all + - action: file:view + domain: + - type: all + - action: hostgroup:view + domain: + - type: all + - action: hostgroup:control + domain: + - type: all + - action: usergroup:view + domain: + - type: all + - action: resource:view + domain: + - type: all + - action: ar:view + domain: + - type: all + - action: ar:control + domain: + - type: all + - action: ar:submit + domain: + - type: all + queue_admin: + permissions: + - action: queue:submit + domain: + - type: queue + values: [normal,priority] + - action: queue:view + domain: + - type: all + - action: queue:control + domain: + - type: self + - action: job:view + domain: + - type: queue + values: [normal] + - action: job:control + domain: + - type: queue + values: [normal] + + diff --git a/src/frontend/apiserver/config/rbac_template.yaml b/src/frontend/apiserver/config/rbac_template.yaml new file mode 100644 index 00000000..b8889465 --- /dev/null +++ b/src/frontend/apiserver/config/rbac_template.yaml @@ -0,0 +1,194 @@ +permissions: + - queue:view # view queue info + - queue:submit # submit job + - queue:control # close, open, active, deactive + - job:view # view job info + - job:control # stop, resume, kill, requeue, peek??, modify + - host:view # view host info + - host:control # close, open + - cluster:view # view cluster info + - cluster:control # shutdown, restart, reconfigure + - user:view # view user info) + - rbac:control # create, update, delete roles + - file:control # upload, download,delete + - file:view # view file info + - hostgroup:view # view hostgroup info + - hostgroup:control # add + - usergroup:view # view usergroup info + - resource:view # view resource info + - ar:view # view advanced reservation info + - ar:submit # submit advanced reservation + - ar:control # delete, modify, create + +roles: + basic_user: + permissions: + - action: cluster:view + domain: + - type: all + - action: queue:view + domain: + - type: all + - action: queue:submit + domain: + - type: all + - action: job:view + domain: + - type: self + - type: self_ug + - action: host:view + domain: + - type: all + - action: user:view + domain: + - type: all + - action: file:view + domain: + - type: self + - action: job:control + domain: + - type: self + - action: hostgroup:view + domain: + - type: all + - action: usergroup:view + domain: + - type: all + - action: resource:view + domain: + - type: all + cluster_admin: + permissions: + - action: cluster:control + domain: + - type: self + - action: cluster:view + domain: + - type: all + - action: queue:submit + domain: + - type: all + - action: queue:control + domain: + - type: all + - action: queue:view + domain: + - type: all + - action: job:control + domain: + - type: all + - action: job:view + domain: + - type: all + - action: host:control + domain: + - type: all + - action: host:view + domain: + - type: all + - action: user:view + domain: + - type: all + - action: rbac:control + domain: + - type: all + - action: file:control + domain: + - type: all + - action: file:view + domain: + - type: all + - action: hostgroup:view + domain: + - type: all + - action: hostgroup:control + domain: + - type: all + - action: usergroup:view + domain: + - type: all + - action: resource:view + domain: + - type: all + - action: ar:view + domain: + - type: all + - action: ar:control + domain: + - type: all + - action: ar:submit + domain: + - type: all + queue_admin: + includes: + - basic_user + permissions: + - action: queue:submit + domain: + - type: queues + values: [normal,priority] + - action: queue:view + domain: + - type: all + - action: queue:control + domain: + - type: self + - action: job:view + domain: + - type: queues + values: [normal] + - action: job:control + domain: + - type: queues + values: [normal] + + host_group_admin: + includes: + - basic_user + permissions: + - action: hostgroup:control + domain: + - type: self_hg + user_group_admin: + includes: + - basic_user + permissions: + - action: job:control + domain: + - type: self_ug + host_group_user: + permissions: + - action: hostgroup:control + domain: + - type: self_hg + mock_host_user: + permissions: + - action: host:control + domain: + - type: hostgroups + values: [hostgroup1,hostgroup2] + mock_job_user: + permissions: + - action: job:view + domain: + - type: usergroups + values: [usergroup1, usergroup2] + +users: + user01: + roles: + - name: cluster_admin + domain: + - type: all + + user02: + roles: + - name: host_group_admin + domain: + - type: hostgroups + values: [hostgroup1, hostgroup2] + user1: + roles: + - name: basic_user + domain: + - type: all \ No newline at end of file diff --git a/src/frontend/apiserver/config/schema/app_schema.json b/src/frontend/apiserver/config/schema/app_schema.json new file mode 100644 index 00000000..a07a4f53 --- /dev/null +++ b/src/frontend/apiserver/config/schema/app_schema.json @@ -0,0 +1,126 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "App Configuration Schema", + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "runmode": { "type": "string", "enum": ["dev", "debug", "release", "test"] }, + "http_port": { "type": "number" }, + "version": { "type": "string" } + }, + "required": ["name", "runmode", "http_port", "version"] + }, + "security": { + "type": "object", + "properties": { + "https": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "port": { "type": "number" }, + "cert_file": { "type": "string" }, + "key_file": { "type": "string" } + }, + "required": ["enabled", "port", "cert_file", "key_file"] + } + }, + "required": ["https"] + }, + "authentication": { + "type": "object", + "properties": { + "token": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "secretKey": { "type": "string" }, + "token_expire_time": { "type": "number" } + }, + "required": ["type", "secretKey", "token_expire_time"] + } + }, + "required": ["token"] + }, + "rate_limit": { + "type": "object", + "properties": { + "global": { + "type": "object", + "properties": { + "rate": { "type": "number" }, + "burst": { "type": "number" }, + "window": { "type": "string", "enum": ["second", "minute"] } + }, + "required": ["rate", "burst", "window"] + }, + "endpoints": { + "type": "object", + "patternProperties": { + "^/": { + "type": "object", + "properties": { + "rate": { "type": "number" }, + "burst": { "type": "number" }, + "window": { "type": "string", "enum": ["second", "minute"] } + }, + "required": ["rate", "burst", "window"] + } + }, + "additionalProperties": false + } + }, + "required": ["global", "endpoints"] + }, + "whitelist": { + "type": "array", + "items": { "type": "string" } + }, + "file_upload": { + "type": "object", + "properties": { + "max_size": { + "type": "string", + "pattern": "^[0-9]+\\s?(MB|GB)$", + "description": "Maximum file size with unit (e.g., '100 MB', '5 GB')" + }, + "allowed_types": { + "type": "array", + "items": { "type": "string" }, + "description": "List of allowed MIME types" + }, + "disallowed_types": { + "type": "array", + "items": { "type": "string" }, + "description": "List of explicitly disallowed MIME types" + } + }, + "required": ["max_size", "allowed_types"] + }, + "job": { + "type":["object","null"], + "properties": { + "max_array_size": { + "type": "integer", + "minimum": 1, + "maximum": 2147483646, + "description": "Default max array size for job submission" + } + } + }, + "cache": { + "type": "object", + "properties": { + "refresh_interval": { + "type": "integer", + "minimum": 1, + "description": "Cache refresh interval in seconds" + } + }, + "required": ["refresh_interval"] + } + }, + "required": ["app", "security", "authentication", "rate_limit", "whitelist", "file_upload", "cache"] +} diff --git a/src/frontend/apiserver/config/schema/logger_schema.json b/src/frontend/apiserver/config/schema/logger_schema.json new file mode 100644 index 00000000..b93c71a0 --- /dev/null +++ b/src/frontend/apiserver/config/schema/logger_schema.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Logger Configuration Schema", + "type": "object", + "properties": { + "logging": { + "type": "object", + "properties": { + "level": { + "type": "string", + "enum": ["debug", "info", "warn", "error", "fatal"] + }, + "encoding": { + "type": "string", + "enum": ["json", "console"] + }, + "outputPath": { + "type": "string" + }, + "file": { + "type": "object", + "properties": { + "maxSize": { + "type": "number", + "description": "megabytes before rotation" + }, + "maxBackups": { + "type": "number", + "description": "number of old files to keep" + }, + "maxAge": { + "type": "number", + "description": "days to keep old files" + }, + "compress": { + "type": "boolean", + "description": "compress rotated files" + } + }, + "required": ["maxSize", "maxBackups", "maxAge", "compress"] + }, + "encoderConfig": { + "type": "object", + "properties": { + "messageKey": { + "type": "string" + }, + "levelKey": { + "type": "string" + }, + "timeKey": { + "type": "string" + }, + "nameKey": { + "type": "string" + }, + "callerKey": { + "type": "string" + }, + "functionKey": { + "type": "string" + }, + "stacktraceKey": { + "type": "string" + }, + "timeEncoder": { + "type": "string" + }, + "levelEncoder": { + "type": "string" + } + }, + "required": [ + "messageKey", + "levelKey", + "timeKey", + "nameKey", + "callerKey", + "functionKey", + "stacktraceKey" + ] + } + }, + "required": ["level", "encoding", "outputPath", "file", "encoderConfig"] + } + }, + "required": ["logging"] +} diff --git a/src/frontend/apiserver/config/ssl/openssl.cnf b/src/frontend/apiserver/config/ssl/openssl.cnf new file mode 100644 index 00000000..f385e741 --- /dev/null +++ b/src/frontend/apiserver/config/ssl/openssl.cnf @@ -0,0 +1,19 @@ +[req] +distinguished_name = req_distinguished_name +x509_extensions = v3_req +prompt = no + +[req_distinguished_name] +CN = *.akrostar-tech.com +C = CN + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = *.akrostar-tech.com +DNS.2 = localhost +IP.1 = 127.0.0.1 +IP.2 = diff --git a/src/frontend/apiserver/config/ssl/server.crt b/src/frontend/apiserver/config/ssl/server.crt new file mode 100644 index 00000000..dd7daba6 --- /dev/null +++ b/src/frontend/apiserver/config/ssl/server.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDYzCCAkugAwIBAgIUX9aDwrjm5WhjOTsU6I55xoLBjZUwDQYJKoZIhvcNAQEL +BQAwMDEhMB8GA1UEAwwYV2VpLUdlLmFrcm9zdGFyLXRlY2guY29tMQswCQYDVQQG +EwJDTjAeFw0yNTAxMDgwOTQ2MzhaFw0yNjAxMDgwOTQ2MzhaMDAxITAfBgNVBAMM +GFdlaS1HZS5ha3Jvc3Rhci10ZWNoLmNvbTELMAkGA1UEBhMCQ04wggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDxYvK9lu7x/OMHMFdYRUtWb7KWxiPTXHgq +y3AG3w2iymSxaQ9SIyxEH+GCyPAO4AP9WVjWX8+ZFHuG5klOjoID6hz1NFQoA60k +XEXhrlPoN8XdFE+aG4+28exOnKK2ZsdNqsYFVUWcJRUbHDwEjeuffEi0y5tgxzDX +WihN1XDSAPEltXstNHehzxmfCt0QyAYnHbVb1N8ec6/qFqzSf2NT6HEZwyC2+PZV +qI6ri0y6qnq0i5kyxLR19VgZKarEsD1FxSgZ7wndIErJ6RAx1mUW++OGh3BNOex8 +T6UXW3Jzgax8h8l/H0ioPRK4VGP4M6j0+6St5z15DcxhSouEOTu7AgMBAAGjdTBz +MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMDoGA1UdEQQzMDGCGFdlaS1HZS5ha3Jv +c3Rhci10ZWNoLmNvbYIJbG9jYWxob3N0hwSsHWQQhwR/AAABMB0GA1UdDgQWBBQE +5YjdDgu+c3ZCGUkASROY+dQwVDANBgkqhkiG9w0BAQsFAAOCAQEA6BGyDOSSFnR9 +j94zvGxL7JfplpzafqK3Aj1hdJ8bUlW3F8bPJOR/BWKkNxUwRwn/9gUBxyXLtb98 +ZzimQCvDX9NdirGpiUiTDykHB3YCmuq3zM+6GlFyMD/fADMmF8hYWjHmub8Hf7TL +0Bo0Ij/RipBckI2acV8eY1lVGyFGzpAce/PY21fLfgFyeTLmJjJv+ez3u7/auHFp +y04fXoXzN4nsfIeVWFhcKXCHfqg0+CIiuCUFV2MfV+BXV4jzmugKtjEqwrLale62 ++l7+o97FcjmCi1FvYgCnu/+8hPSf5lTpmx0MUr3Y5+CnLQNMhkjaVNRf75pm0ZsN +vyXwwDk83Q== +-----END CERTIFICATE----- diff --git a/src/frontend/apiserver/config/ssl/server.key b/src/frontend/apiserver/config/ssl/server.key new file mode 100644 index 00000000..b3e6ced4 --- /dev/null +++ b/src/frontend/apiserver/config/ssl/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDxYvK9lu7x/OMH +MFdYRUtWb7KWxiPTXHgqy3AG3w2iymSxaQ9SIyxEH+GCyPAO4AP9WVjWX8+ZFHuG +5klOjoID6hz1NFQoA60kXEXhrlPoN8XdFE+aG4+28exOnKK2ZsdNqsYFVUWcJRUb +HDwEjeuffEi0y5tgxzDXWihN1XDSAPEltXstNHehzxmfCt0QyAYnHbVb1N8ec6/q +FqzSf2NT6HEZwyC2+PZVqI6ri0y6qnq0i5kyxLR19VgZKarEsD1FxSgZ7wndIErJ +6RAx1mUW++OGh3BNOex8T6UXW3Jzgax8h8l/H0ioPRK4VGP4M6j0+6St5z15Dcxh +SouEOTu7AgMBAAECggEAD3GWOD0M4pycZdxY6kPcsxYIZg8dQaub4JkNfNMOi0jJ +snQXT9kCPwWPJw/2E+2PM8lgnq6YmKGv2YyoSgdiGCAFZEbJz5mUnUvuCKdfpO7U +kPV4tlJnGnxV8x2gZ20h+T0g2DyBbKEOvaJL0LrHxXdtK2wPb9yuqkiRZZFvW3Am +LhfH0U/zQ4Gefbu+dFK1CSUQy+ct8YtsrarN6bJ+apKYTGe0ZTJeS/ZoqJUmilcS +FNGcR32O1zrOQH594NxEcAwCDJomU3ZfFCSYi4zrWIZstlwTTKC5IWeCxRogDJxn +tdn8axyldXFHh6kWNOUn2a0GbsQueDM8hEejcL2EKQKBgQD67ICNzzaQ2ioWDkHQ +VvSNomtRqgMwxv9YBOkLn0kcyeU4caQ6g4Re+3ugrvlJCIZRTBvG85m7Xa+GTN25 +aSPEb5u4zunnHW4a7Hla8s4n5+QVrr/5GP4IhimDJ0ooHxE6lZ9BX7wIQ3sf6GNx +xwAnoQncZ9BsI2+fy68GLt7n1wKBgQD2RQ28ZWS+FwWprB+NTUMOSw4367Mpz1ei +YCxARskak3msmTkqKQeUVyQJ8y4AEE98BV1gfymcD6+JL7VcTY71AGJU0cEgNU4G +cFo8aUDXr1IYpbpMe/ifolfxLdxY4Hg9gHpfo0K8Y50Szu062QRTbOxXhP+aTSY4 +VhoKxk4+vQKBgA38gsnkw/DdCD2Ee+yl/Cms5Ph/VoAwh2kblPXd4hBlvI4h33qX +fBhjQifZuPF2NS5w5Jd5WpdUCIWe4TkFgEXhxrdNaUJFpK6Bgd4s7IrS5SbEP9Ud +rINDHys9VGfaG3a9gkv9jxiS0lkb9fcQlZSf6FHNw5NTglRQ4SeiEzCvAoGBAOUT +DJje89vCDHlsaxkD8kj8Veh8bXaJsKmQrfjIH06COZ7sE/l8mtlISXSb6IEUqurh +Dp+xCWJqUMzaPOf0GPMa/VyKlAtCqiZhIf6ppTA7m7Djj6yPcJAE8aqdDyb4/b+v +n6UQdwZH3pbdNlwT++siGaG7TAsakl+675RwK949AoGBAMn3fEf9TGo9CNbACOUb +eDSjEf6TFd8s5lZ8B5xCP3ZVQLLVUgf1lmvXGjpYwj7QoHrgzpSaoBQIeZO7Bb0J +1eAKN5p58EoDeWpf9P+T7/QZZ1Ao4dZEYkPe9joIY4HUpKllkfyaUHOaOx/wnjuf +PBUe0tjuiXAHR9PDktXk5Z9x +-----END PRIVATE KEY----- diff --git a/src/frontend/apiserver/controllers/apikey.go b/src/frontend/apiserver/controllers/apikey.go new file mode 100644 index 00000000..4467c1ea --- /dev/null +++ b/src/frontend/apiserver/controllers/apikey.go @@ -0,0 +1,61 @@ +package controllers + +import ( + "crypto/rand" + "encoding/hex" + "time" + + "apiserver/common" + "apiserver/models" + + "github.com/gin-gonic/gin" +) + +func generateAPIKey() (string, error) { + bytes := make([]byte, 32) + if _, err := rand.Read(bytes); err != nil { + return "", err + } + return hex.EncodeToString(bytes), nil +} + +type APIKeyController struct{} + +func (a *APIKeyController) GenerateAPIKey(ctx *gin.Context) { + // Check authorization + token := ctx.GetHeader("Authorization") + if token == "" { + ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE, "Unauthorized: No token provided", nil) + return + } + + // Parse request + var req models.APIKeyRequest + if err := ctx.ShouldBindJSON(&req); err != nil { + ReturnError(ctx, common.INVALID_PARAMS_CODE, "Invalid request format: "+err.Error(), nil) + return + } + + // Generate API key + apiKey, err := generateAPIKey() + if err != nil { + ReturnError(ctx, common.ERROR_CODE, "Failed to generate API key", nil) + return + } + + // Set default expiry if not specified + expiryDays := 30 + if req.ExpiryDays > 0 { + expiryDays = req.ExpiryDays + } + + // Create response + response := models.APIKeyResponse{ + APIKey: apiKey, + Username: req.Username, + CreateTime: time.Now(), + ExpiryTime: time.Now().AddDate(0, 0, expiryDays), + } + + ReturnSuccess(ctx, 0, "API key generated successfully", response) +} diff --git a/src/frontend/apiserver/controllers/auth.go b/src/frontend/apiserver/controllers/auth.go new file mode 100644 index 00000000..96000fe1 --- /dev/null +++ b/src/frontend/apiserver/controllers/auth.go @@ -0,0 +1,219 @@ +package controllers + +import ( + "apiserver/backend/auth" + "apiserver/common" + "apiserver/models" + "apiserver/store" + "apiserver/utils" + + logger "apiserver/utils" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v4" + "go.uber.org/zap" +) + +// Define a struct for handling authentication operations +type AuthController struct{} + +// generate JWT token +func generateJWT(username string, IP string) (string, error) { + //appConfig := utils.AppYamlConfig + // Create claims with user info + claims := models.JWTClaims{ + Username: username, + IP: IP, + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Duration(utils.AuthenticationYamlConfig.Token.TokenExpireTime) * time.Second)), + IssuedAt: jwt.NewNumericDate(time.Now()), + NotBefore: jwt.NewNumericDate(time.Now()), + }, + } + + // Create token with claims + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + // Generate signed token + secretKey := []byte(utils.AuthenticationYamlConfig.Token.SecretKey) + // In production, use environment variable + signedToken, err := token.SignedString(secretKey) + if err != nil { + return "", err + } + return signedToken, nil +} + +// applyQueueRBACFilter appends a queue_name:in:(...) filter exactly once +// based on the user's accessible queues. If the user already specified a +// queue_name filter, the RBAC filter is not added. +func applyQueueRBACFilter(filters []utils.FilterCondition, accessQueues []string) []utils.FilterCondition { + if len(accessQueues) == 0 { + return filters + } + for _, f := range filters { + if f.Field == "queue_name" { + return filters + } + } + return append(filters, utils.FilterCondition{ + Field: "queue_name", + Operator: "in", + Value: "(" + strings.Join(accessQueues, ",") + ")", + }) +} +func storeToken(token string, username string) error { + tokenDir := filepath.Join(utils.Global_TopDir, "tokens", username) + if err := os.MkdirAll(tokenDir, 0700); err != nil { + logger.Error("Failed to create token directory", zap.Error(err)) + return err + } + + tokenFile := filepath.Join(tokenDir, "token") + if err := os.WriteFile(tokenFile, []byte(token), 0600); err != nil { + logger.Error("Failed to write token file", zap.Error(err)) + return err + } + return nil +} + +// removeToken removes the JWT file for a user. +func removeToken(username string) error { + tokenDir := filepath.Join(utils.Global_TopDir, "tokens", username) + tokenFile := filepath.Join(tokenDir, "token") + if err := os.Remove(tokenFile); err != nil { + return err + } + // Remove user directory if empty + os.Remove(tokenDir) + return nil +} + +// User logon +func (a AuthController) Logon(ctx *gin.Context) { + var req models.LogonRequest + + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + if auth.GlobalProvider == nil { + logger.Error("No auth provider configured") + ReturnError(ctx, common.ERROR_CODE, "authentication provider not configured", nil) + return + } + + // Authenticate via provider + if err := auth.GlobalProvider.Authenticate(req.Username, req.Password); err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Auto-assign roles from provider + if roles := auth.GlobalProvider.GetUserRoles(req.Username); len(roles) > 0 { + for _, role := range roles { + store.AddRoleForUser(req.Username, role) + } + } + + // Generate JWT token + if utils.AuthenticationYamlConfig.Token.Type == "jwt" { + token, err := generateJWT(req.Username, ctx.ClientIP()) + if err != nil { + logger.Error("Failed to generate JWT token", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_JWT_GENERATION_MSG+err.Error(), nil) + return + } + + // Store token in server-managed directory + if err := storeToken(token, req.Username); err != nil { + logger.Error("Failed to store token", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, "Failed to store token", nil) + return + } + + tokenDir := filepath.Join(utils.Global_TopDir, "tokens", req.Username) + response := models.LogonResponse{ + Token: token, + Path: tokenDir, + IP: ctx.ClientIP(), + TokenExpireTime: utils.AuthenticationYamlConfig.Token.TokenExpireTime, + } + logger.Info("Post /logon successfully from ", zap.String("client_ip", ctx.ClientIP())) + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, response) + } +} + +// User logout +func (a AuthController) Logout(ctx *gin.Context) { + logger.Debug("Enter Logout() function") + // get token from header + token := ctx.GetHeader("Authorization") + + // get username from token + var err error + username, err := GetUsernameFromToken(strings.TrimPrefix(token, "Bearer ")) + if err != nil { + logger.Error(common.ERROR_GET_USERNAME_MSG, zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_GET_USERNAME_MSG+": "+err.Error(), nil) + return + } + logger.Debug("username", zap.String("username", username)) + + // Remove token file from server-managed directory + if err := removeToken(username); err != nil { + logger.Error(common.ERROR_REMOVE_TOKEN_MSG, zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_REMOVE_TOKEN_MSG+": "+err.Error(), nil) + return + } + // Mock successful logout + response := map[string]interface{}{ + "username": username, + "message": "Logout successful", + } + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, response) +} + +func GetUsernameFromToken(tokenString string) (string, error) { + token, err := jwt.Parse(strings.TrimPrefix(tokenString, "Bearer "), func(token *jwt.Token) (interface{}, error) { + // Validate the algorithm + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return []byte(utils.AuthenticationYamlConfig.Token.SecretKey), nil + }) + + if err != nil { + return "", err + } + + if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid { + if username, ok := claims["username"].(string); ok { + return username, nil + } + } + return "", fmt.Errorf("invalid token") +} + +// getAccessQueues returns the list of queues the user can access for the given permission. +// Returns an error if the user has no such permission. +func getAccessQueues(username, permID string) ([]string, error) { + perm, err := GetRequireResourcePermission(username, permID) + if err != nil { + return nil, err + } + switch perm.DomainType { + case models.DomainTypeAll: + return store.GetAllQueueNames(), nil + case models.DomainTypeQueue: + return perm.DomainValues, nil + default: + return nil, nil + } +} diff --git a/src/frontend/apiserver/controllers/clusters.go b/src/frontend/apiserver/controllers/clusters.go new file mode 100644 index 00000000..34d94af2 --- /dev/null +++ b/src/frontend/apiserver/controllers/clusters.go @@ -0,0 +1,219 @@ +package controllers + +import ( + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/utils" + "errors" + + logger "apiserver/utils" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type ClusterController struct{} + +func (c ClusterController) GetClustersInfo(ctx *gin.Context) { + + logger.Debug("Enter GetClusterInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + + var fields []string + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid cluster info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + filterStr := ctx.Query("filter") + fieldsStr := ctx.Query("fields") + sortStr := ctx.Query("sort") // sort=cluster_name:asc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + limitStr := ctx.Query("limit") // eg: limit=100 + + logger.Debug("Cluster query values are:", + zap.String("fieldsStr", fieldsStr), + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + // ---- check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + for _, filter := range filters { + // check the filter is supported. + if !utils.ContainsField(models.CLUSTER_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + var clusters models.ClusterInfo + filtercds, err := utils.ConvertToNewFilter(filters, &clusters) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check fields ---- + // get fields and do check, if not set, use default display columns + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.ClusterInfo{}) + logger.Debug("cluster supportedFields: %v", zap.Any("supportedFields", supportedFields)) + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + // if user not specify cluster_name in fields, add cluster_name as default field. + if field == "cluster_name" { + found = true + } + } + if !found { + logger.Debug("user doesn't specify cluster_name in query fields, add it.") + fields = append(fields, "cluster_name") + } + } else { + fields = models.CLUSTER_DEFAULT_COLUMN + } + + // ---- check sort ---- + // handle sort, when no sortStr specified, need add the primary key of cluster_info + // table into sortStr as default value + if sortStr == "" { + logger.Debug("sort is not specified, set sort=cluster_name:asc as default") + sortStr = "cluster_name:asc" + } + // parse sortStr + sorts, err := utils.ParseSortParam(sortStr, models.CLUSTER_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check paging --- + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + //the primary key of cluster_info table cluster_name should always be added into sortStr if user doesn't specify. + found := false + for _, sortent := range sorts { + if sortent.Column == "cluster_name" { + found = true + break + } + } + // user not specify the cluster_name, only append cluster_name at the end. + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "cluster_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + //fmt.Println("filtercds:", filtercds, "sortOrders:", sortOrders, "limit:", limit, "pagetokenStr:", pagetokenStr) + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_CLUSTERINFO_MSG+": "+err.Error(), nil) + return + } + + clqueryReply, pagetoken, err := services.QueryData(services.TableClusterInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query cluster info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_CLUSTERINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the cluster info query reply:", zap.Any("clqueryReply", clqueryReply), zap.String("pagetoken", pagetoken)) + //convert to readable string. + queryResult := processClusterData(clqueryReply, pagetoken) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) + +} + +func processClusterData(rows []utils.OrderMap, page_token string) utils.QueryResult { + for _, row := range rows { + if v, exist := row.Data["status"]; exist { + if v != nil { + statusVal, _ := utils.IntOrInt64(v) + status, err := utils.GetKeyFromValue(models.CLUSTERSTAT, statusVal) + if err != nil { + logger.Error("cluster status cannot recognized.", zap.Any("status", row.Data["status"])) + row.Data["status"] = "UNKNOWN" + } else { + row.Data["status"] = status + } + } + } + // Process JSONB fields based on predefined list + for _, jsonbField := range models.JSONBCLFields { + if v, exist := row.Data[jsonbField]; exist { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data[jsonbField] = jsonbData + } else { + logger.Error("Failed to parse cluster info JSONB field.", zap.Error(err)) + } + } + } + } + //utils.FormatRowsOutput(row.Data) + } + return utils.QueryResult{ + Rows: rows, + Count: len(rows), + Pagetoken: page_token, + } +} diff --git a/src/frontend/apiserver/controllers/files.go b/src/frontend/apiserver/controllers/files.go new file mode 100644 index 00000000..eafa9f91 --- /dev/null +++ b/src/frontend/apiserver/controllers/files.go @@ -0,0 +1,540 @@ +package controllers + +import ( + "apiserver/common" + "apiserver/utils" + logger "apiserver/utils" + "archive/zip" + "crypto/md5" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type FilesController struct{} + +func (fc *FilesController) UploadFiles(ctx *gin.Context) { + logger.Debug("Enter UploadFiles() function from ", zap.String("client_ip", ctx.ClientIP())) + + // use form-data format to upload files + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Get the home directory of the submitUser + userHomeDir := filepath.Join("/home", submitUser) + + // Check if the directory exists + if _, err := os.Stat(userHomeDir); os.IsNotExist(err) { + ReturnError(ctx, common.ERROR_CODE, fmt.Sprintf("User home directory does not exist: %s", userHomeDir), nil) + return + } + + // Check Content-Type if it contains "multipart/form-data" + contentType := ctx.ContentType() + if !strings.Contains(contentType, "form-data") { + ReturnError(ctx, common.ERROR_CODE, "Invalid Content-Type, expected multipart/form-data", nil) + return + } + + // get all uploaded files + form, err := ctx.MultipartForm() + if err != nil { + ReturnError(ctx, common.ERROR_CODE, "Failed to parse multipart form", nil) + return + } + + files := form.File["files"] + if len(files) == 0 { + ReturnError(ctx, common.ERROR_CODE, "No files uploaded", nil) + return + } + + // get configuration of file upload + fileUploadConfig := utils.FileUploadYamlConfig + if fileUploadConfig == nil { + ReturnError(ctx, common.ERROR_CODE, "File upload configuration is missing", nil) + return + } + + webUrl := utils.GetServerHostPort(ctx) + + // Initialize structures to hold results + + successFiles := []map[string]interface{}{} + failedFiles := []map[string]interface{}{} + + for _, file := range files { + // check disallowed file types + if err := validateFileType(file, fileUploadConfig); err != nil { + logger.Error("File validation failed", zap.String("filename", file.Filename), zap.Error(err)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "File validation failed", + }) + continue + } + + // check uploaded file size + maxSizeMB, err := utils.ParseMaxSize(fileUploadConfig.MaxSize) + if err != nil { + logger.Error("Invalid MaxSize configuration", zap.String("max_size", fileUploadConfig.MaxSize), zap.Error(err)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "Invalid MaxSize configuration", + }) + continue + } + fileSizeMB := file.Size / (1024 * 1024) + if fileSizeMB > maxSizeMB { + logger.Error("File size exceeds limit", zap.String("filename", file.Filename), zap.Int64("size", file.Size), zap.Int64("max_size", maxSizeMB)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "File size exceeds limit", + }) + continue + } + + // Open the uploaded file + srcFile, err := file.Open() + if err != nil { + logger.Error("Failed to open uploaded file", zap.String("filename", file.Filename), zap.Error(err)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "Failed to open file", + }) + continue + } + defer srcFile.Close() + + // Generate MD5 hash for the file content + hasher := md5.New() + if _, err := io.Copy(hasher, srcFile); err != nil { + logger.Error("Failed to hash file content", zap.String("filename", file.Filename), zap.Error(err)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "Failed to hash file content", + }) + continue + } + contentHash := hasher.Sum(nil) + + // Generate UUID based on the MD5 hash + contentUUID := uuid.NewMD5(uuid.NameSpaceOID, contentHash[:]).String() + + // Add submission date (YYYYMMDD) as a prefix to the UUID + submissionDate := time.Now().Format("20060102") // Using Go's standard library to get the current date + contentUUID = submissionDate + "_" + contentUUID + // Create a directory based on the UUID + uuidDir := filepath.Join(userHomeDir, "./apiserver_uploads", submissionDate) + if err := os.MkdirAll(uuidDir, os.ModePerm); err != nil { + logger.Error("Failed to create UUID directory", zap.String("uuid_dir", uuidDir), zap.Error(err)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "Failed to create directory", + }) + continue + } + + // Save file to the UUID directory + savePath := filepath.Join(uuidDir, contentUUID) + + // Check if the file already exists + if _, err := os.Stat(savePath); err == nil { + logger.Warn("File already exists, skipping upload", zap.String("filename", contentUUID)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "File already exists", + }) + continue + } + + if err := ctx.SaveUploadedFile(file, savePath); err != nil { + logger.Error("Failed to save file", zap.String("filename", contentUUID), zap.Error(err)) + failedFiles = append(failedFiles, map[string]interface{}{ + "file_name": file.Filename, + "msg": "Failed to save file", + }) + continue + } + + // Append source, destination, web URL, and encoded path to the result + successFiles = append(successFiles, map[string]interface{}{ + "src_file": file.Filename, + "dest_file": savePath, + "web_url": webUrl + "/" + contentUUID + "/" + file.Filename, + }) + } + + response, successMsg := BuildBatchOperationResponse("Upload", "File", successFiles, failedFiles) + overallStatus := response["status"].(string) + logger.Info("Finished upload files", + zap.String("overallStatus", overallStatus), + zap.String("client_ip", ctx.ClientIP())) + ReturnSuccess(ctx, common.ACCEPTED, successMsg, response) +} + +func validateFileType(file *multipart.FileHeader, config *utils.FileUploadConfig) error { + // check content type + fileType := file.Header.Get("Content-Type") + for _, disallowedType := range config.DisallowedTypes { + if fileType == disallowedType { + return fmt.Errorf("File type is not allowed: %s (type: %s)", file.Filename, fileType) + } + } + + // check file extension + fileExt := strings.ToLower(filepath.Ext(file.Filename)) + if fileExt == ".exe" { + return fmt.Errorf("File extension is not allowed: %s (extension: %s)", file.Filename, fileExt) + } + return nil +} + +func (fc *FilesController) DeleteFiles(ctx *gin.Context) { + logger.Debug("Enter DeleteFiles() function from ", zap.String("client_ip", ctx.ClientIP())) + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Get the home directory of the submitUser + userHomeDir := filepath.Join("/home", submitUser) + + // Check if the directory exists + if _, err := os.Stat(userHomeDir); os.IsNotExist(err) { + ReturnError(ctx, common.ERROR_CODE, fmt.Sprintf("User home directory does not exist: %s", userHomeDir), nil) + return + } + + filePathStr := ctx.Query("path") + if filePathStr == "" { + ReturnError(ctx, common.INVALID_PARAMS_CODE, "File path is required", nil) + return + } + fileList := strings.Split(filePathStr, ",") + + var successfulDeletes []string + var failedDeletes []string + + webUrl := utils.GetServerHostPort(ctx) + + for _, path := range fileList { + trimmedPath := strings.TrimSpace(path) + + // Check if the path start with web_url + if strings.HasPrefix(trimmedPath, webUrl) { + trimmedPath = strings.TrimPrefix(trimmedPath, webUrl+"/") + } + + // Check if the path is absolute, if not, prepend userHomeDir + if !filepath.IsAbs(trimmedPath) { + trimmedPath = filepath.Join(userHomeDir, "./apiserver_uploads", trimmedPath) + } + + // Check if the path exists + fileInfo, err := os.Stat(trimmedPath) + if os.IsNotExist(err) { + logger.Error("The file doesn't exist", zap.Error(err)) + failedDeletes = append(failedDeletes, trimmedPath) + continue + } + + // Delete file or directory + if fileInfo.IsDir() { + // Delete directory and its contents + if err := os.RemoveAll(trimmedPath); err != nil { + logger.Error("Failed to delete directory", zap.String("path", trimmedPath), zap.Error(err)) + failedDeletes = append(failedDeletes, trimmedPath) + continue + } + successfulDeletes = append(successfulDeletes, trimmedPath) + } else { + // Delete file + if err := os.Remove(trimmedPath); err != nil { + logger.Error("Failed to delete file", zap.String("path", trimmedPath), zap.Error(err)) + failedDeletes = append(failedDeletes, trimmedPath) + continue + } + successfulDeletes = append(successfulDeletes, trimmedPath) + } + } + + // Prepare response + var response gin.H + if len(failedDeletes) > 0 { + if len(successfulDeletes) > 0 { + response = gin.H{ + "successful_deletes": successfulDeletes, + "failed_deletes": failedDeletes, + } + } else { + response = gin.H{ + "failed_deletes": failedDeletes, + } + } + ReturnError(ctx, common.StatusPartialContent, common.PARTIAL_DELETE_FILES_MSG, response) + } else { + response = gin.H{ + "successful_deletes": successfulDeletes, + } + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, response) + } +} + +func (fc *FilesController) GetFileLists(ctx *gin.Context) { + logger.Debug("Enter GetFileLists() function from ", zap.String("client_ip", ctx.ClientIP())) + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Get the home directory of the submitUser + userHomeDir := filepath.Join("/home", submitUser) + + // Check if the directory exists + if _, err := os.Stat(userHomeDir); os.IsNotExist(err) { + ReturnError(ctx, common.ERROR_CODE, fmt.Sprintf("User home directory does not exist: %s", userHomeDir), nil) + return + } + + _, err = GetRequireResourcePermission(submitUser, "file:view") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + filePathStr := ctx.Query("path") + if filePathStr == "" { + ReturnError(ctx, common.INVALID_PARAMS_CODE, "File path is required", nil) + return + } + + webUrl := utils.GetServerHostPort(ctx) + if strings.HasPrefix(filePathStr, webUrl) { + filePathStr = strings.TrimPrefix(filePathStr, webUrl+"/") + } + + // if only filename, it will join the real path + if !strings.HasPrefix(filePathStr, "/") && !strings.Contains(filePathStr, string(os.PathSeparator)) { + filePathStr = filepath.Join(userHomeDir, "./apiserver_uploads", filePathStr) + } + + // Check if the path belongs to the current submitUser + if !strings.HasPrefix(filePathStr, userHomeDir) { + logger.Error("Access denied: path does not belong to the user", zap.String("path", filePathStr), zap.String("user_home", userHomeDir)) + ReturnError(ctx, common.ERROR_CODE, "Access denied: path does not belong to the user", nil) + return + } + + // Check if the path is a file or directory + fileInfo, err := os.Stat(filePathStr) + if err != nil { + logger.Error("Failed to get file or directory info", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, "Invalid file or directory path", nil) + return + } + + var fileInfos []gin.H + if fileInfo.IsDir() { + // Read the directory + files, err := os.ReadDir(filePathStr) + if err != nil { + logger.Error("Can't read the directory", zap.Error(err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": "Can't read the directory"}) + return + } + + // Collect file information + for _, file := range files { + info, err := file.Info() + if err != nil { + logger.Error("Failed to get file info", zap.String("file", file.Name()), zap.Error(err)) + continue + } + + // Ensure the name is correct + name := file.Name() + + fileInfos = append(fileInfos, gin.H{ + "name": name, + "size": info.Size(), + "is_dir": file.IsDir(), + "mod_time": info.ModTime().Format("2006-01-02 15:04:05"), + "perm": fmt.Sprintf("%#o", info.Mode().Perm()), + }) + } + } else { + fileInfos = append(fileInfos, gin.H{ + "name": filepath.Base(filePathStr), // Ensure the name is correct + "size": fileInfo.Size(), + "is_dir": fileInfo.IsDir(), + "mod_time": fileInfo.ModTime().Format("2006-01-02 15:04:05"), + "perm": fmt.Sprintf("%#o", fileInfo.Mode().Perm()), + }) + } + + // Return file list as JSON + ctx.JSON(http.StatusOK, gin.H{ + "files": fileInfos, + }) +} + +func (fc *FilesController) DownloadFile(ctx *gin.Context) { + logger.Debug("Enter DownloadFile() function from ", zap.String("client_ip", ctx.ClientIP())) + + // Validate token and get submitUser + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Get the home directory of the submitUser + userHomeDir := filepath.Join("/home", submitUser) + if _, err := os.Stat(userHomeDir); os.IsNotExist(err) { + ReturnError(ctx, common.ERROR_CODE, fmt.Sprintf("User home directory does not exist: %s", userHomeDir), nil) + return + } + + // Get UUID from URL parameter + uuid := ctx.Param("uuid") + if uuid == "" { + ReturnError(ctx, common.INVALID_PARAMS_CODE, "UUID is required", nil) + return + } + // Get filename from URL parameter + filename := ctx.Param("filename") + if filename == "" { + ReturnError(ctx, common.INVALID_PARAMS_CODE, "Filename is required", nil) + return + } + + // Extract the date part from the UUID (assuming format YYYYMMDD_UUID) + parts := strings.SplitN(uuid, "_", 2) + if len(parts) != 2 { + ReturnError(ctx, common.INVALID_PARAMS_CODE, "Invalid UUID format", nil) + return + } + datePart := parts[0] + + // Construct the path based on the extracted date and UUID + uuidPath := filepath.Join(userHomeDir, "./apiserver_uploads", datePart, uuid) + + // Check if the path exists + fileInfo, err := os.Stat(uuidPath) + if os.IsNotExist(err) { + ReturnError(ctx, common.ERROR_NOT_FOUND_CODE, fmt.Sprintf("Path not found: %s", uuidPath), nil) + return + } + + if fileInfo.IsDir() { + // If it's a directory, check its contents + files, err := os.ReadDir(uuidPath) + if err != nil { + logger.Error("Failed to read directory", zap.String("path", uuidPath), zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, "Failed to read directory", nil) + return + } + + if len(files) == 1 && !files[0].IsDir() { + // If the directory contains a single file, download the file directly + filePath := filepath.Join(uuidPath, files[0].Name()) + ctx.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", strconv.Quote(filename))) + ctx.Header("Content-Type", "application/octet-stream") + http.ServeFile(ctx.Writer, ctx.Request, filePath) + + // Log the download event + logger.Info("File downloaded successfully", zap.String("filename", files[0].Name()), zap.String("user", submitUser)) + } else { + // If the directory contains multiple files or subdirectories, compress and download + zipFileName := strconv.Quote(filename) + ".zip" + ctx.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", zipFileName)) + ctx.Header("Content-Type", "application/zip") + + zipWriter := zip.NewWriter(ctx.Writer) + defer zipWriter.Close() + + err := filepath.Walk(uuidPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Create a header for the file or directory + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + header.Name, _ = filepath.Rel(uuidPath, path) // Relative path inside the ZIP + if info.IsDir() { + header.Name += "/" + } else { + header.Method = zip.Deflate + } + + // Write the header to the ZIP + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + + // If it's a file, write its content + if !info.IsDir() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(writer, file) + if err != nil { + return err + } + } + return nil + }) + + if err != nil { + logger.Error("Failed to create ZIP file", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, "Failed to create ZIP file", nil) + return + } + + // Log the download event + logger.Info("Directory downloaded successfully as ZIP", zap.String("directory", uuidPath), zap.String("user", submitUser)) + } + } else { + // If it's a single file, download it directly + ctx.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", strconv.Quote(filename))) + ctx.Header("Content-Type", "application/octet-stream") + http.ServeFile(ctx.Writer, ctx.Request, uuidPath) + + // Log the download event + logger.Info("File downloaded successfully", zap.String("filename", filename), zap.String("user", submitUser)) + } + +} // End of DownloadFile function diff --git a/src/frontend/apiserver/controllers/hosts.go b/src/frontend/apiserver/controllers/hosts.go new file mode 100644 index 00000000..38d4f159 --- /dev/null +++ b/src/frontend/apiserver/controllers/hosts.go @@ -0,0 +1,605 @@ +package controllers + +import ( + "apiserver/backend" + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/store" + "apiserver/utils" + logger "apiserver/utils" + "errors" + "reflect" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type HostHandler struct { +} + +var allFlag bool + +// GetHosts godoc +// @Summary Get host information +// @Description Get host information +// @Tags hosts +// @Accept json +// @Produce json +// @Success 200 +// @Failure 400 {object} models.Error +// @Router /api/v1/hosts [get] +func (h *HostHandler) GetHostsInfo(ctx *gin.Context) { + logger.Debug("Enter GetAllHosts() function from ", zap.String("client_ip", ctx.ClientIP())) + + var fields []string + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid host info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + filterStr := ctx.Query("filter") + fieldsStr := ctx.Query("fields") + sortStr := ctx.Query("sort") // sort=host_name:asc,submit_time:desc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + limitStr := ctx.Query("limit") // eg: limit=100 + + logger.Debug("Host query values are:", + zap.String("fieldsStr", fieldsStr), + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + // ---- check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + //filtercds := make([]utils.Condition, len(filters)) + for _, filter := range filters { + // check the filter is supported. + if !utils.ContainsField(models.HOST_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + var hosts models.HostInfo + filtercds, err := utils.ConvertToNewFilter(filters, &hosts) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check fields ---- + // get fields and do check, if not set, use default display columns + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.HostInfo{}) + logger.Debug("host supportedFields: %v", zap.Any("supportedFields", supportedFields)) + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + // if user not specify host-name in fields, add host_name as default field. + if field == "host_name" { + found = true + } + } + if !found { + logger.Debug("user doesn't specify hostname in query fields, add it.") + fields = append(fields, "host_name") + } + } else { + fields = models.HOST_DEFAULT_COLUMN + } + + // ---- check sort ---- + // handle sort, when no sortStr specified, need add the primary key of host_info + // table into sortStr as default value + if sortStr == "" { + logger.Debug("sort is not specified, set sort=host_name:asc as default") + sortStr = "host_name:asc" + } + // parse sortStr + sorts, err := utils.ParseSortParam(sortStr, models.HOST_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check paging --- + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + //the primary key of host_info table host_id should always be added into sortStr if user doesn't specify. + found := false + for _, sortent := range sorts { + if sortent.Column == "host_name" { + found = true + break + } + } + // user not specify the host_name, only append host_name at the end. + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "host_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + //fmt.Println("filtercds:", filtercds, "sortOrders:", sortOrders, "limit:", limit, "pagetokenStr:", pagetokenStr) + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_HOSTINFO_MSG+": "+err.Error(), nil) + return + } + + hqueryReply, pagetoken, err := services.QueryData(services.TableHostInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query host info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_HOSTINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the host info query reply:", zap.Any("hqueryReply", hqueryReply), zap.String("pagetoken", pagetoken)) + //convert to readable string. + queryResult := processHostData(hqueryReply, pagetoken) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func selectFields(host interface{}, fields []string) map[string]interface{} { + result := make(map[string]interface{}) + v := reflect.ValueOf(host) + t := v.Type() + + // Create a map of lowercase field names to actual field names + fieldMap := make(map[string]string) + + // Handle embedded BasicHost fields + if t.Kind() == reflect.Struct { + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + + // If it's an embedded struct (BasicHost) + if field.Anonymous { + embedValue := v.Field(i) + embedType := embedValue.Type() + for j := 0; j < embedType.NumField(); j++ { + embedField := embedType.Field(j) + fieldMap[strings.ToLower(embedField.Name)] = embedField.Name + } + } + + // Add the field itself + fieldMap[strings.ToLower(field.Name)] = field.Name + } + } + + for _, field := range fields { + lowerField := strings.ToLower(field) + if actualField, ok := fieldMap[lowerField]; ok { + var value interface{} + var jsonTag string + + // If not found in BasicHost, try the main struct + + if v.FieldByName(actualField).IsValid() { + value = v.FieldByName(actualField).Interface() + f, _ := t.FieldByName(actualField) + jsonTag = f.Tag.Get("json") + } + + if value != nil { + result[jsonTag] = value + } + } + } + return result +} + +func (h HostHandler) GetHostByHostname(ctx *gin.Context) { + logger.Debug("Enter GetHostByHostname() function from ", zap.String("client_ip", ctx.ClientIP())) + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid specific host info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + hostNamesStr := ctx.Param("hostname") + fieldsStr := ctx.Query("fields") + + logger.Debug("origin query string hostname, fieldsStr values are:", + zap.String("hostname", hostNamesStr), + zap.String("fields", fieldsStr)) + + //add host_name list as filter. + hostNames := strings.Split(hostNamesStr, ",") + mergeHqueryReply := make([]utils.OrderMap, 0) + var page_token = "" + for _, hostname := range hostNames { + hostname = strings.TrimSpace(hostname) + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + filters = append(filters, utils.FilterCondition{ + Field: "host_name", + Operator: "eq", + Value: hostname, + }) + + //filtercds := make([]utils.Condition, len(filters)) + for _, filter := range filters { + // check the filter is supported. + if !utils.ContainsField(models.HOST_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + var host models.HostInfo + var err error + filtercds, err := utils.ConvertToNewFilter(filters, &host) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + var fields []string + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.HostInfo{}) + logger.Debug("host supportedFields: %v", zap.Any("supportedFields", supportedFields)) + + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // if user not specify host_name in fields, add host_name as default field. + if field == "host_name" { + found = true + } + } + + if !found { + logger.Debug("user doesn't specify host_name in query fields, add it.") + fields = append(fields, "host_name") + } + } else { // default display columns + fields = models.HOST_DEFAULT_COLUMN + } + + // add a default order: sort=host_name:asc + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "host_name", + Ascending: true, + }) + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, utils.PER_PAGE_LIMIT, "") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + + hqueryReply, page_token, err := services.QueryData(services.TableHostInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query the specific host info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_HOSTINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("specific host query reply:", zap.Any("hqueryReply", hqueryReply), zap.String("pagetoken", page_token)) + mergeHqueryReply = append(mergeHqueryReply, hqueryReply...) + } // end for hostname + logger.Debug("the specific host query reply:", zap.Any("hqueryReply", mergeHqueryReply)) + + //convert host_status to readable string. + queryResult := processHostData(mergeHqueryReply, page_token) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func preCheckHostRequest(req *models.ResReqSelectRequest) error { + if strings.TrimSpace(req.Select) != "" { + if len(req.Select) > 4094 { + return errors.New("Select string exceeds the limit of 4094 characters") + } + } else { + return errors.New("Select string cannot be empty") + } + + return nil +} + +func (h *HostHandler) FilterHostByResReqSelect(ctx *gin.Context) { + logger.Debug("Enter FilterHostByResReqSelect() function") + // Resource requirement select filtering is not yet implemented without backend. + queryResult := processHostData([]utils.OrderMap{}, "") + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +// ControlHost +func (h *HostHandler) ControlHost(ctx *gin.Context) { + logger.Debug("Enter ControlHost() function") + var req models.HostControlRequest + + // use strict json format check + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + err := preCheckHostControlRequest(&req) + if err != nil { + logger.Error("Failed to send host control request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + baseAction := req.ControlAction + logger.Debug("ControlHost baseAction is ", zap.Any("baseAction", baseAction)) + + // get the request submit user + token := ctx.GetHeader("Authorization") + _, err = GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + var actionType int + if baseAction == "close" { + actionType = models.HOST_CLOSE + } + if baseAction == "open" { + actionType = models.HOST_OPEN + } + + successHosts := []map[string]interface{}{} + failedHosts := []map[string]interface{}{} + + validatedSuccessHosts, validatedFailedHosts := validateHost(ctx, actionType, req.Hostname) + if ctx.IsAborted() { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + if allFlag { + hostnames := store.GetAllHostNames() + validatedSuccessHosts, validatedFailedHosts = validateHost(ctx, actionType, hostnames) + } + + for _, hostname := range validatedSuccessHosts { + successHosts = append(successHosts, map[string]interface{}{ + "host_name": hostname, + "msg": "host is being controlled", + }) + } + for hostname, reason := range validatedFailedHosts { + failedHosts = append(failedHosts, map[string]interface{}{ + "host_name": hostname, + "msg": reason, + }) + } + + for _, host := range successHosts { + hostname := host["host_name"].(string) + err = backend.ControlHost(hostname, actionType) + if err != nil { + logger.Error("Failed to send host control", zap.Error(err)) + failedHosts = append(failedHosts, map[string]interface{}{ + "host_name": hostname, + "msg": "failed to control host: " + err.Error(), + }) + } + } + response, successMsg := BuildBatchOperationResponse(baseAction, "Host", successHosts, failedHosts) + + overallStatus := response["status"].(string) + logger.Info("Finished processing host control request", + zap.String("action", baseAction), + zap.String("overallStatus", overallStatus), + zap.Int("succeededCount", len(successHosts)), + zap.Int("failedCount", len(failedHosts)), + zap.String("client_ip", ctx.ClientIP())) + + ReturnSuccess(ctx, common.ACCEPTED, successMsg, response) + return +} + +// validateHost validates the hostname in the request +func validateHost(ctx *gin.Context, control int, hostnames []string) (successHosts []string, failedHosts map[string]string) { + logger.Debug("Validating hostnames: %v", zap.Any("hostnames", hostnames)) + + // get logon user + token := ctx.GetHeader("Authorization") + logonUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + successHosts = []string{} + failedHosts = make(map[string]string) + + if len(hostnames) == 0 { + logger.Error("Hostnames cannot be empty") + return + } + + for _, hostname := range hostnames { + if hostname == "" { + logger.Error("Hostname cannot be empty") + continue + } + + hostPermission, err := GetRequireResourcePermission(logonUser, "host:control") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if hostPermission.DomainType == models.DomainTypeHostGroup { + // TODO: check hostgroup values + } + + if store.Hosts.Has(hostname) { + logger.Debug("Found hostname in store", zap.String("hostname", hostname)) + successHosts = append(successHosts, hostname) + /* no need to check host_status + if rawStatus, exists := hostInfo[0].Data["host_status"]; exists { + if hostStatus, ok := rawStatus.(int64); ok { + db_host_status := int(hostStatus) + if isHostControlActionAllowed(control, db_host_status) { + successHosts = append(successHosts, hostname) + } else { + logger.Error("action not allowed for host", zap.String("hostname", hostname)) + failedHosts[hostname] = "action not allowed for current host_status" + } + } else { + logger.Error("host_status is not an int64", zap.Any("host_status", rawStatus)) + failedHosts[hostname] = "invalid host_status type for host" + } + } else { + logger.Error("host_status not found in hostInfo", zap.String("hostname", hostname)) + failedHosts[hostname] = "host_status not found for host" + } */ + } else { + logger.Error("invalid hostname", zap.String("hostname", hostname)) + failedHosts[hostname] = common.ERROR_HOST_NOT_FOUND_MSG + } + } + return +} + +func isHostControlActionAllowed(actionType int, status int) bool { + switch actionType { + case 1: // open strictly + if status&models.HOSTSTAT["CLOSED"] != 0 { + return true + } + case 2: // close + if (status & models.HOSTSTAT["CLOSED"]) != 0 { + return false + } else { + return true + } + default: + return false + } + return false +} +func preCheckHostControlRequest(req *models.HostControlRequest) error { + logger.Debug("Enter preCheckHostControlRequest") + + if err := utils.ValidateControlAction(req.ControlAction, models.SupportedHostActions); err != nil { + return err + } + req.ControlAction = strings.ToLower(strings.TrimSpace(req.ControlAction)) + + if req.Hostname != nil && len(req.Hostname) == 1 && req.Hostname[0] == "all" { + allFlag = true + } else { + allFlag = false + } + + if req.Comment != "" { + if len(req.Comment) > 512 { + return errors.New("comment length exceeds the limit of 512 characters") + } + } + return nil +} + +func processHostData(rows []utils.OrderMap, page_token string) utils.QueryResult { + for _, row := range rows { + if v, exist := row.Data["host_status"]; exist { + if v != nil { + hostStatus, _ := utils.IntOrInt64(v) + row.Data["host_status"] = models.HostStateStr(hostStatus) + } + } + + // Process JSONB fields based on predefined list + for _, jsonbField := range models.JSONBHFields { + if v, exist := row.Data[jsonbField]; exist { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data[jsonbField] = jsonbData + } else { + logger.Error("Failed to parse host info JSONB field.", zap.Error(err)) + } + } + } + } + + //utils.FormatRowsOutput(row.Data) + } + return utils.QueryResult{ + Rows: rows, + Count: len(rows), + Pagetoken: page_token, + } +} diff --git a/src/frontend/apiserver/controllers/jobs.go b/src/frontend/apiserver/controllers/jobs.go new file mode 100644 index 00000000..4de5a37b --- /dev/null +++ b/src/frontend/apiserver/controllers/jobs.go @@ -0,0 +1,3585 @@ +package controllers + +import ( + "apiserver/backend" + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/store" + "apiserver/utils" + logger "apiserver/utils" + "crypto/md5" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "go.uber.org/zap" +) + +const ( + //TODO: Need assign clusterID globally. + clusterID = 1 +) + +var ( + SignalMap = map[string]string{ + "kill": "KILL", + "resume": "CONT", + "stop": "STOP", + } + jobID_Worker = utils.NewWorker(clusterID) + realCopyLocalFiles []string + fakeCopyRemoteFile []string + indices []int +) + +type JobController struct{} + +// GetJobsInfo godoc +// @Summary Get jobs information +// @Description Get jobs information +// @Tags jobs +// @Accept json +// @Produce json +// @Param filter query string "filter=[{user:eq:user1},{queue:eq:queue1}]" +// @Param fields query string "fields=jobid,status,queue" +// @Success 200 {array} models.JobInfo "Response for jobs info" +// @Failure 400 {object} models.Error +// @Router /api/v1/jobs [get] +func (j JobController) GetJobsInfo(ctx *gin.Context) { + logger.Debug("Enter GetJobsInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + var fields []string + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + if ctx.IsAborted() { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + logger.Debug("GetJobsInfo param is ", zap.Any("param", param)) + if !utils.ContainsField(utils.SupportedQueryParams, param) && !utils.ContainsField(utils.TypeQueryParams, param) { + err := errors.New("invalid job info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + filterStr := ctx.Query("filter") + fieldsStr := ctx.Query("fields") + sortStr := ctx.Query("sort") // sort=job_id:asc,submit_time:desc + limitStr := ctx.Query("limit") // limit=100 + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + typeStr := ctx.Query("type") // type=array_stats + + logger.Debug("Fields, filters values are:", + zap.String("filterStr", filterStr), + zap.String("fieldsStr", fieldsStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr), + zap.String("typeStr", typeStr)) + + // ---- check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- check type ---- + typeValue, err := utils.ParseType(typeStr, models.JobAvailableTypes) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } else { + logger.Debug("query filter is not specified, set submit_user as default filter: filter=[{submit_user:eq:" + submitUser) + filters = append(filters, utils.FilterCondition{ + Field: "submit_user", + Operator: "eq", + Value: submitUser, + }) + } + + permission, err := GetRequireResourcePermission(submitUser, "queue:view") + if err != nil { + ReturnError(ctx, common.ERROR_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + var accessQueues []string + if permission.DomainType == models.DomainTypeAll { + accessQueues = store.GetAllQueueNames() + } else if permission.DomainType == models.DomainTypeQueue { + accessQueues = permission.DomainValues + } + logger.Debug("get job info accessQueues", zap.Any("accessQueues", accessQueues), + zap.Any("permission", permission)) + // check job:view permission + permission, err = GetRequireResourcePermission(submitUser, "job:view") + logger.Debug("get job info for job:view permission:", zap.Any("permission", permission)) + if err != nil || len(accessQueues) == 0 { + ReturnError(ctx, common.ERROR_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + // if submit_user is not specified, use query request submission user + var specify_user bool = false + var delindex int = -1 + for i, filter := range filters { + if filter.Field == "submit_user" { + specify_user = true + //handle submit_user = "all" + if filter.Value == models.DomainTypeAll { + delindex = i + } + if permission.DomainType != models.DomainTypeSelf { + submitUser = filter.Value + } + break + } + } + if delindex != -1 { + filters = append(filters[:delindex], filters[delindex+1:]...) + } + if !specify_user || permission.DomainType == models.DomainTypeSelf { + logger.Debug("submit_user is not specified in query filter, add submit_user into filter: filter=[{submit_user:eq:" + submitUser) + filters = append(filters, utils.FilterCondition{ + Field: "submit_user", + Operator: "eq", + Value: submitUser, + }) + } + + // job_id:eq:101, [array_idx:btw:(1,3), array_idx:btw:(6,8) ... ] + compfilters := make(utils.CompFilterCondition) + + //filtercds := make([]utils.Condition, len(filters)) + //subFilters is used to store internal filter extract from job_id:in:(101[1-2, 6-10], 102[1-2, 6-10]) + //will be placed in a OR tree. + + for i, filter := range filters { + // check the filter is supported. + if typeValue == "array_stats" { + if !utils.ContainsField(models.ARRAY_JOB_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } else { + if !utils.ContainsField(models.JOB_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // check whether exist queue_name for rbac + if filter.Field == "queue_name" { + if accessQueues != nil { + found := false + for _, q := range accessQueues { + if filter.Value == q { + found = true + break + } + } + if !found { + logger.Debug("The queue name does not exist in accessible queues", zap.Any("filter.Value", filter.Value)) + } + } + } + + // special handle job_name[index] cases, here index can be [1-3], [1,3,5],[1-3,5-7] situations + if filters[i].Field == "job_name" { + filtCdArrays := make([]utils.FilterConditionArray, 0) + jobNames := extractJobNames(filters[i].Value) + for i := range jobNames { + jobNames[i] = strings.TrimSpace(jobNames[i]) + if typeValue == "array_stats" { + if strings.Contains(jobNames[i], "[") || strings.Contains(jobNames[i], "]") { + err := errors.New("job_name with array index is not supported in statistics type") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + } + logger.Debug("After extractJobNames jobNames is", zap.Any("jobNames", jobNames)) + + // only "in", "between", "notin" support multiple jobnames (include array job name) + if !(filters[i].Operator == "in" || filters[i].Operator == "between" || filters[i].Operator == "notin") && len(jobNames) > 1 { + err := fmt.Errorf("operator doesn't allow to use multiple values: %v", jobNames) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + if !(strings.Contains(filters[i].Value, "[") && strings.Contains(filters[i].Value, "]")) { + continue + } + + if strings.Contains(filters[i].Value, "[") || strings.Contains(filters[i].Value, "]") { + if filters[i].Operator == "ne" { + err := fmt.Errorf("operator doesn't support %v", jobNames) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + for _, jobname := range jobNames { + subORFilters := make([]utils.FilterCondition, 0) + jobnameArry, err := parseArrayName(jobname) + logger.Debug("After parseArrayName jobname is ", zap.Any("jobname", jobnameArry)) + + if err != nil { + logger.Error("job name format is invalid", zap.Error(err)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + if err = isValidArrayName(jobnameArry.KeyID); err != nil { + err := errors.New("bad jobname: " + jobnameArry.KeyID) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + var operator string + if filters[i].Operator == "in" { + operator = "eq" + } else { + operator = filters[i].Operator + } + // treat every section as an OR condition + mainFiltCond := utils.FilterCondition{ + Field: filters[i].Field, + Operator: operator, + Value: jobnameArry.KeyID, + } + if jobnameArry.Indices != nil { + + // array jobid: job_id:in:(101[1-3, 5-6]) + // handle every section in Indices list. + for _, jidxsec := range jobnameArry.Indices { + st, ed, err := parseArrayIndexSection(jidxsec.Index) + if err != nil { + err := errors.New("bad job array index: " + jidxsec.Index) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + if jidxsec.Step == 1 { + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "between", + Value: "(" + strconv.Itoa(st) + "," + strconv.Itoa(ed) + ")", + }) + } else { // handle step > 1 + idxList := make([]string, 0) + for idx := st; idx <= ed; idx += jidxsec.Step { + idxList = append(idxList, strconv.Itoa(idx)) + } + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "in", + Value: "(" + strings.Join(idxList, ",") + ")", + }) + } + } + } + filtCdArrays = append(filtCdArrays, utils.FilterConditionArray{ + MainFiltCond: mainFiltCond, + SubFiltConds: subORFilters, + }) + } + compfilters["job_name"] = filtCdArrays + } + + if filters[i].Field == "job_id" { + filtCdArrays := make([]utils.FilterConditionArray, 0) + jobids := extractJobNames(filters[i].Value) + for i := range jobids { + jobids[i] = strings.TrimSpace(jobids[i]) + if typeValue == "array_stats" { + if strings.Contains(jobids[i], "[") || strings.Contains(jobids[i], "]") { + err := errors.New("job_id with array index is not supported in statistics type") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + } + logger.Debug("After extractJobNames jobids is", zap.Any("jobids", jobids)) + + if !(filters[i].Operator == "in" || filters[i].Operator == "between" || filters[i].Operator == "notin") && len(jobids) > 1 { + err := fmt.Errorf("operator doesn't allow to use multiple values: %v", jobids) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + if !(strings.Contains(filters[i].Value, "[") && strings.Contains(filters[i].Value, "]")) { + continue + } + if strings.Contains(filters[i].Value, "[") || strings.Contains(filters[i].Value, "]") { + if filters[i].Operator == "ne" { + err := fmt.Errorf("operator doesn't support %v", jobids) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + for _, jobid := range jobids { + subORFilters := make([]utils.FilterCondition, 0) + jobidArry, err := parseArrayName(jobid) + logger.Debug("After parseArrayName jobid is ", zap.Any("jobid", jobidArry)) + + if err != nil { + logger.Error("job id format is invalid", zap.Error(err)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + if !utils.IsNumeric(jobidArry.KeyID) { + err := errors.New("bad jobid: " + jobidArry.KeyID) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + var operator string + if filters[i].Operator == "in" { + operator = "eq" + } else { + // for other operator, only one job is supported. + // means job_id:eq:100[1-3,6-10]. + // if job_id:eq:(100[1-3],101[1-2]) should be rejected + // TODO: need check if already handle this case ??? + operator = filters[i].Operator + } + + // single jobid: job_id:in:(100, 101[1-3, 5-6]) + // here, handle the job_id = 100 + mainFiltCond := utils.FilterCondition{ + Field: filters[i].Field, + Operator: operator, + Value: jobidArry.KeyID, + } + // treat every section as an OR condition + if jobidArry.Indices != nil { + // handle every section in Indices list. + for _, jidxsec := range jobidArry.Indices { + st, ed, err := parseArrayIndexSection(jidxsec.Index) + if err != nil { + err := errors.New("bad job array index: " + jidxsec.Index) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + if jidxsec.Step == 1 { + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "between", + Value: "(" + strconv.Itoa(st) + "," + strconv.Itoa(ed) + ")", + }) + } else { // handle step > 1 + idxList := make([]string, 0) + for idx := st; idx <= ed; idx += jidxsec.Step { + idxList = append(idxList, strconv.Itoa(idx)) + } + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "in", + Value: "(" + strings.Join(idxList, ",") + ")", + }) + } + } + } + filtCdArrays = append(filtCdArrays, utils.FilterConditionArray{ + MainFiltCond: mainFiltCond, + SubFiltConds: subORFilters, + }) + } // end of for jobids + compfilters["job_id"] = filtCdArrays + } // filter with arrayjob[] + + // ==================== exec_hosts ================ + // filter is exec_hosts need special handling + // the exec_hosts query request is array, eg: "{exec_hosts:eq:h1,h2,hg1} + if filters[i].Field == "exec_hosts" { + if filters[i].Operator != "eq" { + err := fmt.Errorf("filter exec_hosts doesn't support operator rather than eq") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + hosts := strings.Split(strings.TrimSpace(filters[i].Value), ",") + execHosts, err := GetHostList(hosts) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + //change the filter to get unique_job_id list from db. + subFilters := make([]utils.FilterCondition, 0) + subFilters = append(subFilters, utils.FilterCondition{ + Field: "host_name", + Operator: "in", + Value: "(" + strings.Join(execHosts, ",") + ")", + }) + + uniqueJobIds, err := FetchJobIdList(subFilters) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + if len(uniqueJobIds) == 0 { + logger.Debug("no job found") + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, map[string]interface{}{ + "count": 0, + "rows": []interface{}{}, + "page_token": "", + }) + return + } + + //Replace filter condition from "exec_hosts" to "unique_job_id". + uniqueJobIdStrs := make([]string, len(uniqueJobIds)) + for i, uniqueJobId := range uniqueJobIds { + uniqueJobIdStrs[i] = strconv.FormatInt(uniqueJobId, 10) + } + filters[i].Field = "unique_job_id" + filters[i].Operator = "in" + filters[i].Value = "(" + strings.Join(uniqueJobIdStrs, ",") + } // end of "exec_hosts" filter + + } // end of for { filters } + + // remove filter with jobarray[] from filters, for filter in compfilters need special handling. + if len(compfilters) > 0 { + var filtersNew []utils.FilterCondition + for _, filter := range filters { + found := false + for k, _ := range compfilters { + if filter.Field == k { + found = true + break + } + } + if found { + continue + } + filtersNew = append(filtersNew, filter) + } + filters = filtersNew + } + + var job interface{} + if typeValue == "array_stats" { + job = &models.ArrayJobInfo{} + } else { + job = &models.JobInfo{} + } + + // Apply RBAC queue filter exactly once after the filter loop + filters = applyQueueRBACFilter(filters, accessQueues) + + logger.Debug("before call ConvertToNewFilter for all jobs", zap.Any("filters", filters)) + + filtercds, err := utils.ConvertToNewFilter(filters, job) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // convert filterConditionArray to compound conditions + logger.Debug("before call ConvertToNewFilterArray for all array jobs", zap.Any("filtersarray", compfilters)) + compfiltercd, err := utils.ConvertToNewFilterArray(compfilters, job) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check fields ---- + // get fields and do check, if not set, use default display columns + if typeValue == "array_stats" { + fields = models.ARRAY_JOB_DEFAULT_COLUMN + } else { + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + + found1 := false + found2 := false + found3 := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.JobSubmitRequest{}) + + utils.AddFieldsToSupportedList(&supportedFields, models.JobAdditionalFields{}) + logger.Debug("job supportedFields: %v", zap.Any("supportedFields", supportedFields)) + + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // if user not specify job_id in fields, add job_id as default field. + if field == "job_id" { + found1 = true + } + + if field == "unique_job_id" { + found2 = true + } + + if field == "job_name" { + found3 = true + } + } + if !found1 { + logger.Debug("user doesn't specify job_id in query fields, add it.") + fields = append(fields, "job_id") + } + if !found2 { + fields = append(fields, "unique_job_id") + } + + if found3 { + fields = append(fields, "array_idx") + } + } else { + fields = models.JOB_DEFAULT_COLUMN + } + } + + // Map fields to database fields + for i, field := range fields { + fields[i] = utils.GetMappedField(field) + } + + // add some fields for rbac + if typeValue == "array_stats" { + fields = append(fields, "submit_user") + } else { + fields = append(fields, "submit_user", "queue_name") + } + + // ---- check sort ---- + // handle sort, when no sortStr specified, need add the primary key of job_info + // table into sortStr as default value + // parse sortStr + var sorts []utils.SortOrder + found := false + if sortStr != "" { + if typeValue == "array_stats" { + sorts, err = utils.ParseSortParam(sortStr, models.ARRAY_JOB_SUPPORT_FILTERS) + } else { + sorts, err = utils.ParseSortParam(sortStr, models.JOB_SUPPORT_SORT) + } + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + //the primary key of job_info table job_id should always be added into sortStr if user doesn't specify. + if typeValue == "array_stats" { + for _, sortent := range sorts { + if sortent.Column == "job_id" { + found = true + break + } + } + } else { + for _, sortent := range sorts { + if sortent.Column == "unique_job_id" { + found = true + break + } + } + } + + } + // user not specify the unique_job_id, just append "unique_job_id:asc" at the end. + if !found || sortStr == "" { + if typeValue == "array_stats" { + logger.Debug("job_id is not specified, set sort=job_id:asc as default") + sorts = append(sorts, utils.SortOrder{ + Column: "job_id", + Ascending: true, + }) + } else { + logger.Debug("unique_job_id is not specified, set sort=unique_job_id:asc as default") + sorts = append(sorts, utils.SortOrder{ + Column: "unique_job_id", + Ascending: true, + }) + } + + } + + sortOrders = sorts + + // ---- check paging --- + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + queryOpts, err := utils.BuildQueryOptions(filtercds, compfiltercd, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + + logger.Debug("the job query request:", zap.Any("jquery request", filtercds), zap.Any("jquery request arrayjob:", compfiltercd), zap.Any("sortOrders", sortOrders), zap.Any("limit", limit), + zap.String("pagetoken", pagetokenStr), zap.Any("fields", fields)) + + var jqueryReply []utils.OrderMap // Declare jqueryReply + pagetoken := "" // Initialize pagetoken with an empty string or appropriate value + if typeValue == "array_stats" { + jqueryReply, pagetoken, err = services.QueryData(services.TableArrayJob, queryOpts, fields) + } else { + jqueryReply, pagetoken, err = services.QueryData(services.TableJobInfo, queryOpts, fields) + } + + if err != nil { + logger.Error("Failed to query job.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the job info query reply:", zap.Any("jqueryReply", jqueryReply), zap.String("pagetoken", pagetoken)) + //convert job_status to readable string. + queryResult := processJobData(jqueryReply, pagetoken) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +// GetJobByIDs godoc +// @Summary Get multiple jobs by their IDs +// @Description Retrieve information for multiple jobs in a single request with optional field selection +// @Tags jobs +// @Accept json +// @Produce json +// @Param jobids path string true "Comma-separated list of job IDs (e.g., 1001,1002,1003)" +// @Param fields query string false "Comma-separated list of fields to return (e.g., jobid,status,queue)" +// @Success 200 {object} JobsResponse "Successfully retrieved jobs" +// @Failure 400 INVALID_PARAMS_CODE "invalid parameters" +// @Failure 500 ERROR_MSG +// @Example {curl} Example Usage: +// @Example curl -X GET "http://localhost:8080/jobs/1001,1002,1003?fields=jobid,status,queue" +// @Router /jobs/{jobids} [get] +func (j JobController) GetJobByIDs(ctx *gin.Context) { + logger.Debug("Enter GetJobByIDs() function from ", zap.String("client_ip", ctx.ClientIP())) + + token := ctx.GetHeader("Authorization") + logonUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + if ctx.IsAborted() { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + logger.Debug("GetJobByIDs param is ", zap.Any("param", param)) + if !utils.ContainsField(utils.SpecificSupportedQueryParams, param) && !utils.ContainsField(utils.TypeQueryParams, param) { + err := errors.New("invalid specific job info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get job IDs from URL parameter + jobIDsStr := ctx.Param("jobids") + fieldsStr := ctx.Query("fields") + typeStr := ctx.Query("type") // type=array_stats + + logger.Debug("GetJobByIDs origin query string jobid, Fields, filters values are:", + zap.String("jobid", jobIDsStr), + zap.String("fieldsStr", fieldsStr), + zap.String("typeStr", typeStr)) + + // ---- check type ---- + typeValue, err := utils.ParseType(typeStr, models.JobAvailableTypes) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // Extract job IDs using the new function + jobids := extractJobNames(jobIDsStr) + + for i := range jobids { + jobids[i] = strings.TrimSpace(jobids[i]) + } + logger.Debug("The input jobids", zap.Any("jobids", jobids)) + mergeJqueryReply := make([]utils.OrderMap, 0) + var page_token = "" + for _, jobid := range jobids { + + compfilters := make(utils.CompFilterCondition) + + filtCdArrays := make([]utils.FilterConditionArray, 0) + subORFilters := make([]utils.FilterCondition, 0) + + jobid = strings.TrimSpace(jobid) + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + var err error + // parse the jobid to support array list format + jobid, err := parseArrayName(jobid) + if err != nil { + logger.Error("job id format is invalid", zap.Error(err)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + logger.Debug("GetJobByIDs after parseArrayName", zap.Any("jobid", jobid)) + + if !utils.IsNumeric(jobid.KeyID) { + err := errors.New("bad jobid: " + jobid.KeyID) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + filters = append(filters, utils.FilterCondition{ + Field: "job_id", + Operator: "eq", + Value: jobid.KeyID, + }) + + for _, filter := range filters { + // check the filter is supported. + if typeValue == "array_stats" { + if !utils.ContainsField(models.ARRAY_JOB_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } else { + if !utils.ContainsField(models.JOB_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + } + + mainFiltCond := utils.FilterCondition{ + Field: filters[0].Field, + Operator: filters[0].Operator, + Value: jobid.KeyID, + } + + // treat every section as an OR condition + if jobid.Indices != nil && typeValue != "array_stats" { + // handle every section in Indices list. + for _, jidxsec := range jobid.Indices { + st, ed, err := parseArrayIndexSection(jidxsec.Index) + if err != nil { + err := errors.New("bad job array index: " + jidxsec.Index) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + if jidxsec.Step == 1 { + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "between", + Value: "(" + strconv.Itoa(st) + "," + strconv.Itoa(ed) + ")", + }) + } else { // handle step > 1 + idxList := make([]string, 0) + for idx := st; idx <= ed; idx += jidxsec.Step { + idxList = append(idxList, strconv.Itoa(idx)) + } + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "in", + Value: "(" + strings.Join(idxList, ",") + ")", + }) + } + } + } + filtCdArrays = append(filtCdArrays, utils.FilterConditionArray{ + MainFiltCond: mainFiltCond, + SubFiltConds: subORFilters, + }) + + compfilters["job_id"] = filtCdArrays + + // convert filterConditionArray to compound conditions + logger.Debug("before call ConvertToNewFilterArray for all jobs", zap.Any("compfilters", compfilters)) + var job interface{} + if typeValue == "array_stats" { + job = &models.ArrayJobInfo{} + } else { + job = &models.JobInfo{} + } + compfiltercd, err := utils.ConvertToNewFilterArray(compfilters, job) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + var fields []string + if typeValue == "array_stats" { + fields = models.ARRAY_JOB_DEFAULT_COLUMN + } else { + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + + found1 := false + found2 := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.JobSubmitRequest{}) + utils.AddFieldsToSupportedList(&supportedFields, models.JobAdditionalFields{}) + logger.Debug("job supportedFields: %v", zap.Any("supportedFields", supportedFields)) + + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // if user not specify job_id in fields, add job_id as default field. + if field == "job_id" { + found1 = true + } + if field == "unique_job_id" { + found2 = true + } + } + + if !found1 { + logger.Debug("user doesn't specify job_id in query fields, add it.") + fields = append(fields, "job_id") + } + if !found2 { + fields = append(fields, "unique_job_id") + } + } else { // default display columns + fields = models.JOB_DEFAULT_COLUMN + } + } + + // Map fields to database fields + for i, field := range fields { + fields[i] = utils.GetMappedField(field) + } + // add some fields for rbac + if typeValue == "array_stats" { + fields = append(fields, "submit_user") + } else { + fields = append(fields, "submit_user", "queue_name") + } + + // add a default order: sort=job_id:asc + if typeValue == "array_stats" { + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "job_id", // use unique_job_id as primary key + Ascending: true, + }) + } else { + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "unique_job_id", // use unique_job_id as primary key + Ascending: true, + }) + } + + var bookmark string = "" + var limit int = utils.PER_PAGE_LIMIT_MAX + for { + queryOpts, err := utils.BuildQueryOptions(nil, compfiltercd, sortOrders, limit, bookmark) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the job query request:", zap.Any("jquery request arrayjob:", compfiltercd), zap.Any("sortOrders", sortOrders), zap.Any("limit", limit), + zap.String("pagetoken", bookmark), zap.Any("fields", fields)) + + var jqueryReply []utils.OrderMap // Declare jqueryReply + + if typeValue == "array_stats" { + jqueryReply, page_token, err = services.QueryData(services.TableArrayJob, queryOpts, fields) + } else { + jqueryReply, page_token, err = services.QueryData(services.TableJobInfo, queryOpts, fields) + } + + if err != nil { + logger.Error("Failed to query job info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + // Here, page_token is escaped and will not be recognized by dbapi, + // for internal use, need add Unescape here. + bookmark, _ = url.QueryUnescape(page_token) + mergeJqueryReply = append(mergeJqueryReply, jqueryReply...) + if bookmark == "" { + break + } + } // end for + + } // end for jobid + logger.Debug("the specific jobid query reply after merge:", zap.Any("jqueryReply", mergeJqueryReply)) + + // check rbac + removeFlag := false + for i, row := range mergeJqueryReply { + jobPermission, err := GetRequireResourcePermission(logonUser, "job:view") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + queuePermission, err := GetRequireResourcePermission(logonUser, "queue:view") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if job_owner, exist := row.Data["submit_user"]; exist { + if jobPermission.DomainType == models.DomainTypeSelf { + if logonUser != job_owner.(string) { + removeFlag = true + mergeJqueryReply = append(mergeJqueryReply[:i], mergeJqueryReply[i+1:]...) + } + } + } + if typeValue != "array_stats" { + if job_queue, exist := row.Data["queue_name"]; exist { + queueName, _ := job_queue.(string) + if queuePermission.DomainType == models.DomainTypeQueue { + permissionQueueValues := queuePermission.DomainValues + if !utils.ContainsField(permissionQueueValues, queueName) { + removeFlag = true + mergeJqueryReply = append(mergeJqueryReply[:i], mergeJqueryReply[i+1:]...) + } + } + } + } + + } + if removeFlag && len(mergeJqueryReply) == 0 { + ReturnError(ctx, common.ERROR_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + //convert job_status to readable string. + queryResult := processJobData(mergeJqueryReply, page_token) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func extractJobNames(jobNamesStr string) []string { + // Trim surrounding parentheses if present + if len(jobNamesStr) > 1 && jobNamesStr[0] == '(' && jobNamesStr[len(jobNamesStr)-1] == ')' { + jobNamesStr = jobNamesStr[1 : len(jobNamesStr)-1] + } + + var jobNames []string + currentJobName := "" + bracketCount := 0 + + for _, char := range jobNamesStr { + if char == '[' { + bracketCount++ + } else if char == ']' { + bracketCount-- + } else if char == ',' && bracketCount == 0 { + // If we encounter a comma and we're not inside brackets, we split here + jobNames = append(jobNames, currentJobName) + currentJobName = "" + continue + } + currentJobName += string(char) + } + // Add the last job name if there's any remaining + if currentJobName != "" { + jobNames = append(jobNames, currentJobName) + } + + return jobNames +} + +func (j JobController) RestartJobByID(ctx *gin.Context) { + logger.Debug("Enter RestartJobByID() function from ", zap.String("client_ip", ctx.ClientIP())) + + // get submit user from token + token := ctx.GetHeader("Authorization") + restarter, err := GetUsernameFromToken(token) + if err != nil { + logger.Error("Get username from token error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check jobid value + jobIDStr := ctx.Param("jobids") + logger.Debug("Receive job restart request:", + zap.String("jobId", jobIDStr), zap.String("restarter", restarter)) + + if err = isValidJobId(jobIDStr); err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // check post body whether correct + var req models.JobRestartRequest + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + err = preCheckJobRestartRequest(&req) + if err != nil { + logger.Error("Failed to restart job", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + compfilters := make(utils.CompFilterCondition) + filtCdArrays := make([]utils.FilterConditionArray, 0) + subORFilters := make([]utils.FilterCondition, 0) + + jobid := strings.TrimSpace(jobIDStr) + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + filters = append(filters, utils.FilterCondition{ + Field: "job_id", + Operator: "eq", + Value: jobid, + }) + mainFiltCond := utils.FilterCondition{ + Field: filters[0].Field, + Operator: filters[0].Operator, + Value: jobid, + } + + fields := utils.GetSupportedFields(models.JobInfo{}) + + filtCdArrays = append(filtCdArrays, utils.FilterConditionArray{ + MainFiltCond: mainFiltCond, + SubFiltConds: subORFilters, + }) + + compfilters["job_id"] = filtCdArrays + + var job interface{} + + job = &models.JobInfo{} + + compfiltercd, err := utils.ConvertToNewFilterArray(compfilters, job) + + var limit int = utils.PER_PAGE_LIMIT_MAX + var bookmark string = "" + queryOpts, err := utils.BuildQueryOptions(nil, compfiltercd, sortOrders, limit, bookmark) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the job query request:", zap.Any("jquery request arrayjob:", compfiltercd), zap.Any("sortOrders", sortOrders), zap.Any("limit", limit), + zap.String("pagetoken", bookmark), zap.Any("fields", fields)) + + var jqueryReply []utils.OrderMap + + jqueryReply, page_token, err := services.QueryData(services.TableJobInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query job info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + // Here, page_token is escaped and will not be recognized by dbapi, + bookmark, _ = url.QueryUnescape(page_token) + if len(jqueryReply) == 0 { + logger.Error("No job found for the given job ID", zap.String("jobID", jobIDStr)) + ReturnError(ctx, common.ERROR_NOT_FOUND_CODE, common.ERROR_JOB_NOT_FOUND_MSG, nil) + return + } + + oldJob := jqueryReply[0].Data // Get the first job info + logger.Debug("Old job info:", zap.Any("oldJob", oldJob)) + if v, ok := oldJob["submit_user"].(string); ok { + if v != restarter { + err := fmt.Errorf("job <%s> is not submitted by <%s>, cannot be restarted by <%s>", + jobIDStr, v, restarter) + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + } + + jobIDArray, err := generateJobIDs(indices) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Use a concrete type for job + newJob := models.JobInfo{ + Unique_JobID: jobIDArray[0].Unique_JobID, + JobID: jobIDArray[0].Unique_JobID, + ArrayIdx: 0, + SubmitUser: restarter, + JobStatus: models.JOBSTAT["PEND"], + } + + err = processJobRestartRequest(&newJob, &req) + if err != nil { + logger.Error("Failed to restart job", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + if v, ok := oldJob["job_name"].(string); ok { + newJob.JobName = v + } + + if chkpntStr, ok := oldJob["chkpnt_info"].(string); ok && chkpntStr != "" { + var chkpntInfo map[string]interface{} + if err := json.Unmarshal([]byte(chkpntStr), &chkpntInfo); err == nil { + if restartCmd, ok := chkpntInfo["restart_command"].(string); ok { + logger.Debug("Restart job with checkpoint command", zap.String("restart_command", restartCmd)) + newJob.Command = restartCmd + } + if ckpt_pid, ok := chkpntInfo["ckpt_pid"].(float64); ok { + extendData := map[string]interface{}{} + if v, ok := oldJob["extend_data"].(string); ok && v != "" { + _ = json.Unmarshal([]byte(v), &extendData) + } + if v, ok := extendData["sub_env_vars"]; ok && v != nil { + if sub, ok := v.(map[string]interface{}); ok { + sub["XCE_JOB_CHECKPOINT_PID"] = strconv.FormatFloat(ckpt_pid, 'f', -1, 64) + extendData["sub_env_vars"] = sub + } + } else { + extendData["sub_env_vars"] = map[string]interface{}{ + "XCE_JOB_CHECKPOINT_PID": strconv.FormatFloat(ckpt_pid, 'f', -1, 64), + } + } + extendDataBytes, _ := json.Marshal(extendData) + newJob.ExtendData = models.Jsonb{String: string(extendDataBytes)} + } else { + if v, ok := oldJob["extend_data"].(string); ok && v != "" { + newJob.ExtendData = models.Jsonb{String: v} + } + } + } + } else { + if v, ok := oldJob["command"].(string); ok { + newJob.Command = v + } + } + + newJob.ChkpntInfo = models.Jsonb{} //Reset checkpoint_info field + if v, ok := oldJob["queue_name"].(string); ok { + newJob.QueueName = v + } + if v, ok := oldJob["project_name"].(string); ok { + newJob.ProjectName = v + } + if v, ok := oldJob["user_group"].(string); ok { + newJob.UserGroup = v + } + if v, ok := oldJob["job_group"].(string); ok { + newJob.JobGroup = v + } + if v, ok := oldJob["license_project"].(string); ok { + newJob.LicenseProject = v + } + if v, ok := oldJob["job_description"].(string); ok { + newJob.JobDescription = v + } + if v, ok := oldJob["res_req"].(string); ok { + newJob.ResReq = v + } + if v, ok := oldJob["rsv_id"].(string); ok { + newJob.RsvID = v + } + if v, ok := oldJob["sla"].(string); ok { + newJob.SLA = v + } + if v, ok := oldJob["maxnum_processors"].(int); ok { + newJob.MaxNumProcessors = v + } + if v, ok := oldJob["num_processors"].(int); ok { + newJob.NumProcessors = v + } + if newJob.AskedHosts == "" { + if v, ok := oldJob["asked_hosts"].(string); ok { + newJob.AskedHosts = v + } + } + if v, ok := oldJob["begin_time"].(time.Time); ok { + newJob.BeginTime = models.CustomTime{Time: v} + } + if v, ok := oldJob["term_time"].(time.Time); ok { + newJob.TermTime = models.CustomTime{Time: v} + } + if v, ok := oldJob["job_priority"].(int); ok { + newJob.JobPriority = v + } + if v, ok := oldJob["user_priority"].(int); ok { + newJob.UserPriority = v + } + if v, ok := oldJob["depend_cond"].(string); ok { + newJob.DependCond = v + } + if v, ok := oldJob["cwd"].(string); ok { + newJob.CWD = v + } + if v, ok := oldJob["from_host"].(string); ok { + newJob.FromHost = v + } + if v, ok := oldJob["pre_exec_cmd"].(string); ok { + newJob.PreExecCmd = v + } + if v, ok := oldJob["post_exec_cmd"].(string); ok { + newJob.PostExecCmd = v + } + if v, ok := oldJob["requeue_error_values"].(string); ok { + newJob.RequeueErrorValues = v + } + if v, ok := oldJob["run_time_estimation"].(int); ok { + newJob.RunTimeEstimation = v + } + if v, ok := oldJob["mig_threshold"].(int); ok { + newJob.MigThreshold = v + } + if v, ok := oldJob["chkpnt_period"].(int); ok { + newJob.ChkpntPeriod = v + } + if v, ok := oldJob["chkpnt_dir"].(string); ok { + newJob.ChkpntDir = v + } + if v, ok := oldJob["out_dir"].(string); ok { + newJob.OutDir = v + } + if v, ok := oldJob["in_file"].(string); ok { + newJob.InFile = v + } + if v, ok := oldJob["err_file"].(string); ok { + newJob.ErrFile = v + } + if v, ok := oldJob["out_file"].(string); ok { + newJob.OutFile = v + } + if v, ok := oldJob["login_shell"].(string); ok { + newJob.LoginShell = v + } + logger.Info("Generated job IDs", zap.Any("jobIDArray", jobIDArray), zap.Any("indices", len(indices))) + + // sequential job + err = services.SubmitJob(&newJob) + if err != nil { + logger.Error("Failed to submit job for restart process.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_WRITE_DATA_MSG+": "+err.Error(), nil) + return + } + + response := models.JobSubmitResponse{ + Message: fmt.Sprintf("Job <%s> will be restarted with new job <%d> to default queue <%s>.", jobid, newJob.JobID, newJob.QueueName), + JobID: newJob.JobID, + QueueName: newJob.QueueName, + } + ReturnSuccess(ctx, common.CREATED, common.SUCCESS_MSG, response) + return +} + +// CreateJob godoc +// @Summary Create a new job +// @Description Submit a new job with specified requirements and parameters +// @Tags jobs +// @Accept json +// @Produce json +// @Param job body JobSubmitRequest true "Job submission parameters" +// @Success 201 {object} JobSubmitResponse "Job successfully created" +// @Failure 400 {object} ErrorResponse "Invalid request body or parameters" +// @Failure 401 {object} ErrorResponse "Unauthorized - Invalid or missing credentials" +// @Failure 403 {object} ErrorResponse "Forbidden - User lacks permission" +// @Failure 500 {object} ErrorResponse "Internal server error" +// @Router /jobs [post] +func (j JobController) CreateJob(ctx *gin.Context) { + logger.Debug("Enter CreateJob() function from ", zap.String("client_ip", ctx.ClientIP())) + + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + var req models.JobSubmitRequest + + indices = []int{} + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // save the original job name before parsing + arrayJobName := req.JobName + // precheck the request parameter before generating the job ID + err = preCheckJobRequest(&req) + if err != nil { + logger.Error("Failed to submit job request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check rbac + queuePermission, err := GetRequireResourcePermission(submitUser, "queue:submit") + if err != nil { + logger.Error("Permission check failed for queue", zap.String("queue", req.QueueName), zap.Error(err)) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + if queuePermission.DomainType == models.DomainTypeQueue { + reqQueue := req.QueueName + queuePermValues := queuePermission.DomainValues + if !utils.ContainsField(queuePermValues, reqQueue) { + logger.Error("Permission check failed for queue", zap.String("queue", req.QueueName), zap.Error(err)) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + } + // generate job IDs + jobIDArray, err := generateJobIDs(indices) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + logger.Info("Generated job IDs", zap.Any("jobIDArray", jobIDArray), zap.Any("indices", len(indices))) + + // get the primary jobID + var primaryJobID int64 + if len(jobIDArray) > 0 { + primaryJobID = jobIDArray[0].Unique_JobID + } + + defaultValues := map[string]string{ + "job_id": fmt.Sprintf("%d", primaryJobID), + "user_name": submitUser, + "job_group": req.JobGroup, + "project_name": req.ProjectName, + } + + logger.Debug("realCopyLocalFiles=", zap.Any("realCopyLocalFiles", realCopyLocalFiles)) + logger.Debug("fakeCopyRemoteFile=", zap.Any("fakeCopyRemoteFile", fakeCopyRemoteFile)) + + err = processJobRequest(ctx, &req, defaultValues) + if err != nil { + logger.Error("Failed to submit job request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + jobs := make([]models.JobInfo, len(jobIDArray)) + for i, jobID := range jobIDArray { + logger.Debug("Create new job info for jobID", zap.Int64("jobID", jobID.Unique_JobID)) + + // Create new job info + jobs[i] = models.JobInfo{ + Unique_JobID: jobID.Unique_JobID, + JobID: primaryJobID, + ArrayIdx: jobID.ArrayIdx, + //SubmitTime: time.Now().Unix(), // backend will fill the submit time; + SubmitUser: submitUser, + JobStatus: models.JOBSTAT["PEND"], // new job is pend. + JobName: req.JobName, + Command: req.Command, + QueueName: req.QueueName, //TODO: need set to DEFAULT_QUEUE + ProjectName: req.ProjectName, //TODO: need set "default" if not specify? + UserGroup: req.UserGroup, + LicenseProject: req.LicenseProject, + JobDescription: req.JobDescription, + ResReq: req.Resreq, + RsvID: req.RsvID, + SLA: req.SLA, + MaxNumProcessors: req.MaxNumProcessors, + NumProcessors: req.NumProcessors, + AskedHosts: req.AskedHosts, + LaunchType: req.LaunchType, + //BeginTime: req.BeginTime, + //TermTime: req.TermTime, + BeginTime: models.CustomTime{Time: req.BeginTime.Time}, + TermTime: models.CustomTime{Time: req.TermTime.Time}, + JobPriority: req.JobPriority, + UserPriority: req.JobPriority, + DependCond: req.DependCond, + CWD: req.CWD, + FromHost: ctx.ClientIP(), //use ip now, need use hostname ? + PreExecCmd: req.PreExecCmd, + PostExecCmd: req.PostExecCmd, + RequeueErrorValues: req.RequeueErrorValues, + RunTimeEstimation: req.RunTimeEstimation, + MigThreshold: req.MigThreshold, + ChkpntPeriod: req.ChkpntPeriod, + ChkpntDir: req.ChkpntDir, + OutDir: req.OutDir, + InFile: req.InFile, // transfer file into json with absolute path + ErrFile: req.ErrFile, + OutFile: req.OutFile, + LoginShell: req.LoginShell, + ExtendData: models.Jsonb{String: req.ExtendData}, + } + + err = postJobRequest(&jobs[i]) + if err != nil { + logger.Error("Failed to handle postJobRequest ", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + logger.Debug("jobinfo before submit.", zap.Any("job", jobs[i])) + } + + if len(jobs) == 1 { + // sequential job + err = services.SubmitJob(&jobs[0]) + if err != nil { + logger.Error("Failed to submit new sequential job.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_WRITE_DATA_MSG+": "+err.Error(), nil) + return + } + } else { + // array job + arrayJob := models.ArrayJobInfo{ + JobID: primaryJobID, + ArrayJobName: arrayJobName, + SubmitUser: submitUser, + QueueName: req.QueueName, + UserGroup: req.UserGroup, + JobDescription: req.JobDescription, + JobName: req.JobName, + ProjectName: req.ProjectName, + LicenseProject: req.LicenseProject, + SLA: req.SLA, + } + err = services.SubmitArrayJobs(arrayJob, jobs) + if err != nil { + logger.Error("Failed to write new array jobs.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_WRITE_DATA_MSG+": "+err.Error(), nil) + return + } + } + + logger.Info("Job submission successfully from ", zap.String("client_ip", ctx.ClientIP()), zap.Any("job", jobs[0])) + + response := models.JobSubmitResponse{ + Message: fmt.Sprintf("Job <%d> is submitted to default queue <%s>.", primaryJobID, req.QueueName), + JobID: primaryJobID, + QueueName: req.QueueName, + } + ReturnSuccess(ctx, common.CREATED, common.SUCCESS_MSG, response) + +} + +func generateJobIDs(indices []int) ([]models.JobArrayID, error) { + logger.Debug("Enter generateJobIDs() function", zap.Any("indices", indices)) + // initialize jobIDPairs + jobIDPairs := make([]models.JobArrayID, 0, len(indices)) + + // if indices is empty, generate the primary jobid + if len(indices) == 0 { + jobID, err := jobID_Worker.NextID() + if err != nil { + logger.Error("Failed to generate job id.", zap.Error(err)) + return nil, fmt.Errorf("failed to generate job id: %w", err) + } + jobIDPairs = append(jobIDPairs, models.JobArrayID{ArrayIdx: 0, Unique_JobID: jobID}) + return jobIDPairs, nil + } + + // generate the other job IDs based on indices + for _, index := range indices { + jobID, err := jobID_Worker.NextID() + if err != nil { + logger.Error("Failed to generate job id.", zap.Int("index", index), zap.Error(err)) + return nil, fmt.Errorf("failed to generate job id: %w", err) + } + jobIDPairs = append(jobIDPairs, models.JobArrayID{ArrayIdx: index, Unique_JobID: jobID}) + } + + logger.Debug("Generated jobIDPairs", zap.Any("jobIDPairs", jobIDPairs)) + return jobIDPairs, nil +} + +func preCheckJobRestartRequest(req *models.JobRestartRequest) error { + logger.Debug("Enter preCheckJobRestartRequest") + if err := validateAskedHosts(req.AskedHosts); err != nil { + return err + } + return nil +} + +func preCheckJobRequest(req *models.JobSubmitRequest) error { + logger.Debug("Enter preCheckJobRequest") + if req.QueueName == "" { + req.QueueName = "normal" + } else { + //validate queue_name whether exists + queueInfo, ok := store.Queues.Get(req.QueueName) + if !ok { + logger.Error("Invalid queue name specified", zap.String("queue_name", req.QueueName)) + return fmt.Errorf("invalid queue name: %s", req.QueueName) + } + queue_statusStr := formatQueueStatus(int64(queueInfo.QueueStatus)) + if strings.HasPrefix(strings.ToLower(queue_statusStr), "closed") { + logger.Error("Queue is closed", zap.String("queue_name", req.QueueName)) + return fmt.Errorf("cannot submit job to queue %s: queue is closed", req.QueueName) + } + } + if req.JobName != "" { + arrayName, parsedIndices, slotLimit, err := createArrayName(req.JobName) + if err != nil { + logger.Error("job name format is invalid", zap.Error(err)) + return errors.New("job array list format is invalid: " + err.Error()) + } + + if arrayName != "" { + if err = isValidArrayName(arrayName); err != nil { + logger.Error(err.Error()) + return errors.New("job array list format is invalid: " + err.Error()) + } + } + + indices = parsedIndices + req.JobName = arrayName + logger.Debug("preCheckJobRequest parsed job name is: ", zap.String("jobName", req.JobName), + zap.Any("indices", indices), zap.Any("slotLimit", slotLimit)) + return nil + } else { + if req.Command != "" { + if len(req.Command) > 4094 { + req.JobName = utils.TruncateString(req.Command, 4094) + } else { + req.JobName = req.Command + } + } + } + + if req.JobDescription != "" { + if len(req.JobDescription) > 4094 { + return errors.New("job description length exceeds the limit of 4094 characters") + } + } + + // Validate AskedHosts using the new function + if err := validateAskedHosts(req.AskedHosts); err != nil { + return err + } + + if req.ProjectName != "" { + if len(req.ProjectName) > 511 { + return errors.New("project name length exceeds the limit of 511 characters") + } + } + + if req.Resreq != "" { + if len(req.Resreq) > 4094 { + return errors.New("resreq length exceeds the limit of 4094 characters") + } + } + + if req.ShellLimit.CPULimit != 0 { + req.CPULimit.Seconds = (req.ShellLimit.CPULimit) / 1000 // convert ms to seconds + } + + if req.ShellLimit.FileLimit.MB != 0 { + req.FileLimit = req.ShellLimit.FileLimit + } + if req.ShellLimit.DataLimit.MB != 0 { + req.DataLimit = req.ShellLimit.DataLimit + } + if req.ShellLimit.StackLimit.MB != 0 { + req.StackLimit = req.ShellLimit.DataLimit + } + if req.ShellLimit.StackLimit.MB != 0 { + req.StackLimit = req.ShellLimit.StackLimit + } + if req.ShellLimit.CoreLimit.MB != 0 { + req.CoreLimit = req.ShellLimit.CoreLimit + } + if req.ShellLimit.ProcessLimit.MB != 0 { + req.ProcessLimit = req.ShellLimit.ProcessLimit + } + if req.ShellLimit.ThreadLimit.MB != 0 { + req.ThreadLimit = req.ShellLimit.ThreadLimit + } + if req.ShellLimit.MemLimit.MB != 0 { + req.MemLimit = req.ShellLimit.MemLimit + } + if req.ShellLimit.SwapLimit.MB != 0 { + req.SwapLimit = req.ShellLimit.SwapLimit + } + if req.ShellLimit.NumFiles != 0 { + req.NumFiles = req.ShellLimit.NumFiles + } + // handle environment variables + if req.Env != nil { + if len(req.Env) > 4094 { + return errors.New("environment variable length exceeds the limit of 4094 characters") + } + for _, value := range req.Env { + parseEnvs := strings.SplitN(value, "=", 2) + if len(parseEnvs) != 2 { + return errors.New("environment variable should use key=value format") + } + key := parseEnvs[0] + if strings.TrimSpace(key) == "" { + return errors.New("environment variable key cannot be empty") + } + } + } + if req.TransferFiles != nil { + // handle transfer file and update InFile + realCopyLocalFiles = make([]string, 0) + fakeCopyRemoteFile = make([]string, 0) + var extendData []map[string]string + + for _, transferFileStr := range req.TransferFiles { + localFile, operator, remoteFile, err := parseTransferFiles(transferFileStr) + if err != nil { + return err + } + + /* based on different operator, check the local file and remote file + >: copy local to remote + <> and ><: copy local to remote + <: copy remote to local + <<: append remote to local + */ + if operator == ">" || operator == "<>" || operator == "><" { + if localFile == "" { + return errors.New("local file is required for operator " + operator) + } + // Append to realCopyLocalFiles for later processing + realCopyLocalFiles = append(realCopyLocalFiles, localFile) + } + + if operator == "<" || operator == "<<" { + if remoteFile == "" { + return errors.New("remote file is required for operator " + operator) + } + fakeCopyRemoteFile = append(fakeCopyRemoteFile, remoteFile) + } + + // Add to inFileData + extendData = append(extendData, map[string]string{ + "local_file": localFile, + "op": operator, + "remote_file": remoteFile, + }) + } + // Convert inFileData to JSON and assign to req.InFile + extendDataJSON, err := json.Marshal(extendData) + if err != nil { + return fmt.Errorf("failed to marshal InFile JSON: %v", err) + } + req.ExtendData = string(extendDataJSON) + } + + if req.NumProcessors != 0 && req.MaxNumProcessors != 0 { + minTasks := req.NumProcessors + maxTasks := req.MaxNumProcessors + + minTasks, maxTasks, err := checkNumProcessors(minTasks, maxTasks) + if err != nil { + logger.Error("Failed to parse NumProcessors", zap.Any("numProcessors", req.NumProcessors), zap.Error(err)) + return fmt.Errorf("invalid NumProcessors specification: %w", err) + } + req.NumProcessors = minTasks + req.MaxNumProcessors = maxTasks + } else if req.NumProcessors != 0 { + req.MaxNumProcessors = req.NumProcessors + } + + if req.MigThreshold < 0 || req.MigThreshold > 2147483647 { + return errors.New("Incorrect migration threshold value. Specify a number greater than or equal to zero.") + } + + if !utils.ContainsField(models.ValidInteractiveValues, strconv.Itoa(req.Interactive)) { + return errors.New("Interactive field must be 0, 1, or 2") + } + + if req.LaunchType != "" { + launchType := strings.ToLower(req.LaunchType) + if !utils.ContainsField(models.ValidLaunchTypes, launchType) { + return errors.New("Launch_type must be openmpi or mpich") + } + req.LaunchType = launchType + } + + return nil +} + +// It returns minTasks, maxTasks, and an error if parsing fails or constraints are violated. +func checkNumProcessors(minTasks, maxTasks int) (minValue, maxValue int, err error) { + logger.Debug("Enter checkNumProcessors", zap.Any("minTasks", minTasks), zap.Any("maxTasks", maxTasks)) + if minTasks > maxTasks { + err = fmt.Errorf("minTasks (%d) cannot be greater than maxTasks (%d)", minTasks, maxTasks) + logger.Error("checkNumProcessors validation failed", zap.Error(err)) + return 0, 0, err + } + + return minTasks, maxTasks, nil +} + +func parseArrayName(jobName string) (models.JobArrayInfo, error) { + logger.Debug("Enter parseArrayName", zap.String("jobName", jobName)) + + var jobArrayInfo models.JobArrayInfo + + // Check if the jobName contains brackets + if !strings.Contains(jobName, "[") { + // If no brackets, return the job name as is + jobArrayInfo.KeyID = jobName + return jobArrayInfo, nil + } + + openBracketIndex := strings.Index(jobName, "[") + closeBracketIndex := strings.Index(jobName, "]") + + jobArrayInfo.KeyID = jobName[:openBracketIndex] + if openBracketIndex != -1 && closeBracketIndex != -1 { + if closeBracketIndex <= openBracketIndex { + return models.JobArrayInfo{}, fmt.Errorf("invalid [] format") + } + indicesPart := jobName[openBracketIndex+1 : len(jobName)-1] // Remove the closing bracket + + // Parse the indices part + rangeParts := strings.Split(indicesPart, ",") + for _, part := range rangeParts { + part = strings.TrimSpace(part) + + if strings.Contains(part, "-") { + // Handle ranges like 1-3 + bounds := strings.Split(part, "-") + if len(bounds) != 2 { + return models.JobArrayInfo{}, fmt.Errorf("invalid range format: %s", part) + } + + step := 1 // Default step for ranges + parts := strings.Split(part, ":") + + if len(parts) == 2 { + var err error + step, err = strconv.Atoi(parts[1]) + if err != nil { + return models.JobArrayInfo{}, fmt.Errorf("invalid step value: %v", err) + } + if step != 1 { + return models.JobArrayInfo{}, fmt.Errorf("invalid step value and only support 1 now") + } + } + + part = parts[0] + jobArrayInfo.Indices = append(jobArrayInfo.Indices, models.JobIndex{Index: part, Step: step}) + } else { + // Handle single indices + step := 1 // Default step for single indices + jobArrayInfo.Indices = append(jobArrayInfo.Indices, models.JobIndex{Index: part, Step: step}) + } + } + } else { + return models.JobArrayInfo{}, fmt.Errorf("no valid indices found in array job") + } + return jobArrayInfo, nil +} + +// based on jobName to generate arrayName and individual indices +func createArrayName(jobName string) (arrayName string, indices []int, slimit int, err error) { + logger.Debug("Enter createArrayName", zap.Any("jobName=", jobName)) + + // find the index of the first '[' + slotLimit := "" + slotLimitValue := -1 + openBracketIndex := strings.Index(jobName, "[") + closeBracketIndex := strings.Index(jobName, "]") + stepIndex := strings.Index(jobName, ":") + slotLimitIndex := strings.Index(jobName, "%") + + logger.Debug("Debugging index values", + zap.Any("openBracketIndex", openBracketIndex), + zap.Any("closeBracketIndex", closeBracketIndex), + zap.Any("stepIndex", stepIndex), + zap.Any("slotLimitIndex", slotLimitIndex)) + + if slotLimitIndex != -1 { + if slotLimitIndex+1 != len(jobName) { + slotLimit = jobName[slotLimitIndex+1 : len(jobName)] + } else { + return "", nil, -1, fmt.Errorf("invalid slot limit value '%s': must be a positive integer", slotLimit) + } + + logger.Debug("slotLimit=", zap.Any("slotLimit", slotLimit)) + slotLimitValue, err = strconv.Atoi(strings.TrimSpace(slotLimit)) + if err != nil || slotLimitValue < 1 { + return "", nil, -1, fmt.Errorf("invalid slot limit value '%s': must be a positive integer", slotLimit) + } + } + + if openBracketIndex != -1 && closeBracketIndex != -1 { + if closeBracketIndex <= openBracketIndex { + return "", nil, -1, fmt.Errorf("invalidate job_name format") + } + if stepIndex != -1 && (stepIndex == closeBracketIndex-1 || stepIndex > closeBracketIndex) { + return "", nil, -1, fmt.Errorf("invalidate step value") + } + // extract array_name + arrayName = jobName[:openBracketIndex] + // extract the part inside the brackets + rangePart := jobName[openBracketIndex+1 : closeBracketIndex] + // split the range parts by comma + rangeParts := strings.Split(rangePart, ",") + seenIndices := make(map[int]struct{}) // detect the duplicate element + for _, part := range rangeParts { + if part == "" { + return "", nil, -1, fmt.Errorf("do not use empty range part: %s", part) + } + part = strings.TrimSpace(part) // remove spaces + + // Remove %slotLimitValue suffix if present + if slotLimitValue != -1 { + suffixToRemove := fmt.Sprintf("%%%d", slotLimitValue) + part = strings.TrimSuffix(part, suffixToRemove) + } + + if strings.Contains(part, "-") { + // split by ':' to get the range and optional step + parts := strings.Split(part, ":") + if len(parts) < 1 || len(parts) > 2 { + return "", nil, -1, fmt.Errorf("invalid range format: %s", part) + } + + // parse the range bounds + rangeBounds := strings.Split(parts[0], "-") + if len(rangeBounds) != 2 { + return "", nil, -1, fmt.Errorf("invalid range bounds: %s", part) + } + + start, err := strconv.Atoi(strings.TrimSpace(rangeBounds[0])) + if err != nil || start < 1 { + return "", nil, -1, fmt.Errorf("invalid start value: %v, must be >= 1", err) + } + logger.Debug("start is:", zap.Any("start", start)) + + end, err := strconv.Atoi(strings.TrimSpace(rangeBounds[1])) + if err != nil || end > utils.JobYamlConfig.MaxArraySize { + return "", nil, -1, fmt.Errorf("invalid end value: %v, must be <= %d", err, utils.JobYamlConfig.MaxArraySize) + } + logger.Debug("end is:", zap.Any("end", end)) + // set default step + step := 1 + if len(parts) == 2 { + if stepIndex != -1 { + step, err = strconv.Atoi(jobName[stepIndex+1 : closeBracketIndex]) + if err != nil { + return "", nil, -1, fmt.Errorf("invalid step value: %v", err) + } + } + } + logger.Debug("step value is", zap.Any("step", step)) + if start >= end { + return "", nil, -1, fmt.Errorf("start value (%d) cannot be greater than or equal to the end value (%d)", start, end) + } + + if step > end || step == 0 { + return "", nil, -1, fmt.Errorf("step value (%d) cannot be greater than the end value (%d) and greater than 0", step, end) + } + + // count the indices for this range + for i := start; i <= end; i += step { + // if there is duplicated index and reject + if _, exists := seenIndices[i]; exists { + return "", nil, -1, fmt.Errorf("duplicate index detected: %d", i) + } + seenIndices[i] = struct{}{} + indices = append(indices, i) + } + } else { + // Handle single indices like 1, 2, 3 + if part != "" { + index, err := strconv.Atoi(part) + if err != nil || index < 1 { + return "", nil, -1, fmt.Errorf("index must be a positive integer") + } + if err != nil { + return "", nil, -1, fmt.Errorf("invalid index value: %s", part) + } + if _, exists := seenIndices[index]; !exists { + seenIndices[index] = struct{}{} + indices = append(indices, index) + } + } + + } + + } + } else if openBracketIndex != -1 || closeBracketIndex != -1 { + return "", nil, -1, fmt.Errorf("invalid array job name '%s'", jobName) + } else { + // no "[]" + /* + if slotLimitIndex != -1 { + arrayName = jobName[:slotLimitIndex] + } else { + arrayName = jobName + }*/ + arrayName = jobName + } + + if slotLimit != "" && len(indices) > 0 { + maxIndex := 0 + for i, index := range indices { + if i == 0 || index > maxIndex { + maxIndex = index + } + } + if slotLimitValue > maxIndex { + return "", nil, -1, fmt.Errorf("slotLimit (%d) cannot be greater than the maximum index (%d)", slotLimitValue, maxIndex) + } + } + logger.Debug("Finish createArrayName function", zap.Any("arrayName", arrayName), + zap.Any("indices", indices), zap.Any("slotLimitValue", slotLimitValue)) + return arrayName, indices, slotLimitValue, nil +} + +func isValidArrayName(arrayName string) error { + // define the validate chars + if len(arrayName) > 4094 { + return fmt.Errorf("job name length exceeds the limit of 4094 characters") + } + validArrayName := regexp.MustCompile(`^[a-zA-Z0-9._-]+$`) + if !validArrayName.MatchString(arrayName) { + return fmt.Errorf("arrayName '%s' contains invalid characters. Allowed characters are [a-zA-Z0-9._-]", arrayName) + } + return nil +} + +func isValidJobId(jobID string) error { + matched, _ := regexp.MatchString(`^\d+$`, jobID) + matchedIndex, _ := regexp.MatchString(`^\d+\[\d+\]$`, jobID) + if !matched && !matchedIndex { + return fmt.Errorf("jobid '%s' contains invalid characters. Allowed characters are 0-9 or with index value", jobID) + } + return nil +} + +func validateAskedHosts(askedHosts string) error { + if askedHosts == "" { + return nil // Nothing to validate + } + + // Split the AskedHosts string by spaces + hostList := strings.Fields(askedHosts) + if len(hostList) == 0 { + // Handle case where AskedHosts might be just whitespace + logger.Warn("AskedHosts field contains only whitespace.", zap.String("AskedHosts", askedHosts)) + return nil + } + + /* Validate each hostname against the cache + for _, host := range hostList { + if _, ok := store.Hosts.Get(host); !ok { + err := fmt.Errorf("invalid hostname specified in AskedHosts: '%s'", host) + logger.Error(err.Error()) + return err // Return the specific error + } + }*/ + logger.Debug("AskedHosts validated successfully.", zap.Strings("hosts", hostList)) + return nil // All hosts are valid +} + +// handle TransferFiles field +func parseTransferFiles(transferFileStr string) (localFile, operator, remoteFile string, err error) { + // remove space char + remoteFile = "" + transferFileStr = strings.TrimSpace(transferFileStr) + parts := strings.Fields(transferFileStr) + + if len(parts) < 2 { + return "", "", "", errors.New("TransferFiles format is invalid, expected format: local_file operator [remote_file]") + } + localFile = parts[0] + operator = parts[1] + + if parts[0] == "<" || parts[0] == "<<" { + operator = parts[0] + pathParts := strings.Split(parts[1], "/") + localFile = pathParts[len(pathParts)-1] + remoteFile = parts[1] + } + if len(parts) > 2 { + remoteFile = parts[2] + } + // validate the operator + validOperators := []string{">", "<", "<<", "<>", "><"} + if !utils.ContainsField(validOperators, operator) { + return "", "", "", fmt.Errorf("invalid operator: %s, supported operators are: %v", operator, validOperators) + } + + return localFile, operator, remoteFile, nil +} + +func processJobRestartRequest(newJob *models.JobInfo, req *models.JobRestartRequest) error { + logger.Debug("Enter processJobRestartRequest function", zap.Any("req", req)) + if req.AskedHosts != "" { + newJob.AskedHosts = req.AskedHosts + } + return nil +} +func processJobRequest(ctx *gin.Context, req *models.JobSubmitRequest, defaultValues map[string]string) error { + + if req.InFile != "" { + req.InFile = strings.ReplaceAll(req.InFile, "%J", defaultValues["job_id"]) + } + + if req.OutFile != "" { + req.OutFile = strings.ReplaceAll(req.OutFile, "%J", defaultValues["job_id"]) + } + + if req.ErrFile != "" { + req.ErrFile = strings.ReplaceAll(req.ErrFile, "%J", defaultValues["job_id"]) + } + + if req.OutDir != "" { + if defaultValues["job_group"] != "" { + req.OutDir = strings.ReplaceAll(req.OutDir, "%JG", defaultValues["job_group"]) + } + req.OutDir = strings.ReplaceAll(req.OutDir, "%J", defaultValues["job_id"]) + req.OutDir = strings.ReplaceAll(req.OutDir, "%U", defaultValues["user_name"]) + req.OutDir = strings.ReplaceAll(req.OutDir, "%P", defaultValues["project_name"]) + } + + if req.CWD != "" { + req.CWD = strings.ReplaceAll(req.CWD, "%EJ", defaultValues["job_id"]) + req.CWD = strings.ReplaceAll(req.CWD, "%J", defaultValues["job_id"]) + if defaultValues["job_group"] != "" { + req.CWD = strings.ReplaceAll(req.CWD, "%JG", defaultValues["job_group"]) + } + req.CWD = strings.ReplaceAll(req.CWD, "%P", defaultValues["project_name"]) + req.CWD = strings.ReplaceAll(req.CWD, "%U", defaultValues["user_name"]) + } + // initialize extendDataMap + extendDataMap := make(map[string]interface{}) + if req.ExtendData != "" { + _ = json.Unmarshal([]byte(req.ExtendData), &extendDataMap) + } + + if req.TransferFiles != nil { + // Copy local files and check for errors + result, err := copyLocalFilesConcurrently(ctx, realCopyLocalFiles, defaultValues["user_name"]) + if err != nil { + return fmt.Errorf("failed to process local files: %v", err) + } + + // prepare + // Update ExtendData with copied URI + + var extendData []map[string]interface{} + if err := json.Unmarshal([]byte(req.ExtendData), &extendData); err != nil { + return fmt.Errorf("failed to parse ExtendData JSON: %v", err) + } + if len(result) != 0 { + for i, entry := range extendData { + if localFile, ok := entry["local_file"].(string); ok { + if copiedURI, exists := result[localFile]; exists { + entry["local_file"] = copiedURI + } + } + extendData[i] = entry + } + } + logger.Debug("In processJobRequest function, after CopyLocalFilesConcurrently, result is:", + zap.Any("result", result)) + // copy remote files locally and append to result + remoteResult := copyRemoteFilesLocally(ctx, fakeCopyRemoteFile, defaultValues["user_name"]) + for remoteFile, copiedURI := range remoteResult { + result[remoteFile] = copiedURI + } + if len(remoteResult) != 0 { + for i, entry := range extendData { + if localFile, ok := entry["remote_file"].(string); ok { + if copiedURI, exists := result[localFile]; exists { + entry["remote_file"] = copiedURI + } + } + extendData[i] = entry + } + } + extendDataMap["transfer_files"] = extendData + } + + // handle env into json format + if req.Env != nil { + envJsonb, err := utils.ConvertStringToJsonb(req.Env) + if err != nil { + return fmt.Errorf("Failed to convert env to JSON: %v", err) + } + + var envData interface{} + err = json.Unmarshal([]byte(envJsonb.String), &envData) + if err != nil { + logger.Error("Failed to unmarshal envJsonb string", zap.Error(err), zap.String("envJsonb", envJsonb.String)) + return fmt.Errorf("failed to process environment variables: %v", err) + } + extendDataMap["sub_env_vars"] = envData + } + + // handle limits + limits := make(map[string]interface{}) + if req.CPULimit.Seconds != 0 { + limits["cpu_limit"] = req.CPULimit.Seconds + } + if req.FileLimit.MB != 0 { + limits["fsize_limit"] = req.FileLimit.MB + } + if req.DataLimit.MB != 0 { + limits["data_limit"] = req.DataLimit.MB + } + if req.StackLimit.MB != 0 { + limits["stack_limit"] = req.StackLimit.MB + } + if req.CoreLimit.MB != 0 { + limits["core_limit"] = req.CoreLimit.MB + } + if req.ProcessLimit.MB != 0 { + limits["process_limit"] = req.ProcessLimit.MB + } + if req.ThreadLimit.MB != 0 { + limits["thread_limit"] = req.ThreadLimit.MB + } + if req.MemLimit.MB != 0 { + limits["rss_limit"] = req.MemLimit.MB + } + if req.SwapLimit.MB != 0 { + limits["swap_limit"] = req.SwapLimit.MB + } + if req.NumFiles != 0 { + limits["nofile_limit"] = req.NumFiles + } + if req.RunLimit.Seconds != 0 { + limits["run_limit"] = req.RunLimit.Seconds + } + if len(limits) > 0 { + extendDataMap["limits"] = limits + } + + if req.Interactive != 0 { + extendDataMap["interactive"] = req.Interactive + } + + if len(extendDataMap) > 0 { + updatedExtendDataBytes, err := json.Marshal(extendDataMap) + if err != nil { + return fmt.Errorf("Failed to marshal updated ExtendData: %v", err) + } + req.ExtendData = string(updatedExtendDataBytes) + } + + return nil +} + +func postJobRequest(job *models.JobInfo) error { + if job.InFile != "" { + job.InFile = strings.ReplaceAll(job.InFile, "%I", strconv.Itoa(job.ArrayIdx)) + } + if job.OutFile != "" { + job.OutFile = strings.ReplaceAll(job.OutFile, "%I", strconv.Itoa(job.ArrayIdx)) + } + + if job.ErrFile != "" { + job.ErrFile = strings.ReplaceAll(job.ErrFile, "%I", strconv.Itoa(job.ArrayIdx)) + } + + if job.OutDir != "" { + job.OutDir = strings.ReplaceAll(job.OutDir, "%I", strconv.Itoa(job.ArrayIdx)) + } + + if job.CWD != "" { + job.CWD = strings.ReplaceAll(job.CWD, "%I", strconv.Itoa(job.ArrayIdx)) + } + if len(job.InFile) > 4094 || len(job.OutFile) > 4094 || len(job.ErrFile) > 4094 || len(job.CWD) > 4094 { + return errors.New("file length exceeds the limit of 4094 characters") + } + if len(job.PreExecCmd) > 4094 || len(job.PostExecCmd) > 4094 { + return errors.New("command path exceeds the limit of 4094 characters") + } + if len(job.ChkpntDir) > 4000 { + return errors.New("checkpoint directory length exceeds the limit of 4000 characters") + } + return nil +} + +// ControlJobByID godoc +// @Summary Perform an action on a job +// @Description Submit an action to be performed on a specific job identified by jobids +// @Tags jobs +// @Accept json +// @Produce json +// @Param jobids path string true "Job ID" +// @Param action body JobSignalRequest true "Action to be performed" +// @Success 202 {object} JobControlResponseSummary "Action successfully accepted" +// @Failure 400 {object} ErrorResponse "Invalid request body or parameters" +// @Failure 404 {object} ErrorResponse "Job not found" +// @Failure 500 {object} ErrorResponse "Internal server error" +// @Router /jobs/{jobids}/control [post] +func (j JobController) ControlJobByID(ctx *gin.Context) { + logger.Debug("Enter ControlJobByID() function from ", zap.String("client_ip", ctx.ClientIP())) + + // get submit user from token + token := ctx.GetHeader("Authorization") + submitter, err := GetUsernameFromToken(token) + if err != nil { + logger.Error("Get username from token error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Get job IDs from URL parameter + jobIDsStr := ctx.Param("jobids") + + var req models.JobControlJobIDRequest + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // Validate command + convertedReq := models.JobControlRequest{ + Action: req.Action, + Signal: req.Signal, + Comment: req.Comment, + ForceKill: req.ForceKill, + Method: req.Method, + } + + err = preCheckJobControlRequest(&convertedReq) + if err != nil { + logger.Error("Failed to send job control request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + signalval := convertedReq.Signal + + logger.Debug("Receive job control request:", zap.String("client_ip", ctx.ClientIP()), + zap.String("jobIds", jobIDsStr), zap.Any("control request:", req)) + + //filters := make([]utils.FilterCondition, 0) + + fields := models.JOB_DEFAULT_COLUMN + for i, field := range fields { + fields[i] = utils.GetMappedField(field) + } + + // add a default order: sort=job_id:asc + sortOrders := make([]utils.SortOrder, 0) + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "unique_job_id", // use unique_job_id as primary key + Ascending: true, + }) + + var acttype int + switch convertedReq.Action { + case "kill": + acttype = models.JOB_KILL + case "stop": + acttype = models.JOB_STOP + case "resume": + acttype = models.JOB_RESUME + case "checkpoint": + acttype = models.JOB_CHKPNT + default: + logger.Debug("Unsupported action: " + req.Action) + ReturnError(ctx, common.INVALID_PARAMS_CODE, "Unsupported action: "+req.Action, nil) + return + } + + //1. get user input job list and validate the job existed or not + // Extract job IDs using the new function + jobids := extractJobNames(jobIDsStr) + for i := range jobids { + jobids[i] = strings.TrimSpace(jobids[i]) + } + logger.Debug("The input jobids", zap.Any("jobids", jobids)) + mergeJqueryReply := make([]utils.OrderMap, 0) + + // store all user's input jobid + allFiltCdArrays := make([]utils.FilterConditionArray, 0) + + for _, jobid := range jobids { + + compfilters := make(utils.CompFilterCondition) + + filtCdArrays := make([]utils.FilterConditionArray, 0) + subORFilters := make([]utils.FilterCondition, 0) + + var err error + + jobid = strings.TrimSpace(jobid) + // parse the jobid to support array list format + jobid, err := parseArrayName(jobid) + if err != nil { + logger.Error("job id format is invalid", zap.Error(err)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + logger.Debug("GetJobByIDs after parseArrayName", zap.Any("jobid", jobid)) + + if !utils.IsNumeric(jobid.KeyID) { + err := errors.New("bad jobid: " + jobid.KeyID) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + mainFiltCond := utils.FilterCondition{ + Field: "job_id", + Operator: "eq", + Value: jobid.KeyID, + } + + // treat every section as an OR condition + if jobid.Indices != nil { + // handle every section in Indices list. + for _, jidxsec := range jobid.Indices { + st, ed, err := parseArrayIndexSection(jidxsec.Index) + if err != nil { + err := errors.New("bad job array index: " + jidxsec.Index) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + if jidxsec.Step == 1 { + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "between", + Value: "(" + strconv.Itoa(st) + "," + strconv.Itoa(ed) + ")", + }) + } else { // handle step > 1 + idxList := make([]string, 0) + for idx := st; idx <= ed; idx += jidxsec.Step { + idxList = append(idxList, strconv.Itoa(idx)) + } + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "in", + Value: "(" + strings.Join(idxList, ",") + ")", + }) + } + } + } + filtCdArrays = append(filtCdArrays, utils.FilterConditionArray{ + MainFiltCond: mainFiltCond, + SubFiltConds: subORFilters, + }) + + allFiltCdArrays = append(allFiltCdArrays, filtCdArrays...) + + compfilters["job_id"] = filtCdArrays + var job models.JobInfo + compfiltercd, err := utils.ConvertToNewFilterArray(compfilters, &job) + if err != nil { + logger.Error(err.Error()) + // TODO: the msg used here is not proper + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + var bookmark string = "" + var limit int = utils.PER_PAGE_LIMIT_MAX + for { + queryOpts, err := utils.BuildQueryOptions(nil, compfiltercd, sortOrders, limit, bookmark) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the job query request:", zap.Any("jquery request arrayjob:", compfiltercd), zap.Any("sortOrders", sortOrders), zap.Any("limit", limit), + zap.String("pagetoken", bookmark), zap.Any("fields", fields)) + jqueryReply, page_token, err := services.QueryData(services.TableJobInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query job info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + // Here, page_token is escaped and will not be recognized by dbapi, + // for internal use, need add Unescape here. + bookmark, _ = url.QueryUnescape(page_token) + mergeJqueryReply = append(mergeJqueryReply, jqueryReply...) + if bookmark == "" { + break + } + } // end for fetch job loop + + } // end of for + logger.Debug("job list got:", zap.Any("jobInfoList", mergeJqueryReply)) + + //2. validate job status and control action matching + jobInfoList := mergeJqueryReply + + successJobs := []map[string]interface{}{} + failedJobs := []map[string]interface{}{} + + // check if specific jobids are exist in jobInfoList. + for _, ujobFilterCdArray := range allFiltCdArrays { + var jobid int64 + var arrayIdx int + foundJobs := make([]utils.OrderMap, 0) + + jobid, _ = strconv.ParseInt(ujobFilterCdArray.MainFiltCond.Value, 10, 64) + + if len(ujobFilterCdArray.SubFiltConds) == 0 { // for non index section sepcified; + arrayIdx = 0 + fjobs, err := foundJob(jobid, arrayIdx, jobInfoList) + if err != nil { + failedJobs = append(failedJobs, map[string]interface{}{ + "job_id": jobid, + "array_idx": arrayIdx, + "msg": "job not found", + }) + continue + } + foundJobs = append(foundJobs, fjobs...) + } else { + for _, subFiltCond := range ujobFilterCdArray.SubFiltConds { + if subFiltCond.Operator == "between" { + // (st, ed) + value := strings.Trim(subFiltCond.Value, "()") + values := strings.Split(value, ",") + st, _ := strconv.Atoi(values[0]) + ed, _ := strconv.Atoi(values[1]) + for i := st; i <= ed; i++ { + arrayIdx = i + fjobs, err := foundJob(jobid, arrayIdx, jobInfoList) + if err != nil { + failedJobs = append(failedJobs, map[string]interface{}{ + "job_id": jobid, + "array_idx": arrayIdx, + "msg": "job not found", + }) + } else { + foundJobs = append(foundJobs, fjobs...) + } + } + } else if subFiltCond.Operator == "in" { + // (1,4,7,...) + value := strings.Trim(subFiltCond.Value, "()") + values := strings.Split(value, ",") + for _, v := range values { + arrayidx, _ := strconv.ParseInt(v, 10, 64) + arrayIdx = int(arrayidx) + fjobs, err := foundJob(jobid, arrayIdx, jobInfoList) + if err != nil { + failedJobs = append(failedJobs, map[string]interface{}{ + "job_id": jobid, + "array_idx": arrayIdx, + "msg": "job not found", + }) + } else { + foundJobs = append(foundJobs, fjobs...) + } + } + } + } + + } + + for _, jobInfo := range foundJobs { + // check rbac + if job_owner, exist := jobInfo.Data["submit_user"]; exist { + jobPermission, err := GetRequireResourcePermission(submitter, "job:control") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if jobPermission.DomainType == models.DomainTypeSelf { + if job_owner.(string) != submitter { + logger.Error("Permission denied", zap.String("perm", "job:control"), zap.String("logonUser", submitter)) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + } + } + + if job_queue, exist := jobInfo.Data["queue_name"]; exist { + queuePermission, err := GetRequireResourcePermission(submitter, "queue:control") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + var accessQueues []string + if queuePermission.DomainType == models.DomainTypeAll { + accessQueues = store.GetAllQueueNames() + } + if queuePermission.DomainType == models.DomainTypeQueue { + accessQueues = queuePermission.DomainValues + } + // check job:control permission + if err != nil || len(accessQueues) == 0 || !utils.ContainsField(accessQueues, job_queue.(string)) { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + } + // check job status to see if can do corresponding action + validatedSuccessJobs, validatedFailedJobs := validateJobControl(jobInfo, convertedReq.Action, convertedReq.Signal) + + for _, successJob := range validatedSuccessJobs { + successJobs = append(successJobs, map[string]interface{}{ + "unique_job_id": successJob["unique_job_id"].(int64), + "job_id": successJob["job_id"].(int64), + "array_idx": successJob["array_idx"].(int), + "msg": successJob["msg"].(string), + }) + } + + for _, failedJob := range validatedFailedJobs { + failedJobs = append(failedJobs, map[string]interface{}{ + "unique_job_id": failedJob["unique_job_id"].(int64), + "job_id": failedJob["job_id"].(int64), + "array_idx": failedJob["array_idx"].(int), + "msg": failedJob["msg"].(string), + }) + } + } + + if len(successJobs) > 0 { + for _, successJob := range successJobs { + signalInt, _ := strconv.Atoi(signalval) + jobCtrlEvt := models.JobControlEvent{ + UniqueJobID: successJob["unique_job_id"].(int64), + Signal: signalInt, + Comment: convertedReq.Comment, + ForceKill: convertedReq.ForceKill, + Method: convertedReq.Method, + } + + jobCtrlEvtBytes, err := json.Marshal(jobCtrlEvt) + if err != nil { + logger.Error("encode job control event error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, "encode job control event error"+": "+err.Error(), nil) + return + } + + err = backend.ControlJob(uint64(successJob["unique_job_id"].(int64)), acttype, string(jobCtrlEvtBytes)) + if err != nil { + logger.Error("send job control to backend error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_WRITE_DATA_MSG+": "+err.Error(), nil) + return + } + } + } + } + + // response to user + //logger.Debug("job ctrl accept resps:", zap.Any("jobCtrlAcceptResps", jobCtrlAcceptResps)) + //ReturnSuccess(ctx, common.ACCEPTED, "Job control accepted.", jobCtrlAcceptResps) + response, successMsg := BuildBatchOperationResponse(convertedReq.Action, "Job", successJobs, failedJobs) + + overallStatus := response["status"].(string) + logger.Info("Finished processing job control request", + zap.String("action", convertedReq.Action), + zap.String("overallStatus", overallStatus), + zap.Int("succeededCount", len(successJobs)), + zap.Int("failedCount", len(failedJobs)), + zap.String("client_ip", ctx.ClientIP())) + + ReturnSuccess(ctx, common.ACCEPTED, successMsg, response) + return + +} + +// ControlJob godoc +// @Summary Perform an action on filtered out jobs +// @Description Submit an action to be performed on specific jobs identified by filter in request body. +// @Tags jobs +// @Accept json +// @Produce json +// @Param jobids path string true "Job ID" +// @Param action body JobControlRequest true "Action to be performed" +// @Success 202 {object} JobControlResponseSummary "Action successfully accepted" +// @Failure 400 {object} ErrorResponse "Invalid request body or parameters" +// @Failure 404 {object} ErrorResponse "Job not found" +// @Failure 500 {object} ErrorResponse "Internal server error" +// @Router /jobs/control [post] +func (j JobController) ControlJob(ctx *gin.Context) { + logger.Debug("Enter ControlJob() function from ", zap.String("client_ip", ctx.ClientIP())) + + // Get filter in request body + var req models.JobControlRequest + + // get submit user from token + token := ctx.GetHeader("Authorization") + submitter, err := GetUsernameFromToken(token) + if err != nil { + logger.Error("Get username from token error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + err = preCheckJobControlRequest(&req) + if err != nil { + logger.Error("Failed to send job control request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + signalval := req.Signal + logger.Debug("Receive job control request:", zap.Any("control request:", req), zap.String("client_ip", ctx.ClientIP())) + + filters := make([]utils.FilterCondition, 0) + + fields := models.JOB_DEFAULT_COLUMN + for i, field := range fields { + fields[i] = utils.GetMappedField(field) + } + + // + // TODO: need consider if we need put all job control action together. + // + var acttype int + switch req.Action { + case "kill": + acttype = models.JOB_KILL + case "stop": + acttype = models.JOB_STOP + case "resume": + acttype = models.JOB_RESUME + case "checkpoint": + acttype = models.JOB_CHKPNT + /* -------- + case "requeue": + acttype = models.JOB_REQUEUE + case "mig": + acttype = models.JOB_MIG + case "modify": + acttype = models.JOB_MODIFY + case "switch": + acttype = models.JOB_SWITCH + case "post": + acttype = models.JOB_POST + case "read": + acttype = models.JOB_READ + case "top": + acttype = models.JOB_TOP + case "bot": + acttype = models.JOB_BOT + */ + default: + logger.Debug("Unsupported action: " + req.Action) + ReturnError(ctx, common.INVALID_PARAMS_CODE, "Unsupported action: "+req.Action, nil) + return + } + + //query job according query filter + logger.Debug("request filters user input: ", zap.Any("filters", req.Filter)) + + // TODO: validate the req.Filter + filters = getFilters(req, submitter) + logger.Debug("final request filters: ", zap.Any("filters", filters)) + + //1. fetch jobs from db with req.Filter + jobList, err := FetchJobList(filters) + if err != nil { + logger.Error("fetch job list error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + + if len(jobList) == 0 { + logger.Debug("no job found") + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, map[string]interface{}{ + "count": 0, + "rows": []interface{}{}, + "page_token": "", + }) + return + } + + var jobInfoList []utils.OrderMap + if !req.Filter.AllJobs { + // will query job order by submittime, will get the last submitted job. + if len(jobList) > 0 && jobList[len(jobList)-1].Data["array_idx"] == 0 { + jobInfoList = jobList[len(jobList)-1:] + } else { + jobInfoList = jobList + } + } else { + jobInfoList = jobList + } + logger.Debug("job list got:", zap.Any("jobInfoList", jobInfoList)) + + //2. go through job list and fill in action_control + + successJobs := []map[string]interface{}{} + failedJobs := []map[string]interface{}{} + + for _, job := range jobInfoList { + + // check rbac begin + if job_owner, exist := job.Data["submit_user"]; exist { + jobPermission, err := GetRequireResourcePermission(submitter, "job:control") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if jobPermission.DomainType == models.DomainTypeSelf { + if job_owner.(string) != submitter { + logger.Error("Permission denied", zap.String("perm", "job:control"), zap.String("logonUser", submitter)) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + } + } + + if job_queue, exist := job.Data["queue_name"]; exist { + queuePermission, err := GetRequireResourcePermission(submitter, "queue:control") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + var accessQueues []string + if queuePermission.DomainType == models.DomainTypeAll { + accessQueues = store.GetAllQueueNames() + } + if queuePermission.DomainType == models.DomainTypeQueue { + accessQueues = queuePermission.DomainValues + } + // check job:control permission + if err != nil || len(accessQueues) == 0 || !utils.ContainsField(accessQueues, job_queue.(string)) { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + } + + // check job status to see if can do corresponding action + validatedSuccessJobs, validatedFailedJobs := validateJobControl(job, req.Action, req.Signal) + + for _, successJob := range validatedSuccessJobs { + successJobs = append(successJobs, map[string]interface{}{ + "unique_job_id": successJob["unique_job_id"].(int64), + "job_id": successJob["job_id"].(int64), + "array_idx": successJob["array_idx"].(int), + "msg": successJob["msg"].(string), + }) + } + + for _, failedJob := range validatedFailedJobs { + failedJobs = append(failedJobs, map[string]interface{}{ + "unique_job_id": failedJob["unique_job_id"].(int64), + "job_id": failedJob["job_id"].(int64), + "array_idx": failedJob["array_idx"].(int), + "msg": failedJob["msg"].(string), + }) + } + } + if len(successJobs) > 0 { + signalInt, _ := strconv.Atoi(signalval) + for _, successJob := range successJobs { + jobCtrlEvt := models.JobControlEvent{ + UniqueJobID: successJob["unique_job_id"].(int64), + Signal: signalInt, + Comment: req.Comment, + ForceKill: req.ForceKill, + Method: req.Method, + } + + jobCtrlEvtBytes, err := json.Marshal(jobCtrlEvt) + if err != nil { + logger.Error("encode job control event error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, "encode job control event error:"+": "+err.Error(), nil) + return + } + err = backend.ControlJob(uint64(successJob["unique_job_id"].(int64)), acttype, string(jobCtrlEvtBytes)) + if err != nil { + logger.Error("send job control to backend error", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_WRITE_DATA_MSG+": "+err.Error(), nil) + return + } + } + } + + // response to user + //logger.Debug("job ctrl accept resps:", zap.Any("jobCtrlAcceptResps", jobCtrlAcceptResps)) + //ReturnSuccess(ctx, common.ACCEPTED, "Job control accepted.", jobCtrlAcceptResps) + response, successMsg := BuildBatchOperationResponse(req.Action, "Job", successJobs, failedJobs) + + overallStatus := response["status"].(string) + logger.Info("Finished processing job control request", + zap.String("action", req.Action), + zap.String("overallStatus", overallStatus), + zap.Int("succeededCount", len(successJobs)), + zap.Int("failedCount", len(failedJobs)), + zap.String("client_ip", ctx.ClientIP())) + + ReturnSuccess(ctx, common.ACCEPTED, successMsg, response) + return +} + +func preCheckJobControlRequest(req *models.JobControlRequest) error { + logger.Debug("Enter preCheckJobControlRequest") + + if err := utils.ValidateControlAction(req.Action, models.SupportedJobActions); err != nil { + return err + } + req.Action = strings.ToLower(strings.TrimSpace(req.Action)) + // don't change the calling sequence in here, + // for stop/resume, it doesn't user input signal, + // but it will set default value later + if (req.Action == "stop" || req.Action == "resume") && req.Signal != "" { + return errors.New("signal must be empty for stop or resume actions") + } + + // set default signal value if not provided + if req.Signal == "" { + if defaultSignal, ok := models.DefaultActionSignal[strings.ToLower(req.Action)]; ok { + req.Signal = defaultSignal + } + + } + + // transform signal to numeric value if it is a string + if req.Signal != "" && !utils.IsNumeric(req.Signal) { + if sigNum, ok := models.JobSignalMapTable[strings.ToUpper(req.Signal)]; ok { + req.Signal = strconv.Itoa(sigNum) + } else { + return fmt.Errorf("unsupported signal: %s", req.Signal) + } + } + + if req.Action == "kill" && req.Signal != "" { + found := false + if utils.IsNumeric(req.Signal) { + for _, v := range models.JobSignalMapTable { + if strconv.Itoa(v) == req.Signal { + found = true + break + } + } + if found == false { + return fmt.Errorf("unsupported signal: %s", req.Signal) + } + } else { + if _, ok := models.JobSignalMapTable[req.Signal]; !ok { + return fmt.Errorf("unsupported signal: %s", req.Signal) + } + } + } + if req.Comment != "" { + if len(req.Comment) > 512 { + return errors.New("comment length exceeds the limit of 512 characters") + } + } + if req.Action == "checkpoint" { + if req.Method != "" { + if len(req.Method) > 64 { + return errors.New("method length exceeds the limit of 64 characters") + } + } else { + req.Method = "CRIU" + } + } else { + req.Method = "" + } + return nil +} +func getFilters(req models.JobControlRequest, submitter string) []utils.FilterCondition { + filters := make([]utils.FilterCondition, 0) + if req.Filter.Users == "" { + filters = append(filters, utils.FilterCondition{ + Field: "submit_user", + Operator: "eq", + Value: strings.TrimSpace(submitter), + }) + } else { + if req.Filter.Users != "all" { + filters = append(filters, utils.FilterCondition{ + Field: "submit_user", + Operator: "eq", + Value: strings.TrimSpace(req.Filter.Users), // TODO: need consider usergroup. + }) + } + } + if req.Filter.AllJobs { + filters = append(filters, utils.FilterCondition{ + Field: "job_status", + Operator: "notin", + Value: "(DONE,EXIT)", + }) + } + // TODO: need consider array job ????? + if req.Filter.JobName != "" { + filters = append(filters, utils.FilterCondition{ + Field: "job_name", + Operator: "eq", + Value: strings.TrimSpace(req.Filter.JobName), + }) + } + + if req.Filter.JobGroupName != "" { + filters = append(filters, utils.FilterCondition{ + Field: "job_group", + Operator: "eq", + Value: strings.TrimSpace(req.Filter.JobGroupName), + }) + } + + if req.Filter.QueueName != "" { + filters = append(filters, utils.FilterCondition{ + Field: "queue_name", + Operator: "eq", + Value: strings.TrimSpace(req.Filter.QueueName), + }) + } + + // TODO: need handle exec host and hostgroup as job query. + // + if req.Filter.ExecHosts != "" { + filters = append(filters, utils.FilterCondition{ + Field: "exec_hosts", + Operator: "eq", + Value: strings.TrimSpace(req.Filter.ExecHosts), + }) + } + + if req.Filter.SLA != "" { + filters = append(filters, utils.FilterCondition{ + Field: "sla", + Operator: "eq", + Value: strings.TrimSpace(req.Filter.SLA), + }) + } + return filters + +} + +/* validate if job status and job control action is match: +* eg: stop a non run/pend/ssuspend job (psusp/done/exit/ususpend); +* resume a non ususpend/psusp job(run/pend/done/exit/ssuspend); +* kill a non active job ( done/exit) + */ +func validateJobControl(jobinfo utils.OrderMap, action string, + signal string) (successJobs []map[string]interface{}, failedJobs []map[string]interface{}) { + var actionType int + switch action { + case "stop": + actionType = models.JOB_STOP + case "resume": + actionType = models.JOB_RESUME + case "kill": + actionType = models.JOB_KILL + case "checkpoint": + actionType = models.JOB_CHKPNT + default: + actionType = models.JOB_KILL + } + + if rawStatus, exists := jobinfo.Data["job_status"]; exists { + if jobStatus, ok := rawStatus.(int64); ok { + db_job_status := int(jobStatus) + job_status := models.JobStateStr(db_job_status) + jobStatus = int64(models.JOBSTAT[job_status]) + if utils.IsControlActionAllowed(actionType, jobStatus, models.JobActionControlMap) { + if action == "kill" && signal != "" { + if !utils.IsControlActionAllowed(actionType, jobStatus, models.JobKillSignalControlMap) { + failedJobs = append(failedJobs, map[string]interface{}{ + "unique_job_id": jobinfo.Data["unique_job_id"].(int64), + "job_id": jobinfo.Data["job_id"].(int64), + "array_idx": int(jobinfo.Data["array_idx"].(int64)), + "msg": fmt.Sprintf("job cannot be or "), + }) + } else { + successJobs = append(successJobs, map[string]interface{}{ + "unique_job_id": jobinfo.Data["unique_job_id"].(int64), + "job_id": jobinfo.Data["job_id"].(int64), + "array_idx": int(jobinfo.Data["array_idx"].(int64)), + "msg": fmt.Sprintf("job is being <%s>", action), + }) + } + return successJobs, failedJobs + } + successJobs = append(successJobs, map[string]interface{}{ + "unique_job_id": jobinfo.Data["unique_job_id"].(int64), + "job_id": jobinfo.Data["job_id"].(int64), + "array_idx": int(jobinfo.Data["array_idx"].(int64)), + "msg": fmt.Sprintf("job is being <%s>", action), + }) + } else { + logger.Error("Action not allowed for job", zap.Any("jobStatus", jobStatus)) + failedJobs = append(failedJobs, map[string]interface{}{ + "unique_job_id": jobinfo.Data["unique_job_id"].(int64), + "job_id": jobinfo.Data["job_id"].(int64), + "array_idx": int(jobinfo.Data["array_idx"].(int64)), + "msg": fmt.Sprintf("job cannot be <%s>", action), + }) + } + } else { + logger.Error("job_status is not an int64", zap.Any("job_status", rawStatus)) + failedJobs = append(failedJobs, map[string]interface{}{ + "unique_job_id": jobinfo.Data["unique_job_id"].(int64), + "job_id": jobinfo.Data["job_id"].(int64), + "array_idx": int(jobinfo.Data["array_idx"].(int64)), + "msg": fmt.Sprintf("Invalid job_status"), + }) + } + } else { + logger.Error("job_status not found in jobInfo", zap.Any("rawStatus", rawStatus)) + failedJobs = append(failedJobs, map[string]interface{}{ + "unique_job_id": jobinfo.Data["unique_job_id"].(int64), + "job_id": jobinfo.Data["job_id"].(int64), + "array_idx": int(jobinfo.Data["array_idx"].(int64)), + "msg": fmt.Sprintf("job_status not found for job"), + }) + } + + return successJobs, failedJobs +} + +/* check if the user specify job is found in job list + * when array_idx = 0, may be an array job, but without index section specify, will only compare jobId + */ +func foundJob(jobid int64, array_idx int, jobinfos []utils.OrderMap) ([]utils.OrderMap, error) { + jobs := make([]utils.OrderMap, 0) + for _, jobinfo := range jobinfos { + if array_idx > 0 { + if jobinfo.Data["job_id"].(int64) == jobid && + int(jobinfo.Data["array_idx"].(int64)) == array_idx { + jobs = append(jobs, jobinfo) + } + } else { + if jobinfo.Data["job_id"].(int64) == jobid { + jobs = append(jobs, jobinfo) + } + } + } + if len(jobs) > 0 { + return jobs, nil + } else { + return nil, errors.New("job not found") + } +} + +func processJobData(rows []utils.OrderMap, pagetoken string) utils.QueryResult { + logger.Debug("Enter processJobData() function", zap.Any("rows", rows)) + for _, row := range rows { + if v, exist := row.Data["job_status"]; exist { + if v != nil { + jobStatus, _ := utils.IntOrInt64(v) + row.Data["job_status"] = models.JobStateStr(jobStatus) + } + } + + // Update job_name if array_idx is not 0 + if jobName, exist := row.Data["job_name"]; exist { + if arrayIdx, ok := row.Data["array_idx"].(int); ok && arrayIdx > 0 { + row.Data["job_name"] = fmt.Sprintf("%s[%d]", jobName, arrayIdx) + } else if arrayIdx64, ok := row.Data["array_idx"].(int64); ok && arrayIdx64 > 0 { + row.Data["job_name"] = fmt.Sprintf("%s[%d]", jobName, arrayIdx64) + } else { + row.Data["job_name"] = jobName + } + } + + // Update launch_type if exist + if val, ok := row.Data["launch_type"]; ok && val != nil { + if int64Val, _ := val.(int64); ok { + code := int(int64Val) + if str, exists := models.GetLaunchTypeString(code); exists { + row.Data["launch_type"] = str + } else { + row.Data["launch_type"] = "unknown" + } + } + } + // Process JSONB fields based on predefined list + for _, jsonbField := range models.JSONBJFields { + if v, exist := row.Data[jsonbField]; exist { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data[jsonbField] = jsonbData + } else { + logger.Error("Failed to parse job info JSONB field.", zap.Error(err)) + } + } + } + } + + // Convert database fields back to user-recognizable fields + for userField, dbField := range models.JobFieldMappings { + //if v, exist := row.Data[dbField]; exist && v != nil { + if v, exist := row.Data[dbField]; exist { + row.Data[userField] = v + // Replace the value in row.Fields with userField + for i, field := range row.Fields { + if field == dbField { + row.Fields[i] = userField + break + } + } + // Remove the user field from the data map + delete(row.Data, dbField) + } + } + } + + return utils.QueryResult{ + Rows: rows, + Count: len(rows), + Pagetoken: pagetoken, + } +} + +func copyLocalFilesConcurrently(ctx *gin.Context, files []string, username string) (map[string]string, error) { + logger.Debug("Enter copyLocalFilesConcurrently function files:", zap.Any("files", files)) + + result := make(map[string]string) + + webUrl := utils.GetServerHostPort(ctx) + + for _, localFile := range files { + if strings.HasPrefix(localFile, webUrl) { + localFile = strings.TrimPrefix(localFile, webUrl+"/") + } + // Extract the date from the localFile path + parts := strings.Split(localFile, "/") + if len(parts) < 2 { + logger.Error("Invalid file path format", zap.String("file", localFile)) + return nil, fmt.Errorf("invalid file path format: %s", localFile) + } + dateParts := strings.Split(parts[0], "_") + if len(dateParts) < 1 { + logger.Error("Invalid date format in file path", zap.String("file", localFile)) + return nil, fmt.Errorf("invalid date format in file path: %s", localFile) + } + date := dateParts[0] // The first part is the date + logger.Debug("Extracted date from file path", zap.String("date", date)) + + localFile = filepath.Join("/home/", username, "./apiserver_uploads", date, parts[0]) + fmt.Println("localFile=", localFile) + // Check if the file exists + if _, err := os.Stat(localFile); os.IsNotExist(err) { + logger.Error("File does not exist", zap.String("file", localFile)) + return nil, fmt.Errorf("file does not exist: %s", localFile) + } + + result[parts[1]] = localFile + } + + return result, nil +} + +func copyRemoteFilesLocally(ctx *gin.Context, files []string, username string) map[string]string { + result := make(map[string]string) + webUrl := utils.GetServerHostPort(ctx) + submissionDate := time.Now().Format("20060102") + + for _, localFile := range files { + // Generate MD5 hash for the file content + hasher := md5.New() + contentHash := hasher.Sum(nil) + + // Generate UUID based on the MD5 hash + contentUUID := uuid.NewMD5(uuid.NameSpaceOID, contentHash[:]).String() + copiedURI := webUrl + "/" + submissionDate + "_" + contentUUID + "/" + filepath.Base(localFile) + result[localFile] = copiedURI + } + return result +} + +// parse [1-3] to start = 1, end=3 +func parseArrayIndexSection(idxsec string) (int, int, error) { + parts := strings.Split(idxsec, "-") + if len(parts) != 2 { + start, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + return 0, 0, err + } + return start, start, nil + } + + start, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + return 0, 0, err + } + + end, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return 0, 0, err + } + if start >= end { + return 0, 0, fmt.Errorf("start index should be bigger than end index") + } else { + return start, end, nil + } +} + +// fetch job list from job_info table according filter condition +// need special handling for job_name=arrayjob[1-3] and exec_hosts cases. +func FetchJobList(filters []utils.FilterCondition) ([]utils.OrderMap, error) { + + logger.Debug("Enter fetchJobList() function", + zap.Any("filter", filters)) + + tablename := services.TableJobInfo + var datatype models.JobInfo + limit := utils.PER_PAGE_LIMIT_MAX + fields := models.JOB_DEFAULT_COLUMN + sortOrders := make([]utils.SortOrder, 0) + jobLists := make([]utils.OrderMap, 0) + + compfilters := make(utils.CompFilterCondition) + + for i, filter := range filters { + if filter.Field == "exec_hosts" { + hosts := strings.Split(strings.TrimSpace(filters[i].Value), ",") + execHosts, err := GetHostList(hosts) + if err != nil { + return nil, err + } + + //change the filter to get unique_job_id list from db. + subFilters := make([]utils.FilterCondition, 0) + subFilters = append(subFilters, utils.FilterCondition{ + Field: "host_name", + Operator: "in", + Value: "(" + strings.Join(execHosts, ",") + ")", + }) + + uniqueJobIds, err := FetchJobIdList(subFilters) + if err != nil { + return nil, err + } + if len(uniqueJobIds) == 0 { + return nil, nil + } + + //Replace filter condition from "exec_hosts" to "unique_job_id". + uniqueJobIdStrs := make([]string, len(uniqueJobIds)) + for i, uniqueJobId := range uniqueJobIds { + uniqueJobIdStrs[i] = strconv.FormatInt(uniqueJobId, 10) + } + filters[i].Field = "unique_job_id" + filters[i].Operator = "in" + filters[i].Value = "(" + strings.Join(uniqueJobIdStrs, ",") + } + + // special handle job_name[index] cases, here index can be [1-3], [1,3,5],[1-3,5-7] situations + if filters[i].Field == "job_name" { + filtCdArrays := make([]utils.FilterConditionArray, 0) + jobNames := extractJobNames(filters[i].Value) + for i := range jobNames { + jobNames[i] = strings.TrimSpace(jobNames[i]) + } + logger.Debug("After extractJobNames jobNames is", zap.Any("jobNames", jobNames)) + + if !(strings.Contains(filters[i].Value, "[") && strings.Contains(filters[i].Value, "]")) { + continue + } + + for _, jobname := range jobNames { + subORFilters := make([]utils.FilterCondition, 0) + jobnameArry, err := parseArrayName(jobname) + logger.Debug("After parseArrayName jobname is ", zap.Any("jobname", jobnameArry)) + + if err != nil { + return nil, errors.New("job name format is invalid") + } + + if err = isValidArrayName(jobnameArry.KeyID); err != nil { + err := errors.New("bad jobname: " + jobnameArry.KeyID) + return nil, err + } + + // in job control filter, operator is "eq" + var operator = "eq" + // treat every section as an OR condition + mainFiltCond := utils.FilterCondition{ + Field: filters[i].Field, + Operator: operator, + Value: jobnameArry.KeyID, + } + if jobnameArry.Indices != nil { + + // array jobid: job_id:in:(101[1-3, 5-6]) + // handle every section in Indices list. + for _, jidxsec := range jobnameArry.Indices { + st, ed, err := parseArrayIndexSection(jidxsec.Index) + if err != nil { + err := errors.New("bad job array index: " + jidxsec.Index) + return nil, err + } + if jidxsec.Step == 1 { + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "between", + Value: "(" + strconv.Itoa(st) + "," + strconv.Itoa(ed) + ")", + }) + } else { // handle step > 1 + idxList := make([]string, 0) + for idx := st; idx <= ed; idx += jidxsec.Step { + idxList = append(idxList, strconv.Itoa(idx)) + } + subORFilters = append(subORFilters, utils.FilterCondition{ + Field: "array_idx", + Operator: "in", + Value: "(" + strings.Join(idxList, ",") + ")", + }) + } + } + } + filtCdArrays = append(filtCdArrays, utils.FilterConditionArray{ + MainFiltCond: mainFiltCond, + SubFiltConds: subORFilters, + }) + } + compfilters["job_name"] = filtCdArrays + } + } + + // remove filter with jobarray[] from filters, for filter in compfilters need special handling. + if len(compfilters) > 0 { + var filtersNew []utils.FilterCondition + for _, filter := range filters { + found := false + for k, _ := range compfilters { + if filter.Field == k { + found = true + break + } + } + if found { + continue + } + filtersNew = append(filtersNew, filter) + } + filters = filtersNew + } + + var filtercds []utils.Condition + var compfiltercd utils.CompCondition + var err error + if len(filters) > 0 { + filtercds, err = utils.ConvertToNewFilter(filters, &datatype) + if err != nil { + logger.Error(err.Error()) + return nil, err + } + } + if len(compfilters) > 0 { + compfiltercd, err = utils.ConvertToNewFilterArray(compfilters, &datatype) + if err != nil { + logger.Error(err.Error()) + return nil, err + } + } + + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "unique_job_id", + Ascending: true, + }) + + pagetokenStr := "" + for { + queryOpts, err := utils.BuildQueryOptions(filtercds, compfiltercd, sortOrders, limit, pagetokenStr) + if err != nil { + return nil, err + } + queryReply, pagetoken, err := services.QueryData(tablename, queryOpts, fields) + if err != nil { + return nil, err + } + + jobLists = append(jobLists, queryReply...) + + if pagetoken == "" { + break + } + pagetokenStr = pagetoken + } + + logger.Debug("In fetchJobList() function", + zap.Any("joblist", jobLists)) + return jobLists, nil +} + +/* fetch job id list from job_exec_hosts table with filters + * exec_hosts:in:(h1,h2...) and return unique_job_id list. + */ +func FetchJobIdList(filters []utils.FilterCondition) ([]int64, error) { + + logger.Debug("Enter fetchJobIdList() function", + zap.Any("filter", filters)) + + tablename := services.TableJobExecHosts + var datatype models.JobExecHosts + + limit := utils.PER_PAGE_LIMIT_MAX + fields := []string{"insert_seq", "unique_job_id"} + + sortOrders := make([]utils.SortOrder, 0) + jobLists := make([]utils.OrderMap, 0) + jobIdList := make([]int64, 0) + + filtercds, err := utils.ConvertToNewFilter(filters, &datatype) + if err != nil { + logger.Error(err.Error()) + return nil, err + } + + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "insert_seq", + Ascending: true, + }) + + pagetokenStr := "" + for { + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + return nil, err + } + queryReply, pagetoken, err := services.QueryData(tablename, queryOpts, fields) + if err != nil { + return nil, err + } + + jobLists = append(jobLists, queryReply...) + + if pagetoken == "" { + break + } + pagetokenStr = pagetoken + } + + if len(jobLists) > 0 { + for _, job := range jobLists { + jobIdList = append(jobIdList, job.Data["unique_job_id"].(int64)) + } + } + // when a parrallel job runs on multiple host, different host will map to the same unique_job_id. + // need remove duplicate unique_job_id. + uniqJobIds := utils.RemoveDuplicates(jobIdList) + + logger.Debug("fetchJobIdList() function", + zap.Any("jobIdList", uniqJobIds)) + + return uniqJobIds, nil +} + +// user specified host/hostgroup list, get individual host list by query host/hostgroup cache; +func GetHostList(hosts []string) ([]string, error) { + + execHosts := make([]string, 0) + + if len(hosts) == 0 { + return nil, errors.New("exec_host is empty") + } + + for _, host := range hosts { + host = strings.TrimSpace(host) + if host == "" { + continue + } + + // TODO: handle hostgroup expansion when hostgroup support is added. + if _, ok := store.Hosts.Get(host); ok { + execHosts = append(execHosts, host) + } else { + return nil, fmt.Errorf("invalid host/hostgroup %s", host) + } + + } + if len(execHosts) == 0 { + return nil, errors.New("exec_host is empty") + } + return execHosts, nil +} + diff --git a/src/frontend/apiserver/controllers/permissions.go b/src/frontend/apiserver/controllers/permissions.go new file mode 100644 index 00000000..cecfa914 --- /dev/null +++ b/src/frontend/apiserver/controllers/permissions.go @@ -0,0 +1,24 @@ +package controllers + +import ( + "apiserver/common" + logger "apiserver/utils" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type PermController struct{} + +func (pc *PermController) GetPermsInfo(ctx *gin.Context) { + logger.Debug("Enter GetPermsInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + + queryResult := GetAllPermissions() + if len(queryResult) == 0 { + logger.Error("No permissions found") + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_PERMISSIONS_MSG, nil) + return + } + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} diff --git a/src/frontend/apiserver/controllers/queue_filter_test.go b/src/frontend/apiserver/controllers/queue_filter_test.go new file mode 100644 index 00000000..a39eef47 --- /dev/null +++ b/src/frontend/apiserver/controllers/queue_filter_test.go @@ -0,0 +1,68 @@ +package controllers + +import ( + "testing" + + "apiserver/utils" +) + +func countQueueFilters(filters []utils.FilterCondition) int { + n := 0 + for _, f := range filters { + if f.Field == "queue_name" { + n++ + } + } + return n +} + +// TestQueueFilter_AddedOnce: with multiple non-queue filters, queue_name added exactly once +func TestQueueFilter_AddedOnce(t *testing.T) { + filters := []utils.FilterCondition{ + {Field: "submit_user", Operator: "eq", Value: "user1"}, + {Field: "job_status", Operator: "notin", Value: "(EXIT,DONE)"}, + } + result := applyQueueRBACFilter(filters, []string{"normal", "priority", "short"}) + if n := countQueueFilters(result); n != 1 { + t.Errorf("expected 1 queue_name filter, got %d", n) + } +} + +// TestQueueFilter_NotAddedWhenUserSpecified: user's queue_name filter preserved +func TestQueueFilter_NotAddedWhenUserSpecified(t *testing.T) { + filters := []utils.FilterCondition{ + {Field: "submit_user", Operator: "eq", Value: "user1"}, + {Field: "queue_name", Operator: "eq", Value: "normal"}, + } + result := applyQueueRBACFilter(filters, []string{"normal", "priority"}) + if n := countQueueFilters(result); n != 1 { + t.Errorf("expected 1 queue_name filter (user's), got %d", n) + } + for _, f := range result { + if f.Field == "queue_name" && f.Operator != "eq" { + t.Errorf("user's queue filter was replaced: %+v", f) + } + } +} + +// TestQueueFilter_NoQueues: empty accessQueues — no filter added +func TestQueueFilter_NoQueues(t *testing.T) { + filters := []utils.FilterCondition{ + {Field: "submit_user", Operator: "eq", Value: "user1"}, + } + result := applyQueueRBACFilter(filters, nil) + if n := countQueueFilters(result); n != 0 { + t.Errorf("expected 0 queue_name filters, got %d", n) + } +} + +// TestQueueFilter_SingleFilter: even with one non-queue filter, added exactly once +func TestQueueFilter_SingleFilter(t *testing.T) { + filters := []utils.FilterCondition{ + {Field: "job_name", Operator: "eq", Value: "myjob"}, + } + result := applyQueueRBACFilter(filters, []string{"q1", "q2"}) + if n := countQueueFilters(result); n != 1 { + t.Errorf("expected 1 queue_name filter, got %d", n) + } +} diff --git a/src/frontend/apiserver/controllers/queues.go b/src/frontend/apiserver/controllers/queues.go new file mode 100644 index 00000000..12a4955f --- /dev/null +++ b/src/frontend/apiserver/controllers/queues.go @@ -0,0 +1,615 @@ +package controllers + +import ( + "apiserver/backend" + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/store" + "apiserver/utils" + logger "apiserver/utils" + "errors" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type QueueController struct{} + +func (q QueueController) GetQueuesInfo(ctx *gin.Context) { + // Get all queues + logger.Debug("Enter GetQueuesInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + + // get logon user + token := ctx.GetHeader("Authorization") + logonUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + var fields []string + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid queues info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + filterStr := ctx.Query("filter") + fieldsStr := ctx.Query("fields") + sortStr := ctx.Query("sort") // sort=queue_name:asc,submit_time:desc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + limitStr := ctx.Query("limit") // eg: limit=100 + + logger.Debug("Queues query values are:", + zap.String("fieldsStr", fieldsStr), + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + // ---- check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + //filtercds := make([]utils.Condition, len(filters)) + for _, filter := range filters { + // check the filter is supported. + if !utils.ContainsField(models.QUEUE_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + var queues models.QueueInfo + filtercds, err := utils.ConvertToNewFilter(filters, &queues) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check fields ---- + // get fields and do check, if not set, use default display columns + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.QueueInfo{}) + logger.Debug("queue supportedFields: %v", zap.Any("supportedFields", supportedFields)) + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + // if user not specify queue_name in fields, add queue_name as default field. + if field == "queue_name" { + found = true + } + } + if !found { + logger.Debug("user doesn't specify queue name in query fields, add it.") + fields = append(fields, "queue_name") + } + } else { + fields = models.QUEUE_DEFAULT_COLUMN + } + + // ---- check sort ---- + // handle sort, when no sortStr specified, need add the primary key of queue_info + // table into sortStr as default value + if sortStr == "" { + logger.Debug("sort is not specified, set sort=queue_name:asc as default") + sortStr = "queue_name:asc" + } + // parse sortStr + sorts, err := utils.ParseSortParam(sortStr, models.QUEUE_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- check paging --- + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + //the primary key of queue_info table queue_name should always be added into sortStr if user doesn't specify. + found := false + for _, sortent := range sorts { + if sortent.Column == "queue_name" { + found = true + break + } + } + // user not specify the queue_name, only append queue_name at the end. + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "queue_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + /* + //fill QueryOptions + qqueryNew := utils.QueryOptions{ + Conditions: filtercds, + Orders: sortOrders, + Limits: limit, + Bookmark: pagetokenStr, + } + */ + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_QUEUEINFO_MSG+": "+err.Error(), nil) + return + } + + qqueryReply, pagetoken, err := services.QueryData(services.TableQueueInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query queue info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_QUEUEINFO_MSG+": "+err.Error(), nil) + return + } + // check rbac + accessRow := make([]utils.OrderMap, 0) + for _, row := range qqueryReply { + if queue_name, exist := row.Data["queue_name"]; exist { + queueStr, ok := queue_name.(string) + if !ok { + logger.Error("queue_name is not a string", zap.Any("queue_name", queue_name)) + ReturnError(ctx, common.ERROR_CODE, "queue_name is not a string", nil) + return + } + + queuePermission, err := GetRequireResourcePermission(logonUser, "queue:view") + if err != nil { + ReturnError(ctx, common.ERROR_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if queuePermission.DomainType == models.DomainTypeQueue { + queueDomainValues := queuePermission.DomainValues + if utils.ContainsField(queueDomainValues, queueStr) { + accessRow = append(accessRow, row) + } + } else if queuePermission.DomainType == models.DomainTypeAll { + accessRow = append(accessRow, row) + } + } + } + logger.Debug("After call GetRequireResourcePermission for all queues info", zap.Error(err), zap.Any("accessRow", accessRow)) + + if err != nil && len(accessRow) == 0 { + ReturnError(ctx, common.ERROR_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + + logger.Debug("the queue info query reply:", zap.Any("qqueryReply", qqueryReply), zap.String("pagetoken", pagetoken)) + //convert to readable string. + queryResult := processQueueData(accessRow, pagetoken) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func processQueueData(rows []utils.OrderMap, page_token string) utils.QueryResult { + for _, row := range rows { + if v, exist := row.Data["queue_status"]; exist && v != nil { + stateVal, ok := utils.IntOrInt64(v) + if !ok { + stateVal = -1 + } + if stateVal >= 0 { + row.Data["queue_status"] = formatQueueStatus(int64(stateVal)) + } else { + row.Data["queue_status"] = "UNKNOWN" + } + } + + // Process JSONB fields based on predefined list + for _, jsonbField := range models.JSONBQFields { + if v, exist := row.Data[jsonbField]; exist { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data[jsonbField] = jsonbData + } else { + logger.Error("Failed to parse queue info JSONB field.", zap.Error(err)) + } + } + } + } + + //utils.FormatRowsOutput(row.Data) + } + return utils.QueryResult{ + Rows: rows, + Count: len(rows), + Pagetoken: page_token, + } +} + +func (q QueueController) GetQueueByName(ctx *gin.Context) { + logger.Debug("Enter GetQueueByName() function from ", zap.String("client_ip", ctx.ClientIP())) + + // get logon user + token := ctx.GetHeader("Authorization") + logonUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid specific queue info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + queueNamesStr := ctx.Param("queuename") + fieldsStr := ctx.Query("fields") + + logger.Debug("origin query string queuename, fieldsStr values are:", + zap.String("queuename", queueNamesStr), + zap.String("fields", fieldsStr)) + + //add jobid list as filter. + queueNames := strings.Split(queueNamesStr, ",") // queue_name: q1,q2,q3 + mergeQqueryReply := make([]utils.OrderMap, 0) + var page_token = "" + + for _, queuename := range queueNames { + queuename = strings.TrimSpace(queuename) + // check rbac + queuePermission, rbacErr := GetRequireResourcePermission(logonUser, "queue:view") + + if rbacErr != nil { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if queuePermission.DomainType == models.DomainTypeQueue { + queueDomainValues := queuePermission.DomainValues + if !utils.ContainsField(queueDomainValues, queuename) { + err := errors.New("access forbidden for queue: " + queuename) + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + } + + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + filters = append(filters, utils.FilterCondition{ + Field: "queue_name", + Operator: "eq", + Value: queuename, + }) + + //filtercds := make([]utils.Condition, len(filters)) + for _, filter := range filters { + // check the filter is supported. + if !utils.ContainsField(models.QUEUE_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + var queue models.QueueInfo + var err error + filtercds, err := utils.ConvertToNewFilter(filters, &queue) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + var fields []string + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.QueueInfo{}) + logger.Debug("queue supportedFields: %v", zap.Any("supportedFields", supportedFields)) + + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // if user not specify queue_name in fields, add queue_name as default field. + if field == "queue_name" { + found = true + } + } + + if !found { + logger.Debug("user doesn't specify queue_name in query fields, add it.") + fields = append(fields, "queue_name") + } + } else { // default display columns + fields = models.QUEUE_DEFAULT_COLUMN + } + + // add a default order: sort=queue_name:asc + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "queue_name", + Ascending: true, + }) + /* + qqueryNew := utils.QueryOptions{ + Conditions: filtercds, + Orders: sortOrders, + Limits: utils.PER_PAGE_LIMIT, + Bookmark: "", // for one queue, no need page_token + } + */ + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, utils.PER_PAGE_LIMIT, "") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + + qqueryReply, page_token, err := services.QueryData(services.TableQueueInfo, queryOpts, fields) + if err != nil { + logger.Error("Failed to query the specific queue info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_JOBINFO_MSG+": "+err.Error(), nil) + return + } + logger.Debug("the specific queue query reply:", zap.Any("qqueryReply", qqueryReply), zap.String("pagetoken", page_token)) + mergeQqueryReply = append(mergeQqueryReply, qqueryReply...) + } // end for queue_name + logger.Debug("the specific queue query reply:", zap.Any("qqueryReply", mergeQqueryReply)) + + //convert job_status to readable string. + queryResult := processQueueData(mergeQqueryReply, page_token) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func (q *QueueController) ControlQueue(ctx *gin.Context) { + logger.Debug("Enter ControlQueue() function") + + // get the logon user + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + var req models.QueueControlRequest + + // use strict json format check + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + err = preCheckQueueControlRequest(&req) + if err != nil { + logger.Error("Failed to send queue control request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + baseAction := req.ControlAction + logger.Debug("ControlQueue baseAction is ", zap.Any("baseAction", baseAction)) + + var actionType int + actionType, exists := models.ActionMap[baseAction] + if !exists { + logger.Error("Unsupported action type", zap.String("baseAction", baseAction)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, "Unsupported action type: "+baseAction, nil) + return + } + + logger.Debug("ControlQueue actionType is ", zap.Any("actionType", actionType)) + + successQueues := []map[string]interface{}{} + failedQueues := []map[string]interface{}{} + + validatedSuccessQueues, validatedFailedQueues := validateQueue(actionType, req.QueueName) + + if allFlag { + queueNames := store.GetAllQueueNames() + validatedSuccessQueues, validatedFailedQueues = validateQueue(actionType, queueNames) + } + logger.Debug("validatedSuccessQueues:", zap.Any("validatedSuccessQueues", validatedSuccessQueues)) + for _, queuename := range validatedSuccessQueues { + // check rbac + queuePermission, err := GetRequireResourcePermission(submitUser, "queue:control") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + if queuePermission.DomainType == models.DomainTypeQueue { + queueDomainValues := queuePermission.DomainValues + if !utils.ContainsField(queueDomainValues, queuename) { + failedQueues = append(failedQueues, map[string]interface{}{ + "queue_name": queuename, + "msg": "queue is not allowed to be controlled", + }) + continue + } + } + successQueues = append(successQueues, map[string]interface{}{ + "queue_name": queuename, + "msg": "queue is being controlled", + }) + } + for queuename, reason := range validatedFailedQueues { + failedQueues = append(failedQueues, map[string]interface{}{ + "queue_name": queuename, + "msg": reason, + }) + } + + for _, queue := range successQueues { + queuename := queue["queue_name"].(string) + err = backend.ControlQueue(queuename, actionType) + if err != nil { + logger.Error("Failed to send queue control", zap.Error(err)) + failedQueues = append(failedQueues, map[string]interface{}{ + "queue_name": queuename, + "msg": "failed to control queue: " + err.Error(), + }) + } + } + response, successMsg := BuildBatchOperationResponse(baseAction, "Queue", successQueues, failedQueues) + + overallStatus := response["status"].(string) + logger.Info("Finished processing queue control request", + zap.String("action", baseAction), + zap.String("overallStatus", overallStatus), + zap.Int("succeededCount", len(successQueues)), + zap.Int("failedCount", len(failedQueues)), + zap.String("client_ip", ctx.ClientIP())) + + ReturnSuccess(ctx, common.ACCEPTED, successMsg, response) + return + +} + +func preCheckQueueControlRequest(req *models.QueueControlRequest) error { + logger.Debug("Enter preCheckQueueControlRequest") + + if err := utils.ValidateControlAction(req.ControlAction, models.SupportedQueueActions); err != nil { + return err + } + req.ControlAction = strings.ToLower(strings.TrimSpace(req.ControlAction)) + if req.QueueName != nil && len(req.QueueName) == 1 && req.QueueName[0] == "all" { + allFlag = true + } else { + allFlag = false + } + + if req.Comment != "" { + if len(req.Comment) > 512 { + return errors.New("comment length exceeds the limit of 512 characters") + } + } + return nil +} + +func validateQueue(control int, queuenames []string) (successQueues []string, failedQueues map[string]string) { + logger.Debug("Validating queuenames: %v", zap.Any("queuenames", queuenames)) + + successQueues = []string{} + failedQueues = make(map[string]string) + + if len(queuenames) == 0 { + logger.Error("queuenames cannot be empty") + return + } + + for _, queuename := range queuenames { + queuename = strings.TrimSpace(queuename) + + if queuename == "" { + logger.Error("queuenames cannot be empty") + continue + } + + if store.Queues.Has(queuename) { + logger.Debug("Found queuename in store", zap.String("queuename", queuename)) + successQueues = append(successQueues, queuename) + } else { + logger.Error("Invalid queuename", zap.String("queuename", queuename)) + failedQueues[queuename] = common.ERROR_QUEUE_NOT_FOUND_MSG + } + } + return +} + +func isQueueControlAllowed(controlType int, queueStateStr string) bool { + parts := strings.Split(queueStateStr, ":") + mainState := parts[0] + secondState := parts[1] + + switch controlType { + case models.QUEUE_CLOSE: + return mainState == "open" + case models.QUEUE_OPEN: + return mainState == "closed" + case models.QUEUE_ACT: + return secondState == "inactive_adm" + case models.QUEUE_INACT: + return secondState != "inactive_adm" + default: + logger.Error("Unsupported control type", zap.Int("controlType", controlType)) + return false + } +} + +func formatQueueStatus(state int64) string { + var status string + + if state&(int64(models.QUEUESTAT["CLOSED"])) != 0 { // CLOSED + status = "closed" + } else { + status = "open" + } + + if state&int64(models.QUEUESTAT["INACTIVE_ADM"]) != 0 { // INACT_WINDOW + status += ":inactive_adm" + } else if state&int64(models.QUEUESTAT["INACTIVE_WIN"]) != 0 { // INACT_ADMIN + status += ":inactive_win" + } else { + status += ":active" + } + + return status +} diff --git a/src/frontend/apiserver/controllers/rbac_check.go b/src/frontend/apiserver/controllers/rbac_check.go new file mode 100644 index 00000000..120983b4 --- /dev/null +++ b/src/frontend/apiserver/controllers/rbac_check.go @@ -0,0 +1,416 @@ +package controllers + +import ( + "apiserver/common" + "apiserver/models" + "apiserver/store" + "apiserver/utils" + logger "apiserver/utils" + "errors" + "os" + "sync" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "gopkg.in/yaml.v2" +) + +type RBACConfig struct { + Permissions []string `yaml:"permissions"` + Roles map[string]Role `yaml:"roles"` + Users map[string]User `yaml:"users"` +} + +// define Role +type Role struct { + Includes []string `yaml:"includes,omitempty"` + Permissions []Permission `yaml:"permissions"` +} +type Permission struct { + Action string `yaml:"action"` + Domain []Domain `yaml:"domain"` +} + +// define User +type User struct { + Roles []UserRole `yaml:"roles"` +} +type UserRole struct { + Name string `yaml:"name"` + Domain []Domain `yaml:"domain"` +} +type Domain struct { + Type string `yaml:"type"` + Values []string `yaml:"values,omitempty"` +} + +var ( + rbacConfig RBACConfig + once sync.Once +) + +func LoadRBACConfig(path string) error { + logger.Debug("Enter LoadRBACConfig() function") + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := yaml.Unmarshal(data, &rbacConfig); err != nil { + return err + } + + // Populate store.Roles from parsed config + for roleName, role := range rbacConfig.Roles { + var perms []store.RolePermission + for _, perm := range role.Permissions { + for _, domain := range perm.Domain { + perms = append(perms, store.RolePermission{ + Action: perm.Action, + DomainType: domain.Type, + DomainValues: domain.Values, + }) + } + } + store.AddRoleNames(roleName) + if len(perms) > 0 { + store.AddPermissionsForRole(roleName, perms) + } + } + + // Resolve role inheritance (includes) + resolveRoleIncludes() + + // Load user-role mappings from users section + for userName, user := range rbacConfig.Users { + for _, role := range user.Roles { + store.AddRoleForUser(userName, role.Name) + } + } + + return nil +} + +func resolveRoleIncludes() { + for roleName, rbacRole := range rbacConfig.Roles { + if len(rbacRole.Includes) == 0 { + continue + } + inherited := collectRolePermissions( + UserRole{Name: roleName}, + rbacConfig.Roles, + make(map[string]struct{}), + ) + for _, perm := range inherited { + for _, domain := range perm.Domain { + store.AddPermissionsForRole(roleName, []store.RolePermission{ + {Action: perm.Action, DomainType: domain.Type, DomainValues: domain.Values}, + }) + } + } + } +} + +// validate token and check permission +func RequirePermission(perm string) gin.HandlerFunc { + return func(ctx *gin.Context) { + logger.Debug("Enter RequirePermission() function", zap.String("perm", perm)) + token := ctx.GetHeader("Authorization") + logonUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + logger.Debug("Logon user is ", zap.String("logonUser", logonUser)) + // check if user has cluster_admin role + userRoles := store.GetUserRoleNames(logonUser) + if utils.ContainsField(userRoles, "cluster_admin") { + return + } + permsWithType := GetUserAllPermissionsWithType(logonUser) + logger.Debug("In RequirePermission after call GetUserAllPermissionsWithType from cache", + zap.Any("permsWithType", permsWithType)) + var allPerms []string + for _, perm := range permsWithType { + allPerms = append(allPerms, perm.Permission) + } + found := false + for _, p := range allPerms { + if p == perm { + found = true + ctx.Next() + return + } + } + if !found { + logger.Error("Permission denied", zap.String("perm", perm), zap.String("logonUser", logonUser)) + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + ctx.Abort() + return + } + } +} + +// get all permissions from rbac.yaml file +func GetAllPermissions() []string { + permissions := rbacConfig.Permissions + return permissions +} + +// get role permissions +func GetBuiltinRolePermissions(roleName string) []Permission { + role, ok := rbacConfig.Roles[roleName] + if !ok { + return []Permission{} + } + return role.Permissions +} + +// get all user permission, +// if user not exist in store, use basic_user permission +func GetUserAllPermissionsWithType(userName string) []models.RolePermission { + userRoles := store.GetUserRoleNames(userName) + if len(userRoles) == 0 { + // case 1: user not in store, use basic_user + builtInPerms := GetBuiltinRolePermissions("basic_user") + permissionTransform := make([]models.RolePermission, 0) + for _, p := range builtInPerms { + for _, domain := range p.Domain { + rp := models.RolePermission{ + Permission: p.Action, + DomainType: domain.Type, + DomainValues: domain.Values, + } + permissionTransform = append(permissionTransform, rp) + } + } + logger.Debug("case 1, assign basic_user permission to unregistered user") + return permissionTransform + } + + builtInRoles := utils.IncludeBuiltinRoles(userRoles) + var permissionListsType []models.RolePermission + if len(builtInRoles) > 0 { + permissionTransform := make([]models.RolePermission, 0) + for _, role := range builtInRoles { + builtinPerms := GetBuiltinRolePermissions(role) + for _, p := range builtinPerms { + for _, domain := range p.Domain { + rp := models.RolePermission{ + Permission: p.Action, + DomainType: domain.Type, + DomainValues: domain.Values, + } + permissionTransform = append(permissionTransform, rp) + } + } + } + permissionListsType = append(permissionListsType, permissionTransform...) + logger.Debug("case 2, builtin role permissions", zap.Any("permissionListsType", permissionListsType)) + } + + for _, userRole := range userRoles { + permissionListsType = append(permissionListsType, GetRoleAllPermissionsWithType(userRole)...) + } + logger.Debug("case 3, before remove duplicated permissions", zap.Any("permissionListsType", permissionListsType)) + permissionListsType = DeduplicatePermissions(permissionListsType) + logger.Debug("case 4 DeduplicatePermissions", zap.Any("permissionListsType", permissionListsType)) + return permissionListsType +} + +func GetRoleAllPermissionsWithType(roleName string) []models.RolePermission { + var permissionsType []models.RolePermission + perms := store.GetRolePermissions(roleName) + for _, perm := range perms { + permissionsType = append(permissionsType, models.RolePermission{ + Permission: perm.Action, + DomainType: perm.DomainType, + DomainValues: perm.DomainValues, + }) + } + return permissionsType +} + +// DeduplicatePermissions removes duplicate permissions based on the specified rules: +// - If permission name and domain type are the same, keep only one +// - If domain values contain "all", keep the one without "all" when both exist +// DeduplicatePermissions removes duplicate permissions based on the specified rules: +// - If permission names are the same, keep only one +// - If one has specific domain values and another has "all" domain type, keep the specific one +func DeduplicatePermissions(permissions []models.RolePermission) []models.RolePermission { + permissionMap := make(map[string]models.RolePermission) // key: permission_name + for _, perm := range permissions { + if existingPerm, exists := permissionMap[perm.Permission]; exists { + // Check if one has more specific domain than the other + existingHasAll := (existingPerm.DomainType == models.DomainTypeAll) + currentHasAll := (perm.DomainType == models.DomainTypeAll) + + // If existing has "all" and current has specific domain, replace it + if existingHasAll && !currentHasAll { + permissionMap[perm.Permission] = perm + } else if !existingHasAll && currentHasAll { + // Keep existing specific domain, discard current "all" + // No action needed + } else if existingHasAll && currentHasAll { + // Both have "all", keep existing + // No action needed + } else { + // Both have specific domains, check if current has values and existing doesn't + existingHasValues := len(existingPerm.DomainValues) > 0 + currentHasValues := len(perm.DomainValues) > 0 + + if !existingHasValues && currentHasValues { + permissionMap[perm.Permission] = perm + } else if existingHasValues && !currentHasValues { + // Keep existing with values + // No action needed + } else { + // Both have values or both don't have values, merge or keep existing + mergedValues := mergeDomainValues(existingPerm.DomainValues, perm.DomainValues) + permissionMap[perm.Permission] = models.RolePermission{ + Permission: existingPerm.Permission, + DomainType: existingPerm.DomainType, + DomainValues: mergedValues, + } + } + } + } else { + permissionMap[perm.Permission] = perm + } + } + + // Convert map back to slice + result := make([]models.RolePermission, 0, len(permissionMap)) + for _, perm := range permissionMap { + result = append(result, perm) + } + + return result +} + +// Helper function to check if domain values contain "all" +func hasAllDomain(domainValues []string) bool { + for _, value := range domainValues { + if value == models.DomainTypeAll { + return true + } + } + return false +} + +func mergeDomainValues(values1, values2 []string) []string { + // Use a map to track unique values + valueMap := make(map[string]bool) + + // Add all values from first slice + for _, v := range values1 { + valueMap[v] = true + } + + // Add all values from second slice + for _, v := range values2 { + valueMap[v] = true + } + + // Convert back to slice + merged := make([]string, 0, len(valueMap)) + for v := range valueMap { + merged = append(merged, v) + } + + return merged +} + +func GetRequireResourcePermission(logonUser string, permId string) (models.RolePermission, error) { + logger.Debug("Enter GetRequireResourcePermission() function", + zap.Any("logonUser", logonUser), zap.Any("permId", permId)) + availablePerm := models.RolePermission{} + + perms := GetUserAllPermissionsWithType(logonUser) + logger.Debug("In GetRequireResourcePermission function user permissions are:", + zap.Any("perms", perms)) + if len(perms) == 0 { + return availablePerm, errors.New("no permission") + } + + for _, perm := range perms { + if perm.Permission != permId { + continue + } + availablePerm.Permission = perm.Permission + availablePerm.DomainType = perm.DomainType + availablePerm.DomainValues = perm.DomainValues + return availablePerm, nil + } + + return availablePerm, errors.New("no permission") +} +func collectRolePermissions(userRole UserRole, rbacRoles map[string]Role, visited map[string]struct{}) []Permission { + var perms []Permission + var collect func(roleName string, userDomains []Domain) + collect = func(roleName string, userDomains []Domain) { + if _, ok := visited[roleName]; ok { + return + } + visited[roleName] = struct{}{} + role, ok := rbacRoles[roleName] + if !ok { + return + } + for _, perm := range role.Permissions { + mergedDomains := mergeDomains(perm.Domain, userDomains) + if mergedDomains == nil { + mergedDomains = []Domain{{Type: "all"}} // default to all if no + } + perms = append(perms, Permission{ + Action: perm.Action, + Domain: mergedDomains, + }) + } + for _, inc := range role.Includes { + collect(inc, userDomains) + } + } + collect(userRole.Name, userRole.Domain) + return perms +} +func mergeDomains(roleDomains, userDomains []Domain) []Domain { + domainMap := make(map[string]map[string]struct{}) + if len(userDomains) == 1 && userDomains[0].Type == "all" { + return roleDomains + } + if len(roleDomains) == 1 && roleDomains[0].Type == "all" { + return userDomains + } + + for _, d := range roleDomains { + if d.Type == "all" { + continue + } + if _, ok := domainMap[d.Type]; !ok { + domainMap[d.Type] = make(map[string]struct{}) + } + for _, v := range d.Values { + domainMap[d.Type][v] = struct{}{} + } + } + for _, d := range userDomains { + if d.Type == "all" { + continue + } + if _, ok := domainMap[d.Type]; !ok { + domainMap[d.Type] = make(map[string]struct{}) + } + for _, v := range d.Values { + domainMap[d.Type][v] = struct{}{} + } + } + var merged []Domain + for t, vals := range domainMap { + var values []string + for v := range vals { + values = append(values, v) + } + merged = append(merged, Domain{Type: t, Values: values}) + } + return merged +} diff --git a/src/frontend/apiserver/controllers/rbac_check_test.go b/src/frontend/apiserver/controllers/rbac_check_test.go new file mode 100644 index 00000000..79980a61 --- /dev/null +++ b/src/frontend/apiserver/controllers/rbac_check_test.go @@ -0,0 +1,219 @@ +package controllers + +import ( + "apiserver/store" + "apiserver/utils" + "os" + "path/filepath" + "testing" + + "go.uber.org/zap" +) + +func resetRBACState() { + rbacConfig = RBACConfig{} + store.InitStores() + utils.Log = zap.NewNop() +} + +func writeTestYAML(t *testing.T, content string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "rbac.yaml") + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("write test yaml: %v", err) + } + return path +} + +func TestLoadRBACConfig_RolesPopulated(t *testing.T) { + resetRBACState() + + yaml := ` +permissions: + - queue:view + - job:view +roles: + basic_user: + permissions: + - action: queue:view + domain: + - type: all + - action: job:view + domain: + - type: self + operator: + permissions: + - action: host:view + domain: + - type: all +` + path := writeTestYAML(t, yaml) + if err := LoadRBACConfig(path); err != nil { + t.Fatalf("LoadRBACConfig: %v", err) + } + + // Verify roles exist in store + if !store.Roles.Has("basic_user") { + t.Error("store should contain basic_user role") + } + if !store.Roles.Has("operator") { + t.Error("store should contain operator role") + } + + // Verify basic_user permissions + perms := store.GetRolePermissions("basic_user") + if len(perms) != 2 { + t.Errorf("basic_user permissions count = %d, want 2", len(perms)) + } + found := false + for _, p := range perms { + if p.Action == "queue:view" && p.DomainType == "all" { + found = true + } + } + if !found { + t.Error("basic_user should have queue:view with domain all") + } +} + +func TestLoadRBACConfig_UserRoleMappings(t *testing.T) { + resetRBACState() + + yaml := ` +permissions: + - queue:view +roles: + basic_user: + permissions: + - action: queue:view + domain: + - type: all +users: + alice: + roles: + - name: basic_user + domain: + - type: all + bob: + roles: + - name: basic_user + domain: + - type: all +` + path := writeTestYAML(t, yaml) + if err := LoadRBACConfig(path); err != nil { + t.Fatalf("LoadRBACConfig: %v", err) + } + + roles := store.GetUserRoleNames("alice") + if len(roles) != 1 || roles[0] != "basic_user" { + t.Errorf("alice roles = %v, want [basic_user]", roles) + } + + roles = store.GetUserRoleNames("bob") + if len(roles) != 1 || roles[0] != "basic_user" { + t.Errorf("bob roles = %v, want [basic_user]", roles) + } + + roles = store.GetUserRoleNames("unknown") + if len(roles) != 0 { + t.Errorf("unknown user roles = %v, want empty", roles) + } +} + +func TestLoadRBACConfig_RoleIncludes(t *testing.T) { + resetRBACState() + + yaml := ` +permissions: + - queue:view + - job:view + - queue:control +roles: + basic_user: + permissions: + - action: queue:view + domain: + - type: all + - action: job:view + domain: + - type: self + queue_admin: + includes: + - basic_user + permissions: + - action: queue:control + domain: + - type: self +` + path := writeTestYAML(t, yaml) + if err := LoadRBACConfig(path); err != nil { + t.Fatalf("LoadRBACConfig: %v", err) + } + + // queue_admin should have its own permissions + inherited basic_user permissions + perms := store.GetRolePermissions("queue_admin") + if len(perms) < 3 { + t.Errorf("queue_admin permissions count = %d, want >= 3 (own + inherited)", len(perms)) + } + + // Verify inherited queue:view is present + foundQueueView := false + foundQueueControl := false + for _, p := range perms { + if p.Action == "queue:view" { + foundQueueView = true + } + if p.Action == "queue:control" { + foundQueueControl = true + } + } + if !foundQueueView { + t.Error("queue_admin should inherit queue:view from basic_user") + } + if !foundQueueControl { + t.Error("queue_admin should have its own queue:control") + } +} + +func TestLoadRBACConfig_EmptyRoles(t *testing.T) { + resetRBACState() + + yaml := ` +permissions: + - queue:view +roles: + empty_role: {} +` + path := writeTestYAML(t, yaml) + if err := LoadRBACConfig(path); err != nil { + t.Fatalf("LoadRBACConfig: %v", err) + } + + if !store.Roles.Has("empty_role") { + t.Error("store should contain empty_role even with no permissions") + } + perms := store.GetRolePermissions("empty_role") + if len(perms) != 0 { + t.Errorf("empty_role permissions = %v, want empty", perms) + } +} + +func TestLoadRBACConfig_FileNotFound(t *testing.T) { + resetRBACState() + + err := LoadRBACConfig("/nonexistent/path/rbac.yaml") + if err == nil { + t.Error("expected error for nonexistent file") + } +} + +// TestNoMockRBACMaps ensures the hardcoded mock group/queue maps have been removed. +// This is a compile-time check — if the vars exist, this test won't compile. +func TestNoMockRBACMaps(t *testing.T) { + // If hostGroupMap, userGroupMap, or queueMap still exist as package-level vars, + // the build will fail with "declared and not used" or similar. + // This test documents the intent: those vars must not exist. + t.Log("mock RBAC maps removed — OK") +} diff --git a/src/frontend/apiserver/controllers/rbac_convention_test.go b/src/frontend/apiserver/controllers/rbac_convention_test.go new file mode 100644 index 00000000..2f29d201 --- /dev/null +++ b/src/frontend/apiserver/controllers/rbac_convention_test.go @@ -0,0 +1,108 @@ +package controllers + +import ( + "testing" + + "apiserver/models" + "apiserver/store" +) + +// TestGetRequireResourcePermission_ReturnsPermThenError verifies the Go-standard +// (T, error) return order: permission first, error second. +func TestGetRequireResourcePermission_GoConvention(t *testing.T) { + store.InitStores() + // unknown user — should return error + perm, err := GetRequireResourcePermission("nobody", "job:view") + if err == nil { + t.Error("expected error for user with no roles") + } + if perm.Permission != "" { + t.Errorf("expected empty permission on error, got %+v", perm) + } +} + +func TestGetRequireResourcePermission_FoundPermission(t *testing.T) { + store.InitStores() + store.AddRoleForUser("alice", "myrole") + store.AddPermissionsForRole("myrole", []store.RolePermission{ + {Action: "job:view", DomainType: "all"}, + }) + + perm, err := GetRequireResourcePermission("alice", "job:view") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if perm.Permission != "job:view" { + t.Errorf("expected job:view, got %q", perm.Permission) + } + if perm.DomainType != "all" { + t.Errorf("expected all, got %q", perm.DomainType) + } +} + +// TestDomainTypeConstants ensures domain type constants match expected string values. +// This guards against accidental constant value changes. +func TestDomainTypeConstants(t *testing.T) { + cases := map[string]string{ + "DomainTypeAll": models.DomainTypeAll, + "DomainTypeSelf": models.DomainTypeSelf, + "DomainTypeSelfUG": models.DomainTypeSelfUG, + "DomainTypeQueue": models.DomainTypeQueue, + } + expected := map[string]string{ + "DomainTypeAll": "all", + "DomainTypeSelf": "self", + "DomainTypeSelfUG": "self_ug", + "DomainTypeQueue": "queue", + } + for name, got := range cases { + if got != expected[name] { + t.Errorf("%s = %q, want %q", name, got, expected[name]) + } + } +} + +// TestGetAccessQueues_AllDomain returns all queue names for "all" domain +func TestGetAccessQueues_AllDomain(t *testing.T) { + store.InitStores() + store.Queues.Put("q1", models.QueueInfo{QueueName: "q1"}) + store.Queues.Put("q2", models.QueueInfo{QueueName: "q2"}) + store.AddRoleForUser("alice", "r1") + store.AddPermissionsForRole("r1", []store.RolePermission{ + {Action: "queue:view", DomainType: "all"}, + }) + queues, err := getAccessQueues("alice", "queue:view") + if err != nil { + t.Fatal(err) + } + if len(queues) != 2 { + t.Errorf("expected 2 queues, got %d", len(queues)) + } +} + +// TestGetAccessQueues_QueueDomain returns only permitted queues +func TestGetAccessQueues_QueueDomain(t *testing.T) { + store.InitStores() + store.Queues.Put("normal", models.QueueInfo{QueueName: "normal"}) + store.Queues.Put("priority", models.QueueInfo{QueueName: "priority"}) + store.AddRoleForUser("bob", "r2") + store.AddPermissionsForRole("r2", []store.RolePermission{ + {Action: "queue:view", DomainType: "queue", DomainValues: []string{"normal"}}, + }) + queues, err := getAccessQueues("bob", "queue:view") + if err != nil { + t.Fatal(err) + } + if len(queues) != 1 || queues[0] != "normal" { + t.Errorf("expected [normal], got %v", queues) + } +} + +// TestGetAccessQueues_NoPermission returns error +func TestGetAccessQueues_NoPermission(t *testing.T) { + store.InitStores() + _, err := getAccessQueues("nobody", "queue:view") + if err == nil { + t.Error("expected error for user with no permission") + } +} diff --git a/src/frontend/apiserver/controllers/resources.go b/src/frontend/apiserver/controllers/resources.go new file mode 100644 index 00000000..6c3d50a0 --- /dev/null +++ b/src/frontend/apiserver/controllers/resources.go @@ -0,0 +1,388 @@ +package controllers + +import ( + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/utils" + logger "apiserver/utils" + "errors" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type ResourceController struct{} + +func (c ResourceController) GetResourcesDistributionInfo(ctx *gin.Context) { + logger.Debug("Enter GetResourcesDistributionInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) && !utils.ContainsField(utils.TypeQueryParams, param) { + err := errors.New("invalid resources distribution info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + typeStr := ctx.Query("type") // type=static/dynamic + limitStr := ctx.Query("limit") // eg: limit=100 + filterStr := ctx.Query("filter") + sortStr := ctx.Query("sort") // sort=resource:asc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + + logger.Debug("GetResourcesDistributionInfo query values are:", + zap.String("typeStr", typeStr), + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + // ---- 1. check type ---- + typeValue, err := utils.ParseType(typeStr, models.ResourcesAvailableTypes) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + logger.Debug("GetResourcesDistributionInfo typeValue is ", zap.String("typeValue", typeValue)) + + // ---- 2. check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- 3. check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + if typeValue == "static" { + filters = append(filters, utils.FilterCondition{ + Field: "is_dynamic", + Operator: "eq", + Value: "false", + }) + } + if typeValue == "dynamic" { + filters = append(filters, utils.FilterCondition{ + Field: "is_dynamic", + Operator: "eq", + Value: "true", + }) + } + + for _, filter := range filters { + if !utils.ContainsField(models.RESOURCE_DISTRIBUTION_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + var resources models.ResourceDistributionInfo + filtercds, err := utils.ConvertToNewFilter(filters, &resources) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 4. check sort ---- + if sortStr == "" { + logger.Debug("sort is not specified, set sort=res_name:asc as default") + sortStr = "res_name:asc" + } + sorts, err := utils.ParseSortParam(sortStr, models.RESOURCE_DISTRIBUTION_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 5. handle fields ---- + fields := make([]string, len(models.RESOURCE_DISTRIBUTION_DEFAULT_COLUMN)) + copy(fields, models.RESOURCE_DISTRIBUTION_DEFAULT_COLUMN) + // ---- 6. check paging --- + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + found := false + for _, sortent := range sorts { + if sortent.Column == "res_name" { + found = true + break + } + } + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "res_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_RESINFO_MSG+": "+err.Error(), nil) + return + } + + rqueryReply, pagetoken, err := services.QueryData(services.TableResourceLoc, queryOpts, fields) + if err != nil { + logger.Error("Failed to query resource distribution info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_RESINFO_MSG+": "+err.Error(), nil) + return + } + + logger.Debug("the resource distribution info query reply:", zap.Any("rqueryReply", rqueryReply), + zap.String("pagetoken", pagetoken)) + //convert to readable string. + queryResult := processResourceDistData(rqueryReply, pagetoken) + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func processResourceDistData(rows []utils.OrderMap, page_token string) utils.QueryResult { + for _, row := range rows { + // Process reported field as JSONB (populated by backend notifications) + if v, exist := row.Data["reported"]; exist && v != nil { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data["reported"] = jsonbData + } else { + logger.Error("Failed to parse reported JSONB field.", zap.Error(err)) + } + } + // Non-string values (e.g. float from builtin data) are left as-is + } + + // Convert database fields back to user-recognizable fields + for userField, dbField := range models.ResourceFieldMappings { + if v, exist := row.Data[dbField]; exist && v != nil { + row.Data[userField] = v + // Replace the value in row.Fields with userField + for i, field := range row.Fields { + if field == dbField { + row.Fields[i] = userField + break + } + } + // Remove the user field from the data map + delete(row.Data, dbField) + } + } + } + + return utils.QueryResult{ + Rows: rows, + Count: len(rows), + Pagetoken: page_token, + } +} + +func (c ResourceController) GetResourcesDefinitionInfo(ctx *gin.Context) { + logger.Debug("Enter GetResourcesDefinitionInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid resources definition info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + limitStr := ctx.Query("limit") // eg: limit=100 + filterStr := ctx.Query("filter") + sortStr := ctx.Query("sort") // sort=resource:asc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + + logger.Debug("GetResourcesDefinitionInfo query values are:", + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + // ---- 1. check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- 2. check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + for _, filter := range filters { + if !utils.ContainsField(models.RESOURCE_DEFINITION_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + var resources models.ResourceDefinitionInfo + filtercds, err := utils.ConvertToNewFilter(filters, &resources) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 3. check sort ---- + if sortStr == "" { + logger.Debug("sort is not specified, set sort=res_name:asc as default") + sortStr = "res_name:asc" + } + sorts, err := utils.ParseSortParam(sortStr, models.RESOURCE_DISTRIBUTION_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 5. handle fields ---- + fields := make([]string, len(models.RESOURCE_DEFINITION_DEFAULT_COLUMN)) + copy(fields, models.RESOURCE_DEFINITION_DEFAULT_COLUMN) + // ---- 6. check paging --- + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + found := false + for _, sortent := range sorts { + if sortent.Column == "res_name" { + found = true + break + } + } + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "res_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_RESINFO_MSG+": "+err.Error(), nil) + return + } + + rqueryReply, pagetoken, err := services.QueryData(services.TableResourceDef, queryOpts, fields) + if err != nil { + logger.Error("Failed to query resource definition info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_RESINFO_MSG+": "+err.Error(), nil) + return + } + + logger.Debug("the resource definition info query reply:", zap.Any("rqueryReply", rqueryReply), + zap.String("pagetoken", pagetoken)) + //convert to readable string. + queryResult := processResourceDefData(rqueryReply, pagetoken) + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func processResourceDefData(rows []utils.OrderMap, page_token string) utils.QueryResult { + for _, row := range rows { + if v, exist := row.Data["value_type"]; exist && v != nil { + row.Data["value_type"] = toValueTypeString(v) + } + if v, exist := row.Data["order_type"]; exist && v != nil { + row.Data["order_type"] = toOrderTypeString(v) + } + if v, exist := row.Data["flag"]; exist && v != nil { + row.Data["flag"] = toFlagJSON(v) + } + } + + return utils.QueryResult{ + Rows: rows, + Count: len(rows), + Pagetoken: page_token, + } +} + +func toValueTypeString(v interface{}) string { + val, ok := utils.IntOrInt64(v) + if !ok { + return "" + } + return models.ValueType(val).String() +} + +func toOrderTypeString(v interface{}) string { + val, ok := utils.IntOrInt64(v) + if !ok { + return "" + } + return models.OrderType(val).String() +} + +func toFlagJSON(v interface{}) map[string]bool { + val, ok := utils.IntOrInt64(v) + if !ok { + return nil + } + return models.FlagValue(val).ToJSON() +} diff --git a/src/frontend/apiserver/controllers/response.go b/src/frontend/apiserver/controllers/response.go new file mode 100644 index 00000000..a05d579f --- /dev/null +++ b/src/frontend/apiserver/controllers/response.go @@ -0,0 +1,70 @@ +package controllers + +import ( + "fmt" + + "github.com/gin-gonic/gin" +) + +type JsonStruct struct { + Code int `json:"code"` + Msg interface{} `json:"msg"` + Data interface{} `json:"data"` +} + +type JsonErrStruct struct { + Code int `json:"code"` + Msg interface{} `json:"msg"` + Details interface{} `json:"details,omitempty"` +} + +func ReturnSuccess(ctx *gin.Context, code int, msg interface{}, data interface{}) { + json := &JsonStruct{ + Code: code, + Msg: msg, + Data: data, + } + ctx.JSON(200, json) +} + +func ReturnError(ctx *gin.Context, code int, msg interface{}, details interface{}) { + json := &JsonErrStruct{ + Code: code, + Msg: msg, + Details: details, + } + ctx.JSON(code, json) +} + +// BuildBatchOperationResponse creates a structured response map and summary message +func BuildBatchOperationResponse(action string, + resourceType string, + succeededItems []map[string]interface{}, + failedItems []map[string]interface{}, +) (map[string]interface{}, string) { + overallStatus := "" + summaryMsg := "" + totalRequested := len(succeededItems) + len(failedItems) + + if len(failedItems) == 0 && totalRequested > 0 { + overallStatus = "accepted" + summaryMsg = fmt.Sprintf("%s control <%s> accepted for all %d requested %s.", resourceType, action, totalRequested, resourceType) // Making it more generic + } else if len(succeededItems) == 0 && totalRequested > 0 { + overallStatus = "failed" + summaryMsg = fmt.Sprintf("%s control <%s> failed for all %d requested %s.", resourceType, action, totalRequested, resourceType) + } else if totalRequested > 0 { + overallStatus = "partially_accepted" + summaryMsg = fmt.Sprintf("%s control <%s> partially accepted: %d received, %d failed.", resourceType, action, len(succeededItems), len(failedItems)) + } else { // Should not happen if input validation is correct, but handle defensively + overallStatus = "no_items_processed" + summaryMsg = fmt.Sprintf("No %s were provided for the '%s' action.", resourceType, action) + } + + response := map[string]interface{}{ + "status": overallStatus, + "accepted": succeededItems, + "failed": failedItems, + } + + return response, summaryMsg +} diff --git a/src/frontend/apiserver/controllers/roles.go b/src/frontend/apiserver/controllers/roles.go new file mode 100644 index 00000000..62457ec9 --- /dev/null +++ b/src/frontend/apiserver/controllers/roles.go @@ -0,0 +1,727 @@ +package controllers + +import ( + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/store" + logger "apiserver/utils" + utils "apiserver/utils" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type RoleController struct{} + +func (rc *RoleController) GetRolesInfo(ctx *gin.Context) { + logger.Debug("Enter GetRolesInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + + var fields []string + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + uniqueFlag := false + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) && + !utils.ContainsField(utils.TypeQueryParams, param) && + !utils.ContainsField(utils.UniqueQueryParams, param) { + err := errors.New("invalid role info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + typeStr := ctx.Query("type") // type=builtin/all + filterStr := ctx.Query("filter") + fieldsStr := ctx.Query("fields") + sortStr := ctx.Query("sort") // sort=role_name:asc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + limitStr := ctx.Query("limit") // eg: limit=100 + uniqueStr := ctx.Query("unique") + + logger.Debug("Role query values are:", + zap.String("typeStr", typeStr), + zap.String("fieldsStr", fieldsStr), + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + // ---- 1. check type ---- + typeValue, err := utils.ParseType(typeStr, models.RolesAvailableTypes) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + logger.Debug("GetRolesInfo typeValue is ", zap.String("typeValue", typeValue)) + + var rqueryReply []utils.OrderMap + var pagetoken string + if typeValue != "builtin" { + + // ---- 2. check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- 3. check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + for _, filter := range filters { + if !utils.ContainsField(models.ROLE_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } else { + filterStr = filter.Value + } + } + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check rbac + permission, err := GetRequireResourcePermission(submitUser, "rbac:view") + + logger.Debug("get role info for rbac:view permission:", zap.Any("permission", permission)) + if permission.DomainType != models.DomainTypeAll { + // only query self roles list info + userSelfRoles := store.GetUserRoleNames(submitUser) + // if submit_user is not specified, use query request submission user + var findSelfRole bool = false + var reset_filterRole string + for _, filter := range filters { + if filter.Field == "role_name" { + filterRoles := strings.Split(filter.Value, ",") + for _, filterRole := range filterRoles { + filterRole = strings.TrimSpace(filterRole) + if !utils.ContainsField(models.BUILTIN_ROLE_NAMES, filterRole) { + if utils.ContainsField(userSelfRoles, filterRole) { + findSelfRole = true + reset_filterRole = filterRole + "," + } + } else { + findSelfRole = true + } + } + if !findSelfRole { + err := errors.New("no permission") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + } + } + if filterStr == "" { + filters = append(filters, utils.FilterCondition{ + Field: "role_name", + Operator: "in", + Value: strings.Join(userSelfRoles, ","), + }) + } else { + filters = append(filters, utils.FilterCondition{ + Field: "role_name", + Operator: "in", + Value: reset_filterRole, + }) + } + } + logger.Debug("before call ConvertToNewFilter filters:", zap.Any("filters", filters)) + var role models.RoleInfo + filtercds, err := utils.ConvertToNewFilter(filters, &role) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 4. check sort ---- + if sortStr == "" { + logger.Debug("sort is not specified, set sort=role_name:asc as default") + sortStr = "role_name:asc" + } + sorts, err := utils.ParseSortParam(sortStr, models.ROLE_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 5. handle fields ---- + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.RoleFields{}) + logger.Debug("role supportedFields: %v", zap.Any("supportedFields", supportedFields)) + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + // if user not specify role_name in fields, add role_name as default field. + if field == "role_name" { + found = true + } + } + if !found { + logger.Debug("user doesn't specify role name in query fields, add it.") + fields = append(fields, "role_name") + } + } else { + fields = models.ROLE_DEFAULT_COLUMN + } + + // -- 6. check unique value -- + uniqueFlag = utils.ParseUnique(uniqueStr) + if uniqueFlag { + if fieldsStr == "" || fieldsStr != "role_name" { + err := errors.New("when unique=true is specified, fields must be set to role_name") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + limit = 0 + } + // ---- 7. check paging --- + pagetokenStr = strings.TrimSpace(pagetokenStr) + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + found := false + for _, sortent := range sorts { + if sortent.Column == "role_name" { + found = true + break + } + } + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "role_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_RESINFO_MSG+": "+err.Error(), nil) + return + } + + rqueryReply, pagetoken, err = services.QueryData(services.TableRolePermission, queryOpts, fields) + if err != nil { + logger.Error("Failed to query role info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUEUR_ROLEINFO_MSG+": "+err.Error(), nil) + return + } + + logger.Debug("the role info query reply:", zap.Any("rqueryReply", rqueryReply), + zap.String("pagetoken", pagetoken)) + } + //convert to readable string. + var queryResult utils.QueryResult + if typeValue == "builtin" || typeValue == "all" { + filterStr = "all" + } + builtinResult := processBuiltinRoles(make(map[string]utils.OrderMap), fieldsStr, filterStr) + dbResult := processRoleData(rqueryReply, uniqueFlag, pagetoken) + if builtinResult.Count > 0 { + pagetoken = strings.TrimSpace(pagetoken) + if pagetoken != "" { + queryResult.Rows = dbResult.Rows + queryResult.Count = dbResult.Count + queryResult.Pagetoken = dbResult.Pagetoken + } else { + queryResult.Rows = append(builtinResult.Rows, dbResult.Rows...) + queryResult.Count = builtinResult.Count + dbResult.Count + queryResult.Pagetoken = dbResult.Pagetoken + } + } else { + queryResult.Rows = dbResult.Rows + queryResult.Count = dbResult.Count + queryResult.Pagetoken = dbResult.Pagetoken + } + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func processRoleData(rows []utils.OrderMap, unique bool, pageToken string) utils.QueryResult { + // Map to track seen role names for deduplication when field is specified + seenRoleNames := make(map[string]bool) + var processedRows []utils.OrderMap + + for _, row := range rows { + roleName, _ := row.Data["role_name"].(string) + + // Deduplicate role_name when field parameter is not empty + if unique { + if seenRoleNames[roleName] { + continue // Skip duplicate role names + } + seenRoleNames[roleName] = true + + row1 := utils.OrderMap{ + Fields: []string{"role_name"}, + Data: map[string]interface{}{ + "role_name": roleName, + }, + } + row.Data = row1.Data + } + + // Process JSONBR fields (keep existing logic) + for _, jsonbField := range models.JSONBRFields { + if v, exist := row.Data[jsonbField]; exist { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data[jsonbField] = jsonbData + } else { + logger.Error("Failed to parse role info JSONB field.", zap.Error(err)) + } + } + } + } + + processedRows = append(processedRows, row) + } + + return utils.QueryResult{ + Rows: processedRows, + Count: len(processedRows), + Pagetoken: pageToken, + } +} + +func processBuiltinRoles(rolePermissions map[string]utils.OrderMap, field string, filter string) utils.QueryResult { + var resultRows []utils.OrderMap + for _, roleName := range models.BUILTIN_ROLE_NAMES { + if filter == roleName || filter == "all" { + if field != "" { + row1 := utils.OrderMap{ + Fields: []string{"role_name"}, + Data: map[string]interface{}{ + "role_name": roleName, + }, + } + resultRows = append(resultRows, row1) + } else { + userPerm := GetBuiltinRolePermissions(roleName) + // For each permission, create a separate entry + for _, perm := range userPerm { + // For each domain, create a separate entry + for _, domain := range perm.Domain { + row := utils.OrderMap{ + Fields: []string{"role_name", "action", "domain_type", "domain_values"}, + Data: map[string]interface{}{ + "role_name": roleName, + "action": perm.Action, + "domain_type": domain.Type, + "domain_values": domain.Values, + }, + } + resultRows = append(resultRows, row) + } + } + } + } + } + + return utils.QueryResult{ + Rows: resultRows, + Count: len(resultRows), + } +} + +func (rc RoleController) GetRoleByRolename(ctx *gin.Context) { + logger.Debug("Enter GetRoleByRolename() function from ", zap.String("client_ip", ctx.ClientIP())) + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) { + err := errors.New("invalid specific role info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + roleNamesStr := ctx.Param("rolename") + fieldsStr := ctx.Query("fields") + + logger.Debug("origin query string rolename, fieldsStr values are:", + zap.String("rolename", roleNamesStr), + zap.String("fields", fieldsStr)) + + //add role_name list as filter. + roleNames := strings.Split(roleNamesStr, ",") + mergeRqueryReply := make([]utils.OrderMap, 0) + builtinResult := make([]utils.OrderMap, 0) + var page_token = "" + for _, rolename := range roleNames { + rolename = strings.TrimSpace(rolename) + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + filters = append(filters, utils.FilterCondition{ + Field: "role_name", + Operator: "eq", + Value: rolename, + }) + + //filtercds := make([]utils.Condition, len(filters)) + for _, filter := range filters { + // check the filter is supported. + if !utils.ContainsField(models.ROLE_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + var role models.RoleInfo + var err error + filtercds, err := utils.ConvertToNewFilter(filters, &role) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + var fields []string + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.RoleInfo{}) + logger.Debug("role supportedFields: %v", zap.Any("supportedFields", supportedFields)) + + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // if user not specify role_name in fields, add role_name as default field. + if field == "role_name" { + found = true + } + } + + if !found { + logger.Debug("user doesn't specify role_name in query fields, add it.") + fields = append(fields, "role_name") + } + } else { // default display columns + fields = models.ROLE_DEFAULT_COLUMN + } + + // add a default order: sort=role_name:asc + sortOrders = append(sortOrders, utils.SortOrder{ + Column: "role_name", + Ascending: true, + }) + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, utils.PER_PAGE_LIMIT, "") + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUEUR_ROLEINFO_MSG+": "+err.Error(), nil) + return + } + + builtinQueryResult := processBuiltinRoles(make(map[string]utils.OrderMap), fieldsStr, rolename) + builtinResult = builtinQueryResult.Rows + + rqueryReply, page_token, err := services.QueryData(services.TableRolePermission, queryOpts, fields) + if err != nil { + logger.Error("Failed to query the specific role info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUEUR_ROLEINFO_MSG+": "+err.Error(), nil) + return + } + + logger.Debug("specific role query reply:", zap.Any("rqueryReply", rqueryReply), zap.String("pagetoken", page_token)) + mergeRqueryReply = append(mergeRqueryReply, rqueryReply...) + } // end for rolename + logger.Debug("the specific role query reply:", zap.Any("rqueryReply", mergeRqueryReply)) + uniqueFlag := false + queryResult := processRoleData(mergeRqueryReply, uniqueFlag, page_token) + if len(builtinResult) > 0 { + queryResult.Rows = append(builtinResult, queryResult.Rows...) + queryResult.Count = len(builtinResult) + queryResult.Count + } + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} +func (rc *RoleController) AddPermissionsForRole(ctx *gin.Context) { + logger.Debug("Enter AddPermissionsForRole() function from ", zap.String("client_ip", ctx.ClientIP())) + roleId := ctx.Param("roleid") + var req models.CreateRoleRequest + req.RoleName = roleId + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + err := preCheckRoleRequest(&req, false) + if err != nil { + logger.Error("Failed to create role request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + storePerms := make([]store.RolePermission, 0, len(req.RolePermissions)) + for _, perm := range req.RolePermissions { + domainValuesJSON, _ := json.Marshal(perm.DomainValues) + storePerms = append(storePerms, store.RolePermission{ + Action: perm.Permission, + DomainType: perm.DomainType, + DomainValues: perm.DomainValues, + }) + _ = domainValuesJSON // no longer needed for store + } + store.AddPermissionsForRole(req.RoleName, storePerms) + + response := models.RoleCreateResponse{ + RoleName: req.RoleName, + Message: fmt.Sprintf("Role <%s> updated successfully", req.RoleName), + } + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, response) + return + +} + +// detele a role and ensure no user is using it, +// can't delete built-in role, delete the role from role_permission table +func (rc *RoleController) DeleteRoleAndPermissions(ctx *gin.Context) { + logger.Debug("Enter DeleteRoleAndPermissions() function from ", zap.String("client_ip", ctx.ClientIP())) + req_rolename := ctx.Param("rolename") + req_rolename = strings.TrimSpace(req_rolename) + logger.Debug("DeleteRoleAndPermissions() function from ", zap.String("rolename", req_rolename)) + + if req_rolename == "" { + logger.Error("role name can't be empty", zap.String("req_rolename", req_rolename)) + err := fmt.Errorf("role name can't be empty") + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + // step 1: role name whether exists + allRoleNames := store.GetAllRoleNames(false) + found := false + for _, roleName := range allRoleNames { + if roleName == req_rolename { + found = true + break + } + } + if !found { + logger.Error("The role name not found", zap.String("req_rolename", req_rolename)) + err := fmt.Errorf("role name not found: %s", req_rolename) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // step 2: built-in role + if utils.ContainsField(models.BUILTIN_ROLE_NAMES, req_rolename) { + logger.Error("built-in role name can't be deleted", zap.String("req_rolename", req_rolename)) + err := fmt.Errorf("built-in role name <%s> can't be deleted", req_rolename) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // step 3: check whether there is a user configured with rolename + allUsers := store.GetUsersUnderRoleName(req_rolename) + if len(allUsers) > 0 { + logger.Error("the role is still under using by the user", zap.String("req_rolename", req_rolename)) + err := fmt.Errorf("the role is still under using by the user") + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // step 4: delete the role + store.Roles.Delete(req_rolename) + + // step 6: return response + logger.Debug("Role deleted successfully", zap.String("rolename", req_rolename)) + Msg := fmt.Sprintf("role of <%s> deleted successfully", req_rolename) + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, Msg) + return +} +func (rc *RoleController) CreateRole(ctx *gin.Context) { + logger.Debug("Enter CreateRole() function from ", zap.String("client_ip", ctx.ClientIP())) + + var req models.CreateRoleRequest + + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // precheck the request parameter + err := preCheckRoleRequest(&req, true) + if err != nil { + logger.Error("Failed to create role request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + storePerms := make([]store.RolePermission, 0, len(req.RolePermissions)) + for _, perm := range req.RolePermissions { + domainValuesJSON, _ := json.Marshal(perm.DomainValues) + storePerms = append(storePerms, store.RolePermission{ + Action: perm.Permission, + DomainType: perm.DomainType, + DomainValues: perm.DomainValues, + }) + _ = domainValuesJSON // no longer needed for store + } + store.AddRoleNames(req.RoleName) + store.AddPermissionsForRole(req.RoleName, storePerms) + + response := models.RoleCreateResponse{ + RoleName: req.RoleName, + Message: fmt.Sprintf("Role <%s> created successfully", req.RoleName), + } + + ReturnSuccess(ctx, common.CREATED, common.SUCCESS_MSG, response) + return +} + +func preCheckRole(role_name string) error { + + return nil +} +func preCheckRoleRequest(req *models.CreateRoleRequest, newFlag bool) error { + logger.Debug("Enter preCheckRoleRequest") + if req.RoleName != "" { + if newFlag { + allRoleNames := store.GetAllRoleNames(true) + //validate queue_name whether exists + for _, roleName := range allRoleNames { + if roleName == req.RoleName { + logger.Error("The role name has been existed", zap.String("role_name", req.RoleName)) + return fmt.Errorf("duplicated role name: %s", req.RoleName) + } + } + } else { + allRoleNames := store.GetAllRoleNames(false) + found := false + for _, roleName := range allRoleNames { + if roleName == req.RoleName { + found = true + break + } + } + if !found { + logger.Error("The role name not found", zap.String("role_name", req.RoleName)) + return fmt.Errorf("role name not found: %s", req.RoleName) + } + } + } + + if len(req.RolePermissions) == 0 { + logger.Error("Role permissions cannot be empty") + return fmt.Errorf("role permissions cannot be empty") + } + for _, perm := range req.RolePermissions { + logger.Debug("Checking permission", zap.String("permission", perm.Permission)) + if !utils.ContainsField(GetAllPermissions(), perm.Permission) { + logger.Error("Unsupported permission", zap.String("permission", perm.Permission)) + return fmt.Errorf("unsupported permission: %s", perm.Permission) + } + if !newFlag { // add a new permission + dbExistPerms := store.GetRoleAllPermissions(req.RoleName) + if utils.ContainsField(dbExistPerms, perm.Permission) { + logger.Error("The permission has been existed", zap.String("permission", perm.Permission)) + return fmt.Errorf("duplicated permission: %s", perm.Permission) + } + } + if perm.DomainType == "" { + logger.Error("Domain type cannot be empty") + return fmt.Errorf("domain type cannot be empty") + } else { + parts := strings.Split(perm.Permission, ":") + objectPart := parts[0] //permission first section + allowedDomainTypes := models.SupportedDomainTypes[objectPart] + domainType := strings.ToLower(perm.DomainType) + if !utils.ContainsField(allowedDomainTypes, domainType) { + logger.Error("Unsupported domain type for permission object", + zap.String("object_type", objectPart), zap.String("domain_type", domainType)) + return fmt.Errorf("unsupported domain type for permission object %s: %s, and allowed types are %v", objectPart, domainType, allowedDomainTypes) + } + if domainType == models.DomainTypeAll || + domainType == models.DomainTypeSelf || + domainType == models.DomainTypeSelfHG || + domainType == models.DomainTypeSelfUG { + if len(perm.DomainValues) > 0 { + logger.Error("Domain values should be empty for 'all' or 'self' domain type", + zap.String("domain_type", domainType)) + return fmt.Errorf("domain values should be empty for 'all' or 'self' domain type: %s", domainType) + } + } + if domainType == models.DomainTypeUserGroup || + domainType == models.DomainTypeHostGroup || + domainType == models.DomainTypeQueue { + if len(perm.DomainValues) == 0 { + logger.Error("Domain values should not be empty for 'usergroups', 'hostgroups' or 'queues' domain type", + zap.String("domain_type", domainType)) + return fmt.Errorf("domain values should not be empty for 'usergroups', 'hostgroups' or 'queues' domain type: %s", domainType) + } + } + switch domainType { + case models.DomainTypeQueue: + if !store.IsQueueExist(perm.DomainValues) { + logger.Error("Queue does not exist for specified values", zap.Any("queue_name", perm.DomainValues)) + return fmt.Errorf("queue does not exist for specified values: %v", perm.DomainValues) + } + } //end of switch + + } + } + return nil +} diff --git a/src/frontend/apiserver/controllers/users.go b/src/frontend/apiserver/controllers/users.go new file mode 100644 index 00000000..065c7bca --- /dev/null +++ b/src/frontend/apiserver/controllers/users.go @@ -0,0 +1,513 @@ +package controllers + +import ( + "apiserver/common" + "apiserver/models" + "apiserver/services" + "apiserver/store" + "apiserver/utils" + logger "apiserver/utils" + "errors" + "fmt" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +type UserController struct{} + +func (uc *UserController) GetUserPermInfo(ctx *gin.Context) { + logger.Debug("Enter GetUserPermInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + // get user specified query username + username := ctx.Param("username") + username = strings.TrimSpace(username) + err := utils.IsValidUserName(username) + if err != nil { + logger.Error("Invalid username format", zap.String("username", username)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check rbac + permission, _ := GetRequireResourcePermission(submitUser, "rbac:view") + if permission.DomainType != models.DomainTypeAll { + if username != submitUser { + err := errors.New("no permission") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + } + var builtInRoles []string + // Call service and handle potential errors + rolelist, pqueryReply, _ := services.GetUserPermissions(username) + // check whether include built-in roles + if rolelist == nil || len(rolelist) == 0 { + builtInRoles = append(builtInRoles, "basic_user") // add basic_user as + } else { + builtInRoles = utils.IncludeBuiltinRoles(rolelist) + } + logger.Debug("Get user roles successfully", zap.String("username", username), zap.Any("roles", rolelist)) + var builtInPermissions []utils.OrderMap + if len(builtInRoles) > 0 { + for _, builtinrole := range builtInRoles { + userBuiltPerm := GetBuiltinRolePermissions(builtinrole) + // Convert built-in permissions to OrderMap format + for _, perm := range userBuiltPerm { + permMap := utils.OrderMap{ + Fields: []string{"role_name", "action", "domain_type", "domain_values"}, + Data: map[string]interface{}{ + "role_name": builtinrole, + "action": perm.Action, + "domain_type": perm.Domain[0].Type, + "domain_values": perm.Domain[0].Values, + }, + } + builtInPermissions = append(builtInPermissions, permMap) + } + } + } + + var allPermissions []utils.OrderMap + allPermissions = append(allPermissions, pqueryReply...) + allPermissions = append(allPermissions, builtInPermissions...) + if allPermissions == nil { + logger.Error("Failed to get user permissions - empty result") + ReturnError(ctx, common.ERROR_CODE, "Failed to retrieve permissions", nil) + return + } + queryResult := processRoleData(allPermissions, false, "") + // Step 3: Return all permissions + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) +} + +func (uc *UserController) GetCurrentUserInfo(ctx *gin.Context) { + logger.Debug("Enter GetUserRolesInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + if ctx.IsAborted() { + ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + return + } + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, submitUser) +} +func (uc *UserController) GetUserRolesInfo(ctx *gin.Context) { + logger.Debug("Enter GetUserRolesInfo() function from ", zap.String("client_ip", ctx.ClientIP())) + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + var fields []string + filters := make([]utils.FilterCondition, 0) + sortOrders := make([]utils.SortOrder, 0) + uniqueFlag := false + + // ---- check parameters ---- + for param := range ctx.Request.URL.Query() { + if !utils.ContainsField(utils.SupportedQueryParams, param) && + !utils.ContainsField(utils.UniqueQueryParams, param) { + err := errors.New("invalid user_role info query parameters of : " + param) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + // Get query parameters + filterStr := ctx.Query("filter") + fieldsStr := ctx.Query("fields") + sortStr := ctx.Query("sort") // sort=role_name:asc + pagetokenStr := ctx.Query("page_token") // page_token=Aafjakjf0023A + limitStr := ctx.Query("limit") // eg: limit=100 + uniqueStr := ctx.Query("unique") + + logger.Debug("User role query values are:", + zap.String("fieldsStr", fieldsStr), + zap.String("filterStr", filterStr), + zap.String("sortStr", sortStr), + zap.String("limitStr", limitStr), + zap.String("pagetokenStr", pagetokenStr)) + + var uqueryReply []utils.OrderMap + var pagetoken string + + // ---- 1. check limit value ---- + limit, err := utils.ParseLimit(limitStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + // ---- 2. check filter ---- + if filterStr != "" { + filters = utils.ParseFilters(filterStr) + if filters == nil { + err := errors.New("filter " + filterStr + " has syntax error") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + for _, filter := range filters { + if !utils.ContainsField(models.USER_SUPPORT_FILTERS, filter.Field) { + err := errors.New("unsupported filter: " + filter.Field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } else { + filterStr = filter.Value + } + } + + // check rbac + permission, _ := GetRequireResourcePermission(submitUser, "rbac:view") + logger.Debug("get user info for rbac:view permission:", zap.Any("permission", permission)) + var specify_user bool = false + for i, filter := range filters { + if filter.Field == "user_name" { + specify_user = true + if permission.DomainType != models.DomainTypeAll { + if strings.Contains(filter.Value, submitUser) { + filters[i].Value = submitUser + } else { + err := errors.New("no permission") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + } + break + } + } + if (!specify_user && permission.DomainType != models.DomainTypeAll) || + filterStr == "" && permission.DomainType != models.DomainTypeAll { + filters = append(filters, utils.FilterCondition{ + Field: "user_name", + Operator: "eq", + Value: submitUser, + }) + } + + var user models.UserInfo + filtercds, err := utils.ConvertToNewFilter(filters, &user) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 3. check sort ---- + if sortStr == "" { + logger.Debug("sort is not specified, set sort=user_name:asc as default") + sortStr = "user_name:asc" + } + sorts, err := utils.ParseSortParam(sortStr, models.USER_SUPPORT_SORT) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + // ---- 4. handle fields ---- + if fieldsStr != "" { + fields = utils.ParseFields(fieldsStr) + found := false + // verify the fields is in supported list + supportedFields := utils.GetSupportedFields(models.UserInfo{}) + logger.Debug("user supportedFields: %v", zap.Any("supportedFields", supportedFields)) + for _, field := range fields { + if !utils.ContainsField(supportedFields, field) { + err := errors.New("unsupported query field: " + field) + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + // if user not specify user_name in fields, add user_name as default field. + if field == "user_name" { + found = true + } + } + if !found { + logger.Debug("user doesn't specify user name in query fields, add it.") + fields = append(fields, "user_name") + } + } else { + fields = models.USER_DEFAULT_COLUMN + } + + // ---- 5. check unique value --- + uniqueFlag = utils.ParseUnique(uniqueStr) + if uniqueFlag { + if fieldsStr == "" || fieldsStr != "user_name" { + err := errors.New("when unique=true is specified, fields must be set to user_name") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + limit = 0 + } + + // ---- 6. check paging --- + pagetokenStr = strings.TrimSpace(pagetokenStr) + if pagetokenStr != "" { + pagetokenStr, err = utils.ParsePageToken(pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + } + + found := false + for _, sortent := range sorts { + if sortent.Column == "user_name" { + found = true + break + } + } + if !found { + sorts = append(sorts, utils.SortOrder{ + Column: "user_name", + Ascending: true, + }) + } + + //check all sort column is in fields + err = utils.CheckAllMapKeyInList(sorts, fields) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, common.INVALID_PARAMS_MSG+": "+err.Error(), nil) + return + } + + sortOrders = sorts + + queryOpts, err := utils.BuildQueryOptions(filtercds, nil, sortOrders, limit, pagetokenStr) + if err != nil { + logger.Error(err.Error()) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUERY_RESINFO_MSG+": "+err.Error(), nil) + return + } + + uqueryReply, pagetoken, err = services.QueryData(services.TableUserRole, queryOpts, fields) + if err != nil { + logger.Error("Failed to query user role info.", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, common.ERROR_QUEUR_ROLEINFO_MSG+": "+err.Error(), nil) + return + } + + logger.Debug("the user role info query reply:", zap.Any("uqueryReply", uqueryReply), + zap.String("pagetoken", pagetoken)) + + //convert to readable string. + queryResult := processUserData(uqueryReply, uniqueFlag, pagetoken) + + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, queryResult) + +} + +func processUserData(rows []utils.OrderMap, unique bool, page_token string) utils.QueryResult { + seenUserNames := make(map[string]bool) + var processedRows []utils.OrderMap + for _, row := range rows { + userName, _ := row.Data["user_name"].(string) + // Deduplicate role_name when field parameter is not empty + if unique { + if seenUserNames[userName] { + continue // Skip duplicate role names + } + seenUserNames[userName] = true + + row1 := utils.OrderMap{ + Fields: []string{"user_name"}, + Data: map[string]interface{}{ + "user_name": userName, + }, + } + row.Data = row1.Data + } + + // Process JSONB fields based on predefined list + for _, jsonbField := range models.JSONBUFields { + if v, exist := row.Data[jsonbField]; exist { + if strValue, ok := v.(string); ok { + if jsonbData, err := utils.ParseJSONB(strValue); err == nil { + row.Data[jsonbField] = jsonbData + } else { + logger.Error("Failed to parse user role info JSONB field.", zap.Error(err)) + } + } + } + } + processedRows = append(processedRows, row) + } + return utils.QueryResult{ + Rows: processedRows, + Count: len(processedRows), + Pagetoken: page_token, + } +} + +// detele a user's role from user_role table +func (uc *UserController) DeleteRolesForUser(ctx *gin.Context) { + logger.Debug("Enter DeleteRolesForUser() function from ", zap.String("client_ip", ctx.ClientIP())) + username := ctx.Param("username") + rolename := ctx.Param("rolename") + username = strings.TrimSpace(username) + rolename = strings.TrimSpace(rolename) + if username == "" || rolename == "" { + err := errors.New("user_name and role_name must be specified") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + err := utils.IsValidUserName(username) + if err != nil { + logger.Error("Invalid username format", zap.String("username", username)) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + + logger.Debug("DeleteRolesForUser() function from ", zap.String("username", username), + zap.String("rolename", rolename)) + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check rbac + permission, _ := GetRequireResourcePermission(submitUser, "rbac:control") + if permission.DomainType != models.DomainTypeAll { + if username != submitUser { + err := errors.New("no permission") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + } + + // check whether user and role existed + userRoles := store.GetUserRoleNames(username) + if !utils.ContainsField(userRoles, rolename) { + errMsg := fmt.Sprintf("The user <%s> doesn't have the role <%s>", username, rolename) + logger.Error(errMsg, zap.String("user_name", username), zap.String("role_name", rolename)) + ReturnError(ctx, common.ERROR_CODE, errMsg, nil) + return + } + + store.DeleteRoleForUser(username, rolename) + logger.Debug("Role deleted successfully", zap.String("username", username), zap.String("rolename", rolename)) + Msg := fmt.Sprintf("user <%s> of role <%s> deleted successfully", username, rolename) + ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, Msg) +} +func (uc *UserController) AddRolesForUser(ctx *gin.Context) { + logger.Debug("Enter AddRolesForUser() function from ", zap.String("client_ip", ctx.ClientIP())) + username := ctx.Param("username") + var req models.AddRoleForUserRequest + req.UserName = username + if err := utils.BindJSONStrict(ctx, &req); err != nil { + logger.Error("Invalid request parameters", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + err := preCheckUserRequest(&req) + if err != nil { + logger.Error("Failed to create role request", zap.Error(err)) + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check token + token := ctx.GetHeader("Authorization") + submitUser, err := GetUsernameFromToken(token) + if err != nil { + ReturnError(ctx, common.ERROR_CODE, err.Error(), nil) + return + } + + // check rbac + permission, _ := GetRequireResourcePermission(submitUser, "rbac:control") + if permission.DomainType != models.DomainTypeAll { + if req.UserName != submitUser { + err := errors.New("no permission") + logger.Error(err.Error()) + ReturnError(ctx, common.INVALID_PARAMS_CODE, err.Error(), nil) + return + } + } + + roleNames := make([]string, 0, len(req.RoleName)) + for _, userRole := range req.RoleName { + userRole = strings.TrimSpace(userRole) + store.AddRoleForUser(req.UserName, userRole) + roleNames = append(roleNames, userRole) + } + + response := models.AddRoleForUserResponse{ + UserName: req.UserName, + Message: fmt.Sprintf("User <%s> has been assigned roles <%s> successfully", req.UserName, strings.Join(roleNames, ", ")), + } + + ReturnSuccess(ctx, common.CREATED, common.SUCCESS_MSG, response) + return +} + +func preCheckUserRequest(req *models.AddRoleForUserRequest) error { + // validate username whether is validate + req.UserName = strings.TrimSpace(req.UserName) + err := utils.IsValidUserName(req.UserName) + if err != nil { + return fmt.Errorf("Invalid user name: %s", req.UserName) + } + // Check each role in the UserRoles slice + for _, userRole := range req.RoleName { + userRole := strings.TrimSpace(userRole) + if userRole != "" { + allRoleNames := store.GetAllRoleNames(true) + found := false + for _, roleName := range allRoleNames { + if roleName == userRole { + found = true + break + } + } + if !found { + logger.Error("The role name does not exist", zap.String("role_name", userRole)) + return fmt.Errorf("role name does not exist: %s", userRole) + } + + userRoles := store.GetUserRoleNames(req.UserName) + if utils.ContainsField(userRoles, userRole) { + logger.Error("The user already has the role", zap.String("user_name", req.UserName), zap.String("role_name", userRole)) + return fmt.Errorf("user already has the role: %s", userRole) + } + } + } + return nil +} diff --git a/src/frontend/apiserver/docs/docs.go b/src/frontend/apiserver/docs/docs.go new file mode 100644 index 00000000..1a793dd1 --- /dev/null +++ b/src/frontend/apiserver/docs/docs.go @@ -0,0 +1,344 @@ +// Package docs Code generated by swaggo/swag. DO NOT EDIT +package docs + +import "github.com/swaggo/swag" + +const docTemplate = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": {}, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/api/v1/hosts": { + "get": { + "description": "Get basic or full host information", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "hosts" + ], + "summary": "Get host information", + "parameters": [ + { + "type": "string", + "description": "Host type (basic or full)", + "name": "type", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "Response for full host type", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/models.FullHost" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/models.Error" + } + } + } + } + } + }, + "definitions": { + "models.BasicHost": { + "type": "object", + "properties": { + "DResources": { + "type": "array", + "items": { + "type": "string" + } + }, + "busyThreshold": { + "type": "array", + "items": { + "type": "number" + } + }, + "cores": { + "type": "integer" + }, + "cores_per_proc": { + "type": "integer" + }, + "cpuFactor": { + "type": "number" + }, + "hostAddr": { + "type": "string" + }, + "hostModel": { + "type": "string" + }, + "hostName": { + "type": "string" + }, + "hostType": { + "type": "string" + }, + "isServer": { + "type": "boolean" + }, + "maxCpus": { + "type": "integer" + }, + "maxMem": { + "type": "integer" + }, + "maxSwap": { + "type": "integer" + }, + "maxTmp": { + "type": "integer" + }, + "nDRes": { + "type": "integer" + }, + "nDisks": { + "type": "integer" + }, + "nRes": { + "type": "integer" + }, + "numIndx": { + "type": "integer" + }, + "pprocs": { + "type": "integer" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "threads_per_core": { + "type": "integer" + }, + "windows": { + "type": "string" + } + } + }, + "models.Error": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + }, + "models.FullHost": { + "type": "object", + "properties": { + "DResources": { + "type": "array", + "items": { + "type": "string" + } + }, + "affinityCpulist": { + "type": "string" + }, + "attr": { + "type": "integer" + }, + "busySched": { + "type": "number" + }, + "busyStop": { + "type": "number" + }, + "busyThreshold": { + "type": "array", + "items": { + "type": "number" + } + }, + "chkSig": { + "type": "integer" + }, + "cnsmrAvail": { + "type": "number" + }, + "cnsmrUsage": { + "type": "number" + }, + "cores": { + "type": "integer" + }, + "cores_per_proc": { + "type": "integer" + }, + "cpuFactor": { + "type": "number" + }, + "cycleTimeLeft": { + "type": "integer" + }, + "hCtrlMsg": { + "type": "integer" + }, + "hStatus": { + "type": "string" + }, + "hostAddr": { + "type": "string" + }, + "hostModel": { + "type": "string" + }, + "hostName": { + "type": "string" + }, + "hostType": { + "type": "string" + }, + "idleTime": { + "type": "integer" + }, + "isServer": { + "type": "boolean" + }, + "load": { + "type": "array", + "items": { + "type": "number" + } + }, + "loadSched": { + "type": "array", + "items": { + "type": "number" + } + }, + "loadStop": { + "type": "array", + "items": { + "type": "number" + } + }, + "maxAvail": { + "type": "number" + }, + "maxCpus": { + "type": "integer" + }, + "maxExitRate": { + "type": "integer" + }, + "maxJobs": { + "type": "integer" + }, + "maxMem": { + "type": "integer" + }, + "maxSwap": { + "type": "integer" + }, + "maxTmp": { + "type": "integer" + }, + "mig": { + "type": "integer" + }, + "nDRes": { + "type": "integer" + }, + "nDisks": { + "type": "integer" + }, + "nIdx": { + "type": "integer" + }, + "nRes": { + "type": "integer" + }, + "numExitRate": { + "type": "integer" + }, + "numIndx": { + "type": "integer" + }, + "numJobs": { + "type": "integer" + }, + "numRESERVE": { + "type": "integer" + }, + "numRUN": { + "type": "integer" + }, + "numSSUSP": { + "type": "integer" + }, + "numUSUSP": { + "type": "integer" + }, + "pprocs": { + "type": "integer" + }, + "prvdrAvail": { + "type": "number" + }, + "prvdrUsage": { + "type": "number" + }, + "realLoad": { + "type": "number" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "threads_per_core": { + "type": "integer" + }, + "userJobLimit": { + "type": "integer" + }, + "windows": { + "type": "string" + } + } + } + } +}` + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = &swag.Spec{ + Version: "1.0", + Host: "localhost:8443", + BasePath: "/", + Schemes: []string{}, + Title: "Host Management API", + Description: "API Server", + InfoInstanceName: "swagger", + SwaggerTemplate: docTemplate, + LeftDelim: "{{", + RightDelim: "}}", +} + +func init() { + swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) +} diff --git a/src/frontend/apiserver/docs/swagger.json b/src/frontend/apiserver/docs/swagger.json new file mode 100644 index 00000000..034e8e78 --- /dev/null +++ b/src/frontend/apiserver/docs/swagger.json @@ -0,0 +1,320 @@ +{ + "swagger": "2.0", + "info": { + "description": "API Server", + "title": "Host Management API", + "contact": {}, + "version": "1.0" + }, + "host": "localhost:8443", + "basePath": "/", + "paths": { + "/api/v1/hosts": { + "get": { + "description": "Get basic or full host information", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "hosts" + ], + "summary": "Get host information", + "parameters": [ + { + "type": "string", + "description": "Host type (basic or full)", + "name": "type", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "Response for full host type", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/models.FullHost" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/models.Error" + } + } + } + } + } + }, + "definitions": { + "models.BasicHost": { + "type": "object", + "properties": { + "DResources": { + "type": "array", + "items": { + "type": "string" + } + }, + "busyThreshold": { + "type": "array", + "items": { + "type": "number" + } + }, + "cores": { + "type": "integer" + }, + "cores_per_proc": { + "type": "integer" + }, + "cpuFactor": { + "type": "number" + }, + "hostAddr": { + "type": "string" + }, + "hostModel": { + "type": "string" + }, + "hostName": { + "type": "string" + }, + "hostType": { + "type": "string" + }, + "isServer": { + "type": "boolean" + }, + "maxCpus": { + "type": "integer" + }, + "maxMem": { + "type": "integer" + }, + "maxSwap": { + "type": "integer" + }, + "maxTmp": { + "type": "integer" + }, + "nDRes": { + "type": "integer" + }, + "nDisks": { + "type": "integer" + }, + "nRes": { + "type": "integer" + }, + "numIndx": { + "type": "integer" + }, + "pprocs": { + "type": "integer" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "threads_per_core": { + "type": "integer" + }, + "windows": { + "type": "string" + } + } + }, + "models.Error": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + }, + "models.FullHost": { + "type": "object", + "properties": { + "DResources": { + "type": "array", + "items": { + "type": "string" + } + }, + "affinityCpulist": { + "type": "string" + }, + "attr": { + "type": "integer" + }, + "busySched": { + "type": "number" + }, + "busyStop": { + "type": "number" + }, + "busyThreshold": { + "type": "array", + "items": { + "type": "number" + } + }, + "chkSig": { + "type": "integer" + }, + "cnsmrAvail": { + "type": "number" + }, + "cnsmrUsage": { + "type": "number" + }, + "cores": { + "type": "integer" + }, + "cores_per_proc": { + "type": "integer" + }, + "cpuFactor": { + "type": "number" + }, + "cycleTimeLeft": { + "type": "integer" + }, + "hCtrlMsg": { + "type": "integer" + }, + "hStatus": { + "type": "string" + }, + "hostAddr": { + "type": "string" + }, + "hostModel": { + "type": "string" + }, + "hostName": { + "type": "string" + }, + "hostType": { + "type": "string" + }, + "idleTime": { + "type": "integer" + }, + "isServer": { + "type": "boolean" + }, + "load": { + "type": "array", + "items": { + "type": "number" + } + }, + "loadSched": { + "type": "array", + "items": { + "type": "number" + } + }, + "loadStop": { + "type": "array", + "items": { + "type": "number" + } + }, + "maxAvail": { + "type": "number" + }, + "maxCpus": { + "type": "integer" + }, + "maxExitRate": { + "type": "integer" + }, + "maxJobs": { + "type": "integer" + }, + "maxMem": { + "type": "integer" + }, + "maxSwap": { + "type": "integer" + }, + "maxTmp": { + "type": "integer" + }, + "mig": { + "type": "integer" + }, + "nDRes": { + "type": "integer" + }, + "nDisks": { + "type": "integer" + }, + "nIdx": { + "type": "integer" + }, + "nRes": { + "type": "integer" + }, + "numExitRate": { + "type": "integer" + }, + "numIndx": { + "type": "integer" + }, + "numJobs": { + "type": "integer" + }, + "numRESERVE": { + "type": "integer" + }, + "numRUN": { + "type": "integer" + }, + "numSSUSP": { + "type": "integer" + }, + "numUSUSP": { + "type": "integer" + }, + "pprocs": { + "type": "integer" + }, + "prvdrAvail": { + "type": "number" + }, + "prvdrUsage": { + "type": "number" + }, + "realLoad": { + "type": "number" + }, + "resources": { + "type": "array", + "items": { + "type": "string" + } + }, + "threads_per_core": { + "type": "integer" + }, + "userJobLimit": { + "type": "integer" + }, + "windows": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/src/frontend/apiserver/docs/swagger.yaml b/src/frontend/apiserver/docs/swagger.yaml new file mode 100644 index 00000000..70596bd2 --- /dev/null +++ b/src/frontend/apiserver/docs/swagger.yaml @@ -0,0 +1,212 @@ +basePath: / +definitions: + models.BasicHost: + properties: + DResources: + items: + type: string + type: array + busyThreshold: + items: + type: number + type: array + cores: + type: integer + cores_per_proc: + type: integer + cpuFactor: + type: number + hostAddr: + type: string + hostModel: + type: string + hostName: + type: string + hostType: + type: string + isServer: + type: boolean + maxCpus: + type: integer + maxMem: + type: integer + maxSwap: + type: integer + maxTmp: + type: integer + nDRes: + type: integer + nDisks: + type: integer + nRes: + type: integer + numIndx: + type: integer + pprocs: + type: integer + resources: + items: + type: string + type: array + threads_per_core: + type: integer + windows: + type: string + type: object + models.Error: + properties: + error: + type: string + type: object + models.FullHost: + properties: + DResources: + items: + type: string + type: array + affinityCpulist: + type: string + attr: + type: integer + busySched: + type: number + busyStop: + type: number + busyThreshold: + items: + type: number + type: array + chkSig: + type: integer + cnsmrAvail: + type: number + cnsmrUsage: + type: number + cores: + type: integer + cores_per_proc: + type: integer + cpuFactor: + type: number + cycleTimeLeft: + type: integer + hCtrlMsg: + type: integer + hStatus: + type: string + hostAddr: + type: string + hostModel: + type: string + hostName: + type: string + hostType: + type: string + idleTime: + type: integer + isServer: + type: boolean + load: + items: + type: number + type: array + loadSched: + items: + type: number + type: array + loadStop: + items: + type: number + type: array + maxAvail: + type: number + maxCpus: + type: integer + maxExitRate: + type: integer + maxJobs: + type: integer + maxMem: + type: integer + maxSwap: + type: integer + maxTmp: + type: integer + mig: + type: integer + nDRes: + type: integer + nDisks: + type: integer + nIdx: + type: integer + nRes: + type: integer + numExitRate: + type: integer + numIndx: + type: integer + numJobs: + type: integer + numRESERVE: + type: integer + numRUN: + type: integer + numSSUSP: + type: integer + numUSUSP: + type: integer + pprocs: + type: integer + prvdrAvail: + type: number + prvdrUsage: + type: number + realLoad: + type: number + resources: + items: + type: string + type: array + threads_per_core: + type: integer + userJobLimit: + type: integer + windows: + type: string + type: object +host: localhost:8443 +info: + contact: {} + description: API Server + title: Host Management API + version: "1.0" +paths: + /api/v1/hosts: + get: + consumes: + - application/json + description: Get basic or full host information + parameters: + - description: Host type (basic or full) + in: query + name: type + required: true + type: string + produces: + - application/json + responses: + "200": + description: Response for full host type + schema: + items: + $ref: '#/definitions/models.FullHost' + type: array + "400": + description: Bad Request + schema: + $ref: '#/definitions/models.Error' + summary: Get host information + tags: + - hosts +swagger: "2.0" diff --git a/src/frontend/apiserver/go.mod b/src/frontend/apiserver/go.mod new file mode 100644 index 00000000..02d7c882 --- /dev/null +++ b/src/frontend/apiserver/go.mod @@ -0,0 +1,75 @@ +module apiserver + +go 1.23.4 + +require ( + github.com/AlecAivazis/survey/v2 v2.3.7 + github.com/gin-gonic/gin v1.10.0 + github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/google/uuid v1.6.0 + github.com/spf13/viper v1.19.0 + github.com/swaggo/files v1.0.1 + github.com/swaggo/gin-swagger v1.6.0 + github.com/swaggo/swag v1.16.4 + go.uber.org/zap v1.27.0 + golang.org/x/time v0.9.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/term v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + golang.org/x/tools v0.13.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/src/frontend/apiserver/go.sum b/src/frontend/apiserver/go.sum new file mode 100644 index 00000000..4d397635 --- /dev/null +++ b/src/frontend/apiserver/go.sum @@ -0,0 +1,237 @@ +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= +github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= +github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= +github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= +github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= +github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= +github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/src/frontend/apiserver/meson.build b/src/frontend/apiserver/meson.build new file mode 100644 index 00000000..75d36986 --- /dev/null +++ b/src/frontend/apiserver/meson.build @@ -0,0 +1,64 @@ +# frontend/apiserver — Go REST API server with CGO backend integration. +# +# CGO is required for two reasons: +# 1. pam_linux.go: dlopen/dlsym for PAM (handled by -ldl in the source file) +# 2. backend/clib.go: libxce C client library for IPC with the controller +# +# Include/link flags for libxce and its transitive deps are gathered from +# pkg-config and injected into CGO_CFLAGS/CGO_LDFLAGS so the go_build script +# can substitute path placeholders before invoking go build. + +cgo_cflags = '' +cgo_ldflags = '' + +if with_debug + cgo_cflags += '-fsanitize=address -g -DWITH_DEBUG=1 ' + cgo_ldflags += '-fsanitize=address -g ' +endif + +# Deps whose cflags/ldflags must be passed into CGO (transitive libxce deps). +apiserver_cgo_deps = [ + librtti_dep, + c_capnproto_dep, + ssl_dep, + tinylog_dep, + math_dep, + rt_dep, +] + +foreach dep : apiserver_cgo_deps + cf = run_command(pkgconfig, '--cflags', dep.name(), check: false) + if cf.returncode() == 0 + cgo_cflags += ' ' + cf.stdout().strip() + endif + lf = run_command(pkgconfig, '--libs', dep.name(), check: false) + if lf.returncode() == 0 + cgo_ldflags += ' ' + lf.stdout().strip() + endif +endforeach + +# Add the source and build-directory include paths with meson placeholders. +# The go_build script resolves @CURRENT_SOURCE_DIR@ and @BUILD_ROOT@ at build time. +cgo_cflags += ' -I@CURRENT_SOURCE_DIR@/../../include -I@BUILD_ROOT@/lib' +cgo_ldflags += ' -L@BUILD_ROOT@/lib -lxce' + +go_env = environment() +go_env.set('CGO_CFLAGS', cgo_cflags) +go_env.set('CGO_LDFLAGS', cgo_ldflags) +go_env.set('BUILDTYPE', get_option('buildtype')) +go_env.set('FIX_ENVARS', 'CGO_CFLAGS CGO_LDFLAGS') +go_env.set('ENVARS', 'CURRENT_SOURCE_DIR BUILD_ROOT') +go_env.set('DEPENDENCIES','thread math') + +custom_target('apiserver', + input: 'cmd/main.go', + output: 'xce_apiserver', + command: [env_program, + 'CURRENT_SOURCE_DIR="@CURRENT_SOURCE_DIR@"', + 'BUILD_ROOT="@BUILD_ROOT@"', + go_build_program, + '@INPUT@', '@OUTPUT@'], + env: go_env, + depends: [libxce], + install: true, + install_dir: get_option('prefix') / 'bin') diff --git a/src/frontend/apiserver/middleware/README b/src/frontend/apiserver/middleware/README new file mode 100644 index 00000000..20884202 --- /dev/null +++ b/src/frontend/apiserver/middleware/README @@ -0,0 +1 @@ +go test -v load_test.go diff --git a/src/frontend/apiserver/middleware/cors.go b/src/frontend/apiserver/middleware/cors.go new file mode 100644 index 00000000..4971cdf2 --- /dev/null +++ b/src/frontend/apiserver/middleware/cors.go @@ -0,0 +1,58 @@ +package middleware + +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" +) + +// CORSMiddleware returns a middleware that sets CORS headers. +// allowedOrigins: list of allowed origins; pass nil to disable CORS entirely. +// Pass []string{"*"} to allow all origins. +func CORSMiddleware(allowedOrigins []string) gin.HandlerFunc { + return func(c *gin.Context) { + if len(allowedOrigins) == 0 { + c.Next() + return + } + + origin := c.Request.Header.Get("Origin") + allowed := "" + for _, o := range allowedOrigins { + if o == "*" || o == origin { + allowed = o + break + } + } + + if allowed != "" { + c.Header("Access-Control-Allow-Origin", allowed) + c.Header("Access-Control-Allow-Methods", "GET, POST, DELETE, OPTIONS") + c.Header("Access-Control-Allow-Headers", "Origin, Content-Type, Authorization") + c.Header("Access-Control-Max-Age", "86400") + } + + if c.Request.Method == http.MethodOptions { + c.AbortWithStatus(http.StatusNoContent) + return + } + + c.Next() + } +} + +// CORSOriginsFromConfig builds the allowed origins list from a config string slice. +// Returns nil if CORS is disabled (empty slice). +func CORSOriginsFromConfig(enabled bool, origins []string) []string { + if !enabled || len(origins) == 0 { + return nil + } + result := make([]string, 0, len(origins)) + for _, o := range origins { + if s := strings.TrimSpace(o); s != "" { + result = append(result, s) + } + } + return result +} diff --git a/src/frontend/apiserver/middleware/cors_test.go b/src/frontend/apiserver/middleware/cors_test.go new file mode 100644 index 00000000..7f05d927 --- /dev/null +++ b/src/frontend/apiserver/middleware/cors_test.go @@ -0,0 +1,60 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" +) + +func TestCORSMiddleware_SetsHeaders(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(CORSMiddleware([]string{"*"})) + r.GET("/ping", func(c *gin.Context) { c.Status(200) }) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/ping", nil) + req.Header.Set("Origin", "http://example.com") + r.ServeHTTP(w, req) + + if w.Header().Get("Access-Control-Allow-Origin") == "" { + t.Error("expected Access-Control-Allow-Origin header") + } +} + +func TestCORSMiddleware_Preflight(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(CORSMiddleware([]string{"http://example.com"})) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("OPTIONS", "/xce/v1/jobs", nil) + req.Header.Set("Origin", "http://example.com") + req.Header.Set("Access-Control-Request-Method", "POST") + r.ServeHTTP(w, req) + + if w.Code != http.StatusNoContent { + t.Errorf("preflight: got %d want 204", w.Code) + } + if w.Header().Get("Access-Control-Allow-Methods") == "" { + t.Error("expected Access-Control-Allow-Methods header") + } +} + +func TestCORSMiddleware_Disabled(t *testing.T) { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(CORSMiddleware(nil)) // nil = disabled + r.GET("/ping", func(c *gin.Context) { c.Status(200) }) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/ping", nil) + req.Header.Set("Origin", "http://example.com") + r.ServeHTTP(w, req) + + if w.Header().Get("Access-Control-Allow-Origin") != "" { + t.Error("CORS headers must not be set when disabled") + } +} diff --git a/src/frontend/apiserver/middleware/load_test.go b/src/frontend/apiserver/middleware/load_test.go new file mode 100644 index 00000000..c43f08ee --- /dev/null +++ b/src/frontend/apiserver/middleware/load_test.go @@ -0,0 +1,111 @@ +package middleware + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync" + "testing" + "time" +) + +type AuthResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Token string `json:"token"` + Path string `json:"path"` + IP string `json:"ip"` + } `json:"data"` +} + +// login +func login(username, password string) (string, error) { + const loginURL = "http://localhost:8080/xce/v1/auth/logon" //replace with your login url + + // create login request json data + loginData := map[string]string{ + "username": username, + "password": password, + } + jsonData, err := json.Marshal(loginData) + if err != nil { + return "", err + } + + // send post request + resp, err := http.Post(loginURL, "application/json", bytes.NewBuffer(jsonData)) + if err != nil { + return "", err + } + defer resp.Body.Close() + + // check response status + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("login failed with status: %s", resp.Status) + } + + // parse token from response + var authResponse AuthResponse + if err := json.NewDecoder(resp.Body).Decode(&authResponse); err != nil { + return "", err + } + + // check if code is 200 + if authResponse.Code != 200 { + return "", fmt.Errorf("login failed: %s", authResponse.Msg) + } + + return authResponse.Data.Token, nil +} + +// TestLoadJobs test function +func TestLoadJobs(t *testing.T) { + const numRequests = 100 + const username = "user01" // replace with your username + const password = "Welcome@1212" // replace with your password + fmt.Println("before login") + // login to get token + token, err := login(username, password) + if err != nil { + t.Fatalf("Failed to log in: %v", err) + } + fmt.Println("Obtained Bearer Token:", token) + + var wg sync.WaitGroup + wg.Add(numRequests) + + startTime := time.Now() + + for i := 0; i < numRequests; i++ { + go func() { + defer wg.Done() + req, err := http.NewRequest("GET", "http://localhost:8080/xce/v1/jobs", nil) // replace with your actual url + if err != nil { + t.Error("Error creating request:", err) + return + } + + // set bearer token + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Error("Error:", err) + return + } + defer resp.Body.Close() + + // optional: check response status + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status OK, got %v", resp.Status) + } + }() + } + + wg.Wait() // wait for all requests to complete + + elapsedTime := time.Since(startTime) + fmt.Printf("Completed %d requests in %s\n", numRequests, elapsedTime) +} diff --git a/src/frontend/apiserver/middleware/rate_limit.go b/src/frontend/apiserver/middleware/rate_limit.go new file mode 100644 index 00000000..f2f72bf0 --- /dev/null +++ b/src/frontend/apiserver/middleware/rate_limit.go @@ -0,0 +1,136 @@ +package middleware + +import ( + "apiserver/common" + "apiserver/controllers" + "apiserver/utils" + logger "apiserver/utils" + "strings" + "sync" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "golang.org/x/time/rate" +) + +var ( + limiters = make(map[string]*rate.Limiter) + limitersMux sync.RWMutex +) + +// RateLimiter struct to hold the limiter +type RateLimiter struct { + limiters map[string]*utils.EndpointLimit + globalLimiter *utils.EndpointLimit +} + +// Initialize RateLimiter with configuration +func NewRateLimiter(cfg *utils.RateLimitConfig) *RateLimiter { + logger.Debug("Enter NewRateLimiter function", zap.Any("cfg", cfg)) + + rl := &RateLimiter{ + limiters: make(map[string]*utils.EndpointLimit), + globalLimiter: &cfg.Global, + } + + for path, endpoint := range cfg.Endpoints { + rl.limiters[path] = &endpoint + } + + return rl +} + +// Middleware function to enforce rate limiting +func (r *RateLimiter) Limit() gin.HandlerFunc { + logger.Debug("Enter Limit() function") + return func(ctx *gin.Context) { + path := ctx.FullPath() + limiter := getRateLimiter(path, ctx) + if limiter == nil { + ctx.Next() + return + } + + if !limiter.Allow() { + logger.Error("Rate limit exceeded", zap.String("path", path), zap.String("client_ip", ctx.ClientIP()), zap.Int("errorcode", common.ERROR_TOO_MANY_REQUESTS_CODE)) + controllers.ReturnError(ctx, common.ERROR_TOO_MANY_REQUESTS_CODE, common.ERROR_TOO_MANY_REQUESTS_MSG, nil) + ctx.Abort() + return + } + ctx.Next() + } +} + +// getRateLimiter returns a rate limiter for the given endpoint +func getRateLimiter(endpoint string, ctx *gin.Context) *rate.Limiter { + // get submit user from token + token := ctx.GetHeader("Authorization") + if token != "" { + submitUser, err := controllers.GetUsernameFromToken(strings.TrimPrefix(token, "Bearer ")) + if err != nil { + logger.Error("Failed to get submit user from token", zap.Any("err", err)) + } + logger.Debug("Enter getRateLimiter() function", zap.Any("endpoint", endpoint), zap.Any("client_ip", ctx.ClientIP()), zap.Any("submitUser", submitUser)) + } else { + logger.Debug("Enter getRateLimiter() function", zap.Any("endpoint", endpoint), zap.Any("client_ip", ctx.ClientIP())) + } + + limitersMux.RLock() + limiter, exists := limiters[endpoint] + limitersMux.RUnlock() + if exists { + return limiter + } + + limitersMux.Lock() + defer limitersMux.Unlock() + + // Double check after acquiring write lock + if limiter, exists = limiters[endpoint]; exists { + return limiter + } + + // Get rate limit configuration + config := utils.RateLimitYamlConfig + if config == nil { + return nil + } + + var limit utils.EndpointLimit + if endpointLimit, ok := config.Endpoints[endpoint]; ok { + limit = endpointLimit + } else { + // Check for wildcard matches + matchFlag := false + for path, endpointLimit := range config.Endpoints { + // Remove trailing slash for comparison + cleanedEndpoint := endpoint + cleanedPath := path + // Compare cleaned paths + if cleanedPath == "*" || (len(cleanedPath) > 1 && string(cleanedPath[len(cleanedPath)-1]) == "*" && cleanedEndpoint == string(cleanedPath[:len(cleanedPath)-2])) { + limit = endpointLimit + matchFlag = true + break + } + } + if !matchFlag { + limit = config.Global + } + } + + // Calculate rate based on window + var r rate.Limit + switch limit.Window { + case "minute": + r = rate.Limit(float64(limit.Rate) / 60.0) + case "second": + r = rate.Limit(limit.Rate) + default: + r = rate.Limit(limit.Rate) + } + logger.Debug("double check rate limit setting for endpoint", zap.Any("endpoint", endpoint), zap.Any("r", r), + zap.Any("burst", limit.Burst)) + limiter = rate.NewLimiter(r, limit.Burst) + limiters[endpoint] = limiter + return limiter +} diff --git a/src/frontend/apiserver/middleware/startserver.go b/src/frontend/apiserver/middleware/startserver.go new file mode 100644 index 00000000..9c47f61f --- /dev/null +++ b/src/frontend/apiserver/middleware/startserver.go @@ -0,0 +1,121 @@ +package middleware + +import ( + "apiserver/utils" + logger "apiserver/utils" + "fmt" + "net" + "os" + "os/user" + "path/filepath" + "runtime" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout server.key -out server.crt -config openssl.cnf -extensions v3_req +func StartServer(r *gin.Engine) error { + logger.Debug("Enter StartServer() function") + + if runtime.GOOS != "windows" { + currentUser, err := user.Current() + if err != nil { + logger.Error("Failed to get current user:", zap.Error(err)) + os.Exit(1) + } + if currentUser.Uid != "0" { + logger.Warn("Not running as root — PAM authentication will not work. OK for dev/testing.") + } + } + + ips, err := GetIPAddresses() + if err != nil { + logger.Error("Failed to get server IP address:", zap.Error(err)) + os.Exit(1) + } + ip := ips[0] + logger.Debug("API Server IP address:", zap.String("ip", ip)) + + httpsConfig := utils.HttpsYamlConfig //get only once + if httpsConfig.Enabled { + addr := ":" + httpsConfig.Port + logger.Info("Starting HTTPS server on " + addr) + + fmt.Println("[GIN-debug] The API server will be run on https://" + ip + addr) + if err := r.RunTLS( + addr, + filepath.Join(utils.Global_TopDir, httpsConfig.CertFile), + filepath.Join(utils.Global_TopDir, httpsConfig.KeyFile), + ); err != nil { + logger.Error("HTTPS Server fails to start:", zap.Error(err)) + os.Exit(1) + } + } else { + addr := ":" + utils.AppYamlConfig.HTTPPort + logger.Info("Starting HTTP server on " + addr) + + fmt.Println("[GIN-debug] The API server will be run on http://" + ip + addr) + if err := r.Run(addr); err != nil { + logger.Error("API Server fails to start:", zap.Error(err)) + os.Exit(1) + } + } + return nil +} + +func getServerIP() (string, error) { + // get the server IP address + addrs, err := net.InterfaceAddrs() + if err != nil { + logger.Error("Failed to get server IP address:", zap.Error(err)) + return "", err + } + + for _, addr := range addrs { + if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() != nil { + return ipnet.IP.String(), nil // real ip address + } + } + return "", nil +} + +func GetIPAddresses() ([]string, error) { + var ips []string + + interfaces, err := net.Interfaces() + if err != nil { + return nil, err + } + + for _, iface := range interfaces { + if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 { + // Skip down or loopback interfaces + continue + } + + addrs, err := iface.Addrs() + if err != nil { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + // Skip IPv6 link-local or loopback addresses + if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() { + continue + } + + ips = append(ips, ip.String()) + } + } + + return ips, nil +} diff --git a/src/frontend/apiserver/middleware/test_rate_limit.bat b/src/frontend/apiserver/middleware/test_rate_limit.bat new file mode 100644 index 00000000..658c4bf8 --- /dev/null +++ b/src/frontend/apiserver/middleware/test_rate_limit.bat @@ -0,0 +1,12 @@ +@echo off +echo Testing Rate Limit for /ping +echo ======================== + +for /l %%x in (1, 1, 15) do ( + echo Request %%x: + curl -i http://localhost:8443/ping + echo. + echo ======================== +) + +pause \ No newline at end of file diff --git a/src/frontend/apiserver/middleware/validatehandler.go b/src/frontend/apiserver/middleware/validatehandler.go new file mode 100644 index 00000000..ccdf9629 --- /dev/null +++ b/src/frontend/apiserver/middleware/validatehandler.go @@ -0,0 +1,143 @@ +package middleware + +import ( + "apiserver/common" + "apiserver/controllers" + "apiserver/models" + "apiserver/utils" + logger "apiserver/utils" + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v4" + "go.uber.org/zap" +) + +func ValidateHandler() gin.HandlerFunc { + logger.Debug("Enter ValidateHandler() function") + return func(ctx *gin.Context) { + if isExcludedPath(ctx.Request.URL.Path) { + ctx.Next() + return + } + + validateHeader()(ctx) + if ctx.IsAborted() { + return + } + flag := !strings.Contains("multipart/form-data", ctx.Request.Header.Get("Content-Type")) + if ctx.Request.Method == http.MethodPost && flag == false { + isValidBody, err := validateRequestBody(ctx) + if err != nil || !isValidBody { + controllers.ReturnError(ctx, common.INVALID_PARAMS_CODE, + common.ERROR_READ_REQUEST_BODY_MSG+": "+err.Error(), nil) + return + } + } + if ctx.IsAborted() { + return + } + ctx.Next() + } +} + +func isExcludedPath(path string) bool { + + allowed := false + // define some excluded paths + excludedPaths := []string{ + "/ping", + "/xce/v1/auth/logon", + "/swagger/*any", + "/xce/v1/auth/logout", + } + for _, excludedPath := range excludedPaths { + if strings.HasPrefix(path, excludedPath[:len(excludedPath)-1]) { // 去掉最后的 * + allowed = true + break + } + } + return allowed +} + +// Token validation middleware +func validateHeader() gin.HandlerFunc { + logger.Debug("Enter validateHeader() function") + return func(ctx *gin.Context) { + //continue to validate token + token := ctx.GetHeader("Authorization") + if token == "" { + logger.Error("Token is empty") + controllers.ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE, + common.ERROR_AUTH_CHECK_TOKEN_EMPTY_MSG, nil) + ctx.Abort() + return + } + + // Validate token (this is a placeholder for your actual token validation logic) + isValid, err := isValidToken(strings.TrimPrefix(token, "Bearer "), ctx.ClientIP()) + if err != nil { + logger.Error("Validate token error", zap.Error(err)) + controllers.ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE, + common.ERROR_AUTH_MSG+": "+err.Error(), nil) + ctx.Abort() + return + } + if !isValid { + logger.Error("Token is invalid") + controllers.ReturnError(ctx, common.ERROR_AUTH_CHECK_TOKEN_CODE, + common.ERROR_AUTH_MSG, nil) + ctx.Abort() + return + } + } +} + +func isValidToken(tokenString string, IP string) (bool, error) { + token, err := jwt.ParseWithClaims(tokenString, &models.JWTClaims{}, func(token *jwt.Token) (interface{}, error) { + return []byte(utils.AuthenticationYamlConfig.Token.SecretKey), nil + }) + + if err != nil { + return false, err + } + + if claims, ok := token.Claims.(*models.JWTClaims); ok && token.Valid { + if claims.IP == IP { + return true, nil + } + } + return false, nil +} + +func validateRequestBody(ctx *gin.Context) (bool, error) { + logger.Debug("Enter validateRequestBody function") + + // read request body and resolve EOF error + body, err := ctx.GetRawData() + if err != nil { + logger.Error("Failed to read request body", zap.Error(err)) + ctx.Abort() + return false, err + } + if len(body) == 0 { + logger.Error("Request body is empty") + ctx.Abort() + return false, errors.New(common.ERROR_REQUEST_BODY_EMPTY_MSG) + } + + // reset request body + ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body)) + // check request body is valid + if !json.Valid(body) { + logger.Error("Request body is not valid", zap.Error(err)) + ctx.Abort() + return false, errors.New(common.ERROR_REQUEST_BODY_NOT_JSON_MSG) + } + return true, nil +} diff --git a/src/frontend/apiserver/middleware/whitelist.go b/src/frontend/apiserver/middleware/whitelist.go new file mode 100644 index 00000000..63ee1684 --- /dev/null +++ b/src/frontend/apiserver/middleware/whitelist.go @@ -0,0 +1,162 @@ +package middleware + +import ( + "apiserver/common" + "apiserver/controllers" + logger "apiserver/utils" + "net" + "strings" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +var WhitelistGETURL = []string{ + "/ping", // ping test + "/swagger", // Swagger interface for document generation + "/xce/v1/cluster", // cluster (legacy) + "/xce/v1/clusters", // clusters + "/xce/v1/hosts", // hosts + "/xce/v1/hosts/*", // hosts + "/xce/v1/jobs", // jobs + "/xce/v1/jobs/*", // jobs + "/xce/v1/queues", // queues + "/xce/v1/queues/*", // queues + "/xce/v1/files", // get all files + "/xce/v1/files/*", // get download file + "/xce/v1/resources/definition", // resources definition + "/xce/v1/resources/distribution", // resources distribution + "/xce/v1/roles", // roles list + "/xce/v1/roles/*", // role detail + "/xce/v1/permissions", // permissions + "/xce/v1/users", // users list + "/xce/v1/users/*", // user detail / current +} + +var WhitelistPOSTURL = []string{ + "/xce/v1/auth/logon", // logon + "/xce/v1/auth/logout", // logout + "/xce/v1/hosts/control", // control hosts + "/xce/v1/hosts/select", // filter host with -R select string + "/xce/v1/queues/control", // control queues + "/xce/v1/jobs/create", // create job + "/xce/v1/jobs/control", // control job + "/xce/v1/jobs/*", // control specified job + "/xce/v1/files/upload", // upload file + "/xce/v1/roles/create", // create role + "/xce/v1/roles/*", // add permissions for role + "/xce/v1/users/*", // add roles for user +} + +var WhitelistDELETEURL = []string{ + "/xce/v1/files", + "/xce/v1/files/*", + "/xce/v1/users/*", + "/xce/v1/roles/*", +} + +// pathMatchesPattern checks if path matches a whitelist pattern. +// Patterns ending in /* match any path with that prefix. +// Exact patterns match only that exact path. +func pathMatchesPattern(path, pattern string) bool { + if len(pattern) == 0 { + return false + } + if pattern[len(pattern)-1] == '*' { + return strings.HasPrefix(path, pattern[:len(pattern)-1]) + } + return path == pattern +} + +// WhitelistURLMiddleware returns a middleware that checks if the URL path is in the whitelist. +func WhitelistURLMiddleware() gin.HandlerFunc { + return func(ctx *gin.Context) { + path := ctx.Request.URL.Path + method := ctx.Request.Method + allowed := false + var whitelist []string + + if method == "GET" { + whitelist = WhitelistGETURL + } else if method == "POST" { + whitelist = WhitelistPOSTURL + } else if method == "DELETE" { + whitelist = WhitelistDELETEURL + } + + for _, pattern := range whitelist { + if pathMatchesPattern(path, pattern) { + allowed = true + break + } + } + if !allowed { + logger.Error("URL is not allowed", zap.String("path", path)) + controllers.ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_REQUEST_URL_NOT_FOUND, nil) + ctx.Abort() + return + } + ctx.Next() + } +} + +// WhitelistMiddleware returns a middleware that checks if the client IP is in the whitelist, +// supporting both single IP addresses and CIDR notation. +func WhitelistIPMiddleware(allowedIPs []string) gin.HandlerFunc { + // parse the allowed IPs + var allowedNetworks []*net.IPNet + var allowedIPList []net.IP + + for _, s := range allowedIPs { + // parse CIDR format eg："172.29.6.0/24" + if _, network, err := net.ParseCIDR(s); err == nil { + allowedNetworks = append(allowedNetworks, network) + logger.Debug("allowedNetworks", zap.String("allowedNetworks", network.String())) + } else { + // parse single IP format + ip := net.ParseIP(s) + if ip != nil { + allowedIPList = append(allowedIPList, ip) + logger.Debug("allowedIPList", zap.String("allowedIPList", ip.String())) + } + } + } + + return func(ctx *gin.Context) { + clientIPStr := ctx.ClientIP() + clientIP := net.ParseIP(clientIPStr) + logger.Debug("clientIP", zap.String("clientIP", clientIPStr)) + if clientIP == nil { + logger.Error("invalid client IP", zap.String("clientIP", clientIPStr)) + controllers.ReturnError(ctx, common.INVALID_PARAMS_CODE, common.ERROR_INVALID_CLIENT_IP_MSG, nil) + return + } + + allowed := false + // single IP + for _, ip := range allowedIPList { + if ip.Equal(clientIP) { + allowed = true + break + } + } + + // ip array + if !allowed { + for _, network := range allowedNetworks { + if network.Contains(clientIP) { + allowed = true + break + } + } + } + + if !allowed { + logger.Error("access forbidden", zap.String("clientIP", clientIPStr)) + controllers.ReturnError(ctx, common.ERROR_AUTH_CODE, common.ERROR_ACCESS_FORBIDDEN_MSG, nil) + ctx.Abort() + return + } + ctx.Next() + } +} diff --git a/src/frontend/apiserver/middleware/whitelist_test.go b/src/frontend/apiserver/middleware/whitelist_test.go new file mode 100644 index 00000000..afc26613 --- /dev/null +++ b/src/frontend/apiserver/middleware/whitelist_test.go @@ -0,0 +1,33 @@ +package middleware + +import "testing" + +func TestPathMatchesPattern(t *testing.T) { + cases := []struct { + pattern string + path string + want bool + }{ + // wildcard patterns + {"/xce/v1/hosts/*", "/xce/v1/hosts/host1", true}, + {"/xce/v1/jobs/*", "/xce/v1/jobs/123/control", true}, + {"/xce/v1/roles/*", "/xce/v1/roles/cluster_admin", true}, + {"/xce/v1/users/*", "/xce/v1/users/current", true}, + {"/xce/v1/users/*", "/xce/v1/users/admin/basic_user", true}, + // exact patterns + {"/xce/v1/hosts", "/xce/v1/hosts", true}, + {"/xce/v1/hosts", "/xce/v1/host", false}, + {"/xce/v1/roles", "/xce/v1/roles", true}, + {"/xce/v1/roles", "/xce/v1/role", false}, + {"/xce/v1/clusters", "/xce/v1/clusters", true}, + {"/ping", "/ping", true}, + {"/ping", "/pin", false}, + } + + for _, tc := range cases { + got := pathMatchesPattern(tc.path, tc.pattern) + if got != tc.want { + t.Errorf("pathMatchesPattern(%q, %q) = %v, want %v", tc.path, tc.pattern, got, tc.want) + } + } +} diff --git a/src/frontend/apiserver/models/apikey.go b/src/frontend/apiserver/models/apikey.go new file mode 100644 index 00000000..0e25d7ec --- /dev/null +++ b/src/frontend/apiserver/models/apikey.go @@ -0,0 +1,18 @@ +package models + +import "time" + +type APIKeyController struct{} + +type APIKeyRequest struct { + Username string `json:"username" binding:"required"` + Description string `json:"description,omitempty"` + ExpiryDays int `json:"expiry_days,omitempty"` +} + +type APIKeyResponse struct { + APIKey string `json:"api_key"` + Username string `json:"username"` + CreateTime time.Time `json:"create_time"` + ExpiryTime time.Time `json:"expiry_time"` +} diff --git a/src/frontend/apiserver/models/auth.go b/src/frontend/apiserver/models/auth.go new file mode 100644 index 00000000..85a2886d --- /dev/null +++ b/src/frontend/apiserver/models/auth.go @@ -0,0 +1,37 @@ +package models + +import ( + "github.com/golang-jwt/jwt/v4" +) + +type LogonRequest struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` +} + +type LogoutRequest struct { + Username string `json:"username" binding:"required"` +} + +type JWTClaims struct { + Username string `json:"username"` + IP string `json:"ip"` + jwt.RegisteredClaims +} + +type LogonResponse struct { + Token string `json:"token"` + Path string `json:"path"` + IP string `json:"ip"` + TokenExpireTime int `json:"token_expire_time"` +} + +type LogoutResponse struct { + Username string `json:"username"` + Message string `json:"message"` +} + +type User struct { + Username string `json:"username"` + Password string `json:"password"` +} diff --git a/src/frontend/apiserver/models/clusters.go b/src/frontend/apiserver/models/clusters.go new file mode 100644 index 00000000..dc056558 --- /dev/null +++ b/src/frontend/apiserver/models/clusters.go @@ -0,0 +1,45 @@ +package models + +const ( + CLUSTER_RECONFIG = 1 + CLUSTER_RESTART = 2 + CLUSTER_SHUTDOWN = 3 +) + +var CLUSTERSTAT = map[string]int{ + "DISCONNECT": 0x0, + "CONNECTED": 0x01, +} + +var JSONBCLFields = []string{"resources", "host_types", "host_models", "cluster_admins"} + +type ClusterInfo struct { + ClusterName string `json:"cluster_name"` + Version string `json:"version"` + Status int `json:"status"` + ControllerName string `json:"controller_name"` + NumServers int `json:"num_servers"` + Resources string `json:"resources"` + HostTypes string `json:"host_types"` + HostModels string `json:"host_models"` + ClusterAdmins string `json:"cluster_admins"` +} + +var CLUSTER_SUPPORT_FILTERS = []string{ + "cluster_name", +} + +var CLUSTER_DEFAULT_COLUMN = []string{ + "cluster_name", + "version", + "status", + "controller_name", + "resources", + "host_types", + "host_models", + "cluster_admins", +} + +var CLUSTER_SUPPORT_SORT = []string{ + "cluster_name", +} diff --git a/src/frontend/apiserver/models/common.go b/src/frontend/apiserver/models/common.go new file mode 100644 index 00000000..7bbdb6cd --- /dev/null +++ b/src/frontend/apiserver/models/common.go @@ -0,0 +1,19 @@ +package models + +import "time" + +const ( + CONTROL_JOB = 1 + CONTROL_HOST = 2 + CONTROL_QUEUE = 3 + CONTROL_CLUSTER = 4 + CONTROL_DEBUG = 5 +) + +type ActionControl struct { + TargetType int `json:"target_type"` + ActionType int `json:"action_type"` + Content Jsonb `json:"content"` + SubmitTime time.Time `json:"submit_time"` + Submitter string `json:"submitter"` +} diff --git a/src/frontend/apiserver/models/files.go b/src/frontend/apiserver/models/files.go new file mode 100644 index 00000000..4ae62351 --- /dev/null +++ b/src/frontend/apiserver/models/files.go @@ -0,0 +1,7 @@ +package models + +type FileUploadRequest struct { + InFile string `json:"in_file" binding:"required"` +} + +var SupportedFileParams = []string{"path"} diff --git a/src/frontend/apiserver/models/hosts.go b/src/frontend/apiserver/models/hosts.go new file mode 100644 index 00000000..ab30636f --- /dev/null +++ b/src/frontend/apiserver/models/hosts.go @@ -0,0 +1,134 @@ +package models + +const ( + HOST_OPEN = 1 + HOST_CLOSE = 2 + HOST_TAG = 3 + HOST_UNTAG = 4 +) + +var SupportedHostActions = []string{"open", "close"} + +var HOSTSTAT = map[string]int{ + "UNAVAIL": 0x00010000, + "LOCKEDU": 0x00020000, + "LOCKEDW": 0x00040000, + "BUSY": 0x00080000, + "CLOSED": 0x00100000, +} + +func HostStateStr(state int) string { + if (state & HOSTSTAT["UNAVAIL"]) != 0 { + return "UNAVAIL" + } else if (state & HOSTSTAT["CLOSED"]) != 0 { + return "CLOSED" + } else if (state & HOSTSTAT["BUSY"]) != 0 { + return "BUSY" + } else if (state & HOSTSTAT["LOCKEDU"]) != 0 { + return "LOCKU" + } else if (state & HOSTSTAT["LOCKEDW"]) != 0 { + return "LOCKW" + } + return "OK" +} + +// JSONBFields contains the list of fields that are in JSONB format +var JSONBHFields = []string{"topology", "sched_stop", "real_load", "dispt_window"} + +var HOST_DEFAULT_COLUMN = []string{ + "host_name", + //"ip_addr", // not supported yet + "host_status", + "host_model", + "host_type", + "host_group", + "flag", + "cpu_factor", + "num_cpus", + "num_cores", + "num_threads", + "dispt_window", + "max_slot", + "max_mem", + "max_swap", + "max_tmp", + "num_disk", + "num_run", + "num_ssusp", + "num_ususp", + //"num_rsv", + "num_done", + "topology", + "sched_stop", + //"real_load", +} + +type HostInfo struct { + HostID int64 `json:"host_id"` + HostName string `json:"host_name"` + HostStatus int `json:"host_status"` + HostModel string `json:"host_model"` + HostType string `json:"host_type"` + HostGroup string `json:"host_group"` + Flag int `json:"flag"` + CPUFactor float64 `json:"cpu_factor"` + DisptWindow string `json:"dispt_window"` + NumCPUs int `json:"num_cpus"` + NumCores int `json:"num_cores"` + NumThreads int `json:"num_threads"` + MaxSlot int `json:"max_slot"` + MaxMem int `json:"max_mem"` + MaxSwap int `json:"max_swap"` + MaxTmp int `json:"max_tmp"` + NumDisk int `json:"num_disk"` + NumRun int `json:"num_run"` + NumSSusp int `json:"num_ssusp"` + NumUSusp int `json:"num_ususp"` + //NumRsv int `json:"num_rsv"` + NumDone int `json:"num_done"` + Topology string `json:"topology"` // JSONB + SchedStop string `json:"sched_stop"` // JSONB + //RealLoad string `json:"real_load"` // JSONB +} + +// host info query supported filter +var HOST_SUPPORT_FILTERS = []string{ + "host_name", + "host_model", + "host_type", + "host_group", + "flag", + "cpu_factor", + "num_cpus", + "max_slot", + "max_mem", + "max_swap", + "max_tmp", + "num_disk", + "num_run", + "num_ssusp", + "num_ususp", + //"num_rsv", + "num_done", +} + +var HOST_SUPPORT_SORT = []string{ + "host_name", +} + +type HostControlRequest struct { + ControlAction string `json:"action" binding:"required"` + Hostname []string `json:"host_name" binding:"required"` + Comment string `json:"comment"` +} +type ResReqSelectRequest struct { + Select string `json:"select" binding:"required"` + Order string `json:"order,omitempty"` + Limit int `json:"limit,omitempty"` + Pagetoken string `json:"page_token,omitempty"` +} + +type HostControlEvent struct { + Hostname string `json:"host_name" binding:"required"` + Comment string `json:"comment,omitempty"` +} diff --git a/src/frontend/apiserver/models/jobs.go b/src/frontend/apiserver/models/jobs.go new file mode 100644 index 00000000..3b44d8aa --- /dev/null +++ b/src/frontend/apiserver/models/jobs.go @@ -0,0 +1,756 @@ +package models + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" +) + +const RLIM_NLIMITS = 12 + +const ( + JOB_STOP = 1 + JOB_RESUME = 2 + JOB_KILL = 3 + JOB_MIG = 4 + JOB_REQUEUE = 5 + JOB_MODIFY = 6 + JOB_SWITCH = 7 + JOB_CHKPNT = 8 + JOB_POST = 9 + JOB_READ = 10 + JOB_TOP = 11 + JOB_BOT = 12 +) + +var JobSignalMapTable = map[string]int{ + "HUP": 1, // Hangup + "INT": 2, // Interrupt + "QUIT": 3, // Quit + "ILL": 4, // Illegal instruction + "TRAP": 5, // Trace trap + "ABRT": 6, // ABRT instruction + "BUS": 7, // Bus error + "FPE": 8, // Floating point exception + "KILL": 9, // Kill (cannot be caught or ignored) + "USR1": 10, // User-defined signal 1 + "SEGV": 11, // Segmentation violation + "USR2": 12, // User-defined signal 2 + "PIPE": 13, // Write on a pipe with no one to read it + "ALRM": 14, // Alarm clock + "TERM": 15, // Termination signal + "STKFLT": 16, // Stack fault (historical, often unused or same as SEGV) + "CHLD": 17, // Child process terminated, stopped, or continued + "CONT": 18, // Continue executing, if stopped + "STOP": 19, // Stop executing (cannot be caught or ignored) + "TSTP": 20, // Terminal stop signal + "TTIN": 21, // Background process attempting read + "TTOU": 22, // Background process attempting write + "URG": 23, // Urgent condition (often unused or same as PWR) + "XCPU": 24, // CPU time limit exceeded + "XFSZ": 25, // File size limit exceeded + "VTALRM": 26, // Virtual timer expired + "PROF": 27, // Profiling timer expired + "WINCH": 28, // Window changed + "IO": 29, // I/O possible (often same as POLL) + "PWR": 30, // Power failure (historical, often unused or same as USR1) + "SYS": 31, // Bad system call (historical, often unused or same as USR2) +} + +var JOBSTAT = map[string]int{ + "PEND": 0x01, + "RUN": 0x02, + "EXIT": 0x04, + "DONE": 0x08, + "PSUSP": 0x10, + "USUSP": 0x20, + "SSUSP": 0x40, + "PDONE": 0x80, + "PERR": 0x100, + "RUNKWN": 0x8000, + "UNKNOWN": 0x1000, +} + +var LAUNCHTYPE = map[string]int{ + "openmpi": 1, + "mpich": 2, +} + +func JobStateStr(state int) string { + if (state & JOBSTAT["PEND"]) != 0 { + if (state & JOBSTAT["PSUSP"]) != 0 { + return "PSUSP" + } + return "PEND" + } else if (state & JOBSTAT["RUN"]) != 0 { + if (state & JOBSTAT["USUSP"]) != 0 { + return "USUSP" + } else if (state & JOBSTAT["SSUSP"]) != 0 { + return "SSUSP" + } + return "RUN" + } else if (state & JOBSTAT["EXIT"]) != 0 { + return "EXIT" + } else if (state & JOBSTAT["DONE"]) != 0 { + return "DONE" + } + return "ERR" +} + +var SupportedJobActions = []string{"kill", "stop", "resume", "checkpoint"} + +var ValidInteractiveValues = []string{"0", "1", "2", "3"} + +var ValidLaunchTypes = []string{"openmpi", "mpich"} + +var JobActionControlMap = map[int][]int{ + JOB_CHKPNT: {JOBSTAT["RUN"]}, + JOB_STOP: {JOBSTAT["RUN"], JOBSTAT["PEND"]}, + JOB_RESUME: {JOBSTAT["USUSP"], JOBSTAT["PSUSP"]}, + JOB_KILL: {JOBSTAT["RUN"], JOBSTAT["PEND"], JOBSTAT["SSUSP"], JOBSTAT["USUSP"], JOBSTAT["PSUSP"], JOBSTAT["UNKNOWN"]}, +} +var JobKillSignalControlMap = map[int][]int{ + JOB_KILL: {JOBSTAT["RUN"], JOBSTAT["PEND"], JOBSTAT["SSUSP"], JOBSTAT["USUSP"], JOBSTAT["PSUSP"]}, +} + +var DefaultActionSignal = map[string]string{ + "kill": "TERM", + "stop": "TSTP", + "resume": "CONT", +} + +var JobAvailableTypes = []string{"array_stats"} + +// JSONBFields contains the list of fields that are in JSONB format +var JSONBJFields = []string{ + "exec_hosts", + "sub_env_vars", + "pending_reason", + "job_rusage", + "license_names", + "submit_ext", + "extend_data", + "tags", + "exit_status", + "pid_info", + "transfer_files", + "pids", + "pgids", +} + +var ARRAY_JOB_SUPPORT_FILTERS = []string{ + "job_id", + "submit_user", + "queue_name", + "job_description", + "job_name", + //"user_group", + //"job_group", + //"project_name", + //"license_project", + //"sla", +} + +// job info query supported filter +var JOB_SUPPORT_FILTERS = []string{ + "submit_user", + "queue_name", + "job_status", + "array_idx", + "exec_hosts", // may be []string + "job_description", + "job_name", + "job_id", + //"user_group", + //"job_group", + //"project_name", + //"license_project", + //"sla", + "submit_time", + "queue_time", + "start_time", + "end_time", + "reserve_time", +} + +var JOB_SUPPORT_SORT = []string{ + "job_id", + "job_status", + "queue_name", + "submit_time", + "submit_user", +} + +// when query jobinfo, the default display columns. +var JOB_DEFAULT_COLUMN = []string{ + "job_id", + "array_idx", + "unique_job_id", + "submit_user", + "job_status", + "queue_name", + "from_host", + "exec_hosts", + "job_name", + "command", + "submit_time", +} + +var ARRAY_JOB_DEFAULT_COLUMN = []string{ + "job_id", + "array_job_name", + "submit_user", + "queue_name", + "user_group", + "job_group", + "job_description", + "job_name", + "project_name", + "license_project", + "sla", + "num_jobs", + "num_done", + "num_exit", + "num_run", + "num_pend", + "num_ssusp", + "num_ususp", + "num_psusp", +} + +type JobArrayID struct { + ArrayIdx int `json:"array_idx"` + Unique_JobID int64 `json:"job_id"` +} +type CustomTime struct { + Time time.Time +} + +type CustomInt struct { + Seconds int +} + +type Jsonb struct { + String string +} + +type TimeStamp struct { + TimeStamp int64 +} + +func (jsb *Jsonb) IsZero() bool { + return jsb.String == "" +} + +func (tm *CustomTime) Tm() time.Time { + return tm.Time +} + +type CustomLimit struct { + MB float64 +} + +func (cl *CustomLimit) String() string { + switch { + case cl.MB >= 1<<30: + return fmt.Sprintf("%.2f PB", cl.MB/(1<<30)) + case cl.MB >= 1<<20: + return fmt.Sprintf("%.2f TB", cl.MB/(1<<20)) + case cl.MB >= 1<<10: + return fmt.Sprintf("%.2f GB", cl.MB/(1<<10)) + default: + return fmt.Sprintf("%.2f MB", cl.MB) + } +} + +func (cl *CustomLimit) UnmarshalJSON(b []byte) error { + s := strings.Trim(string(b), "\"") + if s == "null" || s == "" { + cl.MB = 0 + return nil + } + + // Expecting format like "1024 MB", "1 GB", etc. + parts := strings.Split(s, " ") + if len(parts) != 1 && len(parts) != 2 { + return fmt.Errorf("invalid limit format, expected ' '") + } + + value, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return fmt.Errorf("invalid limit format, expected ' '") + } + + unit := "MB" + + if len(parts) == 2 && parts[1] != "" { + unit = strings.ToUpper(parts[1]) + } + switch unit { + case "MB", "M": + cl.MB = value + case "GB", "G": + cl.MB = value * 1024 + case "TB", "T": + cl.MB = value * 1024 * 1024 + case "PB", "P": + cl.MB = value * 1024 * 1024 * 1024 + default: + return fmt.Errorf("unsupported unit: %s", unit) + } + return nil +} + +func (sl *ShellLimit) UnmarshalJSON(b []byte) error { + type Alias ShellLimit + aux := &struct { + *Alias + }{ + Alias: (*Alias)(sl), + } + + if err := json.Unmarshal(b, &aux); err != nil { + return err + } + + limits := []*CustomLimit{ + &sl.FileLimit, + &sl.DataLimit, + &sl.StackLimit, + &sl.CoreLimit, + &sl.ProcessLimit, + &sl.ThreadLimit, + &sl.MemLimit, + &sl.SwapLimit, + } + + for _, limit := range limits { + if limit != nil { + if err := limit.UnmarshalJSON([]byte(limit.String())); err != nil { + return err + } + } + } + + return nil +} + +func (tm *CustomTime) IsZero() bool { + return tm.Time.IsZero() +} + +func (ci *CustomInt) String() string { + hours := ci.Seconds / 3600 + minutes := (ci.Seconds % 3600) / 60 + return fmt.Sprintf("%d:%02d", hours, minutes) +} + +func (ci *CustomInt) Int() int { + return ci.Seconds +} + +func (ci *CustomInt) Float64() float64 { + return float64(ci.Seconds) +} + +func (ci *CustomInt) UnmarshalJSON(b []byte) error { + var hours, minutes int + var err error + s := strings.Trim(string(b), "\"") + if s == "null" || s == "" { + ci.Seconds = 0 + return nil + } + // If the input is already in seconds format + if !strings.Contains(s, ":") { + minutes, err = strconv.Atoi(s) + if err != nil { + return fmt.Errorf("invalid minutes value: %v", err) + } + ci.Seconds = minutes * 60 + return nil + } + // Handle hours:minutes format + parts := strings.Split(s, ":") + + if len(parts) == 2 { + hours, err = strconv.Atoi(parts[0]) + if err != nil || hours < 0 || hours > 24 { + return errors.New("hour must be between 0 and 24") + } + if hours == 24 && parts[1] != "00" { + return errors.New("hour must be between 0 and 24, and minute must be 00") + } + minutes, err = strconv.Atoi(parts[1]) + } else { + minutes, err = strconv.Atoi(parts[0]) + } + + if err != nil || (hours != 0 && (minutes < 0 || minutes > 59)) { + return errors.New("minute must be between 0 and 59") + } + ci.Seconds = hours*3600 + minutes*60 + if ci.Seconds > 2147483647 { + return errors.New("limit exceeds the value of 2147483647") + } + return nil +} + +// UnmarshalJSON handle time.time format data +func (ct *CustomTime) UnmarshalJSON(b []byte) error { + s := strings.Trim(string(b), "\"") + if s == "null" || s == "" { + ct.Time = time.Time{} + return nil + } + + // support multiple date formats + formats := []string{ + "2006-01-02 15:04:05 MST", + "2006-01-02 15:04:05", + "2006-01-02 15:04 MST", + "2006-01-02 15:04", + //"2006-01-02", + //"15:04", + } + + var lastErr error + for _, format := range formats { + t, err := time.Parse(format, s) + if err == nil { + // If only time is provided (format "15:04"), use current date + /* + if format == "15:04" { + now := time.Now() + t = time.Date(now.Year(), now.Month(), + now.Day(), t.Hour(), t.Minute(), 0, 0, time.Local) + } + */ + // change time into minute level + ct.Time = t.Truncate(time.Minute) + return nil + } + lastErr = err + } + return fmt.Errorf("cannot parse time string: %s, last error: %v", s, lastErr) +} + +// JRusage represents job resource usage information +// TODO: need determine +type JRusage struct { + Mem int64 `json:"mem"` + Swap int64 `json:"swap"` + Utime int64 `json:"utime"` + Stime int64 `json:"stime"` + NPgids int `json:"npgids"` + Pgid []int `json:"pgid"` + NThreads int `json:"nthreads"` + CpuUT float32 `json:"cpuUT"` + IO int64 `json:"io"` + IORate float32 `json:"iorate"` + DiskIO int64 `json:"diskio"` + DiskIORate float32 `json:"diskiorate"` +} + +var JobFieldMappings = map[string]string{ + // "user_used": "db_filed_name", + "file_limit": "fsize_limit", + "mem_limit": "rss_limit", + "num_files": "nofile_limit", + "env": "sub_env_vars", + // more fields can be added here +} + +type JobInfo struct { + Unique_JobID int64 `json:"unique_job_id"` + JobID int64 `json:"job_id"` + ArrayIdx int `json:"array_idx"` //new + SubmitUser string `json:"submit_user"` + FromHost string `json:"from_host"` + SubmitTime time.Time `json:"submit_time"` + JobStatus int `json:"job_status"` + JobName string `json:"job_name"` + Command string `json:"command"` + QueueName string `json:"queue_name"` + ProjectName string `json:"project_name"` + JobGroup string `json:"job_group"` // new + UserGroup string `json:"user_group"` + LicenseProject string `json:"license_project"` + LicenseNames Jsonb `json:"license_names"` // JSONB as []byte + JobDescription string `json:"job_description"` + ResReq string `json:"resreq"` + RsvID string `json:"rsv_id"` + SLA string `json:"sla"` + MaxNumProcessors int `json:"maxnum_processors"` + NumProcessors int `json:"num_processors"` + AskedHosts string `json:"asked_hosts"` + LaunchType string `json:"launch_type"` + BeginTime CustomTime `json:"begin_time"` // CustomTime + TermTime CustomTime `json:"term_time"` // CustomTime + UserPriority int `json:"user_priority"` + DependCond string `json:"depend_cond"` + Umask int `json:"umask"` + CWD string `json:"cwd"` + SubmitHomeDir string `json:"submit_home_dir"` + JobPriority int `json:"job_priority"` + PreExecCmd string `json:"pre_exec_cmd"` + PostExecCmd string `json:"post_exec_cmd"` + MigThreshold int `json:"mig_threshold"` + ChkpntPeriod int `json:"chkpnt_period"` + ChkpntDir string `json:"chkpnt_dir"` + ChkpntInfo Jsonb `json:"chkpnt_info"` // JSONB as []byte + OutDir string `json:"out_dir"` + InFile string `json:"in_file"` + OutFile string `json:"out_file"` + ErrFile string `json:"err_file"` + LoginShell string `json:"login_shell"` + ExceptList string `json:"except_list"` + MailUser string `json:"mail_user"` + RequeueErrorValues string `json:"requeue_error_values"` + GPUResReq string `json:"gpu_resreq"` + SubmitExt Jsonb `json:"submit_ext"` // new, JSONB as []byte + ExtendData Jsonb `json:"extend_data"` // new, JSONB as []byte + Tags Jsonb `json:"tags"` // new, JSONB as []byte + EffectiveResReq string `json:"effective_resreq"` //new + EffectiveUserGroup string `json:"effective_user_group"` //new + QueueTime time.Time `json:"queue_time"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + ReserveTime time.Time `json:"reserve_time"` + Duration int `json:"duration"` + CPUTime int `json:"cpu_time"` + ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte + ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte + ExecUID int `json:"exec_uid"` + ExecHome string `json:"exec_home"` + ExecCWD string `json:"exec_cwd"` + ExecUser string `json:"exec_user"` //name change + ChargedSaap string `json:"charged_saap"` + RunTime int `json:"run_time"` + RunTimeEstimation int `json:"run_time_estimation"` + PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte + PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte + JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte +} + +type JobExecHosts struct { + InsertSeq int64 `json:"insert_seq"` + HostName string `json:"host_name"` + UniqueJobID int64 `json:"unique_job_id"` + SlotNum int `json:"slot_num"` + JobStatus int `json:"job_status"` +} + +type ArrayJobInfo struct { + JobID int64 `json:"job_id"` + ArrayJobName string `json:"array_job_name"` + SubmitUser string `json:"submit_user"` + QueueName string `json:"queue_name"` + UserGroup string `json:"user_group"` + JobGroup string `json:"job_group"` + JobDescription string `json:"job_description"` + JobName string `json:"job_name"` + ProjectName string `json:"project_name"` + LicenseProject string `json:"license_project"` + SLA string `json:"sla"` + NumJobs int `json:"num_jobs"` + NumDone int `json:"num_done"` + NumExit int `json:"num_exit"` + NumRun int `json:"num_run"` + NumPend int `json:"num_pend"` + NumSSusp int `json:"num_ssusp"` + NumUSusp int `json:"num_ususp"` + NumPSusp int `json:"num_psusp"` +} + +type JobAdditionalFields struct { + JobID int64 `json:"job_id"` + SubmitUser string `json:"submit_user"` + JobStatus int `json:"job_status"` + SubmitTime time.Time `json:"submit_time"` + FromHost string `json:"from_host"` + ExecHosts string `json:"exec_hosts"` + + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + JobRusage Jsonb `json:"job_rusage"` + MaxMem float64 `json:"max_mem"` + AvgMem float64 `json:"avg_mem"` + UsedMem float64 `json:"used_mem"` + CPUTime int `json:"cpu_time"` + PeakCPUTime int `json:"peak_cpu_time"` + PeakCPUDuration float64 `json:"peak_cpu_duration"` + MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage + AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage + PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage + NumHosts int `json:"num_hosts"` // Number of hosts used by the job + FirstHost string `json:"first_host"` // First host where the job ran + AllocSlot string `json:"alloc_slots"` // Allocated slot information + PIDInfo Jsonb `json:"pid_info"` + Pids []int `json:"pids"` // List of process IDs associated with the job + Pgids []int `json:"pgids"` // List of process group IDs associated with the job + InteractiveJobURL string `json:"interactive_job_url"` +} + +type JobSubmitRequest struct { + JobName string `json:"job_name,omitempty" ` + Command string `json:"command" binding:"required"` + QueueName string `json:"queue_name,omitempty"` // set to DEFAULT_QUEUE in config + ProjectName string `json:"project_name,omitempty"` // default project: "default" + UserGroup string `json:"user_group,omitempty"` + LicenseProject string `json:"license_project,omitempty"` + JobDescription string `json:"job_description,omitempty"` + Resreq string `json:"resreq,omitempty"` // default: "type == local" + Affinity string `json:"affinity,omitempty"` + Gpureq string `json:"gpureq,omitempty"` + RsvID string `json:"rsv_id,omitempty"` + SLA string `json:"sla,omitempty"` + MaxNumProcessors int `json:"maxnum_processors,omitempty"` + NumProcessors int `json:"num_processors,omitempty"` + AskedHosts string `json:"asked_hosts,omitempty"` + LaunchType string `json:"launch_type,omitempty"` + BeginTime CustomTime `json:"begin_time,omitempty"` //k change type + TermTime CustomTime `json:"term_time,omitempty"` //k change type + JobPriority int `json:"job_priority,omitempty"` + DependCond string `json:"depend_cond,omitempty"` + JobGroup string `json:"job_group,omitempty"` + HostSpec string `json:"host_spec,omitempty"` + Env []string `json:"env,omitempty"` + Tags []string `json:"tags,omitempty"` + Exclusive bool `json:"exclusive,omitempty"` + Rerunable bool `json:"rerunable,omitempty"` + Interactive int `json:"interactive,omitempty"` + WarningTimePeriod int `json:"warning_time_period,omitempty"` + WarningAction string `json:"warning_action,omitempty"` + DataSpecFile string `json:"data_spec_file,omitempty"` + TransferFiles []string `json:"transfer_files,omitempty"` + PreExecCmd string `json:"pre_exec_cmd,omitempty"` + PostExecCmd string `json:"post_exec_cmd,omitempty"` + RequeueErrorValues string `json:"requeue_error_values,omitempty"` + RunTimeEstimation int `json:"run_time_estimation,omitempty"` + MigThreshold int `json:"mig_threshold,omitempty"` + ChkpntPeriod int `json:"chkpnt_period,omitempty"` + ChkpntDir string `json:"chkpnt_dir,omitempty"` + InitChkpntPeriod int `json:"init_chkpnt_period,omitempty"` + CWD string `json:"cwd,omitempty"` + OutDir string `json:"out_dir,omitempty"` + InFile string `json:"in_file,omitempty"` + OutFile string `json:"out_file,omitempty"` + ErrFile string `json:"err_file,omitempty"` + Mailuser string `json:"mailuser,omitempty"` + LoginShell string `json:"login_shell,omitempty"` + ExceptList string `json:"except_list,omitempty"` + CPULimit CustomInt `json:"cpu_limit,omitempty"` // k change type + RunLimit CustomInt `json:"run_limit,omitempty"` //k change type + FileLimit CustomLimit `json:"file_limit,omitempty"` // fsize + DataLimit CustomLimit `json:"data_limit,omitempty"` + StackLimit CustomLimit `json:"stack_limit,omitempty"` + CoreLimit CustomLimit `json:"core_limit,omitempty"` + MemLimit CustomLimit `json:"mem_limit,omitempty"` + NumFiles int `json:"num_files,omitempty"` + SwapLimit CustomLimit `json:"swap_limit,omitempty"` + ProcessLimit CustomLimit `json:"process_limit,omitempty"` + ThreadLimit CustomLimit `json:"thread_limit,omitempty"` + ShellLimit ShellLimit `json:"shell_limit,omitempty"` + ExtendData string `json:"extend_data,omitempty"` +} + +type JobRestartRequest struct { + AskedHosts string `json:"asked_hosts,omitempty"` +} +type ShellLimit struct { + CPULimit int `json:"cpu_limit,omitempty"` // in milliseconds + FileLimit CustomLimit `json:"file_limit,omitempty"` + DataLimit CustomLimit `json:"data_limit,omitempty"` + StackLimit CustomLimit `json:"stack_limit,omitempty"` + CoreLimit CustomLimit `json:"core_limit,omitempty"` + ProcessLimit CustomLimit `json:"process_limit,omitempty"` + ThreadLimit CustomLimit `json:"thread_limit,omitempty"` + MemLimit CustomLimit `json:"mem_limit,omitempty"` + SwapLimit CustomLimit `json:"swap_limit,omitempty"` + NumFiles int `json:"num_files,omitempty"` +} + +type JobSubmitResponse struct { + JobID int64 `json:"jobid"` + QueueName string `json:"queue_name"` + Message string `json:"message"` +} + +// Job query struct in C lib +// TODO: need consider more filter conditions and "0"(means all satisfied jobs) +type JobQueryFilters struct { + Submit_user string `json:"submit_user,omitempty"` + Queue_name string `json:"queue_name,omitempty"` + User_group string `json:"user_group,omitempty"` + Job_status int `json:"job_status,omitempty"` + Exec_hosts string `json:"exec_hosts,omitempty"` + Job_name string `json:"job_name,omitempty"` + Job_group string `json:"job_group,omitempty"` + Project_name string `json:"project_name,omitempty"` + License_project string `json:"license_project,omitempty"` + Job_id int64 `json:"job_id,omitempty"` +} + +type JobIndex struct { + Index string // e.g., "1-3" + Step int // Step for the range +} + +type JobArrayInfo struct { + KeyID string // e.g., "316499657728" + Indices []JobIndex // List of indices with steps +} + +// ------- job control ------------- +type JobControlFilter struct { + JobName string `json:"job_name,omitempty"` + JobGroupName string `json:"job_group_name,omitempty"` + QueueName string `json:"queue_name,omitempty"` + Users string `json:"users,omitempty"` + ExecHosts string `json:"exec_hosts,omitempty"` + SLA string `json:"sla,omitempty"` + AllJobs bool `json:"all_jobs,omitempty"` +} + +type JobControlRequest struct { + Filter JobControlFilter `json:"filter" binding:"required"` + Action string `json:"action" binding:"required"` + Signal string `json:"signal,omitempty"` + Comment string `json:"comment,omitempty"` + ForceKill bool `json:"force_kill,omitempty"` + Method string `json:"method,omitempty"` // "criu" as default +} + +type JobControlJobIDRequest struct { + Action string `json:"action" binding:"required"` + Signal string `json:"signal,omitempty"` // signal value + Comment string `json:"comment,omitempty"` + ForceKill bool `json:"force_kill,omitempty"` + Method string `json:"method,omitempty"` // "criu" as default +} + +type JobControlEvent struct { + UniqueJobID int64 `json:"unique_job_id" binding:"required"` + Signal int `json:"signal,omitempty"` // signal value + Comment string `json:"comment,omitempty"` + ForceKill bool `json:"force_kill,omitempty"` + Method string `json:"method,omitempty"` // "criu" as default +} + +var launchTypeReverse map[int]string + +func init() { + launchTypeReverse = make(map[int]string, len(LAUNCHTYPE)) + for k, v := range LAUNCHTYPE { + launchTypeReverse[v] = k + } +} + +func GetLaunchTypeString(code int) (string, bool) { + str, exists := launchTypeReverse[code] + return str, exists +} diff --git a/src/frontend/apiserver/models/queues.go b/src/frontend/apiserver/models/queues.go new file mode 100644 index 00000000..81a06bf6 --- /dev/null +++ b/src/frontend/apiserver/models/queues.go @@ -0,0 +1,82 @@ +package models + +const ( + QUEUE_OPEN = 1 + QUEUE_CLOSE = 2 + QUEUE_INACT = 3 + QUEUE_ACT = 4 +) + +var SupportedQueueActions = []string{"open", "close", "deactive", "active"} + +var ActionMap = map[string]int{ + "close": QUEUE_CLOSE, + "open": QUEUE_OPEN, + "active": QUEUE_ACT, + "deactive": QUEUE_INACT, +} + +// queues info query supported filter +var QUEUE_SUPPORT_FILTERS = []string{ + "queue_name", + "priority", + "description", + "num_pend", + "num_run", + "num_ssusp", + "num_ususp", + "num_done", +} + +var QUEUE_SUPPORT_SORT = []string{ + "queue_name", +} +var QUEUESTAT = map[string]int{ + "OPEN": 0x00, + "CLOSED": 0x01, + "INACTIVE_WIN": 0x02, + "INACTIVE_ADM": 0x04, + "NOPERM": 0x08, + "DISC": 0x10, + "FAIRSHARE_DEFAULT": 0x20, +} + +// JSONBFields contains the list of fields that are in JSONB format +var JSONBQFields = []string{"fs_info"} + +var QUEUE_DEFAULT_COLUMN = []string{ + "queue_name", + "queue_status", + "priority", + "description", + "num_pend", + "num_run", + "num_ssusp", + "num_ususp", + "num_done", + "fs_info", +} + +type QueueInfo struct { + QueueName string `json:"queue_name"` + QueueStatus int `json:"queue_status"` + Priority int `json:"priority"` + Description string `json:"description"` + NumPend int `json:"num_pend"` + NumRun int `json:"num_run"` + NumSSusp int `json:"num_ssusp"` + NumUSusp int `json:"num_ususp"` + NumDone int `json:"num_done"` + FSInfo string `json:"fs_info"` // JSONB +} + +type QueueControlRequest struct { + ControlAction string `json:"action" binding:"required"` + QueueName []string `json:"queue_name" binding:"required"` + Comment string `json:"comment"` +} + +type QueueControlEvent struct { + Queuename string `json:"queue_name" binding:"required"` + Comment string `json:"comment,omitempty"` +} diff --git a/src/frontend/apiserver/models/resources.go b/src/frontend/apiserver/models/resources.go new file mode 100644 index 00000000..f2cf868c --- /dev/null +++ b/src/frontend/apiserver/models/resources.go @@ -0,0 +1,139 @@ +package models + +type ValueType int + +const ( + STRING ValueType = iota // Bool value + NUMERIC // Numeric value + BOOLEAN // String value +) + +// String returns the string representation of ValueType +func (v ValueType) String() string { + switch v { + case BOOLEAN: + return "Boolean" + case NUMERIC: + return "Numeric" + case STRING: + return "String" + default: + return "Unknown" + } +} + +type OrderType int + +const ( + INCR OrderType = iota + DECR + NA +) + +// String returns the string representation of OrderType +func (o OrderType) String() string { + switch o { + case INCR: + return "Incr" + case DECR: + return "Decr" + case NA: + return "NA" + default: + return "Unknown" + } +} + +var RESOURCE_DEFAULT_COLUMN = []string{ + "res_name", + "description", + "value_type", + "order_type", + "flag", + "interval", +} + +var RESOURCE_DISTRIBUTION_DEFAULT_COLUMN = []string{ + "res_name", + "numeric_value", + "location", + "is_dynamic", +} + +var RESOURCE_DEFINITION_DEFAULT_COLUMN = []string{ + "res_name", + "description", + "value_type", + "order_type", + "flag", + "interval", +} + +var ResourceFieldMappings = map[string]string{ + // API field name → struct field name + "value": "numeric_value", +} + +var ResourcesAvailableTypes = []string{"static", "dynamic"} + +const ( + BUILTIN = 0x01 + DYNAMIC = 0x02 + RELEASE = 0x20 + CONSUMABLE = 0x80 +) + +type FlagValue int + +// ToJSON returns the JSON object representation of FlagValue +func (f FlagValue) ToJSON() map[string]bool { + return map[string]bool{ + "builtin": f&BUILTIN != 0, + "dynamic": f&DYNAMIC != 0, + "release": f&RELEASE != 0, + "consumable": f&CONSUMABLE != 0, + } +} + +type ResourceInfo struct { + ResName string + Description string + ValueType ValueType + OrderType OrderType + Flag int + Interval int +} + +// queues info query supported filter +var RESOURCE_DISTRIBUTION_SUPPORT_FILTERS = []string{ + "res_name", + "is_dynamic", + "location", +} + +var RESOURCE_DEFINITION_SUPPORT_FILTERS = []string{ + "res_name", +} + +type ResourceDistributionInfo struct { + ResourceName string `json:"res_name"` + IsDynamic bool `json:"is_dynamic"` + ValueType int `json:"value_type"` + BooleanValue bool `json:"boolean_value"` + StringValue string `json:"string_value"` + NumericValue float64 `json:"numeric_value"` + Location string `json:"location"` +} + +type ResourceDefinitionInfo struct { + ResourceName string `json:"res_name"` + Description string `json:"description"` + ValueType int `json:"value_type"` + OrderType int `json:"order_type"` + Flag int `json:"flag"` + Interval int `json:"interval"` +} + +var RESOURCE_DISTRIBUTION_SUPPORT_SORT = []string{ + "res_name", +} diff --git a/src/frontend/apiserver/models/roles.go b/src/frontend/apiserver/models/roles.go new file mode 100644 index 00000000..015fbd78 --- /dev/null +++ b/src/frontend/apiserver/models/roles.go @@ -0,0 +1,84 @@ +package models + +var RolesAvailableTypes = []string{"builtin", "all"} + +var BUILTIN_ROLE_NAMES = []string{ + "basic_user", + "cluster_admin", + "queue_admin", +} + +var ROLE_SUPPORT_FILTERS = []string{ + "role_name", + "action", +} + +var ROLE_SUPPORT_SORT = []string{ + "role_name", + "action", +} + +var ROLE_DEFAULT_COLUMN = []string{ + "role_name", + "action", + "domain_type", + "domain_values", +} + +var JSONBRFields = []string{"domain_values"} + +type RoleFields struct { + RoleName string `json:"role_name"` +} +type RoleInfo struct { + RoleName string `json:"role_name"` + Action string `json:"action"` + DomainType string `json:"domain_type"` + DomainValues Jsonb `json:"domain_values"` +} + +type RolePermission struct { + Permission string `json:"permission_name" binding:"required"` + DomainType string `json:"domain_type" binding:"required"` + DomainValues []string `json:"domain_values"` +} + +type CreateRoleRequest struct { + RoleName string `json:"role_name" binding:"required"` + RolePermissions []RolePermission `json:"permissions" binding:"required"` +} + +const ( + DomainTypeAll = "all" + DomainTypeSelf = "self" + DomainTypeCluster = "cluster" + DomainTypeJob = "job" + DomainTypeQueue = "queue" + DomainTypeHost = "host" + DomainTypeUser = "user" + DomainTypeFile = "file" + DomainTypeHostGroup = "hostgroup" + DomainTypeUserGroup = "usergroup" + DomainTypeResource = "resource" + DomainTypeSelfUG = "self_ug" + DomainTypeSelfHG = "self_hg" + DomainTypeRBAC = "rbac" +) + +var SupportedDomainTypes = map[string][]string{ + DomainTypeCluster: {DomainTypeAll}, + DomainTypeJob: {DomainTypeAll, DomainTypeSelf, DomainTypeSelfUG}, + DomainTypeQueue: {DomainTypeAll, DomainTypeQueue}, + DomainTypeHost: {DomainTypeAll, DomainTypeHostGroup}, + DomainTypeUser: {DomainTypeAll, DomainTypeSelf}, + DomainTypeFile: {DomainTypeAll, DomainTypeSelf}, + DomainTypeHostGroup: {DomainTypeAll, DomainTypeSelfHG, DomainTypeHostGroup}, + DomainTypeUserGroup: {DomainTypeAll, DomainTypeSelfUG, DomainTypeUserGroup}, + DomainTypeResource: {DomainTypeAll, DomainTypeSelf}, + DomainTypeRBAC: {DomainTypeAll, DomainTypeSelf}, +} + +type RoleCreateResponse struct { + RoleName string `json:"role_name"` + Message string `json:"message"` +} diff --git a/src/frontend/apiserver/models/users.go b/src/frontend/apiserver/models/users.go new file mode 100644 index 00000000..a58ad8ed --- /dev/null +++ b/src/frontend/apiserver/models/users.go @@ -0,0 +1,33 @@ +package models + +var USER_SUPPORT_FILTERS = []string{ + "user_name", + "role_name", +} + +var USER_SUPPORT_SORT = []string{ + "user_name", + "role_name", +} + +var JSONBUFields = []string{"domain_values"} + +type AddRoleForUserRequest struct { + UserName string `json:"user_name" binding:"required"` + RoleName []string `json:"role_name" binding:"required"` +} + +type UserInfo struct { + UserName string `json:"user_name"` + RoleName string `json:"role_name"` +} + +type AddRoleForUserResponse struct { + UserName string `json:"user_name"` + Message string `json:"message"` +} + +var USER_DEFAULT_COLUMN = []string{ + "user_name", + "role_name", +} diff --git a/src/frontend/apiserver/router/routers.go b/src/frontend/apiserver/router/routers.go new file mode 100644 index 00000000..e999813f --- /dev/null +++ b/src/frontend/apiserver/router/routers.go @@ -0,0 +1,154 @@ +package router + +import ( + "apiserver/common" + "apiserver/controllers" + "apiserver/middleware" + "apiserver/utils" + logger "apiserver/utils" + "path/filepath" + + "github.com/gin-gonic/gin" + swaggerFiles "github.com/swaggo/files" + ginSwagger "github.com/swaggo/gin-swagger" + "go.uber.org/zap" +) + +func Router() *gin.Engine { + logger.Debug("Enter Router() function") + r := gin.Default() + + // check access URL whitelist + r.Use(middleware.WhitelistURLMiddleware()) + + // check client IP whitelist + whitelistConfig := utils.WhitelistYamlConfig + logger.Debug("whitelistConfig", zap.Any("whitelistConfig", whitelistConfig)) + r.Use(middleware.WhitelistIPMiddleware(whitelistConfig.Whitelist)) + + // Initialize rate limiter: 10 requests per second with a burst of 20 + rateLimitConfig := utils.RateLimitYamlConfig + globalRL := middleware.NewRateLimiter(rateLimitConfig) + r.Use(globalRL.Limit()) + r.Use(middleware.ValidateHandler()) //validate token + + // CORS + corsOrigins := middleware.CORSOriginsFromConfig( + utils.CORSYamlConfig != nil && utils.CORSYamlConfig.Enabled, + func() []string { + if utils.CORSYamlConfig != nil { + return utils.CORSYamlConfig.AllowedOrigins + } + return nil + }(), + ) + r.Use(middleware.CORSMiddleware(corsOrigins)) + + // load default rbac built-in roles + rbacPath := filepath.Join(utils.Global_TopDir, "config", "rbac.yaml") + if err := controllers.LoadRBACConfig(rbacPath); err != nil { + logger.Error("Failed to load RBAC config: " + err.Error()) + } + + // ################ case: ping route + r.GET("/ping", func(ctx *gin.Context) { + logger.Debug("Get /ping successfully from ", zap.String("client_ip", ctx.ClientIP())) + controllers.ReturnSuccess(ctx, common.SUCCESS_CODE, common.SUCCESS_MSG, "pong success") + }) + + // ################ case: swagger route + r.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler)) + + // ################ case: auth route + auth := r.Group("/xce/v1/auth") + { + authHandler := &controllers.AuthController{} + auth.POST("/logon", authHandler.Logon) + auth.POST("/logout", authHandler.Logout) + } + + // ################ case: permissions route + permsApi := r.Group("/xce/v1/permissions") + { + permHandler := &controllers.PermController{} + permsApi.GET("", controllers.RequirePermission("rbac:view"), permHandler.GetPermsInfo) + } + // ################ case: roles route + rolesApi := r.Group("/xce/v1/roles") + { + roleHandler := &controllers.RoleController{} + rolesApi.GET("", controllers.RequirePermission("rbac:view"), roleHandler.GetRolesInfo) + rolesApi.GET("/:rolename", controllers.RequirePermission("rbac:view"), roleHandler.GetRoleByRolename) + rolesApi.POST("/create", controllers.RequirePermission("rbac:control"), roleHandler.CreateRole) + rolesApi.POST("/:roleid/permissions", controllers.RequirePermission("rbac:control"), roleHandler.AddPermissionsForRole) + // update role_permission table + rolesApi.DELETE("/:rolename", controllers.RequirePermission("rbac:control"), roleHandler.DeleteRoleAndPermissions) + } + // ################ case: users route + usersApi := r.Group("/xce/v1/users") + { + userHandler := &controllers.UserController{} + usersApi.GET("", controllers.RequirePermission("rbac:view"), userHandler.GetUserRolesInfo) + usersApi.GET("/current", controllers.RequirePermission("rbac:view"), userHandler.GetCurrentUserInfo) + usersApi.GET("/:username/permissions", controllers.RequirePermission("rbac:view"), userHandler.GetUserPermInfo) + usersApi.POST("/:username/roles", controllers.RequirePermission("rbac:control"), userHandler.AddRolesForUser) + // update user_role table + usersApi.DELETE("/:username/:rolename", controllers.RequirePermission("rbac:control"), userHandler.DeleteRolesForUser) + } + // ################ case: clusters route + clusterApi := r.Group("/xce/v1/clusters") + { + clusterHandler := &controllers.ClusterController{} + clusterApi.GET("", controllers.RequirePermission("cluster:view"), clusterHandler.GetClustersInfo) + } + // ################ case: hosts route + hostsApi := r.Group("xce/v1/hosts") + { + hostHandler := &controllers.HostHandler{} + hostsApi.GET("", controllers.RequirePermission("host:view"), hostHandler.GetHostsInfo) + hostsApi.GET("/:hostname", controllers.RequirePermission("host:view"), hostHandler.GetHostByHostname) + // for select res_req contains &&, ||, >,<, !, == charactars, so use POST method. + hostsApi.POST("/select", controllers.RequirePermission("host:view"), hostHandler.FilterHostByResReqSelect) + hostsApi.POST("/control", controllers.RequirePermission("host:control"), hostHandler.ControlHost) + } + + // ################ case: queues route + queueApi := r.Group("/xce/v1/queues") + { + queueHandler := &controllers.QueueController{} + queueApi.GET("", controllers.RequirePermission("queue:view"), queueHandler.GetQueuesInfo) + queueApi.GET("/:queuename", controllers.RequirePermission("queue:view"), queueHandler.GetQueueByName) + queueApi.POST("/control", controllers.RequirePermission("queue:control"), queueHandler.ControlQueue) + } + + // ################ case: jobs route + jobsApi := r.Group("/xce/v1/jobs") + { + jobHandler := &controllers.JobController{} + jobsApi.GET("", controllers.RequirePermission("job:view"), jobHandler.GetJobsInfo) + jobsApi.GET("/:jobids", controllers.RequirePermission("job:view"), jobHandler.GetJobByIDs) + jobsApi.POST("/create", controllers.RequirePermission("queue:submit"), jobHandler.CreateJob) + jobsApi.POST("/:jobids/control", controllers.RequirePermission("job:control"), jobHandler.ControlJobByID) + jobsApi.POST("/control", controllers.RequirePermission("job:control"), jobHandler.ControlJob) + jobsApi.POST("/:jobids/restart", controllers.RequirePermission("queue:submit"), jobHandler.RestartJobByID) + } + + // ################ case: resources route + resApi := r.Group("/xce/v1/resources") + { + resHandler := &controllers.ResourceController{} + resApi.GET("/distribution", controllers.RequirePermission("resource:view"), resHandler.GetResourcesDistributionInfo) + resApi.GET("/definition", controllers.RequirePermission("resource:view"), resHandler.GetResourcesDefinitionInfo) + } + // ################ case: files route + filesApi := r.Group("/xce/v1/files") + { + filesHandler := &controllers.FilesController{} + filesApi.POST("/upload", controllers.RequirePermission("file:control"), filesHandler.UploadFiles) //upload one or more files + //filesApi.DELETE("", filesHandler.DeleteFiles) //delete + filesApi.GET("/:uuid/:filename", controllers.RequirePermission("file:view"), filesHandler.DownloadFile) //download + filesApi.GET("", controllers.RequirePermission("file:view"), filesHandler.GetFileLists) //query + } + + return r +} diff --git a/src/frontend/apiserver/services/data_service.go b/src/frontend/apiserver/services/data_service.go new file mode 100644 index 00000000..e9805f38 --- /dev/null +++ b/src/frontend/apiserver/services/data_service.go @@ -0,0 +1,117 @@ +package services + +import ( + "apiserver/store" + "apiserver/utils" + "fmt" + + logger "apiserver/utils" +) + +// Store table name constants. +const ( + TableJobInfo = "job_info" + TableQueueInfo = "queue_info" + TableHostInfo = "host_info" + TableResourceDef = "resource_def" + TableResourceLoc = "resource_loc" + TableClusterInfo = "cluster_info" + TableArrayJob = "array_job" + TableJobExecHosts = "job_exec_hosts" + TableRolePermission = "role_permission" + TableUserRole = "user_role" +) + +// QueryData queries in-memory store by table name, applies filter/sort/pagination. +func QueryData(tableName string, queryOpts *utils.QueryOptions, fields []string) ([]utils.OrderMap, string, error) { + var rows []utils.OrderMap + + switch tableName { + case TableJobInfo: + rows = store.StoreToOrderMaps(store.Jobs) + case TableQueueInfo: + rows = store.StoreToOrderMaps(store.Queues) + case TableHostInfo: + rows = store.StoreToOrderMaps(store.Hosts) + case TableResourceDef: + rows = store.StoreToOrderMaps(store.Resources) + case TableResourceLoc: + rows = store.StoreToOrderMaps(store.ResourceLocations) + case TableClusterInfo: + rows = store.StoreToOrderMaps(store.Clusters) + case TableArrayJob: + rows = store.StoreToOrderMaps(store.ArrayJobs) + case TableJobExecHosts: + rows = store.StoreToOrderMaps(store.JobExecHosts) + case TableRolePermission: + rows = roleStoreToOrderMaps() + case TableUserRole: + rows = userRoleStoreToOrderMaps() + default: + return nil, "", fmt.Errorf("unknown store: %s", tableName) + } + + result, pageToken := store.ExecuteQuery(rows, queryOpts, fields) + return result, pageToken, nil +} + +// InsertEntity inserts an entity into the named store. +func InsertEntity(tableName string, key string, entity interface{}) error { + logger.Debug("InsertEntity: " + tableName + " key=" + key) + // For now, this is a no-op since job submission goes through backend. + // Queue/host/resource updates come from notifications. + return nil +} + +// DeleteEntity deletes an entity from the named store. +func DeleteEntity(tableName string, key string) error { + logger.Debug("DeleteEntity: " + tableName + " key=" + key) + switch tableName { + case TableRolePermission: + store.Roles.Delete(key) + case TableUserRole: + // key format: "username:rolename" + // handled by store helper functions + } + return nil +} + +// roleStoreToOrderMaps converts role store to OrderMap format. +func roleStoreToOrderMaps() []utils.OrderMap { + rows := make([]utils.OrderMap, 0, store.Roles.Count()*4) + store.Roles.Range(func(roleName string, entry store.RoleEntry) bool { + for _, perm := range entry.Permissions { + row := utils.OrderMap{ + Fields: []string{"role_name", "action", "domain_type", "domain_values"}, + Data: map[string]interface{}{ + "role_name": roleName, + "action": perm.Action, + "domain_type": perm.DomainType, + "domain_values": perm.DomainValues, + }, + } + rows = append(rows, row) + } + return true + }) + return rows +} + +// userRoleStoreToOrderMaps converts user-role store to OrderMap format. +func userRoleStoreToOrderMaps() []utils.OrderMap { + rows := make([]utils.OrderMap, 0, store.UserRoles.Count()*2) + store.UserRoles.Range(func(userName string, roleNames []string) bool { + for _, roleName := range roleNames { + row := utils.OrderMap{ + Fields: []string{"user_name", "role_name"}, + Data: map[string]interface{}{ + "user_name": userName, + "role_name": roleName, + }, + } + rows = append(rows, row) + } + return true + }) + return rows +} diff --git a/src/frontend/apiserver/services/job_service.go b/src/frontend/apiserver/services/job_service.go new file mode 100644 index 00000000..c7957ada --- /dev/null +++ b/src/frontend/apiserver/services/job_service.go @@ -0,0 +1,16 @@ +package services + +import ( + "apiserver/backend" + "apiserver/models" +) + +// SubmitJob sends a job to the backend. The store is updated via notification callback. +func SubmitJob(job *models.JobInfo) error { + return backend.SubmitJob(job) +} + +// SubmitArrayJobs sends an array job to the backend. +func SubmitArrayJobs(arrayjob models.ArrayJobInfo, joblist []models.JobInfo) error { + return backend.SubmitArrayJobs(&arrayjob, joblist) +} diff --git a/src/frontend/apiserver/services/rbac_service.go b/src/frontend/apiserver/services/rbac_service.go new file mode 100644 index 00000000..6c43fbaa --- /dev/null +++ b/src/frontend/apiserver/services/rbac_service.go @@ -0,0 +1,29 @@ +package services + +import ( + "apiserver/store" + "apiserver/utils" +) + +// GetUserPermissions returns the user's roles and their permissions. +func GetUserPermissions(username string) ([]string, []utils.OrderMap, error) { + roleNames := store.GetUserRoleNames(username) + + var permRows []utils.OrderMap + for _, roleName := range roleNames { + perms := store.GetRolePermissions(roleName) + for _, perm := range perms { + row := utils.OrderMap{ + Fields: []string{"role_name", "action", "domain_type", "domain_values"}, + Data: map[string]interface{}{ + "role_name": roleName, + "action": perm.Action, + "domain_type": perm.DomainType, + "domain_values": perm.DomainValues, + }, + } + permRows = append(permRows, row) + } + } + return roleNames, permRows, nil +} diff --git a/src/frontend/apiserver/store/audit_test.go b/src/frontend/apiserver/store/audit_test.go new file mode 100644 index 00000000..a576e4cc --- /dev/null +++ b/src/frontend/apiserver/store/audit_test.go @@ -0,0 +1,148 @@ +package store + +import ( + "bufio" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// forbiddenPatterns lists symbols that must NOT appear in Go source files +// (dangling references from the DB removal refactor). +var forbiddenPatterns = []struct { + name string + pattern *regexp.Regexp +}{ + {"DBConnPool/DBHandle/DBPool", regexp.MustCompile(`\b(DBConnPool|DBHandle|DBPool)\b`)}, + {"InitDB/QueryFromDB/InsertToDB", regexp.MustCompile(`\b(InitDB|QueryFromDB|InsertToDB)\b`)}, + {"DBService functions", regexp.MustCompile(`\b(QueryFromDBService|GetDBConnection|InsertOneEntityToDBService|InsertBatchEntitiesToDBService|GetUserPermissionsFromDBService)\b`)}, + {"old cache types", regexp.MustCompile(`\b(HostInfoCache|QueueInfoCache|RoleInfoCache|UserInfoCache)\b`)}, + {"old cache functions", regexp.MustCompile(`\b(LoadCaches|GetInfoCache|LSInfoResource)\b`)}, + {"CGo import \"C\"", regexp.MustCompile(`import\s+"C"`)}, + {"CDBQueryOptions", regexp.MustCompile(`\bCDBQueryOptions\b`)}, +} + +// allowedFiles maps pattern names to files where the symbol is permitted. +var allowedFiles = map[string][]string{ + "CGo import \"C\"": {"cgo_stub.go"}, +} + +func TestNoDanglingReferences(t *testing.T) { + // Walk all .go files under the apiserver source tree + root := "." + entries, err := os.ReadDir(root) + if err != nil { + t.Fatalf("read dir: %v", err) + } + // Walk one level: skip vendor, result, etc. + dirs := []string{"."} + for _, e := range entries { + if e.IsDir() && !strings.HasPrefix(e.Name(), ".") && e.Name() != "result" { + dirs = append(dirs, e.Name()) + } + } + + for _, dir := range dirs { + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + if !strings.HasSuffix(path, ".go") { + return nil + } + // Skip test files for this audit — we only check production code + if strings.HasSuffix(path, "_test.go") { + return nil + } + + content, err := os.ReadFile(path) + if err != nil { + return err + } + + for _, fp := range forbiddenPatterns { + matches := fp.pattern.FindAllString(string(content), -1) + if len(matches) == 0 { + continue + } + // Check if this file is in the allowed list + allowed := false + if allowList, ok := allowedFiles[fp.name]; ok { + base := filepath.Base(path) + for _, a := range allowList { + if base == a { + allowed = true + break + } + } + } + if allowed { + continue + } + t.Errorf("%s: found forbidden symbol %q in %s", fp.name, matches, path) + } + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + } +} + +// TestNoUnusedImports scans production .go files for common unused-import +// indicators (blank identifiers with import paths that suggest dead code). +func TestNoDBImportsInProductionCode(t *testing.T) { + root := "." + entries, err := os.ReadDir(root) + if err != nil { + t.Fatalf("read dir: %v", err) + } + dirs := []string{"."} + for _, e := range entries { + if e.IsDir() && !strings.HasPrefix(e.Name(), ".") && e.Name() != "result" { + dirs = append(dirs, e.Name()) + } + } + + for _, dir := range dirs { + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return err + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + inImport := false + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if strings.HasPrefix(line, "import (") || strings.HasPrefix(line, "import\t(") { + inImport = true + continue + } + if inImport { + if line == ")" { + inImport = false + continue + } + // Check for DB-related import paths + if strings.Contains(line, "database/sql") || + strings.Contains(line, "lib/pq") || + strings.Contains(line, "pgx") || + strings.Contains(line, "gorm") { + t.Errorf("%s: found DB driver import: %s", path, line) + } + } + } + return scanner.Err() + }) + if err != nil { + t.Fatalf("walk %s: %v", dir, err) + } + } +} diff --git a/src/frontend/apiserver/store/builtin.go b/src/frontend/apiserver/store/builtin.go new file mode 100644 index 00000000..612cdc33 --- /dev/null +++ b/src/frontend/apiserver/store/builtin.go @@ -0,0 +1,195 @@ +package store + +import ( + "apiserver/models" + "time" +) + +// LoadBuiltinData populates all stores with test data. +func LoadBuiltinData() error { + LoadBuiltinQueues() + LoadBuiltinHosts() + LoadBuiltinResources() + LoadBuiltinResourceLocations() + LoadBuiltinJobs() + return nil +} + +func LoadBuiltinQueues() { + queues := []models.QueueInfo{ + { + QueueName: "normal", + QueueStatus: 0x00, // OPEN:active + Priority: 30, + Description: "Default queue for normal jobs", + NumPend: 5, + NumRun: 3, + NumDone: 100, + }, + { + QueueName: "priority", + QueueStatus: 0x00, + Priority: 50, + Description: "High priority queue", + NumPend: 2, + NumRun: 5, + NumDone: 50, + }, + { + QueueName: "short", + QueueStatus: 0x00, + Priority: 20, + Description: "Queue for short-running jobs", + NumPend: 10, + NumRun: 8, + NumDone: 200, + }, + } + for _, q := range queues { + Queues.Put(q.QueueName, q) + } +} + +func LoadBuiltinHosts() { + hosts := []models.HostInfo{ + { + HostName: "host1", + HostStatus: 0, // OK + HostModel: "x86_64", + HostType: "linux", + NumCPUs: 16, + NumCores: 8, + NumThreads: 16, + MaxSlot: 16, + MaxMem: 65536, + MaxSwap: 32768, + MaxTmp: 102400, + CPUFactor: 1.0, + }, + { + HostName: "host2", + HostStatus: 0, + HostModel: "x86_64", + HostType: "linux", + NumCPUs: 32, + NumCores: 16, + NumThreads: 32, + MaxSlot: 32, + MaxMem: 131072, + MaxSwap: 65536, + MaxTmp: 204800, + CPUFactor: 2.0, + }, + { + HostName: "host3", + HostStatus: 0, + HostModel: "x86_64", + HostType: "linux", + NumCPUs: 64, + NumCores: 32, + NumThreads: 64, + MaxSlot: 64, + MaxMem: 262144, + MaxSwap: 131072, + MaxTmp: 409600, + CPUFactor: 4.0, + }, + } + for _, h := range hosts { + Hosts.Put(h.HostName, h) + } +} + +func LoadBuiltinResources() { + resources := []models.ResourceDefinitionInfo{ + {ResourceName: "ncpus", Description: "Number of CPUs", ValueType: int(models.NUMERIC), OrderType: int(models.INCR), Flag: models.BUILTIN, Interval: 0}, + {ResourceName: "mem", Description: "Memory in MB", ValueType: int(models.NUMERIC), OrderType: int(models.DECR), Flag: models.BUILTIN | models.CONSUMABLE, Interval: 0}, + {ResourceName: "tmp", Description: "Temp space in MB", ValueType: int(models.NUMERIC), OrderType: int(models.DECR), Flag: models.BUILTIN, Interval: 0}, + {ResourceName: "swp", Description: "Swap space in MB", ValueType: int(models.NUMERIC), OrderType: int(models.DECR), Flag: models.BUILTIN, Interval: 0}, + {ResourceName: "nslots", Description: "Number of slots", ValueType: int(models.NUMERIC), OrderType: int(models.INCR), Flag: models.BUILTIN | models.CONSUMABLE, Interval: 0}, + {ResourceName: "gpu", Description: "GPU devices", ValueType: int(models.NUMERIC), OrderType: int(models.DECR), Flag: models.BUILTIN | models.CONSUMABLE | models.DYNAMIC, Interval: 60}, + {ResourceName: "matlab_lic", Description: "MATLAB network license", ValueType: int(models.NUMERIC), OrderType: int(models.DECR), Flag: models.BUILTIN | models.DYNAMIC, Interval: 60}, + {ResourceName: "vcs_lic", Description: "VCS simulator license", ValueType: int(models.NUMERIC), OrderType: int(models.DECR), Flag: models.BUILTIN | models.DYNAMIC, Interval: 60}, + } + for _, r := range resources { + Resources.Put(r.ResourceName, r) + } +} + +func LoadBuiltinResourceLocations() { + locations := []models.ResourceDistributionInfo{ + // Host-based: each host has its own value + {ResourceName: "ncpus", IsDynamic: false, Location: "host1", NumericValue: 16}, + {ResourceName: "ncpus", IsDynamic: false, Location: "host2", NumericValue: 32}, + {ResourceName: "ncpus", IsDynamic: false, Location: "host3", NumericValue: 64}, + {ResourceName: "mem", IsDynamic: false, Location: "host1", NumericValue: 65536}, + {ResourceName: "mem", IsDynamic: false, Location: "host2", NumericValue: 131072}, + {ResourceName: "mem", IsDynamic: false, Location: "host3", NumericValue: 262144}, + {ResourceName: "swp", IsDynamic: false, Location: "host1", NumericValue: 32768}, + {ResourceName: "swp", IsDynamic: false, Location: "host2", NumericValue: 65536}, + {ResourceName: "swp", IsDynamic: false, Location: "host3", NumericValue: 131072}, + {ResourceName: "tmp", IsDynamic: false, Location: "host1", NumericValue: 102400}, + {ResourceName: "tmp", IsDynamic: false, Location: "host2", NumericValue: 204800}, + {ResourceName: "tmp", IsDynamic: false, Location: "host3", NumericValue: 409600}, + // Global: license server accessible by all hosts (location = "*") + {ResourceName: "matlab_lic", IsDynamic: true, Location: "*", NumericValue: 100}, + // Partial: GPU only on specific hosts + {ResourceName: "gpu", IsDynamic: true, Location: "host2", NumericValue: 4}, + {ResourceName: "gpu", IsDynamic: true, Location: "host3", NumericValue: 8}, + // Partial: license only accessible by some hosts + {ResourceName: "vcs_lic", IsDynamic: true, Location: "lic_group_a", NumericValue: 50}, + } + for _, loc := range locations { + key := loc.ResourceName + ":" + loc.Location + ResourceLocations.Put(key, loc) + } +} + +func LoadBuiltinJobs() { + now := time.Now() + jobs := []models.JobInfo{ + { + Unique_JobID: 1001, + JobID: 1001, + SubmitUser: "user1", + JobName: "test_job_1", + JobStatus: 0x02, // RUN + QueueName: "normal", + Command: "sleep 100", + SubmitTime: now.Add(-1 * time.Hour), + }, + { + Unique_JobID: 1002, + JobID: 1002, + SubmitUser: "user1", + JobName: "test_job_2", + JobStatus: 0x01, // PEND + QueueName: "normal", + Command: "echo hello", + SubmitTime: now.Add(-30 * time.Minute), + }, + { + Unique_JobID: 1003, + JobID: 1003, + SubmitUser: "user2", + JobName: "priority_job", + JobStatus: 0x02, // RUN + QueueName: "priority", + Command: "python train.py", + SubmitTime: now.Add(-2 * time.Hour), + }, + { + Unique_JobID: 1004, + JobID: 1004, + SubmitUser: "user2", + JobName: "done_job", + JobStatus: 0x08, // DONE + QueueName: "short", + Command: "ls -la", + SubmitTime: now.Add(-3 * time.Hour), + }, + } + for _, j := range jobs { + Jobs.Put(uint64(j.Unique_JobID), j) + } +} diff --git a/src/frontend/apiserver/store/convert.go b/src/frontend/apiserver/store/convert.go new file mode 100644 index 00000000..0a6f432a --- /dev/null +++ b/src/frontend/apiserver/store/convert.go @@ -0,0 +1,50 @@ +package store + +import ( + "apiserver/utils" + "reflect" + "strings" +) + +// StructToOrderMap converts any struct to an OrderMap using json tags. +func StructToOrderMap(v interface{}) utils.OrderMap { + val := reflect.ValueOf(v) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + typ := val.Type() + + fields := make([]string, 0, typ.NumField()) + data := make(map[string]interface{}, typ.NumField()) + + for i := 0; i < typ.NumField(); i++ { + ft := typ.Field(i) + jsonTag := ft.Tag.Get("json") + if jsonTag == "" || jsonTag == "-" { + continue + } + fieldName := strings.Split(jsonTag, ",")[0] + fields = append(fields, fieldName) + data[fieldName] = val.Field(i).Interface() + } + + return utils.OrderMap{ + Fields: fields, + Data: data, + } +} + +// StructsToOrderMaps converts a slice of structs to []OrderMap. +func StructsToOrderMaps[V any](items []V) []utils.OrderMap { + result := make([]utils.OrderMap, len(items)) + for i, item := range items { + result[i] = StructToOrderMap(item) + } + return result +} + +// StoreToOrderMaps converts all items in a DataStore to []OrderMap. +func StoreToOrderMaps[K comparable, V any](s *DataStore[K, V]) []utils.OrderMap { + items := s.List() + return StructsToOrderMaps(items) +} diff --git a/src/frontend/apiserver/store/query.go b/src/frontend/apiserver/store/query.go new file mode 100644 index 00000000..508dbc45 --- /dev/null +++ b/src/frontend/apiserver/store/query.go @@ -0,0 +1,370 @@ +package store + +import ( + "apiserver/utils" + "encoding/base64" + "encoding/json" + "fmt" + "sort" + "strings" +) + +const PER_PAGE_LIMIT = 100 + +// ExecuteQuery runs the full filter → sort → paginate → select pipeline. +func ExecuteQuery(rows []utils.OrderMap, opts *utils.QueryOptions, fields []string) ([]utils.OrderMap, string) { + if opts == nil { + return SelectFields(rows, fields), "" + } + + // 1. Filter + if len(opts.Conditions) > 0 { + rows = FilterRows(rows, opts.Conditions) + } + + // 2. Sort + if len(opts.Orders) > 0 { + SortRows(rows, opts.Orders) + } + + // 3. Paginate + limit := opts.Limit + if limit <= 0 { + limit = PER_PAGE_LIMIT + } + rows, nextToken := PaginateRows(rows, limit, opts.Bookmark, opts.Orders) + + // 4. Select fields + rows = SelectFields(rows, fields) + + return rows, nextToken +} + +// MatchesCondition evaluates a single filter condition against a row. +func MatchesCondition(row utils.OrderMap, cond utils.Condition) bool { + fieldVal := row.Data[cond.Column] + + switch cond.Operator { + case utils.OpEQ: + return compareValues(fieldVal, cond.Value.Value) == 0 + case utils.OpNE: + return compareValues(fieldVal, cond.Value.Value) != 0 + case utils.OpLT: + return compareValues(fieldVal, cond.Value.Value) < 0 + case utils.OpLTE: + return compareValues(fieldVal, cond.Value.Value) <= 0 + case utils.OpGT: + return compareValues(fieldVal, cond.Value.Value) > 0 + case utils.OpGTE: + return compareValues(fieldVal, cond.Value.Value) >= 0 + case utils.OpIN: + return valueInList(fieldVal, cond.Value.Value) + case utils.OpNOTIN: + return !valueInList(fieldVal, cond.Value.Value) + case utils.OpLIKE: + return matchLike(fieldVal, cond.Value.Value) + case utils.OpNOTLIKE: + return !matchLike(fieldVal, cond.Value.Value) + case utils.OpBETWEEN: + return valueBetween(fieldVal, cond.Value.Value) + case utils.OpISNULL: + return fieldVal == nil + case utils.OpNOTNULL: + return fieldVal != nil + } + return false +} + +// MatchesBitmaskCondition handles bitmask fields (job_status, host_status). +// For bitmask EQ: (val & mask) != 0 means the status is set. +// For bitmask NE: (val & mask) == 0 means the status is NOT set. +func MatchesBitmaskCondition(row utils.OrderMap, field string, bitmask int64, op utils.OperatorType) bool { + val := toInt64(row.Data[field]) + result := val & bitmask + switch op { + case utils.OpNE: // bitmask "eq" is stored as OpNE (val & mask != 0) + return result != 0 + case utils.OpEQ: // bitmask "ne" is stored as OpEQ (val & mask == 0) + return result == 0 + } + return false +} + +// IsBitmaskField returns true for fields that use bitmask-based filtering. +func IsBitmaskField(field string) bool { + return field == "job_status" || field == "host_status" +} + +// FilterRows applies all conditions (AND logic) to rows. +func FilterRows(rows []utils.OrderMap, conditions []utils.Condition) []utils.OrderMap { + result := make([]utils.OrderMap, 0, len(rows)) + for _, row := range rows { + match := true + for _, cond := range conditions { + if IsBitmaskField(cond.Column) { + bitmask, ok := cond.Value.Value.(int64) + if !ok { + match = false + break + } + if !MatchesBitmaskCondition(row, cond.Column, bitmask, cond.Operator) { + match = false + break + } + } else { + if !MatchesCondition(row, cond) { + match = false + break + } + } + } + if match { + result = append(result, row) + } + } + return result +} + +// SortRows sorts by multiple fields with asc/desc. +func SortRows(rows []utils.OrderMap, orders []utils.SortOrder) { + sort.SliceStable(rows, func(i, j int) bool { + for _, ord := range orders { + cmp := compareValues(rows[i].Data[ord.Column], rows[j].Data[ord.Column]) + if cmp == 0 { + continue + } + if ord.Ascending { + return cmp < 0 + } + return cmp > 0 + } + return false + }) +} + +// PaginateRows applies limit + page_token (bookmark-based pagination). +func PaginateRows(rows []utils.OrderMap, limit int, pageToken string, orders []utils.SortOrder) ([]utils.OrderMap, string) { + startIdx := 0 + if pageToken != "" { + startIdx = findPageTokenIndex(rows, pageToken, orders) + } + if limit <= 0 { + limit = PER_PAGE_LIMIT + } + endIdx := startIdx + limit + if endIdx > len(rows) { + endIdx = len(rows) + } + if startIdx >= len(rows) { + return []utils.OrderMap{}, "" + } + + result := rows[startIdx:endIdx] + nextToken := "" + if endIdx < len(rows) { + nextToken = encodePageToken(result[len(result)-1], orders) + } + return result, nextToken +} + +// SelectFields projects only requested fields from rows. +func SelectFields(rows []utils.OrderMap, fields []string) []utils.OrderMap { + if len(fields) == 0 { + return rows + } + result := make([]utils.OrderMap, len(rows)) + for i, row := range rows { + newData := make(map[string]interface{}) + for _, f := range fields { + if v, ok := row.Data[f]; ok { + newData[f] = v + } + } + result[i] = utils.OrderMap{Fields: fields, Data: newData} + } + return result +} + +// --- Helper functions --- + +// compareValues does type-aware comparison. Returns -1, 0, or 1. +func compareValues(a, b interface{}) int { + if a == nil && b == nil { + return 0 + } + if a == nil { + return -1 + } + if b == nil { + return 1 + } + + // Try int64 comparison + aInt, aIsInt := toInt64Safe(a) + bInt, bIsInt := toInt64Safe(b) + if aIsInt && bIsInt { + if aInt < bInt { + return -1 + } + if aInt > bInt { + return 1 + } + return 0 + } + + // Try float64 comparison + aFloat, aIsFloat := toFloat64Safe(a) + bFloat, bIsFloat := toFloat64Safe(b) + if aIsFloat && bIsFloat { + if aFloat < bFloat { + return -1 + } + if aFloat > bFloat { + return 1 + } + return 0 + } + + // String comparison + aStr := fmt.Sprintf("%v", a) + bStr := fmt.Sprintf("%v", b) + if aStr < bStr { + return -1 + } + if aStr > bStr { + return 1 + } + return 0 +} + +func toInt64(v interface{}) int64 { + val, _ := toInt64Safe(v) + return val +} + +func toInt64Safe(v interface{}) (int64, bool) { + switch val := v.(type) { + case int64: + return val, true + case int: + return int64(val), true + case int32: + return int64(val), true + case uint64: + return int64(val), true + case float64: + return int64(val), true + } + return 0, false +} + +func toFloat64Safe(v interface{}) (float64, bool) { + switch val := v.(type) { + case float64: + return val, true + case float32: + return float64(val), true + case int64: + return float64(val), true + case int: + return float64(val), true + } + return 0, false +} + +// valueInList checks if val is in a list of Values. +func valueInList(val interface{}, list interface{}) bool { + switch l := list.(type) { + case []utils.Value: + for _, item := range l { + if compareValues(val, item.Value) == 0 { + return true + } + } + case []interface{}: + for _, item := range l { + if compareValues(val, item) == 0 { + return true + } + } + } + return false +} + +// matchLike implements SQL LIKE with Go string matching. +func matchLike(fieldVal, pattern interface{}) bool { + fieldStr := fmt.Sprintf("%v", fieldVal) + patternStr := fmt.Sprintf("%v", pattern) + + // If pattern contains SQL wildcards, translate them + if strings.Contains(patternStr, "%") { + // Simple wildcard matching + if strings.HasPrefix(patternStr, "%") && strings.HasSuffix(patternStr, "%") { + return strings.Contains(strings.ToLower(fieldStr), strings.ToLower(strings.Trim(patternStr, "%"))) + } + if strings.HasPrefix(patternStr, "%") { + return strings.HasSuffix(strings.ToLower(fieldStr), strings.ToLower(strings.TrimPrefix(patternStr, "%"))) + } + if strings.HasSuffix(patternStr, "%") { + return strings.HasPrefix(strings.ToLower(fieldStr), strings.ToLower(strings.TrimSuffix(patternStr, "%"))) + } + } + + // Default: substring match (contains) + return strings.Contains(strings.ToLower(fieldStr), strings.ToLower(patternStr)) +} + +// valueBetween checks if val is between the first two elements of a Value array. +func valueBetween(val interface{}, rangeVal interface{}) bool { + switch r := rangeVal.(type) { + case []utils.Value: + if len(r) >= 2 { + return compareValues(val, r[0].Value) >= 0 && compareValues(val, r[1].Value) <= 0 + } + } + return false +} + +// Page token encoding/decoding using base64 JSON. +func encodePageToken(lastRow utils.OrderMap, orders []utils.SortOrder) string { + tokenData := make(map[string]interface{}) + for _, ord := range orders { + if v, ok := lastRow.Data[ord.Column]; ok { + tokenData[ord.Column] = v + } + } + jsonBytes, err := json.Marshal(tokenData) + if err != nil { + return "" + } + return base64.URLEncoding.EncodeToString(jsonBytes) +} + +func findPageTokenIndex(rows []utils.OrderMap, pageToken string, orders []utils.SortOrder) int { + jsonBytes, err := base64.URLEncoding.DecodeString(pageToken) + if err != nil { + return 0 + } + var tokenData map[string]interface{} + if err := json.Unmarshal(jsonBytes, &tokenData); err != nil { + return 0 + } + + for i, row := range rows { + match := true + for _, ord := range orders { + tokenVal, ok := tokenData[ord.Column] + if !ok { + continue + } + if compareValues(row.Data[ord.Column], tokenVal) != 0 { + match = false + break + } + } + if match { + return i + 1 // Start after the matched row + } + } + return 0 +} diff --git a/src/frontend/apiserver/store/store.go b/src/frontend/apiserver/store/store.go new file mode 100644 index 00000000..3930eb4c --- /dev/null +++ b/src/frontend/apiserver/store/store.go @@ -0,0 +1,104 @@ +package store + +import "sync" + +// DataStore is a generic thread-safe in-memory key-value store. +type DataStore[K comparable, V any] struct { + mu sync.RWMutex + data map[K]V +} + +func NewDataStore[K comparable, V any]() *DataStore[K, V] { + return &DataStore[K, V]{ + data: make(map[K]V), + } +} + +func (s *DataStore[K, V]) Get(key K) (V, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + v, ok := s.data[key] + return v, ok +} + +func (s *DataStore[K, V]) Has(key K) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.data[key] + return ok +} + +func (s *DataStore[K, V]) Put(key K, val V) { + s.mu.Lock() + defer s.mu.Unlock() + s.data[key] = val +} + +func (s *DataStore[K, V]) Delete(key K) bool { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.data[key] + if ok { + delete(s.data, key) + } + return ok +} + +func (s *DataStore[K, V]) List() []V { + s.mu.RLock() + defer s.mu.RUnlock() + result := make([]V, 0, len(s.data)) + for _, v := range s.data { + result = append(result, v) + } + return result +} + +func (s *DataStore[K, V]) Keys() []K { + s.mu.RLock() + defer s.mu.RUnlock() + result := make([]K, 0, len(s.data)) + for k := range s.data { + result = append(result, k) + } + return result +} + +func (s *DataStore[K, V]) Count() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.data) +} + +// Range iterates over a snapshot of all entries. Return false from fn to stop early. +// The callback runs outside the lock, so it is safe to call other store methods from fn. +func (s *DataStore[K, V]) Range(fn func(K, V) bool) { + s.mu.RLock() + type kv struct { + k K + v V + } + snap := make([]kv, 0, len(s.data)) + for k, v := range s.data { + snap = append(snap, kv{k, v}) + } + s.mu.RUnlock() + for _, e := range snap { + if !fn(e.k, e.v) { + break + } + } +} + +// Filter returns all values matching the predicate. +func (s *DataStore[K, V]) Filter(fn func(V) bool) []V { + s.mu.RLock() + defer s.mu.RUnlock() + result := make([]V, 0, len(s.data)/4+1) + for _, v := range s.data { + if fn(v) { + result = append(result, v) + } + } + return result +} diff --git a/src/frontend/apiserver/store/store_test.go b/src/frontend/apiserver/store/store_test.go new file mode 100644 index 00000000..d9052626 --- /dev/null +++ b/src/frontend/apiserver/store/store_test.go @@ -0,0 +1,137 @@ +package store + +import ( + "sync" + "testing" +) + +func TestDataStore_BasicOperations(t *testing.T) { + s := NewDataStore[string, int]() + + // Get on empty store + if _, ok := s.Get("a"); ok { + t.Error("Get on empty store should return false") + } + + // Put and Get + s.Put("a", 1) + if v, ok := s.Get("a"); !ok || v != 1 { + t.Errorf("Get(a) = %d, %v; want 1, true", v, ok) + } + + // Put overwrites + s.Put("a", 2) + if v, ok := s.Get("a"); !ok || v != 2 { + t.Errorf("Get(a) after overwrite = %d, %v; want 2, true", v, ok) + } + + // Has + if !s.Has("a") { + t.Error("Has(a) should be true") + } + if s.Has("missing") { + t.Error("Has(missing) should be false") + } + + // Delete + if !s.Delete("a") { + t.Error("Delete(a) should return true") + } + if s.Delete("a") { + t.Error("Delete(a) again should return false") + } + if _, ok := s.Get("a"); ok { + t.Error("Get after delete should return false") + } +} + +func TestDataStore_ListKeysCount(t *testing.T) { + s := NewDataStore[string, int]() + s.Put("x", 1) + s.Put("y", 2) + s.Put("z", 3) + + if s.Count() != 3 { + t.Errorf("Count() = %d, want 3", s.Count()) + } + + keys := s.Keys() + if len(keys) != 3 { + t.Errorf("Keys() length = %d, want 3", len(keys)) + } + + list := s.List() + if len(list) != 3 { + t.Errorf("List() length = %d, want 3", len(list)) + } +} + +func TestDataStore_Filter(t *testing.T) { + s := NewDataStore[string, int]() + s.Put("a", 1) + s.Put("b", 2) + s.Put("c", 3) + s.Put("d", 1) + + result := s.Filter(func(v int) bool { return v == 1 }) + if len(result) != 2 { + t.Errorf("Filter(v==1) count = %d, want 2", len(result)) + } +} + +func TestDataStore_Range(t *testing.T) { + s := NewDataStore[string, int]() + s.Put("a", 10) + s.Put("b", 20) + + var sum int + s.Range(func(k string, v int) bool { + sum += v + return true + }) + if sum != 30 { + t.Errorf("Range sum = %d, want 30", sum) + } + + // Test early stop + count := 0 + s.Range(func(k string, v int) bool { + count++ + return false + }) + if count != 1 { + t.Errorf("Range early stop count = %d, want 1", count) + } +} + +func TestDataStore_ConcurrentAccess(t *testing.T) { + s := NewDataStore[int, int]() + var wg sync.WaitGroup + + // Concurrent writes + for i := 0; i < 100; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + s.Put(n, n) + }(i) + } + wg.Wait() + + if s.Count() != 100 { + t.Errorf("Count after concurrent writes = %d, want 100", s.Count()) + } + + // Concurrent reads + for i := 0; i < 100; i++ { + wg.Add(1) + go func(n int) { + defer wg.Done() + v, ok := s.Get(n) + if !ok || v != n { + t.Errorf("Get(%d) = %d, %v; want %d, true", n, v, ok, n) + } + }(i) + } + wg.Wait() +} diff --git a/src/frontend/apiserver/store/stores.go b/src/frontend/apiserver/store/stores.go new file mode 100644 index 00000000..81b7a3a1 --- /dev/null +++ b/src/frontend/apiserver/store/stores.go @@ -0,0 +1,176 @@ +package store + +import "apiserver/models" + +// Global store instances. +var ( + Jobs *DataStore[uint64, models.JobInfo] + ArrayJobs *DataStore[uint64, models.ArrayJobInfo] + Queues *DataStore[string, models.QueueInfo] + Hosts *DataStore[string, models.HostInfo] + Resources *DataStore[string, models.ResourceDefinitionInfo] + ResourceLocations *DataStore[string, models.ResourceDistributionInfo] + Clusters *DataStore[string, models.ClusterInfo] + JobExecHosts *DataStore[int64, models.JobExecHosts] + Roles *DataStore[string, RoleEntry] + UserRoles *DataStore[string, []string] // user_name → role_names +) + +// RoleEntry holds a role's permissions. +type RoleEntry struct { + RoleName string + Permissions []RolePermission +} + +// RolePermission holds a single permission entry for a role. +type RolePermission struct { + Action string + DomainType string + DomainValues []string +} + +// InitStores creates all global store instances. +func InitStores() { + Jobs = NewDataStore[uint64, models.JobInfo]() + ArrayJobs = NewDataStore[uint64, models.ArrayJobInfo]() + Queues = NewDataStore[string, models.QueueInfo]() + Hosts = NewDataStore[string, models.HostInfo]() + Resources = NewDataStore[string, models.ResourceDefinitionInfo]() + ResourceLocations = NewDataStore[string, models.ResourceDistributionInfo]() + Clusters = NewDataStore[string, models.ClusterInfo]() + JobExecHosts = NewDataStore[int64, models.JobExecHosts]() + Roles = NewDataStore[string, RoleEntry]() + UserRoles = NewDataStore[string, []string]() +} + +// Helper functions for RBAC. + +func GetUserRoleNames(username string) []string { + roles, ok := UserRoles.Get(username) + if !ok { + return []string{} + } + return roles +} + +func GetRolePermissions(roleName string) []RolePermission { + role, ok := Roles.Get(roleName) + if !ok { + return []RolePermission{} + } + return role.Permissions +} + +func AddRoleForUser(username string, roleName string) { + roles, _ := UserRoles.Get(username) + for _, r := range roles { + if r == roleName { + return // already has this role + } + } + roles = append(roles, roleName) + UserRoles.Put(username, roles) +} + +func DeleteRoleForUser(username string, roleName string) { + roles, ok := UserRoles.Get(username) + if !ok { + return + } + newRoles := make([]string, 0, len(roles)) + for _, r := range roles { + if r != roleName { + newRoles = append(newRoles, r) + } + } + if len(newRoles) == 0 { + UserRoles.Delete(username) + } else { + UserRoles.Put(username, newRoles) + } +} + +func AddPermissionsForRole(roleName string, perms []RolePermission) { + role, ok := Roles.Get(roleName) + if !ok { + role = RoleEntry{RoleName: roleName} + } + role.Permissions = append(role.Permissions, perms...) + Roles.Put(roleName, role) +} + +// GetAllQueueNames returns all queue names in the store. +func GetAllQueueNames() []string { + return Queues.Keys() +} + +// GetAllHostNames returns all host names in the store. +func GetAllHostNames() []string { + return Hosts.Keys() +} + +// GetAllRoleNames returns all role names. +func GetAllRoleNames(includeBuiltin bool) []string { + roleMap := make(map[string]bool) + for _, k := range Roles.Keys() { + roleMap[k] = true + } + if includeBuiltin { + for _, name := range models.BUILTIN_ROLE_NAMES { + roleMap[name] = true + } + } + names := make([]string, 0, len(roleMap)) + for k := range roleMap { + names = append(names, k) + } + return names +} + +// GetUsersUnderRoleName returns all user names that have the given role. +func GetUsersUnderRoleName(roleName string) []string { + var users []string + UserRoles.Range(func(userName string, roles []string) bool { + for _, r := range roles { + if r == roleName { + users = append(users, userName) + break + } + } + return true + }) + return users +} + +// GetRoleAllPermissions returns permission action names for a role. +func GetRoleAllPermissions(roleName string) []string { + role, ok := Roles.Get(roleName) + if !ok { + return []string{} + } + perms := make([]string, 0, len(role.Permissions)) + for _, p := range role.Permissions { + perms = append(perms, p.Action) + } + return perms +} + +// AddRoleNames ensures a role entry exists (creates empty if not). +func AddRoleNames(roleName string) { + if !Roles.Has(roleName) { + Roles.Put(roleName, RoleEntry{RoleName: roleName}) + } +} + +// IsQueueExist checks if all queue names exist in the store. +func IsQueueExist(queueNames []string) bool { + if len(queueNames) == 0 { + return false + } + for _, q := range queueNames { + if !Queues.Has(q) { + return false + } + } + return true +} diff --git a/src/frontend/apiserver/tools/encrypt/crypter.go b/src/frontend/apiserver/tools/encrypt/crypter.go new file mode 100644 index 00000000..ca0d0e16 --- /dev/null +++ b/src/frontend/apiserver/tools/encrypt/crypter.go @@ -0,0 +1,117 @@ +package main + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + + survey "github.com/AlecAivazis/survey/v2" // survey package +) + +// admin need set env: ENCRYPTION_KEY +// the default value: ")ZLxR6X@h5tYGhkwin [options] + + Options: + -action enc|dec + -password password (if empty, will be prompted) + `) + } + + action := flag.String("action", "", "action to perform: enc or dec") + password := flag.String("password", "", "password") + + flag.Parse() + + if *action != "enc" && *action != "dec" { + fmt.Println("Invalid action. Use -action=enc or -action=dec") + os.Exit(1) + } + + // use survey to prompt for missing values + var inputPassword string + if *password == "" { + survey.AskOne(&survey.Password{Message: "Enter password:\n"}, password) + inputPassword = *password + } + inputPassword = *password + + if *action == "enc" { + encryptedPassword, err := Encrypt(inputPassword, "") + if err != nil { + log.Fatalf("Failed to encrypt password: %v", err) + } + fmt.Println(encryptedPassword) + } + + if *action == "dec" { + realPassword, err := Decrypt(inputPassword, "") + if err != nil { + log.Fatalf("Failed to decrypt password: %v", err) + } + fmt.Println(realPassword) + } +} + +// use sha256 to get 32 bytes key +func getAESKey(key string) []byte { + selfSecretKey := os.Getenv("ENCRYPTION_KEY") + if selfSecretKey != "" { + key = selfSecretKey + } else { + key = EncryptionKey + } + hash := sha256.Sum256([]byte(key)) + return hash[:] +} + +func Encrypt(plaintext, secret string) (string, error) { + key := getAESKey(secret) + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + ciphertext := make([]byte, aes.BlockSize+len(plaintext)) + iv := ciphertext[:aes.BlockSize] + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + + stream := cipher.NewCFBEncrypter(block, iv) + stream.XORKeyStream(ciphertext[aes.BlockSize:], []byte(plaintext)) + return base64.StdEncoding.EncodeToString(ciphertext), nil +} + +func Decrypt(encryptedText, secret string) (string, error) { + key := getAESKey(secret) + ciphertext, err := base64.StdEncoding.DecodeString(encryptedText) + if err != nil { + return "", err + } + if len(ciphertext) < aes.BlockSize { + return "", errors.New("the ciphertext is too short") + } + iv := ciphertext[:aes.BlockSize] + ciphertext = ciphertext[aes.BlockSize:] + block, err := aes.NewCipher(key) + if err != nil { + return "", err + } + stream := cipher.NewCFBDecrypter(block, iv) + stream.XORKeyStream(ciphertext, ciphertext) + return string(ciphertext), nil +} diff --git a/src/frontend/apiserver/utils/cgo_stub.go b/src/frontend/apiserver/utils/cgo_stub.go new file mode 100644 index 00000000..9a7fc2fc --- /dev/null +++ b/src/frontend/apiserver/utils/cgo_stub.go @@ -0,0 +1,5 @@ +package utils + +// This file is reserved for future CGo bindings to the backend C library. +// The CGo infrastructure (compiler flags, linker flags) is kept in meson.build +// and will be used when the backend C library integration is implemented. diff --git a/src/frontend/apiserver/utils/config.go b/src/frontend/apiserver/utils/config.go new file mode 100644 index 00000000..ceb8820a --- /dev/null +++ b/src/frontend/apiserver/utils/config.go @@ -0,0 +1,293 @@ +package utils + +import ( + "errors" + "fmt" + "log" + "os" + "runtime" + "strconv" + "strings" + + "github.com/spf13/viper" +) + +type BackendConfig struct { + Address string `yaml:"address"` + Port int `yaml:"port"` + From string `yaml:"from"` + Key string `yaml:"key"` +} + +type AppServerConfig struct { + AppConfig AppConfig `yaml:"app"` + Security Security `yaml:"security"` + RateLimit RateLimitConfig `yaml:"rate_limit"` + Authentication Authentication `yaml:"authentication"` + JobConfig JobConfig `yaml:"job"` + CacheConfig CacheConfig `yaml:"cache"` + Backend BackendConfig `yaml:"backend"` +} + +type AppConfig struct { + AppName string `yaml:"name"` + AppMode string `yaml:"mode"` + HTTPPort string `yaml:"http_port"` + AppVersion string `yaml:"version"` +} + +// TokenConfig is the configuration for the token +type TokenConfig struct { + Type string `yaml:"type"` + SecretKey string `yaml:"secret_key"` + TokenExpireTime int `yaml:"token_expire_time"` +} + +// Authentication is the configuration for the authentication +type Authentication struct { + Token TokenConfig +} + +type Security struct { + HTTPS HTTPSConfig `yaml:"https"` +} +type HTTPSConfig struct { + Enabled bool `yaml:"enabled"` + Port string `yaml:"port"` + CertFile string `yaml:"cert_file"` + KeyFile string `yaml:"key_file"` +} + +type RateLimitConfig struct { + Global EndpointLimit `yaml:"global"` + Endpoints map[string]EndpointLimit `yaml:"endpoints"` +} + +type EndpointLimit struct { + Rate int `yaml:"rate"` // requests per second + Burst int `yaml:"burst"` // requests per minute + Window string `yaml:"window"` // Time window (second, minute) +} + +type WhitelistConfig struct { + Whitelist []string `yaml:"whitelist"` +} + +type FileUploadConfig struct { + MaxSize string `yaml:"max_size"` // maximum file size in MB + AllowedTypes []string `yaml:"allowed_types"` // allowed file types + DisallowedTypes []string `yaml:"disallowed_types"` // disallowed file types +} + +type JobConfig struct { + MaxArraySize int `yaml:"max_array_size"` +} + +type CORSConfig struct { + Enabled bool `yaml:"enabled"` + AllowedOrigins []string `yaml:"allowed_origins"` +} + +type CacheConfig struct { + RefreshInterval int `yaml:"refresh_interval"` +} + +var DirPerm os.FileMode + +// Add global variables to store configs +var ( + AppYamlConfig *AppConfig + AuthenticationYamlConfig *Authentication + HttpsYamlConfig *HTTPSConfig + RateLimitYamlConfig *RateLimitConfig + WhitelistYamlConfig *WhitelistConfig + FileUploadYamlConfig *FileUploadConfig + JobYamlConfig *JobConfig + CacheYamlConfig *CacheConfig + CORSYamlConfig *CORSConfig + BackendYamlConfig *BackendConfig +) + +func init() { + if runtime.GOOS == "windows" { + DirPerm = 0666 + } else { + DirPerm = 0700 + } +} + +func InitConf() error { + Debug("Enter InitConf() function") + + viper.SetConfigType("yaml") + viper.AddConfigPath(Global_TopDir + "/config") + viper.SetConfigName("app") + // validate the app.yaml file whether is validate + data, err := ValidateYamlFormat(Global_TopDir + "/config/app.yaml") + if err != nil { + Error("Fail to call DiagnoseLogCreationError with error") + return err + } + if data == nil { + Error("Fail to call ValidateYamlFormat and data is nil") + return err + } + + err1 := viper.ReadInConfig() + if err1 != nil { + _, ok := err1.(viper.ConfigFileNotFoundError) + if ok { + Error("Can't find the configuration files of app.yaml") + return err1 + } else { + log.Println("The configuration file of app.yaml with some error") + return err1 + } + } + // parse app section + AppYamlConfig = &AppConfig{ + AppName: viper.GetString("app.name"), + AppMode: viper.GetString("app.runmode"), + HTTPPort: viper.GetString("app.http_port"), + AppVersion: viper.GetString("app.version"), + } + // parse authentication section + AuthenticationYamlConfig = &Authentication{ + Token: TokenConfig{ + Type: viper.GetString("authentication.token.type"), + SecretKey: resolveJWTSecret(viper.GetString("authentication.token.secretKey")), + TokenExpireTime: int(viper.GetFloat64("authentication.token.token_expire_time")), + }, + } + // parse security section + HttpsYamlConfig = &HTTPSConfig{ + Enabled: viper.GetBool("security.https.enabled"), + Port: viper.GetString("security.https.port"), + CertFile: viper.GetString("security.https.cert_file"), + KeyFile: viper.GetString("security.https.key_file"), + } + + // parse rate limit section + RateLimitYamlConfig = &RateLimitConfig{ + Global: EndpointLimit{ + Rate: viper.GetInt("rate_limit.global.rate"), + Burst: viper.GetInt("rate_limit.global.burst"), + Window: viper.GetString("rate_limit.global.window"), + }, + Endpoints: make(map[string]EndpointLimit), + } + // parse endpoints + endpoints := viper.GetStringMap("rate_limit.endpoints") + for path := range endpoints { + RateLimitYamlConfig.Endpoints[path] = EndpointLimit{ + Rate: viper.GetInt("rate_limit.endpoints." + path + ".rate"), + Burst: viper.GetInt("rate_limit.endpoints." + path + ".burst"), + Window: viper.GetString("rate_limit.endpoints." + path + ".window"), + } + } + + WhitelistYamlConfig = &WhitelistConfig{ + Whitelist: viper.GetStringSlice("whitelist"), + } + // parse file upload section + rawMaxSize := viper.GetString("file_upload.max_size") + fileMaxSizeMB, err := ParseMaxSize(rawMaxSize) + if err != nil { + return fmt.Errorf("failed to parse max_size: %v", err) + } + FileUploadYamlConfig = &FileUploadConfig{ + MaxSize: strconv.FormatInt(fileMaxSizeMB, 10) + " MB", + AllowedTypes: viper.GetStringSlice("file_upload.allowed_types"), + DisallowedTypes: viper.GetStringSlice("file_upload.disallowed_types"), + } + + // parse job config section + // parse job config section + if !viper.IsSet("job") { + // if job section is not set, use default values + JobYamlConfig = &JobConfig{ + MaxArraySize: 1000, + } + } else { + maxArraySize := viper.GetInt("job.max_array_size") + if maxArraySize == 0 { + maxArraySize = 1000 + } + if maxArraySize < 1 || maxArraySize > 2147483646 { + return fmt.Errorf("invalid job.max_array_size value: %d, must be between 1 and 2147483646", maxArraySize) + } + JobYamlConfig = &JobConfig{ + MaxArraySize: maxArraySize, + } + } + + // parse cache config section + if !viper.IsSet("cache") { + // if cache section is not set, use default values + CacheYamlConfig = &CacheConfig{ + RefreshInterval: 300, + } + } else { + refreshInterval := viper.GetInt("cache.refresh_interval") + if refreshInterval == 0 { + refreshInterval = 300 + } + if refreshInterval < 1 || refreshInterval > 300 { + return fmt.Errorf("invalid cache.refresh_interval value: %d, must be between 1 and 300", refreshInterval) + } + CacheYamlConfig = &CacheConfig{ + RefreshInterval: refreshInterval, + } + } + + CORSYamlConfig = &CORSConfig{ + Enabled: viper.GetBool("cors.enabled"), + AllowedOrigins: viper.GetStringSlice("cors.allowed_origins"), + } + + BackendYamlConfig = &BackendConfig{ + Address: viper.GetString("backend.address"), + Port: viper.GetInt("backend.port"), + From: viper.GetString("backend.from"), + Key: viper.GetString("backend.key"), + } + + return nil +} + +func ParseMaxSize(maxSize string) (int64, error) { + parts := strings.Fields(maxSize) + if len(parts) != 2 { + return 0, errors.New("invalid max_size format, expected ' '") + } + + size, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return 0, errors.New("invalid number in max_size") + } + + unit := strings.ToUpper(parts[1]) + switch unit { + case "MB": + size *= 1 // MB remains as is + case "GB": + size *= 1024 // convert GB to MB + default: + return 0, errors.New("unsupported unit in max_size, use MB or GB") + } + + if size > 10240 { + return 0, errors.New("max_size cannot exceed 10GB (10240 MB)") + } + + return size, nil +} + +// resolveJWTSecret returns the JWT secret from XCE_JWT_SECRET env var if set and non-empty, +// otherwise falls back to the value from config file. +func resolveJWTSecret(configValue string) string { + if env := os.Getenv("XCE_JWT_SECRET"); env != "" { + return env + } + return configValue +} diff --git a/src/frontend/apiserver/utils/config_test.go b/src/frontend/apiserver/utils/config_test.go new file mode 100644 index 00000000..2bb86b7c --- /dev/null +++ b/src/frontend/apiserver/utils/config_test.go @@ -0,0 +1,35 @@ +package utils + +import ( + "os" + "testing" +) + +func TestResolveJWTSecret_EnvOverridesConfig(t *testing.T) { + os.Setenv("XCE_JWT_SECRET", "env-secret") + defer os.Unsetenv("XCE_JWT_SECRET") + + got := resolveJWTSecret("config-secret") + if got != "env-secret" { + t.Errorf("expected env-secret, got %q", got) + } +} + +func TestResolveJWTSecret_FallsBackToConfig(t *testing.T) { + os.Unsetenv("XCE_JWT_SECRET") + + got := resolveJWTSecret("config-secret") + if got != "config-secret" { + t.Errorf("expected config-secret, got %q", got) + } +} + +func TestResolveJWTSecret_EmptyEnvUsesConfig(t *testing.T) { + os.Setenv("XCE_JWT_SECRET", "") + defer os.Unsetenv("XCE_JWT_SECRET") + + got := resolveJWTSecret("config-secret") + if got != "config-secret" { + t.Errorf("expected config-secret, got %q", got) + } +} diff --git a/src/frontend/apiserver/utils/crypto.go b/src/frontend/apiserver/utils/crypto.go new file mode 100644 index 00000000..d7ba643f --- /dev/null +++ b/src/frontend/apiserver/utils/crypto.go @@ -0,0 +1,63 @@ +package utils + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "io" + "os" +) + +var EncryptionKey = ")ZLxR6X@h5tYGhkwin 0 { + mainCond = mainConds[0] + } + + // Build sub OR conditions + subConds, err := BuildFilterConditions(fca.SubFiltConds, datatype) + if err != nil { + return nil, err + } + + condArrays = append(condArrays, ConditionArray{ + MainCondition: mainCond, + SubORConditions: subConds, + }) + } + result[key] = condArrays + } + return result, nil +} + +// ConvertToNewFilter is an alias for BuildFilterConditions. +func ConvertToNewFilter(filters []FilterCondition, datatype interface{}) ([]Condition, error) { + return BuildFilterConditions(filters, datatype) +} + +// init suppresses unused import warnings +func init() { + _ = zap.String +} diff --git a/src/frontend/apiserver/utils/utils.go b/src/frontend/apiserver/utils/utils.go new file mode 100644 index 00000000..f97b6150 --- /dev/null +++ b/src/frontend/apiserver/utils/utils.go @@ -0,0 +1,646 @@ +package utils + +import ( + "apiserver/models" + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" + "gopkg.in/yaml.v3" +) + +var ( + Global_TopDir string + PER_PAGE_LIMIT = 50 + PER_PAGE_LIMIT_MAX = 1000 + FileChunkSize = 1024 * 1024 // 1 MB +) + +type FilterCondition struct { + Field string + Operator string + Value string +} + +// to store filter condition for array job. 101[1-3, 6-8] +// mainFiltCond: job_id=101 +// SubFiltConds: [ array_idx:btw:(1,3), array_idx:btw:(6,8) ] +type FilterConditionArray struct { + MainFiltCond FilterCondition + SubFiltConds []FilterCondition +} + +type CompFilterCondition map[string][]FilterConditionArray + +// check the starting top directory whether correctly +func PromptForTopDir(topDir *string) string { + if *topDir == "" { + fmt.Fprintln(os.Stderr, "Error: -d parameter is required") + flag.Usage() + os.Exit(1) + } + + // convert to absolute path if the directory is relative + absPath, err := filepath.Abs(*topDir) + if err != nil { + fmt.Fprintf(os.Stderr, "Error: failed to convert to absolute path: %v\n", err) + os.Exit(1) + } + *topDir = absPath + + // check if the directory exists + if _, err := os.Stat(*topDir); os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "Error: directory %s does not exist, fails to run the command\n", *topDir) + os.Exit(1) + } + + // check if the config directory exists + configDir := filepath.Join(*topDir, "config") + if _, err := os.Stat(configDir); os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "Error: config directory %s does not exist, fails to run the command\n", configDir) + os.Exit(1) + } + + return *topDir +} + +// bindJSONStrict strict check the json format, if the json format is not correct, return error. +func BindJSONStrict(ctx *gin.Context, obj interface{}) error { + // First read the raw body to check for duplicate fields + body, err := io.ReadAll(ctx.Request.Body) + if err != nil { + return fmt.Errorf("error reading request body: %v", err) + } + + // Reset the request body so it can be read again + ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body)) + + // Check for duplicate fields using json.Decoder + dec := json.NewDecoder(bytes.NewBuffer(body)) + seenFields := make(map[string]bool) + + // Read the opening brace + t, err := dec.Token() + if err != nil { + return fmt.Errorf("error reading JSON: %v", err) + } + if delim, ok := t.(json.Delim); !ok || delim != '{' { + return fmt.Errorf("expected JSON object") + } + + // Read key-value pairs + for dec.More() { + // Read key + t, err := dec.Token() + if err != nil { + return fmt.Errorf("error reading JSON key: %v", err) + } + key, ok := t.(string) + if !ok { + return fmt.Errorf("expected string key") + } + + // Check for duplicate key + if seenFields[key] { + return fmt.Errorf("duplicate field found in JSON: %s", key) + } + seenFields[key] = true + + // Skip the value + var value interface{} + if err := dec.Decode(&value); err != nil { + return fmt.Errorf("error reading JSON value: %v", err) + } + } + + // Read the closing brace + t, err = dec.Token() + if err != nil { + return fmt.Errorf("error reading JSON: %v", err) + } + if delim, ok := t.(json.Delim); !ok || delim != '}' { + return fmt.Errorf("expected closing brace") + } + + // Now decode into the target object + dec = json.NewDecoder(bytes.NewBuffer(body)) + dec.DisallowUnknownFields() + if err := dec.Decode(obj); err != nil { + var syntaxError *json.SyntaxError + var unmarshalTypeError *json.UnmarshalTypeError + var invalidUnmarshalError *json.InvalidUnmarshalError + + switch { + case errors.As(err, &syntaxError): + return fmt.Errorf("request body contains badly-formed JSON (at position %d)", syntaxError.Offset) + + case errors.Is(err, io.ErrUnexpectedEOF): + return errors.New("request body contains badly-formed JSON") + + case errors.As(err, &unmarshalTypeError): + if unmarshalTypeError.Field != "" { + return fmt.Errorf("request body contains incorrect JSON type for field %q", unmarshalTypeError.Field) + } + return fmt.Errorf("request body contains incorrect JSON type (at position %d)", unmarshalTypeError.Offset) + + case errors.Is(err, io.EOF): + return errors.New("request body must not be empty") + + case strings.HasPrefix(err.Error(), "json: unknown field "): + fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ") + return fmt.Errorf("request body contains unknown field %s", fieldName) + + case errors.As(err, &invalidUnmarshalError): + return fmt.Errorf("error unmarshalling json: %v", err) + + default: + return errors.New(err.Error()) + } + } + + val := reflect.ValueOf(obj) + if val.Kind() == reflect.Ptr { + val = val.Elem() + } + + for i := 0; i < val.NumField(); i++ { + field := val.Type().Field(i) + if field.Tag.Get("binding") == "required" { + if val.Field(i).Kind() == reflect.String { + trimmedValue := strings.TrimSpace(val.Field(i).String()) + val.Field(i).SetString(trimmedValue) + } + if isZero(val.Field(i)) { + // get json tag name + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + jsonTag = field.Name // if no tag name, using field name + } + return fmt.Errorf("json key of %s is required", jsonTag) + } + } + } + return nil +} + +func isZero(v reflect.Value) bool { + return v.IsZero() +} + +// ParseFilters parses filter string into FilterConditions +func ParseFilters(filterStr string) []FilterCondition { + Debug("Enter ParseFilters function", zap.String("filterStr", filterStr)) + // Remove brackets + //filterStr = strings.TrimPrefix(filterStr, "[") + //filterStr = strings.TrimSuffix(filterStr, "]") // since job_name will add index + + // remove double quotes + filterStr = strings.ReplaceAll(filterStr, "\"", "") + filterStr = strings.ReplaceAll(filterStr, "}, {", "},{") + + if filterStr == "" { + Debug("filterStr is empty, return nil") + return nil + } + + conditions := strings.Split(filterStr, "},{") + filters := make([]FilterCondition, 0) + + for _, condition := range conditions { + condition = strings.TrimPrefix(condition, "{") + condition = strings.TrimSuffix(condition, "}") + + // need handle submit_time:eq:2025-03-31 14:34:07} + parts := strings.SplitN(condition, ":", 3) + if len(parts) != 3 { + continue + } + + // check if there are same filter key and operator + exists := false + for _, filter := range filters { + if filter.Field == parts[0] && filter.Operator == parts[1] { + exists = true + break + } + } + if exists { + Error("There are duplicated filter key and operator, please check the input.") + return nil // return error for filter + } + + filters = append(filters, FilterCondition{ + Field: parts[0], + Operator: parts[1], + Value: strings.TrimSpace(parts[2]), + }) + Debug("In function of ParseFilters, Field, Operator, Value are:", zap.Any("Field", parts[0]), + zap.Any("Op", parts[1]), zap.Any("Value", strings.TrimSpace(parts[2]))) + } + if len(filters) > 0 { + Debug("Finish calling ParseFilters function, filter results", zap.Any("filters", filters)) + return filters + } else { + Debug("The format of filter is not correct!") + return nil + } +} + +// parseFields will parse the fields string +// eg: field=col1,col2,col3 +func ParseFields(fieldsStr string) []string { + // Parse fields + var fields []string + + if fieldsStr == "" { + return nil + } + fields = strings.Split(fieldsStr, ",") + // Trim spaces + result := make([]string, 0, len(fields)) + for _, field := range fields { + field = strings.TrimSpace(field) + if field != "" { + result = append(result, field) + } + } + return result +} + +func ParsePageToken(pagingStr string) (string, error) { + if pagingStr != "" { + pagingStr = strings.TrimPrefix(pagingStr, "\"") + pagingStr = strings.TrimSuffix(pagingStr, "\"") + pagingStr = strings.TrimSpace(pagingStr) + return pagingStr, nil + } + return "", nil +} + +// Parse sort string in query parameters +// sort=job_id:asc,submit_time:desc +func ParseSortParam(s string, supportSortList []string) ([]SortOrder, error) { + if s == "" { + return nil, fmt.Errorf("the sort params is empty") + } + + sortStr := strings.TrimSpace(s) + sorts := strings.SplitN(sortStr, ",", -1) + sortList := make([]SortOrder, 0) + + // Convert supportSortList to map for quick lookup + sortMap := make(map[string]bool) + for _, item := range supportSortList { + sortMap[item] = true + } + + if len(sorts) > 0 { + for _, sort := range sorts { + var col, order string + if strings.Contains(sort, ":") { + col = strings.SplitN(sort, ":", -1)[0] + order = strings.SplitN(sort, ":", -1)[1] + if order != "asc" && order != "desc" { + return nil, fmt.Errorf("the order string %s not in \"asc\" or \"desc\" ", order) + } + } else { + col = sort + order = "asc" // by default + } + if _, ok := sortMap[col]; !ok { + return nil, fmt.Errorf("unsupported sort column: %s", col) + } + sortList = append(sortList, SortOrder{ + Column: col, + Ascending: order == "asc", + }) + } + } else { + return nil, fmt.Errorf("the sort params is empty") + } + return sortList, nil +} + +func CheckAllMapKeyInList(sorts []SortOrder, fields []string) error { + // check the sort column is among the query fields + for _, col := range sorts { + found := ContainsField(fields, col.Column) + if !found { + err := fmt.Errorf("key \"%s\" is not in fields:%v ", col.Column, fields) + return err + } + } + return nil +} + +// judge if a field exist in supported fields list +func ContainsField(fields []string, field string) bool { + for _, v := range fields { + if v == field { + return true + } + } + return false +} + +// judge if a tagName exist in structType json tag ; +func JsonTagExists(structType interface{}, tagName string) bool { + val := reflect.ValueOf(structType) + for i := 0; i < val.NumField(); i++ { + field := val.Type().Field(i) + if field.Tag.Get("json") == tagName { + return true + } + } + return false +} + +// in a map, get value's corresponding key +func GetKeyFromValue(m map[string]int, value int) (string, error) { + for k, v := range m { + if v == value { + return k, nil + } + } + return "", errors.New("value is not recognized.") +} + +// GetMappedField returns the mapped field if it exists, otherwise returns the original field. +func GetMappedField(jsonTag string) string { + if mappedField, exists := models.JobFieldMappings[jsonTag]; exists { + return mappedField + } + return jsonTag +} + +// GetSupportedFields returns the supported fields for a given model, applying any field mappings. +func GetSupportedFields(model interface{}) []string { + var fields []string + t := reflect.TypeOf(model) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag != "" { + // Only take the first part before any comma + jsonTag = strings.Split(jsonTag, ",")[0] + // Apply field mapping + fields = append(fields, jsonTag) + } + } + return fields +} + +// IntOrInt64 converts an interface{} value that is either int or int64 to int. +// Returns (0, false) for any other type. +func IntOrInt64(v interface{}) (int, bool) { + switch t := v.(type) { + case int: + return t, true + case int64: + return int(t), true + default: + return 0, false + } +} + +// ValidateYamlFormat reads and validates a YAML file, returning its raw bytes. +func ValidateYamlFormat(filePath string) ([]byte, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read YAML file %s: %v", filePath, err) + } + var out interface{} + if err := yaml.Unmarshal(data, &out); err != nil { + return nil, fmt.Errorf("invalid YAML format in %s: %v", filePath, err) + } + return data, nil +} + +func ParseLimit(limitStr string) (int, error) { + if limitStr == "" { + return PER_PAGE_LIMIT, nil + } + limit, err := strconv.Atoi(limitStr) + if err != nil { + return 0, err + } + if limit > PER_PAGE_LIMIT_MAX { + return 0, fmt.Errorf("limit=%d exceed MAX=%d", limit, PER_PAGE_LIMIT_MAX) + } + return limit, nil +} + +func ParseType(typeStr string, availableTypes []string) (string, error) { + if typeStr == "" { + return "", nil + } + typeStr = strings.TrimSpace(typeStr) + for _, t := range availableTypes { + if typeStr == t { + return typeStr, nil + } + } + return "", fmt.Errorf("type=%s is not supported, available types: %v", typeStr, availableTypes) +} + +func ParseUnique(uniqueStr string) bool { + if uniqueStr == "" { + return false + } + + // Convert to lowercase and check if it's exactly "true" + return strings.ToLower(strings.TrimSpace(uniqueStr)) == "true" +} + +func ConvertStringToJSON(str []string) (string, error) { + // eg:"VAR1=value1","VAR2=value2" + jsonMap := make(map[string]string) + for _, e := range str { + parts := strings.SplitN(e, "=", 2) + if len(parts) == 2 { + jsonMap[parts[0]] = parts[1] + } + } + strIntoJson, err := json.Marshal(jsonMap) + if err != nil { + return "", err + } + return string(strIntoJson), nil +} + +func ConvertStringToJsonb(str []string) (models.Jsonb, error) { + // eg:"VAR1=value1","VAR2=value2" + jsonMap := make(map[string]string) + for _, e := range str { + parts := strings.SplitN(e, "=", 2) + if len(parts) == 2 { + jsonMap[parts[0]] = parts[1] + } + } + strIntoJson, err := json.Marshal(jsonMap) + if err != nil { + return models.Jsonb{}, err + } + jsonb := models.Jsonb{ + String: string(strIntoJson), + } + return jsonb, nil +} + +func AddFieldsToSupportedList(target *[]string, fields interface{}) { + t := reflect.TypeOf(fields) + for i := 0; i < t.NumField(); i++ { + jsonTag := t.Field(i).Tag.Get("json") + if jsonTag != "" && jsonTag != "-" { + *target = append(*target, jsonTag) + } + } +} + +func GetServerHostPort(ctx *gin.Context) string { + scheme := "http" + if ctx.Request.TLS != nil { + scheme = "https" + } + return fmt.Sprintf("%s://%s", scheme, ctx.Request.Host) +} + +// convert timestamp(s) to local time string with TZ. +func convertTimestampToLocalTime(timestamp int64) string { + utcTime := time.Unix(timestamp, 0).UTC() + localTime := utcTime.Local() + return localTime.Format("2006-01-02 15:04:05 MST") +} + +// convert time string with TZ to timestamp +func convertTimestrToTimestamp(timeStr string) (int64, error) { + // Define the layout for parsing the time string. + // "2006-01-02 15:04:05 MST" is a reference time in Go. + layout := "2006-01-02 15:04:05 MST" + + // Parse the time string into a time.Time object. + parsedTime, err := time.Parse(layout, timeStr) + if err != nil { + return 0, fmt.Errorf("failed to parse time: %w", err) + } + // Get the timestamp in seconds. + timestamp := parsedTime.Unix() + + return timestamp, nil +} + +func IsNumeric(s string) bool { + _, err := strconv.ParseFloat(s, 64) + return err == nil +} + +func ValidateControlAction(controlAction string, supportedActions []string) error { + controlAction = strings.ToLower(strings.TrimSpace(controlAction)) + for _, action := range supportedActions { + if controlAction == action { + return nil + } + } + + return errors.New("unsupported control action: " + controlAction) +} + +func IsControlActionAllowed(actionType int, status int64, actionControlMap map[int][]int) bool { + Debug("Enter IsActionAllowed function", + zap.Any("actionType", actionType), + zap.Any("status", status)) + + allowedStatuses, exists := actionControlMap[actionType] + if !exists { + Error("Unsupported action type", zap.Any("actionType", actionType)) + return false + } + + for _, allowedStatus := range allowedStatuses { + if allowedStatus == int(status) { + Debug("Action is allowed", + zap.Any("status", status), + zap.Any("allowedStatus", allowedStatus)) + return true + } + } + Debug("Action is not allowed", zap.Any("status", status)) + return false +} + +// RemoveDuplicates removes duplicate elements from any slice. +func RemoveDuplicates[T comparable](input []T) []T { + seen := make(map[T]bool) + result := []T{} + + for _, value := range input { + if !seen[value] { + seen[value] = true + result = append(result, value) + } + } + return result +} + +func TruncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + runes := []rune(s) + if len(runes) > maxLen { + return string(runes[:maxLen]) + } + return s +} + +func IncludeBuiltinRoles(roleNames []string) (builtinIncluded []string) { + for _, roleName := range roleNames { + if ContainsField(models.BUILTIN_ROLE_NAMES, roleName) { + builtinIncluded = append(builtinIncluded, roleName) + } + } + return builtinIncluded +} + +func IsValidUserName(userName string) error { + // Check if name is empty + if userName == "" { + return errors.New("username cannot be empty") + } + + // Check maximum length (32 characters) + if len(userName) > 32 { + return errors.New("username cannot exceed 32 characters") + } + + // Check if it begins with a letter + if !((userName[0] >= 'a' && userName[0] <= 'z') || (userName[0] >= 'A' && userName[0] <= 'Z')) { + return errors.New("username must begin with a letter") + } + + // Check for valid characters (0-9, a-z, A-Z, -, _, space) + for i := 0; i < len(userName); i++ { + c := userName[i] + if !((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '-' || + c == '_' || + c == ' ') { + return errors.New("username can only contain letters, numbers, hyphens, underscores, or spaces") + } + } + + return nil +} diff --git a/src/frontend/apiserver/utils/utils_test.go b/src/frontend/apiserver/utils/utils_test.go new file mode 100644 index 00000000..a7b840f0 --- /dev/null +++ b/src/frontend/apiserver/utils/utils_test.go @@ -0,0 +1,37 @@ +package utils + +import ( + "testing" +) + +func TestIntOrInt64(t *testing.T) { + tests := []struct { + name string + input interface{} + want int + ok bool + }{ + {"int zero", int(0), 0, true}, + {"int positive", int(42), 42, true}, + {"int negative", int(-1), -1, true}, + {"int64 zero", int64(0), 0, true}, + {"int64 positive", int64(42), 42, true}, + {"int64 negative", int64(-1), -1, true}, + {"int64 large", int64(9223372036854775807), int(9223372036854775807), true}, // int==int64 on 64-bit + {"string", "42", 0, false}, + {"float64", 42.0, 0, false}, + {"nil", nil, 0, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := IntOrInt64(tt.input) + if ok != tt.ok { + t.Errorf("IntOrInt64(%v) ok = %v, want %v", tt.input, ok, tt.ok) + } + if ok && got != tt.want { + t.Errorf("IntOrInt64(%v) = %d, want %d", tt.input, got, tt.want) + } + }) + } +} diff --git a/src/frontend/docs/analysis.md b/src/frontend/docs/analysis.md new file mode 100644 index 00000000..cf9627c2 --- /dev/null +++ b/src/frontend/docs/analysis.md @@ -0,0 +1,244 @@ +# API Server & Client Analysis + +> Generated: 2026-04-07 +> Scope: `apiserver/` ↔ `xcecli/` communication layer only. +> Backend integration is excluded. + +--- + +## 1. Component Overview + +```mermaid +graph LR + subgraph xcecli + CMD["cobra commands"] + APICLIENT["APIClient\nresty.dev/v3"] + CFG["~/.xce/config.json\ntoken · url · expiry"] + CMD --> APICLIENT + APICLIENT --> CFG + end + + subgraph apiserver + MW["middleware stack\nIP whitelist → URL whitelist\n→ rate limit → JWT validate"] + ROUTER["Gin router\n/xce/v1/..."] + CTRL["controllers"] + STORE["in-memory DataStore\n(sync.RWMutex)"] + MW --> ROUTER --> CTRL --> STORE + end + + APICLIENT -- "HTTP/HTTPS Bearer JWT" --> MW +``` + +--- + +## 2. API Completeness + +### 2.1 Endpoint Coverage + +All routes registered in `router/routers.go` have a corresponding CLI command. The table below marks the few gaps. + +| Endpoint | Server | CLI | Gap | +|---|:---:|:---:|---| +| POST `/xce/v1/auth/logon` | ✅ | ✅ | — | +| POST `/xce/v1/auth/logout` | ✅ | ✅ | — | +| GET `/xce/v1/permissions` | ✅ | ✅ | — | +| GET/POST/DELETE `/xce/v1/roles/...` | ✅ | ✅ | — | +| GET/POST/DELETE `/xce/v1/users/...` | ✅ | ✅ | — | +| GET `/xce/v1/clusters` | ✅ | ✅ | — | +| GET/POST `/xce/v1/hosts/...` | ✅ | ✅ | — | +| GET/POST `/xce/v1/queues/...` | ✅ | ✅ | — | +| GET/POST `/xce/v1/jobs/...` | ✅ | ✅ | — | +| GET `/xce/v1/resources/...` | ✅ | ✅ | — | +| POST `/xce/v1/files/upload` | ✅ | ✅ | — | +| GET `/xce/v1/files` | ✅ | ✅ | — | +| GET `/xce/v1/files/:uuid/:filename` | ✅ | ✅ | — | +| DELETE `/xce/v1/files` | ❌ commented out | ❌ no command | Route removed from server but still in URL whitelist | +| API key CRUD | ❌ model exists, no route | ❌ no command | `models/apikey.go` + `controllers/apikey.go` are dead code | +| `?type=array_stats` on jobs | ✅ | ⚠️ partial | Server supports it; CLI has no dedicated display path for array stats | + +### 2.2 Missing Functional Areas + +- **File delete** — commented out on server, no CLI command. Either implement or remove the dead code and whitelist entry. +- **API key management** — controller and model exist but are unreachable. Wire a route or delete the files. +- **CORS** — middleware is commented out in `router/routers.go`. Any web-based client will be blocked. + +--- + +## 3. Client–Server Communication Gaps + +```mermaid +sequenceDiagram + participant CLI as xcecli + participant SRV as apiserver + + CLI->>SRV: POST /xce/v1/auth/logon + SRV-->>CLI: {code, msg, data:{token, token_expire_time, ...}} + Note over CLI: stores token in ~/.xce/config.json + + CLI->>SRV: GET /xce/v1/jobs Authorization: Bearer + Note over SRV: JWT validated, IP checked + SRV-->>CLI: {code, msg, data:{count, rows:[], page_token}} + + CLI->>SRV: POST /xce/v1/jobs/create {command, job_name, ...} + SRV-->>CLI: {code, msg, data:{jobid, queue_name}} +``` + +### 3.1 Type Mismatches (will cause silent data loss or parse errors) + +| # | Field | Server type | Client type | Effect | +|---|---|---|---|---| +| **M1** | `QueueInfo.queue_status` | `int` (bitmask) | `string` in `DBQueueInfo` | JSON unmarshal error — queue status always blank | +| **M2** | `JobInfo.exit_status` | `Jsonb` (raw JSON string) | `int` in `QueryJob` | Parse error — exit status always 0 | +| **M3** | `HostInfo.num_cpus` | field `num_cpus` | mapped to `num_cores` in `XCEHost` | CPU count always 0 in CLI output | +| **M4** | `ResourceDistributionInfo` | separate fields: `numeric_value`, `boolean_value`, `string_value` | `ResourceDist.ResourceValue` reads from `"value"` key | Resource values always empty | +| **M5** | `JobSubmitRequest.job_name` | `omitempty`, not required | `binding:"required"` | Client enforces job name; server does not — inconsistent validation | +| **M6** | `JobControlJobIDRequest` | has `method` field (checkpoint method) | no `method` field | Checkpoint method cannot be set from CLI | + +### 3.2 Token / Auth Issues + +| # | Issue | Location | Effect | +|---|---|---|---| +| **A1** | Leading space in Authorization header | `request.go` `SubmitJob`: `" " + token` vs `GetRequest`: `"" + token` | Inconsistent — some endpoints may reject the token | +| **A2** | Token expiry checked client-side only | `config.go` `IsAuthenticated()` | Server-side revocation (logout) not reflected until client's local timer expires | +| **A3** | `SetRootCAs` always sets `InsecureSkipVerify: true` | `request.go:SetRootCAs` | TLS cert is never actually verified regardless of `certPath` | +| **A4** | JWT IP binding | `validatehandler.go` `isValidToken` | Token tied to login IP — fails behind NAT/proxy or on IP change | + +### 3.3 Response Envelope Inconsistency + +The server wraps all responses in `{code, msg, data}`. Most client models match this, but: + +- `LogonResponse` on the server returns `data` as a flat object (`token`, `path`, `ip`, `token_expire_time`). + The client's `LogonResponse` wraps it in a `LogonData` sub-struct — this matches, but `token_expire_time` is `int` on server and `int64` on client. Works in practice but is fragile. +- `ReturnError` in `response.go` calls `ctx.JSON(code, ...)` using the error code as the HTTP status. The client checks `httpResp.StatusCode() != http.StatusOK` — this is correct, but means any non-200 internal code (e.g. `500`, `400`) is also the HTTP status, which is actually good practice. Just needs to be documented. + +--- + +## 4. Server-Side Issues (apiserver only) + +### 4.1 Security + +| # | Issue | File | Severity | +|---|---|---|---| +| **S1** | JWT secret hardcoded in config file | `config/app.yaml` `secretKey: "xce-api-key"` | 🔴 High — anyone with the file can forge tokens | +| **S2** | Token stored as plain file on disk | `controllers/auth.go` `storeToken()` | 🟠 Medium — long-lived (86400s), not cleared on restart | +| **S3** | Hardcoded mock RBAC group maps | `controllers/rbac_check.go` lines 19–31 | 🟠 Medium — test fixtures in production code affect access control | +| **S4** | Whitelist pattern truncation bug | `middleware/whitelist.go` `pattern[:len(pattern)-1]` | 🟠 Medium — strips last char from all patterns, not just `*`-suffixed ones; can block or allow wrong paths | + +### 4.2 Correctness + +| # | Issue | File | Detail | +|---|---|---|---| +| **C1** | `validateRequestBody` condition is inverted | `middleware/validatehandler.go` line ~30 | `strings.Contains("multipart/form-data", contentType)` checks if the literal string contains the header value, not the other way around — always `false`. Works by accident. | +| **C2** | Rate limiter `limiters` map grows unboundedly | `middleware/rate_limit.go` | New entries added per unique route path, never evicted | +| **C3** | `ioutil.ReadFile` (deprecated Go 1.16) | `controllers/rbac_check.go` | Replace with `os.ReadFile` | +| **C4** | `AlecAivazis/survey` in server `go.mod` | `go.mod` | Interactive prompt library has no use in a REST server — dead dependency | + +### 4.3 Swagger Docs Are Stale + +`docs/swagger.yaml` references: +- Old host: `localhost:8443` +- Old path: `/api/v1/hosts` +- Old models: `models.BasicHost`, `models.FullHost` + +None of these exist in the current codebase. The Swagger UI will show completely wrong information. + +--- + +## 5. Client-Side Issues (xcecli only) + +| # | Issue | File | Detail | +|---|---|---|---| +| **CL1** | `resty.dev/v3` is a beta release | `go.mod` | Not production-stable; consider `github.com/go-resty/resty/v2` | +| **CL2** | TLS verification disabled | `request.go` `SetRootCAs` | `InsecureSkipVerify: true` always set; `loadRootCAs` is defined but never called | +| **CL3** | Token header space inconsistency | `request.go` | `SubmitJob` uses `" "+token`; all other methods use `""+token` | +| **CL4** | No retry on transient errors | `request.go` | Single attempt; network blips cause immediate failure | +| **CL5** | `xce jobs` array stats has no display | `cmd/jobs/list.go` | `?type=array_stats` is sent but output formatter doesn't handle `ArrayJobStatsData` | + +--- + +## 6. Prioritized Fix List + +```mermaid +quadrantChart + title Fix Priority (Impact vs Effort) + x-axis Low Effort --> High Effort + y-axis Low Impact --> High Impact + quadrant-1 Do First + quadrant-2 Plan + quadrant-3 Later + quadrant-4 Quick Wins + M1 QueueStatus type: [0.15, 0.90] + M2 ExitStatus type: [0.15, 0.85] + M3 num_cpus mapping: [0.10, 0.70] + M4 Resource value fields: [0.20, 0.75] + A1 Token header space: [0.05, 0.60] + S1 JWT secret in config: [0.15, 0.95] + S3 Mock RBAC data: [0.20, 0.80] + S4 Whitelist pattern bug: [0.15, 0.75] + A3 TLS InsecureSkipVerify: [0.25, 0.65] + C1 validateRequestBody: [0.10, 0.40] + C3 ioutil deprecated: [0.05, 0.20] + CL1 resty beta: [0.30, 0.50] + Swagger regen: [0.35, 0.55] + CORS enable: [0.10, 0.45] +``` + +### Ordered Action Items + +**Do immediately (bugs that break functionality):** + +1. **M1** — Change `DBQueueInfo.QueueStatus` from `string` to `int` in xcecli, add a `QueueStateStr()` helper for display +2. **M2** — Change `QueryJob.ExitStatus` from `int` to `string` (or parse the Jsonb on server before sending) +3. **M3** — Fix `XCEHost.NumCPUs` json tag from `"num_cores"` to `"num_cpus"` +4. **M4** — Align resource distribution response: server should return a single `value` field, or client should read the typed fields +5. **A1** — Standardize token header to `"Bearer "+token` in all `request.go` methods +6. **S4** — Fix whitelist pattern matching: only strip trailing `*`, not all patterns + +**Do before any production deployment:** + +7. **S1** — Load JWT secret from environment variable, not config file +8. **S3** — Remove hardcoded `hostGroupMap`, `userGroupMap`, `queueMap` from `rbac_check.go` +9. **A3** — Wire `loadRootCAs` properly; only use `InsecureSkipVerify` when explicitly opted in +10. **Swagger** — Run `swag init` after fixing controller annotations; add to CI + +**Cleanup / low risk:** + +11. **C1** — Fix `validateRequestBody` condition (currently works by accident) +12. **C3** — Replace `ioutil.ReadFile` with `os.ReadFile` +13. **C4** — Remove `AlecAivazis/survey` from server `go.mod` +14. **CL1** — Upgrade resty to stable `v2` +15. **File delete** — Implement or remove the dead route + whitelist entry + +--- + +## 7. Dependencies Summary + +### apiserver + +| Package | Purpose | Note | +|---|---|---| +| `gin-gonic/gin` v1.10 | HTTP framework | stable | +| `golang-jwt/jwt/v4` v4.5.1 | JWT auth | stable | +| `google/uuid` v1.6 | File upload UUIDs | stable | +| `lib/pq` v1.10.9 | PostgreSQL driver | in go.mod, unused (store is in-memory) | +| `msteinert/pam` v1.2 | PAM auth (Linux) | build-tag guarded | +| `spf13/viper` v1.19 | Config loading | stable | +| `swaggo/gin-swagger` v1.6 | Swagger UI | docs are stale | +| `xeipuuv/gojsonschema` v1.2 | Config validation | stable | +| `uber-go/zap` v1.27 | Logging | stable | +| `golang.org/x/time` v0.9 | Rate limiter | stable | +| `lumberjack.v2` v2.2.1 | Log rotation | stable | +| `AlecAivazis/survey/v2` v2.3.7 | Interactive prompts | **unused in server — remove** | + +### xcecli + +| Package | Purpose | Note | +|---|---|---| +| `spf13/cobra` v1.9.1 | CLI framework | stable | +| `spf13/viper` v1.20.1 | Config loading | stable | +| `xeipuuv/gojsonschema` v1.2 | Config validation | stable | +| `uber-go/zap` v1.27 | Logging | stable | +| `golang.org/x/term` v0.27 | Password input | stable | +| `lumberjack.v2` v2.2.1 | Log rotation | stable | +| `resty.dev/v3` v3.0.0-beta.2 | HTTP client | **⚠️ beta — replace with go-resty/resty/v2** | +| `gopkg.in/yaml.v2` v2.4 | YAML parsing | stable | diff --git a/src/frontend/docs/analysis.pdf b/src/frontend/docs/analysis.pdf new file mode 100644 index 00000000..f025bd4e Binary files /dev/null and b/src/frontend/docs/analysis.pdf differ diff --git a/src/frontend/docs/improvement_suggestions.md b/src/frontend/docs/improvement_suggestions.md new file mode 100644 index 00000000..c86aeeee --- /dev/null +++ b/src/frontend/docs/improvement_suggestions.md @@ -0,0 +1,163 @@ +# Code Improvement Suggestions + +> Date: 2026-04-09 +> Based on current codebase state after all bug fixes. + +--- + +## 1. Critical Bugs Still Present + +### 1.1 `queue_name` filter added N times in `GetJobsInfo` (apiserver) + +**File:** `controllers/jobs.go` ~line 226 +**Problem:** The `queue_name:in:(...)` RBAC filter is appended inside the per-filter loop — once for every non-queue filter in the request. With 2 filters (e.g. `job_status` + `submit_user`), it gets added twice. Seen in server logs: `queue_name:in:(normal,priority,short)` appears twice in every query. + +**Fix:** Move the `accessQueues` append **outside** the loop, after it completes. + +```go +// BEFORE (inside loop — adds queue_name once per non-queue filter) +for i, filter := range filters { + ... + } else { + if len(accessQueues) > 0 { + filters = append(filters, ...) // BUG: runs N times + } + } +} + +// AFTER (outside loop — adds once) +for i, filter := range filters { ... } +if len(accessQueues) > 0 && !queueFilterPresent { + filters = append(filters, utils.FilterCondition{ + Field: "queue_name", Operator: "in", + Value: "(" + strings.Join(accessQueues, ",") + ")", + }) +} +``` + +--- + +## 2. Design Issues + +### 2.1 `controllers/jobs.go` is 3590 lines — needs splitting + +The file handles job query, submit, control, restart, array jobs, and verbose output. Each is a distinct concern. + +**Suggested split:** +``` +controllers/ + jobs_query.go — GetJobsInfo, GetJobByIDs (~600 lines) + jobs_submit.go — CreateJob, RestartJobByID (~500 lines) + jobs_control.go — ControlJob, ControlJobByID (~400 lines) + jobs_array.go — array job handling (~300 lines) + jobs_helpers.go — shared: parseArrayName, processJobData, etc. +``` + +### 2.2 RBAC permission check duplicated across controllers + +`GetRequireResourcePermission` + `accessQueues` logic is copy-pasted in `GetJobsInfo`, `ControlJob`, `ControlJobByID`, `RestartJobByID`. Extract to a helper: + +```go +// controllers/rbac_helpers.go +func getJobAccessQueues(username string) ([]string, error) { ... } +func checkJobViewPermission(username, queueName string) error { ... } +``` + +### 2.3 `xcecli/cmd/jobs/submit.go` is 734 lines + +`JobSubmitter` mixes CLI flag parsing, file upload, interactive job handling, and HTTP submission. Split: + +``` +cmd/jobs/ + submit.go — cobra command + flag binding only + submitter.go — JobSubmitter struct + BuildJobRequest + submit_files.go — file upload logic (already in files.go ✓) + submit_interactive.go — interactive/TTY/xjob handling +``` + +--- + +## 3. Code Quality + +### 3.1 `ioutil.ReadFile` deprecated (Go 1.16) + +**File:** `controllers/rbac_check.go:54` +**Fix:** `os.ReadFile(path)` — one-line change, tracked as Q3. + +### 3.2 Inconsistent error return convention + +Some functions return `(error, T)`, others return `(T, error)`. Go convention is always `(T, error)`. + +```go +// Non-standard (apiserver/controllers/rbac_check.go) +func GetRequireResourcePermission(...) (error, models.RolePermission) + +// Should be +func GetRequireResourcePermission(...) (models.RolePermission, error) +``` + +This affects ~8 call sites in `jobs.go` that do `err, perm := GetRequire...`. + +### 3.3 Magic strings for domain types + +`"all"`, `"self"`, `"self_ug"`, `"queue"` appear as raw strings in multiple files. They're defined as constants in `models/roles.go` but not always used. + +**Fix:** Replace all raw string comparisons with the constants (`models.DomainTypeAll`, etc.). + +### 3.4 `xcecli` output printers have no shared time formatting + +`page_print_jobs.go` now has `parseJobTime`/`formatJobTime`. The same pattern will be needed for `start_time`, `end_time` in other printers. Move to `common/output/time.go`. + +--- + +## 4. Architecture + +### 4.1 In-memory store has no persistence + +After restart, all submitted jobs (non-builtin) are lost. The `lib/pq` PostgreSQL driver is already in `go.mod`. When backend integration happens, the notification handlers (`OnJobCreated`, etc.) should write through to PostgreSQL, not just the in-memory store. + +### 4.2 `UserRoleSeeder` only implemented for `config_file` + +After the startup fix, PAM/LDAP/OAuth2 providers don't implement `AllUsers()` — so after restart with those providers, users still need to re-login to get their roles. Consider storing user-role assignments in a persistent store (DB or file) so they survive restarts regardless of provider. + +### 4.3 Rate limiter `limiters` map grows unboundedly + +**File:** `middleware/rate_limit.go` +The per-endpoint `limiters` map is never evicted. Under normal operation with fixed routes this is fine, but worth noting for future dynamic routing. + +--- + +## 5. Priority Order + +```mermaid +quadrantChart + title Impact vs Effort + x-axis Low Effort --> High Effort + y-axis Low Impact --> High Impact + quadrant-1 Do First + quadrant-2 Plan + quadrant-3 Later + quadrant-4 Quick Wins + queue_name double-add: [0.10, 0.90] + ioutil fix: [0.05, 0.20] + error convention: [0.25, 0.60] + split jobs.go: [0.60, 0.70] + split submit.go: [0.50, 0.50] + RBAC helper: [0.35, 0.65] + time.go shared: [0.15, 0.35] + persistence: [0.80, 0.85] + UserRoleSeeder all providers: [0.30, 0.55] +``` + +**Do immediately:** +1. Fix `queue_name` double-add bug in `GetJobsInfo` (10 min, high impact) +2. `ioutil` → `os.ReadFile` (1 min) + +**Next sprint:** +3. Fix `(error, T)` → `(T, error)` return convention +4. Extract RBAC helpers to reduce duplication in `jobs.go` +5. Split `jobs.go` into focused files + +**Backend integration phase:** +6. PostgreSQL persistence via notification handlers +7. `UserRoleSeeder` for PAM/LDAP providers diff --git a/src/frontend/docs/improvement_tasks.md b/src/frontend/docs/improvement_tasks.md new file mode 100644 index 00000000..7f17a0eb --- /dev/null +++ b/src/frontend/docs/improvement_tasks.md @@ -0,0 +1,16 @@ +# Code Improvement Tasks + +> Last updated: 2026-04-09 + +| ID | Status | Category | Component | Task | Effort | +|---|---|---|---|---|---| +| I1 | ✅ done | 🔴 Bug | apiserver | `queue_name:in:(...)` filter added N times in `GetJobsInfo` loop | 10 min | +| I2 | ✅ done | 🟠 Quality | apiserver | Fix `(error, T)` → `(T, error)` return convention in `GetRequireResourcePermission` and call sites | 1h | +| I3 | ⏳ open | 🟠 Quality | apiserver | Replace `ioutil.ReadFile` with `os.ReadFile` in `rbac_check.go` | 5 min | +| I4 | ⏳ open | 🟠 Quality | apiserver | Replace raw domain strings (`"all"`, `"self"`) with `models.DomainTypeAll` constants | 30 min | +| I5 | ⏳ open | 🟠 Quality | xcecli | Move `parseJobTime`/`formatJobTime` to `common/output/time.go` for reuse | 15 min | +| I6 | ⏳ open | 🟡 Refactor | apiserver | Extract RBAC `accessQueues` logic — duplicated in `GetJobsInfo`, `ControlJob`, `ControlJobByID`, `RestartJobByID` | 2h | +| I7 | ⏳ open | 🟡 Refactor | apiserver | Split `controllers/jobs.go` (3590 lines) into query / submit / control / array / helpers | 4h | +| I8 | ⏳ open | 🟡 Refactor | xcecli | Split `cmd/jobs/submit.go` (734 lines) — cobra command / BuildJobRequest / interactive | 2h | +| I10 | ⏳ open | 🔵 Arch | apiserver | `UserRoleSeeder` for PAM/LDAP — users need re-login after restart with non-config_file providers | 2h | +| I11 | ⏳ open | 🔵 Arch | apiserver | Rate limiter `limiters` map grows unboundedly — add eviction or use fixed route keys | 1h | diff --git a/src/frontend/docs/integration_test.md b/src/frontend/docs/integration_test.md new file mode 100644 index 00000000..57c0860e --- /dev/null +++ b/src/frontend/docs/integration_test.md @@ -0,0 +1,278 @@ +# Integration Test Report: xcecli ↔ apiserver + +> Date: 2026-04-08 +> Server: `apiserver` built from `apiserver/` with builtin test data (3 hosts, 3 queues, 4 jobs, 8 resources) +> Client: `xcecli` built from `xcecli/` (CGO_ENABLED=0 static binary) +> Auth: `config_file` provider — users: `admin` (cluster_admin), `user1`, `user2` (basic_user) + +> **Updated: 2026-04-08** — All critical bugs fixed. See resolution status table below. + +--- + +## Summary (at time of initial test) + +| Category | Count | +|---|---| +| ✅ Commands working correctly | 13 | +| ⚠️ Commands partially working (wrong/empty data) | 11 | +| ❌ Commands totally broken | 3 | +| Server endpoints not covered by CLI | 2 | + +## Summary (after fixes) + +| Category | Count | +|---|---| +| ✅ Commands working correctly | ~27 | +| ⚠️ Minor issues remaining | 2 (clusters empty — no backend; jobs list no filter = current user only, by design) | +| ❌ Broken | 0 | + +--- + +## Test Results by Command + +### Auth + +| Command | Result | Notes | +|---|---|---| +| `xce login -u admin -p admin --url http://...` | ✅ Works | Token saved to `~/.xce/config.json` | +| `xce logout --url http://...` | ✅ Works | Token cleared | +| `xce list` | ✅ Works | Shows server URL and default marker | +| `xce users whoami` | ✅ Works | Returns current username from token | + +**Bug found:** `xce logout` requires `--url` flag — if omitted it errors with `required flag(s) "url" not set`. Should default to the active server. + +--- + +### Permissions + +| Command | Result | Notes | +|---|---|---| +| `xce permissions list` | ✅ Works | Returns all 21 permissions correctly | + +--- + +### Roles + +| Command | Result | Notes | +|---|---|---| +| `xce roles list` | ❌ Broken | Server rejects with `invalid role info query parameters of : unique` | +| `xce roles view -r ` | ⚠️ Partial | Request succeeds, returns correct row count, but all fields display as `-` | +| `xce roles add -r -p "job:view=all"` | ✅ Works | Role created successfully | +| `xce roles delete -r ` | ✅ Works | Role deleted successfully | + +**Root cause — `roles list`:** Client sends `params["unique"] = "true"` as a query parameter. Server's `SupportedQueryParams` does not include `unique` → rejected with 400. + +**Root cause — `roles view` empty data:** Server returns `role_name`, `action`, `domain_type`, `domain_values` fields. Client `RolesResponse` model maps to `models.RoleInfo` which has different field names — JSON unmarshal silently produces zero values. + +--- + +### Users + +| Command | Result | Notes | +|---|---|---| +| `xce users list` | ❌ Broken | Server rejects with `invalid user_role info query parameters of : unique` | +| `xce users view -u ` | ⚠️ Partial | Returns correct row count, all fields display as `-` | +| `xce users whoami` | ✅ Works | Returns `admin` | +| `xce users assignrole -u user1 -r basic_user` | ✅ Works | Role assigned | +| `xce users unassignrole -u user1 -r basic_user` | ✅ Works | Role removed | +| `xce users permissions4user -u admin` | ⚠️ Partial | Returns correct row count, all fields display as `-` | + +**Root cause — `users list`:** Same `unique` query param issue as `roles list`. + +**Root cause — `users view` / `permissions4user` empty data:** Same field name mismatch as roles view. + +--- + +### Hosts + +| Command | Result | Notes | +|---|---|---| +| `xce hosts list` | ⚠️ Partial | Returns 3 rows (correct count), but `HOST_NAME`, `STATUS`, `HOST_TYPE` all show `-`; `NCPUS` shows `0` | +| `xce hosts open ` | ✅ Works | `host is being ` | +| `xce hosts close ` | ✅ Works | `host is being ` | + +**Root cause — `hosts list` empty fields:** +1. `HostStatus` is `int` on server, `string` in client `XCEHost` → JSON unmarshal fails silently, `HostStatus = ""` +2. `XCEHost.NumCPUs` has json tag `"num_cores"` but server sends `"num_cpus"` → always 0 +3. `HostName` and `HostType` are present in server response but `HostsResponse.Data.Rows` is `[]XCEHost` — the field names match (`host_name`, `host_type`) so these should work. Actual cause: server serializes `HostInfo` via `StructToOrderMap` using reflect + json tags, then re-serializes to JSON. The `HostStatus int` becomes a JSON number; client `XCEHost.HostStatus string` cannot unmarshal a number → entire row decode fails, leaving all string fields empty. + +--- + +### Queues + +| Command | Result | Notes | +|---|---|---| +| `xce queues list` | ⚠️ Partial | Returns 3 rows (correct count), all fields show `-` or `0` | +| `xce queues open ` | ✅ Works | `queue is being ` | +| `xce queues close ` | ✅ Works | `queue is being ` | +| `xce queues active ` | ✅ Works | `queue is being ` | +| `xce queues deactive ` | ✅ Works | `queue is being ` | + +**Root cause — `queues list` empty fields:** `QueueInfo.QueueStatus` is `int` on server, `string` in client `DBQueueInfo` → same unmarshal failure as hosts. + +--- + +### Jobs + +| Command | Result | Notes | +|---|---|---| +| `xce jobs list` (no filter) | ⚠️ Partial | Returns "no job found" — server filters by current user AND accessible queues; admin has no jobs | +| `xce jobs list -u all` | ⚠️ Partial | Returns 3 rows (correct count for builtin data), but `JOBID=0`, all string fields `-` | +| `xce jobs list -u user1` | ⚠️ Partial | Returns 2 rows, same display issue | +| `xce jobs submit -J name -q normal ` | ✅ Works | Job ID returned, submitted to queue | +| `xce jobs submit -q normal ` (no `-J`) | ✅ Works | Server accepts without job name (client `binding:"required"` not enforced server-side) | +| `xce jobs submit -f ...` | ⚠️ Partial | Job submitted but `parse file transfer config failed: []` printed — file upload path parsing error | +| `xce jobs kill ` | ⚠️ Partial | Request reaches server, server finds job by `job_id` but submitted job IDs (Snowflake) don't match builtin IDs (1001-1004) → `job not found` | +| `xce jobs stop ` | ⚠️ Partial | Same as kill | +| `xce jobs resume ` | ⚠️ Partial | Same as kill | +| `xce jobs checkpoint ` | ⚠️ Partial | Same as kill | +| `xce jobs restart ` | ⚠️ Partial | Reaches server, but fails: `job not submitted by , cannot be restarted by ` — correct RBAC behavior | + +**Root cause — `jobs list` display issue:** `QueryJob.JobStatus` is `string` in client but server sends `int` (bitmask). `QueryJob.JobID` is `int64` but server sends `unique_job_id` as the primary key — client maps `job_id` which is the array-level ID (same as `unique_job_id` for non-array jobs in builtin data, but Snowflake-generated IDs differ). The `JobStatus` type mismatch causes the entire row JSON decode to fail. + +**Root cause — `jobs list` no results for admin:** Server `GetJobsInfo` adds `submit_user = ` filter unless user has `job:view` permission with `all` domain. Admin's RBAC domain is not configured for `all` in the default `rbac.yaml`. + +**Root cause — `jobs submit -f` parse error:** `xcecli/cmd/jobs/submit.go` tries to parse the `-f` flag value as a data spec file format (`src > dst`) before treating it as a plain file path. Plain paths without `>` fail the parse but the job still submits. + +--- + +### Clusters + +| Command | Result | Notes | +|---|---|---| +| `xce id` | ⚠️ Partial | Returns "no cluster found" — builtin data has no cluster entries in store | + +--- + +### Resources + +| Command | Result | Notes | +|---|---|---| +| `xce resources distribution list` | ⚠️ Partial | Returns 16 rows (correct count), all values show `-` | +| `xce resources definition list` | ⚠️ Partial | Returns 8 rows (correct count), all values show `-` | + +**Root cause:** Server `ResourceDistributionInfo` has fields `numeric_value`, `boolean_value`, `string_value`. Client `ResourceDist` reads from a single `"value"` key via `ResourceValue.UnmarshalJSON`. Server never sends a `"value"` key → always empty. Same for definition: server sends `value_type` as `int`, client `ResourceDef.ResourceType` is `string`. + +--- + +## Server Endpoints Not Covered by CLI + +| Endpoint | Method | Notes | +|---|---|---| +| `/xce/v1/hosts/select` | POST | Filter hosts by resource requirement string (`-R select(...)`) — no CLI command | +| `/xce/v1/files` | GET | List uploaded files — no `xce files list` command (file ops are embedded in `jobs submit`) | +| `/xce/v1/files/:uuid/:filename` | GET | Download file — no `xce files download` command | +| `/xce/v1/jobs/:jobids` | GET | Get specific jobs by ID — CLI uses the general list with filter instead | + +--- + +## Root Cause Summary + +```mermaid +graph TD + A[Display shows all dashes / zeros] --> B{Type mismatch\nserver int → client string} + B --> C[QueueStatus: int vs string] + B --> D[HostStatus: int vs string] + B --> E[JobStatus: int vs string] + B --> F[ResourceType: int vs string] + B --> G[ResourceValue: typed fields vs single value key] + + H[roles list / users list broken] --> I[Client sends unique=true\nServer rejects unknown param] + + J[roles view / users view / permissions4user\nshow dashes] --> K[Field name mismatch\nserver action/domain_type vs\nclient RoleInfo struct] + + L[jobs list no results for admin] --> M[RBAC: admin role missing\njob:view with all domain] + + N[jobs control not found] --> O[Snowflake job IDs don't\nmatch builtin test IDs 1001-1004] + + P[jobs submit -f parse error] --> Q[File path parsed as\ndata spec format src>dst] +``` + +--- + +## New Bugs Found — Resolution Status + +| ID | Severity | Component | Description | Status | +|---|---|---|---|---| +| T1 | 🔴 | xcecli | `roles list`/`users list`: remove `unique=true` param; deduplicate client-side | ✅ fixed | +| T2 | 🔴 | xcecli | `QueueStatus`/`HostStatus`/`JobStatus` type mismatch | ✅ fixed (server sends strings; T3b root cause) | +| T3 | 🔴 | xcecli | Resource distribution value fields | ✅ fixed (part of T3b) | +| T3b | 🔴 | xcecli | All rows wrapped in `{Fields,Data}` OrderMap — `UnmarshalOrderMapRow` added | ✅ fixed | +| T4 | 🔴 | xcecli | `ResourceDef` display alignment | ✅ verified OK | +| T5 | 🔴 | xcecli | Column truncation in roles/users view | ✅ fixed | +| T6 | 🟠 | apiserver | Admin `job:view` RBAC | ✅ already correct in rbac.yaml | +| T7 | 🟠 | xcecli | `xce logout --url` optional | ✅ fixed | +| T8 | 🟠 | xcecli | `jobs submit -f` plain path parse error | ✅ fixed | +| T9 | 🟡 | xcecli | Add `xce hosts select` command | ⏳ pending | +| T10 | 🟡 | xcecli | Add `xce files list`/`download` commands | ⏳ pending | +| M2 | 🔴 | xcecli | `QueryJob.ExitStatus` type mismatch | ✅ fixed | +| M3 | 🔴 | xcecli | `XCEHost.NumCPUs` json tag wrong | ✅ fixed (part of T3b) | +| A1 | 🔴 | xcecli | Token header leading space | ✅ fixed | +| A3 | 🟠 | xcecli | TLS `InsecureSkipVerify` always set | ✅ fixed | +| S1 | 🔴 | apiserver | JWT secret hardcoded in config | ✅ fixed (env var override) | +| S3 | 🟠 | apiserver | Hardcoded mock RBAC maps | ✅ fixed | +| S4 | 🟠 | apiserver | Whitelist pattern truncation bug | ✅ fixed | + +| ID | Severity | Component | Description | +|---|---|---|---| +| **T1** | 🔴 | xcecli | `roles list` / `users list`: remove `params["unique"] = "true"` — server rejects this param. Implement client-side deduplication instead | +| **T2** | 🔴 | xcecli | `QueueStatus`, `HostStatus`, `JobStatus`: change client model fields from `string` → `int`, add display helper functions to convert bitmask to string | +| **T3** | 🔴 | xcecli | `ResourceDist`: read `numeric_value`/`boolean_value`/`string_value` from server response instead of `"value"` | +| **T4** | 🔴 | xcecli | `ResourceDef.ResourceType`: change from `string` → `int`, add display helper | +| **T5** | 🔴 | xcecli | `roles view` / `users view` / `permissions4user`: fix field name mapping — server returns `action`, `domain_type`, `domain_values`; client struct must match | +| **T6** | 🟠 | apiserver | `rbac.yaml` default config: `cluster_admin` role needs `job:view` with `all` domain so admin can see all jobs | +| **T7** | 🟠 | xcecli | `xce logout`: make `--url` optional, default to active server URL from config | +| **T8** | 🟠 | xcecli | `jobs submit -f`: fix file path parsing — plain paths (no `>`) should not trigger data spec parse error | +| **T9** | 🟡 | xcecli | Add `xce hosts select` command to expose `POST /xce/v1/hosts/select` | +| **T10** | 🟡 | xcecli | Add `xce files list` and `xce files download` commands | + +--- + +## Updated Full Todo List + +### 🔴 Critical — breaks functionality + +| ID | Component | Task | Source | +|---|---|---|---| +| T1 | xcecli | Remove `unique=true` query param from roles/users list; deduplicate client-side | New | +| T2 | xcecli | Fix `QueueStatus`/`HostStatus`/`JobStatus` type: `string` → `int` + display helpers | New (confirms M1, M2) | +| T3 | xcecli | Fix resource distribution value fields: read `numeric_value`/`boolean_value`/`string_value` | New (confirms M4) | +| T4 | xcecli | Fix `ResourceDef.ResourceType`: `string` → `int` + display helper | New | +| T5 | xcecli | Fix roles/users view field name mapping to match server response | New | +| M3 | xcecli | Fix `XCEHost.NumCPUs` json tag: `"num_cores"` → `"num_cpus"` | Existing | +| A1 | xcecli | Fix token header: standardize to `"Bearer "+token` in all request methods | Existing | +| S4 | apiserver | Fix whitelist pattern truncation bug | Existing | + +### 🟠 High — security / usability + +| ID | Component | Task | Source | +|---|---|---|---| +| T6 | apiserver | Fix `rbac.yaml`: add `job:view` with `all` domain to `cluster_admin` role | New | +| T7 | xcecli | Make `xce logout --url` optional, default to active server | New | +| T8 | xcecli | Fix `jobs submit -f` file path parsing (plain path triggers data spec error) | New | +| S1 | apiserver | Move JWT secret out of `config/app.yaml` into environment variable | Existing | +| S3 | apiserver | Remove hardcoded mock RBAC maps from `rbac_check.go` | Existing | +| A3 | xcecli | Fix TLS: wire `loadRootCAs`, only use `InsecureSkipVerify` when opted in | Existing | + +### 🟡 Missing features + +| ID | Component | Task | Source | +|---|---|---|---| +| T9 | xcecli | Add `xce hosts select` command (`POST /xce/v1/hosts/select`) | New | +| T10 | xcecli | Add `xce files list` and `xce files download` commands | New | +| F1 | server+cli | Implement file delete endpoint + `xce files delete` command | Existing | +| F2 | server+cli | Decide on API key management: implement or remove dead code | Existing | +| F3 | xcecli | Add array job stats display (`?type=array_stats`) | Existing | +| F4 | apiserver | Enable CORS middleware | Existing | + +### 🔵 Cleanup + +| ID | Component | Task | Source | +|---|---|---|---| +| Q1 | apiserver | Regenerate Swagger docs (`swag init`) | Existing | +| Q2 | apiserver | Fix `validateRequestBody` inverted condition | Existing | +| Q3 | apiserver | Replace `ioutil.ReadFile` with `os.ReadFile` | Existing | +| Q4 | apiserver | Remove unused `AlecAivazis/survey` dependency | Existing | +| Q5 | xcecli | Upgrade `resty.dev/v3` beta to stable `go-resty/resty/v2` | Existing | +| Q6 | apiserver | Fix rate limiter map unbounded growth | Existing | +| Q7 | xcecli | Add retry logic for transient network errors | Existing | diff --git a/src/frontend/docs/plan_t9_f1_f4.md b/src/frontend/docs/plan_t9_f1_f4.md new file mode 100644 index 00000000..dde50bf6 --- /dev/null +++ b/src/frontend/docs/plan_t9_f1_f4.md @@ -0,0 +1,151 @@ +# Implementation Plan: T9, F1, F4 + +> Created: 2026-04-08 +> Scope: Three remaining feature items before backend integration + +--- + +## Overview + +| ID | Item | Effort | Order | +|---|---|---|---| +| F4 | CORS middleware (apiserver) | ~2h | 1st | +| F1 | File delete (server route + CLI command) | ~2.5h | 2nd | +| T9 | `xce hosts select` CLI command | ~3h | 3rd | + +--- + +## F4 — Enable CORS Middleware + +**Problem:** CORS middleware is commented out in `router/routers.go`. Any web-based client will be blocked by browser same-origin policy. + +**Approach:** Inline CORS middleware (no new dependency), configurable via `app.yaml`. + +### Files to change + +| File | Change | +|---|---| +| `apiserver/middleware/cors.go` | New: `CORSMiddleware(cfg CORSConfig)` | +| `apiserver/utils/config.go` | Add `CORSConfig` struct + parse from viper | +| `apiserver/config/app.yaml` | Add `cors:` section | +| `apiserver/router/routers.go` | Replace commented block with `r.Use(middleware.CORSMiddleware(...))` | + +### Implementation steps (TDD) + +1. Write `TestCORSHeaders` — OPTIONS preflight returns `Access-Control-Allow-Origin` +2. Implement `CORSMiddleware()` setting response headers +3. Wire into router, add config to `app.yaml` + +### Config shape + +```yaml +cors: + enabled: true + allowed_origins: + - "*" + allowed_methods: + - "GET" + - "POST" + - "DELETE" + - "OPTIONS" + allowed_headers: + - "Origin" + - "Content-Type" + - "Authorization" +``` + +--- + +## F1 — File Delete + +**Problem:** `DeleteFiles` controller is fully implemented but the route is commented out. No CLI command exists. + +### Server side + +| File | Change | +|---|---| +| `apiserver/router/routers.go` | Uncomment `filesApi.DELETE("", ...)` | + +`DELETE /xce/v1/files` is already in `WhitelistDELETEURL` — no whitelist change needed. + +### Client side + +| File | Change | +|---|---| +| `xcecli/cmd/files/files.go` | New: root `files` cobra command | +| `xcecli/cmd/files/delete.go` | New: `xce files delete --path ` | +| `xcecli/router/router.go` | Register `files.NewFilesCmd(configManager)` | + +### CLI usage + +``` +xce files delete --path http://server/xce/v1/files/uuid/filename +xce files delete --path uuid/filename,uuid2/filename2 +``` + +### Implementation steps (TDD) + +1. Write `TestRunFilesDelete_RequiresPath` — errors when `--path` missing +2. Uncomment server route +3. Implement `xce files delete` command +4. Add regression test case to `tests/regression/run.sh` + +--- + +## T9 — `xce hosts select` + +**Problem:** `POST /xce/v1/hosts/select` exists on the server (returns empty list pre-backend) but has no CLI command. + +**Server note:** `FilterHostByResReqSelect` is a stub — returns empty until backend integration. The CLI command should work end-to-end; it will return empty results until the backend is connected. + +### Client side + +| File | Change | +|---|---| +| `xcecli/models/hosts.go` | Add `ResReqSelectRequest` struct | +| `xcecli/cmd/hosts/select.go` | New: `xce hosts select -R "select_string"` | +| `xcecli/cmd/hosts/hosts.go` | Register new subcommand | + +### CLI usage + +``` +xce hosts select -R "type==linux && ncpus>=8" +xce hosts select -R "select(!mg)" --order "ncpus" --limit 10 +``` + +### ResReqSelectRequest model (mirrors server) + +```go +type ResReqSelectRequest struct { + Select string `json:"select"` + Order string `json:"order,omitempty"` + Limit int `json:"limit,omitempty"` + PageToken string `json:"page_token,omitempty"` +} +``` + +### Implementation steps (TDD) + +1. Write `TestParseSelectArgs` — `-R` flag required, max 4094 chars +2. Add `ResReqSelectRequest` to `xcecli/models/hosts.go` +3. Implement `xce hosts select` — POST to `/xce/v1/hosts/select`, display with existing `HostsPagePrinter` +4. Add regression test case + +--- + +## Execution Order + +```mermaid +gantt + title Implementation Schedule + dateFormat HH:mm + section F4 CORS + Test + implement middleware :f4, 00:00, 2h + section F1 File Delete + Uncomment server route :f1a, after f4, 30m + CLI command + test :f1b, after f1a, 2h + section T9 Hosts Select + Model + CLI command + test :t9, after f1b, 3h +``` + +**Total estimated effort: ~7.5h** diff --git a/src/frontend/docs/status.md b/src/frontend/docs/status.md new file mode 100644 index 00000000..c7fa7976 --- /dev/null +++ b/src/frontend/docs/status.md @@ -0,0 +1,94 @@ +# Project Status + +> Last updated: 2026-04-09 + +--- + +## Overall Progress + +```mermaid +pie title Component Completion + "Done" : 99 + "Missing features" : 1 +``` + +--- + +## Integration Test Summary + +Live test: `xcecli` against running `apiserver` with builtin test data. + +| Status | Commands (after all fixes) | +|---|---| +| ✅ Working | `login`, `logout`, `list`, `whoami`, `permissions list`, `roles list/view/add/delete`, `users list/view/assignrole/unassignrole/permissions4user`, `hosts list/open/close`, `queues list/open/close/active/deactive`, `jobs submit/list/kill/stop/resume/checkpoint`, `resources distribution/definition list` | +| ⚠️ Expected | `jobs list` (no filter = current user only, by design); `clusters list` (empty — no backend data) | +| ❌ Broken | none | + +--- + +## Done + +### apiserver +- [x] Gin HTTP server with config loading (Viper + JSON schema validation) +- [x] JWT authentication — logon / logout, token stored server-side per user +- [x] Middleware stack: IP whitelist → URL whitelist → rate limiter → JWT validator +- [x] RBAC engine — roles, permissions, domain-scoped access control +- [x] Auth provider abstraction — PAM (Linux), config-file, LDAP stub, OAuth2 stub +- [x] Full REST API surface (30 endpoints across jobs, hosts, queues, clusters, resources, files, roles, users, permissions) +- [x] In-memory generic `DataStore[K,V]` with filter / sort / paginate pipeline +- [x] Job ID generator (Snowflake-style, cluster-aware) +- [x] File upload with MIME type validation and size limits +- [x] Structured logging with `zap` + log rotation (`lumberjack`) +- [x] Per-endpoint rate limiting (token bucket, configurable window) +- [x] Backend stub + notification handler interfaces ready for integration +- [x] **S1** — JWT secret loaded from `XCE_JWT_SECRET` env var; falls back to config file +- [x] **S3** — Removed hardcoded mock RBAC maps from `rbac_check.go` +- [x] **S4** — Whitelist pattern matching: `pathMatchesPattern()` — exact paths no longer truncated; wildcards fixed for all sub-paths +- [x] **F4** — CORS middleware: configurable via `app.yaml`, handles preflight (OPTIONS 204) + +### xcecli +- [x] Cobra CLI with command groups; auth-check middleware on all commands +- [x] Multi-server config (`~/.xce/config.json`) with token + expiry tracking +- [x] All command groups: auth, permissions, roles, users, jobs, hosts, queues, clusters, resources +- [x] Paginated table output for all resource types +- [x] **T1** — `roles list` / `users list`: removed `unique=true` param; client-side cross-page dedup +- [x] **T3b/M3** — `UnmarshalOrderMapRow`: all row types unwrap server `{Fields,Data}` wrapper; all list commands show real data; `num_cpus` tag fixed +- [x] **T5** — Column truncation fixed in `roles view` / `users view` / `permissions4user` +- [x] **T7** — `xce logout --url` now optional; defaults to active server +- [x] **T8** — `jobs submit -f /path` plain paths no longer error +- [x] **A1** — Authorization header: removed leading space; all 5 request methods consistent +- [x] **A3** — `SetRootCAs`: HTTPS + empty cert → `InsecureSkipVerify`; HTTPS + cert path → load pool +- [x] **T9** — `xce hosts select -R "expr"` command; POSTs to `/xce/v1/hosts/select` (returns empty until backend) +- [x] **Job time display** — `SUBMIT_TIME` now shows `Apr 08 17:00` instead of `-`; `parseJobTime()` supports RFC3339 and space formats +- [x] **Build** — xcecli builds static (no libssl.so.3) via `CGO_ENABLED=0`; pam/openssl removed from nix shell until backend integration + +### Infrastructure +- [x] Nix flake dev shell (`nix develop`) — Go 1.25, PAM, OpenSSL, meson, gnumake, swag, gopls, delve +- [x] Root `Makefile`: `make test`, `make test-integration`, `make test-all` +- [x] Regression suite: `tests/regression/run.sh` — 29 tests, all pass +- [x] Unit tests across: `apiserver/store`, `apiserver/utils`, `apiserver/controllers`, `xcecli/cmd/auth`, `xcecli/cmd/jobs`, `xcecli/cmd/roles`, `xcecli/cmd/users`, `xcecli/common/request`, `xcecli/models` + +--- + +## Todo + +### 🟡 Missing features + +| ID | Component | Task | +|---|---|---| +| F1 | server+cli | File delete: uncomment server route + `xce files delete` command | + +### 🔵 Cleanup + +| ID | Component | Task | +|---|---|---| +| Q1 | apiserver | Regenerate Swagger docs (`swag init`) | + +--- + +## Upcoming: Backend Integration Prep + +- [ ] Register `DefaultHandler` with backend notification system so store is populated from real data +- [ ] Replace `backend/stub.go` no-ops with real C library / gRPC calls +- [ ] Decide persistence strategy — `lib/pq` (PostgreSQL) in `go.mod` but unused; store is pure in-memory +- [ ] Make `clusterID` configurable (hardcoded `const clusterID = 1` in `controllers/jobs.go`) diff --git a/src/frontend/meson.build b/src/frontend/meson.build new file mode 100644 index 00000000..ec801099 --- /dev/null +++ b/src/frontend/meson.build @@ -0,0 +1,2 @@ +subdir('apiserver') +subdir('xcecli') diff --git a/src/frontend/tests/regression/run.sh b/src/frontend/tests/regression/run.sh new file mode 100755 index 00000000..debc45c2 --- /dev/null +++ b/src/frontend/tests/regression/run.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# Regression test suite: xcecli ↔ apiserver +# Usage: +# ./run.sh # uses defaults below +# XCE=/path/to/xce URL=http://host:8080 ./run.sh +# +# Requires a running apiserver with builtin test data and config_file auth. +# Exit code: 0 = all passed, non-zero = failures found. + +set -euo pipefail + +XCE="${XCE:-/tmp/xce_static}" +URL="${URL:-http://localhost:8080}" +PASS=0 +FAIL=0 +SKIP=0 + +# ── helpers ────────────────────────────────────────────────────────────────── + +GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; NC='\033[0m' + +pass() { echo -e "${GREEN}PASS${NC} $1"; PASS=$((PASS+1)); } +fail() { echo -e "${RED}FAIL${NC} $1 — $2"; FAIL=$((FAIL+1)); } +skip() { echo -e "${YELLOW}SKIP${NC} $1 — $2"; SKIP=$((SKIP+1)); } + +# Run command, assert stdout contains expected string +assert_contains() { + local desc="$1" expected="$2"; shift 2 + local out + out=$("$@" 2>&1) || true + if echo "$out" | grep -qF "$expected"; then + pass "$desc" + else + fail "$desc" "expected '${expected}' in output, got: $(echo "$out" | head -3)" + fi +} + +# Run command, assert stdout does NOT contain string +assert_not_contains() { + local desc="$1" unexpected="$2"; shift 2 + local out + out=$("$@" 2>&1) || true + if echo "$out" | grep -qF "$unexpected"; then + fail "$desc" "unexpected '${unexpected}' found in output" + else + pass "$desc" + fi +} + +# Assert command exits successfully (any output) +assert_ok() { + local desc="$1"; shift + if "$@" >/dev/null 2>&1; then + pass "$desc" + else + fail "$desc" "command exited non-zero" + fi +} + +# ── setup ──────────────────────────────────────────────────────────────────── + +echo "=== Regression Suite ===" +echo "XCE : $XCE" +echo "URL : $URL" +echo "" + +if [[ ! -x "$XCE" ]]; then + echo "ERROR: xce binary not found at $XCE" + echo "Build with: cd xcecli && CGO_ENABLED=0 go build -o /tmp/xce_static ./cmd/main.go" + exit 1 +fi + +# Check server reachable +if ! "$XCE" login -u admin -p admin --url "$URL" 2>&1 | grep -q "login successfully"; then + echo "ERROR: cannot login to $URL — is apiserver running?" + exit 1 +fi + +echo "--- Auth ---" + +# ── auth ───────────────────────────────────────────────────────────────────── + +assert_contains "login admin" "login successfully" $XCE login -u admin -p admin --url "$URL" +assert_contains "list shows url" "$URL" $XCE list +assert_contains "whoami returns user" "admin" $XCE users whoami +assert_contains "logout clears session" "clear login info" $XCE logout --url "$URL" + +# Re-login for remaining tests +$XCE login -u admin -p admin --url "$URL" >/dev/null 2>&1 + +echo "" +echo "--- Permissions ---" + +assert_contains "permissions list returns data" "job:view" $XCE permissions list +assert_not_contains "permissions list no error" "failed" $XCE permissions list + +echo "" +echo "--- Roles ---" + +# T1 fix: roles list must not send unknown param +assert_not_contains "roles list no 400 error" "invalid parameters" $XCE roles list +assert_not_contains "roles list no unique error" "unique" $XCE roles list + +# roles add / delete round-trip +$XCE roles delete -r regression_test_role >/dev/null 2>&1 || true +assert_contains "roles add" "created successfully" $XCE roles add -r regression_test_role -p "job:view=all" +assert_contains "roles delete" "deleted successfully" $XCE roles delete -r regression_test_role + +echo "" +echo "--- Users ---" + +# T1 fix: users list must not send unknown param +assert_not_contains "users list no 400 error" "invalid parameters" $XCE users list +assert_not_contains "users list no unique error" "unique" $XCE users list + +assert_contains "users whoami" "admin" $XCE users whoami +# ensure clean state before assign +$XCE users unassignrole -u user1 -r basic_user >/dev/null 2>&1 || true +assert_contains "users assignrole" "assigned roles" $XCE users assignrole -u user1 -r basic_user +assert_contains "users unassignrole" "deleted successfully" $XCE users unassignrole -u user1 -r basic_user + +echo "" +echo "--- Hosts ---" + +assert_not_contains "hosts list no error" "failed" $XCE hosts list +assert_contains "hosts open" "open" $XCE hosts open host1 +assert_contains "hosts close" "close" $XCE hosts close host1 + +echo "" +echo "--- Queues ---" + +assert_not_contains "queues list no error" "failed" $XCE queues list +assert_contains "queues open" "open" $XCE queues open normal +assert_contains "queues close" "close" $XCE queues close normal +assert_contains "queues active" "active" $XCE queues active normal +assert_contains "queues deactive" "deactive" $XCE queues deactive normal + +echo "" +echo "--- Jobs ---" + +# submit returns a job ID +SUBMIT_OUT=$($XCE jobs submit -J regression_test -q normal sleep 1 2>&1) +if echo "$SUBMIT_OUT" | grep -qE "Job <[0-9]+> is submitted"; then + pass "jobs submit" + JOB_ID=$(echo "$SUBMIT_OUT" | grep -oE '[0-9]{10,}' | head -1) +else + fail "jobs submit" "unexpected output: $SUBMIT_OUT" + JOB_ID="" +fi + +assert_not_contains "jobs list -u all no error" "failed" $XCE jobs list -u all + +# Verify submit time is parsed (not showing '-') +$XCE login -u user1 -p user1 --url "$URL" >/dev/null 2>&1 +assert_not_contains "jobs list submit_time not dash" "- " $XCE jobs list +$XCE login -u admin -p admin --url "$URL" >/dev/null 2>&1 + +if [[ -n "$JOB_ID" ]]; then + assert_not_contains "jobs kill by id no crash" "panic" $XCE jobs kill "$JOB_ID" +else + skip "jobs kill" "no job ID from submit" +fi + +echo "" +echo "--- Resources ---" + +assert_not_contains "resources distribution list no error" "failed" $XCE resources distribution list +assert_not_contains "resources definition list no error" "failed" $XCE resources definition list + +echo "" +echo "--- Clusters ---" + +assert_not_contains "clusters list no error" "failed" $XCE id + +echo "" +echo "─────────────────────────────────────────" +echo -e "Results: ${GREEN}${PASS} passed${NC} ${RED}${FAIL} failed${NC} ${YELLOW}${SKIP} skipped${NC}" +echo "─────────────────────────────────────────" + +[[ $FAIL -eq 0 ]] diff --git a/src/frontend/xcecli/cmd/auth/cert/cert.go b/src/frontend/xcecli/cmd/auth/cert/cert.go new file mode 100755 index 00000000..a9d030e1 --- /dev/null +++ b/src/frontend/xcecli/cmd/auth/cert/cert.go @@ -0,0 +1,82 @@ +package cert + +import ( + "crypto/tls" + "encoding/pem" + "log" + "os" + "os/user" + "path" + logger "xcecli/config" + + "go.uber.org/zap" +) + +func GeneratorCert(serverAddress, configPath string) error { + serverAddress, err := extractHostAndPort(serverAddress) + logger.Debug("Server address", zap.String("serverAddress", serverAddress)) + if err != nil { + return err + } + // The HTTPS server's address (replace with the server you want to check) + //serverAddress := "172.16.10.10:8443" + + // Create a tls.Config with InsecureSkipVerify set to true + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + + // Dial the server using TLS with the custom tls.Config + conn, err := tls.Dial("tcp", serverAddress, tlsConfig) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + + // Retrieve the certificate chain + certs := conn.ConnectionState().PeerCertificates + + // Print the certificate chain + // for i, cert := range certs { + // fmt.Printf("Certificate %d:\n", i) + // fmt.Printf(" Subject: %s\n", cert.Subject) + // fmt.Printf(" Issuer: %s\n", cert.Issuer) + // fmt.Printf(" Not Before: %s\n", cert.NotBefore) + // fmt.Printf(" Not After: %s\n", cert.NotAfter) + // } + + // The root CA is usually the last certificate in the chain + //rootCert := certs[len(certs)-1] + // fmt.Println("\nRoot CA Certificate:") + // fmt.Printf(" Subject: %s\n", rootCert.Subject) + // fmt.Printf(" Issuer: %s\n", rootCert.Issuer) + // fmt.Printf(" Not Before: %s\n", rootCert.NotBefore) + // fmt.Printf(" Not After: %s\n", rootCert.NotAfter) + + // Save the server certificate to a file + certOut, err := os.Create(configPath) + if err != nil { + return err + } + defer certOut.Close() + + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: certs[0].Raw}) + //fmt.Println("Server certificate saved to server.crt") + return nil +} + +func GetCertPath() (string, error) { + usr, err := user.Current() + if err != nil { + return "", err + } + + configDir := path.Join(usr.HomeDir, ".xce") + configPath := path.Join(configDir, "server.crt") + + // create the config directory if it doesn't exist + if err := os.MkdirAll(configDir, 0700); err != nil { + return "", err + } + return configPath, nil +} diff --git a/src/frontend/xcecli/cmd/auth/cert/parser.go b/src/frontend/xcecli/cmd/auth/cert/parser.go new file mode 100755 index 00000000..8285c8bd --- /dev/null +++ b/src/frontend/xcecli/cmd/auth/cert/parser.go @@ -0,0 +1,37 @@ +package cert + +import ( + "fmt" + "net/url" + "xcecli/common/request" +) + +func extractHostAndPort(rawURL string) (string, error) { + // parse the URL + parsedURL, err := url.Parse(rawURL) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return "", fmt.Errorf("failed to parse URL: [%s]", errMsg) + } + + // get the host and port + host := parsedURL.Hostname() + port := parsedURL.Port() + + // if the port is empty, use the default port (HTTP is 80, HTTPS is 443) + if port == "" { + if parsedURL.Scheme == "https" { + port = "443" + } else if parsedURL.Scheme == "http" { + port = "80" + } + } + + // concatenate the host and port + result := host + if port != "" { + result = fmt.Sprintf("%s:%s", host, port) + } + + return result, nil +} diff --git a/src/frontend/xcecli/cmd/auth/list.go b/src/frontend/xcecli/cmd/auth/list.go new file mode 100755 index 00000000..58324f69 --- /dev/null +++ b/src/frontend/xcecli/cmd/auth/list.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "fmt" + "os" + "text/tabwriter" + "xcecli/common/request" + "xcecli/config" + + "github.com/spf13/cobra" +) + +// NewListCmd create the list command +func NewListCmd(configManager *config.ConfigManager) *cobra.Command { + return &cobra.Command{ + Use: "list", + GroupID: "api", + Short: "List the API Servers", + Long: "List all the configured API Servers and the default API Server", + RunE: func(cmd *cobra.Command, args []string) error { + return runList(configManager) + + }, + } +} + +func runList(cm *config.ConfigManager) error { + cfg, err := cm.GetConfig() + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get the config: [%s]", errMsg) + } + + // use tabwriter to format the output + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "Default\tVersion\tURL") + + for _, server := range cfg.APIServerInfo { + isDefault := " " + if server.Token == "" { + continue + } + + fmt.Fprintf(w, "%s\t%s\n", + isDefault, + server.URL) + } + + return w.Flush() +} diff --git a/src/frontend/xcecli/cmd/auth/logon.go b/src/frontend/xcecli/cmd/auth/logon.go new file mode 100755 index 00000000..e2f59d14 --- /dev/null +++ b/src/frontend/xcecli/cmd/auth/logon.go @@ -0,0 +1,189 @@ +package cmd + +import ( + "fmt" + "net/url" + "os" + "strings" + "syscall" + "xcecli/cmd/auth/cert" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + "xcecli/models" + + "go.uber.org/zap" + "golang.org/x/term" + + "github.com/spf13/cobra" +) + +// updateServerConfig +func updateServerConfig(cfg *config.Config, url, certPath string, loginResp *models.LogonResponse) error { + logger.Debug("updateServerConfig", zap.String("url", url), zap.String("certPath", certPath)) + logger.Info("updateServerConfig", zap.String("url", url), zap.String("certPath", certPath)) + + // check if the server with the same URL already exists + Found := false + for i, server := range cfg.APIServerInfo { + if server.URL == url { + // update the existing server info + cfg.APIServerInfo[i].Token = loginResp.Data.Token + cfg.APIServerInfo[i].Path = loginResp.Data.Path //save the path + cfg.APIServerInfo[i].CreateTime = loginResp.Data.CreateTime + cfg.APIServerInfo[i].CACert = certPath + Found = true + break + } + } + + // if it's a new server, add it to the list + if !Found { + + newServer := config.APIServerInfo{ + URL: url, + Token: loginResp.Data.Token, + Path: loginResp.Data.Path, + CreateTime: loginResp.Data.CreateTime, + CACert: certPath, + } + + // add it to the end of the array + cfg.APIServerInfo = append(cfg.APIServerInfo, newServer) + logger.Debug("newServer", zap.Any("newServer", newServer)) + logger.Info("newServer", zap.Any("newServer", newServer)) + } + + return nil +} + +func NewLogonCmd(configManager *config.ConfigManager) *cobra.Command { + var opts struct { + username string + password string + url string + } + logger.Debug("NewLogonCmd", zap.Any("opts", opts)) + logger.Info("NewLogonCmd", zap.Any("opts", opts)) + + cmd := &cobra.Command{ + Use: "login", + GroupID: "api", + Short: "Login to API Server", + Long: `Login to the specified API Server, and save the login info to the config file. +Example: + xce login -u USERNAME -p PASSWORD --url http://IP:8080 + xce login -u USERNAME -p PASSWORD --url https://IP:8443`, + Run: func(cmd *cobra.Command, args []string) { + // if the password is not provided, prompt the user to enter it + if opts.password == "" { + fmt.Print("enter password: ") + bytePassword, err := term.ReadPassword(int(syscall.Stdin)) + fmt.Println() // new line + if err != nil { + errMsg := request.ExtractErrorMessage(err) + fmt.Printf("error reading password: [%s]", errMsg) + os.Exit(1) + } + opts.password = string(bytePassword) + } + + err := runLogon(opts, configManager) + if err != nil { + fmt.Println(strings.ToLower(err.Error())) + } + }, + } + + // set the command line parameters + flags := cmd.Flags() + flags.StringVarP(&opts.username, "username", "u", "", "username") + flags.StringVarP(&opts.password, "password", "p", "", "password (optional, will prompt if not provided)") + flags.StringVar(&opts.url, "url", "", "API Server URL") + + // required parameters + cmd.MarkFlagRequired("username") + + //cmd.MarkFlagRequired("password") + cmd.MarkFlagRequired("url") + + logger.Debug("cmd", zap.Any("cmd", cmd)) + logger.Info("cmd", zap.Any("cmd", cmd)) + + return cmd +} + +func runLogon(opts struct{ username, password, url string }, cm *config.ConfigManager) error { + if cm == nil { + return fmt.Errorf("fail to initialize the config manager") + } + logger.Debug("runLogon", zap.Any("opts", opts)) + logger.Info("runLogon", zap.Any("opts", opts)) + + //get the config + cfg, err := cm.GetConfig() + if err != nil { + cfg = &config.Config{} + } + logger.Debug("cfg", zap.Any("cfg", cfg)) + logger.Info("cfg", zap.Any("cfg", cfg)) + + // create the API client + apiClient := request.NewAPIClient(opts.url) + certPath := "" + // Only setup certs for HTTPS connections + if u, err := url.Parse(opts.url); err == nil && u.Scheme == "https" { + certPath, err = cert.GetCertPath() + apiClient.SetRootCAs(certPath) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get cert path: [%s]", errMsg) + } + if err := cert.GeneratorCert(opts.url, certPath); err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to generate cert: [%s]", errMsg) + } + } + logger.Debug("apiClient", zap.Any("apiClient", apiClient)) + logger.Info("apiClient", zap.Any("apiClient", apiClient)) + + // perform the login + loginResp, err := apiClient.Logon(opts.username, opts.password) + if err != nil { + // If login fails, remove the server entry with the same URL if it exists + for i, server := range cfg.APIServerInfo { + if server.URL == opts.url { + // Remove this server from the slice + cfg.APIServerInfo = append(cfg.APIServerInfo[:i], cfg.APIServerInfo[i+1:]...) + // Save the updated config + if saveErr := cm.SaveConfig(cfg); saveErr != nil { + logger.Error("Failed to save the config after removing server entry", zap.Error(saveErr)) + } + break + } + } + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to login: [%s]", errMsg) + } + logger.Debug("loginResp", zap.Any("loginResp", loginResp)) + logger.Info("loginResp", zap.Any("loginResp", loginResp)) + + // update the server config + if err := updateServerConfig(cfg, opts.url, certPath, loginResp); err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to update the server config: [%s]", errMsg) + } + logger.Debug("cfg", zap.Any("cfg", cfg)) + logger.Info("cfg", zap.Any("cfg", cfg)) + + // save the config + if err := cm.SaveConfig(cfg); err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to save the config: [%s]", errMsg) + } + logger.Debug("cfg", zap.Any("cfg", cfg)) + logger.Info("cfg", zap.Any("cfg", cfg)) + + fmt.Println("login successfully") + return nil +} diff --git a/src/frontend/xcecli/cmd/auth/logout.go b/src/frontend/xcecli/cmd/auth/logout.go new file mode 100755 index 00000000..1d62ca94 --- /dev/null +++ b/src/frontend/xcecli/cmd/auth/logout.go @@ -0,0 +1,74 @@ +package cmd + +import ( + "fmt" + "xcecli/common/request" + "xcecli/config" + + "github.com/spf13/cobra" +) + +// NewLogoutCmd create logout command +func NewLogoutCmd(configManager *config.ConfigManager) *cobra.Command { + var url string + + cmd := &cobra.Command{ + Use: "logout", + GroupID: "api", + Short: "Logout from API Server", + Long: `Logout from specified API Server, clear local saved token. +Example: + xce logout --url http://tt1.test.com:8080 + xce logout --url https://tt1.test.com:8443`, + RunE: func(cmd *cobra.Command, args []string) error { + return runLogout(url, configManager) + }, + } + + // add url arg + cmd.Flags().StringVar(&url, "url", "", "specify API Server url (defaults to active server)") + + return cmd +} + +func runLogout(url string, cm *config.ConfigManager) error { + cfg, err := cm.GetConfig() + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("get config failed: [%s]", errMsg) + } + // default to active server when --url not provided + if url == "" { + if len(cfg.APIServerInfo) == 0 { + return fmt.Errorf("not logged in") + } + url = cfg.APIServerInfo[0].URL + } + apiServerInfo := make([]config.APIServerInfo, 0) + // find server with specified url + found := false + for _, server := range cfg.APIServerInfo { + switch server.URL { + case "": + continue + case url: + found = true + default: + apiServerInfo = append(apiServerInfo, server) + } + + } + //fmt.Printf("apiServerInfo: %+v\n", apiServerInfo) + if !found { + return fmt.Errorf("server not found: %s", url) + } + cfg.APIServerInfo = apiServerInfo + // save config + if err := cm.SaveConfig(cfg); err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("save config failed: [%s]", errMsg) + } + + fmt.Printf("clear login info for server %s\n", url) + return nil +} diff --git a/src/frontend/xcecli/cmd/auth/logout_test.go b/src/frontend/xcecli/cmd/auth/logout_test.go new file mode 100644 index 00000000..27c00b1b --- /dev/null +++ b/src/frontend/xcecli/cmd/auth/logout_test.go @@ -0,0 +1,53 @@ +package cmd + +import ( + "testing" + "xcecli/config" +) + +func TestRunLogout_WithURL(t *testing.T) { + cm, cfg := newTestConfig(t, "http://server1:8080", "http://server2:8080") + if err := runLogout("http://server1:8080", cm); err != nil { + t.Fatal(err) + } + cfg, _ = cm.GetConfig() + if len(cfg.APIServerInfo) != 1 || cfg.APIServerInfo[0].URL != "http://server2:8080" { + t.Errorf("expected server2 to remain, got %+v", cfg.APIServerInfo) + } +} + +func TestRunLogout_DefaultsToActiveServer(t *testing.T) { + // empty url should use the first (active) server + cm, _ := newTestConfig(t, "http://server1:8080") + if err := runLogout("", cm); err != nil { + t.Fatal(err) + } + cfg, _ := cm.GetConfig() + if len(cfg.APIServerInfo) != 0 { + t.Errorf("expected empty config, got %+v", cfg.APIServerInfo) + } +} + +func TestRunLogout_NotFound(t *testing.T) { + cm, _ := newTestConfig(t, "http://server1:8080") + err := runLogout("http://notexist:8080", cm) + if err == nil { + t.Fatal("expected error for unknown server") + } +} + +// newTestConfig creates a temp ConfigManager with the given server URLs. +func newTestConfig(t *testing.T, urls ...string) (*config.ConfigManager, *config.Config) { + t.Helper() + dir := t.TempDir() + cm := config.NewConfigManagerWithPath(dir + "/config.json") + servers := make([]config.APIServerInfo, len(urls)) + for i, u := range urls { + servers[i] = config.APIServerInfo{URL: u, Token: "tok"} + } + cfg := &config.Config{APIServerInfo: servers} + if err := cm.SaveConfig(cfg); err != nil { + t.Fatal(err) + } + return cm, cfg +} diff --git a/src/frontend/xcecli/cmd/clusters/list.go b/src/frontend/xcecli/cmd/clusters/list.go new file mode 100755 index 00000000..07aeaceb --- /dev/null +++ b/src/frontend/xcecli/cmd/clusters/list.go @@ -0,0 +1,80 @@ +package clusters + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +var ( + cluster string + fields string + Verbose bool +) + +// NewClustersCmd create cluster query command +func NewClustersCmd(configManager *config.ConfigManager) *cobra.Command { + cmd := &cobra.Command{ + Use: "id", + GroupID: "query", + Short: "Query cluster info", + Long: `Query cluster info, support filter by cluster name . +example: + xce clusters # query all clusters`, + + RunE: func(cmd *cobra.Command, args []string) error { + err := runClusters(configManager) + if err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + //cmd.Flags().BoolVarP(&Verbose, "verbose", "l", false, "show cluster detail") + return cmd +} + +func runClusters(cm *config.ConfigManager) error { + // get the config + serverInfo, _ := cm.GetServerInfo() + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // validate the parameters + cluster = strings.ToLower(cluster) + + // create the query parameters + params := make(map[string]string) + + // handle field selection + if fields != "" || cluster != "" { + params["fields"] = fields + } + + // create the API client and query the host info + var resp models.ClusterResponse + err := apiClient.GetRequestFromAPIServer(serverInfo.Token, "", serverInfo.CACert, "/xce/v1/clusters", &resp, params) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get clusters: [%s]", errMsg) + } + + if resp.Data.Count == 0 { + fmt.Println("no cluster found") + return nil + } + + // print result + output := output.NewOutput(fields, false, false) + output.SetVerbose(Verbose) + output.ClusterPrintData(&resp.Data.Rows[0]) + return nil +} diff --git a/src/frontend/xcecli/cmd/hosts/controller.go b/src/frontend/xcecli/cmd/hosts/controller.go new file mode 100755 index 00000000..dcdc4ece --- /dev/null +++ b/src/frontend/xcecli/cmd/hosts/controller.go @@ -0,0 +1,179 @@ +package hosts + +import ( + "encoding/json" + "fmt" + "strings" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +const ( + HOST_OPEN = 1 + HOST_CLOSE = 2 +) + +// HostController host control controller +type HostController struct { + configManager *config.ConfigManager + action string + hostnames []string + comment string + allHosts bool +} + +// NewHostController create host control controller +func NewHostController(configManager *config.ConfigManager, action string) *HostController { + return &HostController{ + configManager: configManager, + action: action, + hostnames: []string{}, + } +} + +// Run execute host control +func (c *HostController) RunHostsControl() error { + // get server info and verify authentication + serverInfo, _ := c.configManager.GetServerInfo() + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // handle all hosts case + if c.allHosts { + c.hostnames = []string{"all"} + } + + // if no hostnames specified, return error + if len(c.hostnames) == 0 { + return fmt.Errorf("must specify hostnames or use --all option") + } + + // create request + req := models.HostControlRequest{ + ControlAction: c.action, + Hostname: c.hostnames, + Comment: c.comment, + } + + // encode request to JSON bytes + reqData, err := json.Marshal(req) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + + var resp models.HostControlResponse + err = apiClient.PostRequestToAPIServer( + serverInfo.Token, + serverInfo.CACert, + "/xce/v1/hosts/control", + reqData, + &resp, + ) + + if err != nil { + return err + } + + if len(c.hostnames) == 1 && c.hostnames[0] == "all" { + for _, data := range resp.Data.Accepted { + fmt.Printf("host <%s> is being <%s>\n", data.HostName, c.action) + } + for _, host := range resp.Data.Failed { + fmt.Printf("host <%s> cannot be <%s> for reason <%s>\n", host.HostName, c.action, host.Msg) + } + } else { + for _, data := range resp.Data.Accepted { + fmt.Printf("host <%s> is being <%s>\n", data.HostName, c.action) + } + for _, host := range resp.Data.Failed { + fmt.Printf("host <%s> cannot be <%s> for reason <%s>\n", host.HostName, c.action, host.Msg) + } + } + return nil +} + +// Parse parse command line arguments +func (c *HostController) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // handle hostnames + if len(args) > 0 { + c.hostnames = append(c.hostnames, args...) + } + + return nil +} + +// NewOpenHostsCmd create open hosts command +func NewOpenHostsCmd(configManager *config.ConfigManager) *cobra.Command { + + controller := NewHostController(configManager, "open") + + cmd := &cobra.Command{ + Use: "open [hostname1 hostname2...]", + Short: "Open hosts", + Long: `Open hosts by hostname. +Open hosts changes hosts from DOWN, UNREACH, or UNAVAIL status to OK status. + +Examples: + xce hosts open host1,host2,... # open specific hosts + xce hosts open --all # open all hosts`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunHostsControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.BoolVarP(&controller.allHosts, "all", "a", false, "apply to all hosts") + flags.StringVarP(&controller.comment, "comment", "c", "", "add comment for this operation") + + return cmd +} + +// NewCloseHostsCmd create close hosts command +func NewCloseHostsCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewHostController(configManager, "close") + + cmd := &cobra.Command{ + Use: "close [hostname1 hostname2...]", + Short: "Close hosts", + Long: `Close hosts by hostname. +Close hosts changes hosts from OK, BUSY, or IDLE status to UNAVAIL status. + +Examples: + xce hosts close host1 host2 # close specific hosts + xce hosts close --all # close all hosts`, + RunE: func(cmd *cobra.Command, args []string) error { + + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunHostsControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.BoolVarP(&controller.allHosts, "all", "a", false, "apply to all hosts") + flags.StringVarP(&controller.comment, "comment", "c", "", "add comment for this operation") + + return cmd +} diff --git a/src/frontend/xcecli/cmd/hosts/hosts.go b/src/frontend/xcecli/cmd/hosts/hosts.go new file mode 100644 index 00000000..17c63737 --- /dev/null +++ b/src/frontend/xcecli/cmd/hosts/hosts.go @@ -0,0 +1,32 @@ +package hosts + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +// NewHostsCmd creates a command for querying host information +func NewHostsCmd(configManager *config.ConfigManager) *cobra.Command { + cmd := &cobra.Command{ + Use: "hosts", + GroupID: "query", + Short: "Manage hosts", + Long: `Manage hosts in the system. +Use 'xce hosts list' to query host information. +Use 'xce hosts open' to open hosts. +Use 'xce hosts close' to close hosts.`, + + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + // Add list subcommand + cmd.AddCommand(NewListHostsCmd(configManager)) + cmd.AddCommand(NewOpenHostsCmd(configManager)) + cmd.AddCommand(NewCloseHostsCmd(configManager)) + cmd.AddCommand(NewSelectHostsCmd(configManager)) + + return cmd +} diff --git a/src/frontend/xcecli/cmd/hosts/list.go b/src/frontend/xcecli/cmd/hosts/list.go new file mode 100755 index 00000000..5d1b4d09 --- /dev/null +++ b/src/frontend/xcecli/cmd/hosts/list.go @@ -0,0 +1,199 @@ +package hosts + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + + host_filter "xcecli/filters" + "xcecli/models" + + "go.uber.org/zap" + + "github.com/spf13/cobra" +) + +// HostsQuerier struct for querying hosts +type HostsQuerier struct { + configManager *config.ConfigManager + filter *host_filter.HostsFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool // wide format +} + +// NewHostsQuerier creates a new hosts querier +func NewHostsQuerier(configManager *config.ConfigManager) *HostsQuerier { + return &HostsQuerier{ + configManager: configManager, + filter: &host_filter.HostsFilter{}, + } +} + +// Parse parses command line arguments, simplified version that doesn't need to process flags already bound to filter +func (hq *HostsQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if len(args) > 0 { + hq.filter.HostName = strings.Join(args, ",") + } + + // Define mutual exclusions for verbose flag + if hq.verbose { + conflicts := make([]string, 0) + + if hq.wideFormat { + conflicts = append(conflicts, "--wided (-w)") + } + + if hq.fields != "" { + conflicts = append(conflicts, "--output (-o)") + } + + if len(conflicts) > 0 { + return fmt.Errorf("flag --verbose (-l) cannot be used with: %s\n\n%s", + strings.Join(conflicts, ", "), cmd.UsageString()) + } + } + return nil +} + +// Run executes the hosts query command +func (hq *HostsQuerier) RunHostsList() error { + // Get configuration + + // Get server information + serverInfo, _ := hq.configManager.GetServerInfo() + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Get filter string + filterStr, err := hq.filter.HandleHostFilter() + + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get filter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + // Field mapping + fieldsMap := models.HostFieldsMap + + if hq.verbose && hq.fields == "" { + for header := range fieldsMap { + hq.fields += header + "," + } + hq.fields = strings.TrimSuffix(hq.fields, ",") + } + // Process field selection + if hq.fields != "" { + hq.fields = request.CleanFields(hq.fields) + mappedFields, err := request.CheckAndMapFields(hq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeHosts, output.WithFields(hq.fields), + output.WithFormatJson(hq.jsonOutput), output.WithNoHeader(hq.noHeader), + output.WithVerbose(hq.verbose), output.WithWideFormat(hq.wideFormat)) + + // Record if the first page has been processed + isFirstPage := true + + var ( + resp models.HostsResponse + loopCount int + pageToken string + ) + + // Loop to get all pages of data + for { + if loopCount > 0 && pageToken == "" { + break + } + + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/hosts", &resp, params) + if err != nil { + return err + } + + if len(resp.Data.Rows) == 0 { + if isFirstPage { + fmt.Println("no host found") + return nil + } + } + + // Process the first page + if isFirstPage { + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + + // Print the current page + pagePrinter.PrintPage(&resp) + + // Update page token and loop counter + pageToken = resp.Data.Page_token + loopCount++ + } + + // Finish printing + pagePrinter.Finish() + + logger.Debug("hosts response completed", zap.Int("pages", loopCount)) + return nil +} + +func NewListHostsCmd(configManager *config.ConfigManager) *cobra.Command { + hostsQuerier := NewHostsQuerier(configManager) + + cmd := &cobra.Command{ + Use: "list", + Short: "Query the host info", + Long: `Query the host info, support basic info and detailed info. +Example: + xce hosts list # query the full info + xce hosts list -a # query the detailed info for all hosts + xce hosts list -o "host_name status max_cpus .." # query the specified fields + xce hosts list -o "host_name status max_cpus .." --json # query the specified fields and output in json format`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := hostsQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := hostsQuerier.RunHostsList(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.BoolVarP(&hostsQuerier.verbose, "host_info", "a", false, "host information") + flags.StringVarP(&hostsQuerier.filter.HostType, "host_type", "t", "", "host type (X86_64 or ARM)") + flags.StringVarP(&hostsQuerier.fields, "output", "o", "", "output fields for hosts") + flags.BoolVarP(&hostsQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&hostsQuerier.verbose, "verbose", "l", false, "show host detail") + flags.BoolVarP(&hostsQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&hostsQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} diff --git a/src/frontend/xcecli/cmd/hosts/select.go b/src/frontend/xcecli/cmd/hosts/select.go new file mode 100644 index 00000000..215ce72b --- /dev/null +++ b/src/frontend/xcecli/cmd/hosts/select.go @@ -0,0 +1,95 @@ +package hosts + +import ( + "encoding/json" + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +// HostSelector handles the hosts select command. +type HostSelector struct { + configManager *config.ConfigManager + resReq string + order string + limit int +} + +func (s *HostSelector) validate() error { + if strings.TrimSpace(s.resReq) == "" { + return fmt.Errorf("-R/--resreq is required") + } + if len(s.resReq) > 4094 { + return fmt.Errorf("select string exceeds the limit of 4094 characters") + } + return nil +} + +func (s *HostSelector) run() error { + serverInfo, err := s.configManager.GetServerInfo() + if err != nil { + return err + } + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + req := models.ResReqSelectRequest{ + Select: s.resReq, + Order: s.order, + Limit: s.limit, + } + body, _ := json.Marshal(req) + + var resp models.HostsResponse + if err := apiClient.PostRequestToAPIServer(serverInfo.Token, serverInfo.CACert, + "/xce/v1/hosts/select", body, &resp); err != nil { + return err + } + + if resp.Data.Count == 0 { + fmt.Println("no host found") + return nil + } + + pagePrinter := output.NewPagePrinter(output.DataTypeHosts) + pagePrinter.StartPrinting() + pagePrinter.PrepareForPrinting(&resp) + pagePrinter.PrintPage(&resp) + pagePrinter.Finish() + return nil +} + +func NewSelectHostsCmd(configManager *config.ConfigManager) *cobra.Command { + s := &HostSelector{configManager: configManager} + + cmd := &cobra.Command{ + Use: "select", + Short: "Select hosts by resource requirement", + Long: `Select hosts matching a resource requirement expression. +Examples: + xce hosts select -R "type==linux" + xce hosts select -R "ncpus>=8" --order ncpus --limit 10`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := s.validate(); err != nil { + fmt.Println(err.Error()) + return nil + } + if err := s.run(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + } + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&s.resReq, "resreq", "R", "", "resource requirement select expression (required)") + flags.StringVar(&s.order, "order", "", "order field") + flags.IntVar(&s.limit, "limit", 0, "max results") + + return cmd +} diff --git a/src/frontend/xcecli/cmd/hosts/select_test.go b/src/frontend/xcecli/cmd/hosts/select_test.go new file mode 100644 index 00000000..9c7ef6ad --- /dev/null +++ b/src/frontend/xcecli/cmd/hosts/select_test.go @@ -0,0 +1,28 @@ +package hosts + +import ( + "testing" +) + +func TestParseSelectArgs_RequiresResReq(t *testing.T) { + s := &HostSelector{} + err := s.validate() + if err == nil { + t.Error("expected error when -R not provided") + } +} + +func TestParseSelectArgs_MaxLength(t *testing.T) { + s := &HostSelector{resReq: string(make([]byte, 4095))} + err := s.validate() + if err == nil { + t.Error("expected error when select string exceeds 4094 chars") + } +} + +func TestParseSelectArgs_Valid(t *testing.T) { + s := &HostSelector{resReq: "type==linux"} + if err := s.validate(); err != nil { + t.Errorf("unexpected error: %v", err) + } +} diff --git a/src/frontend/xcecli/cmd/jobs/controller.go b/src/frontend/xcecli/cmd/jobs/controller.go new file mode 100755 index 00000000..71e53bdf --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/controller.go @@ -0,0 +1,287 @@ +package jobs + +import ( + "encoding/json" + "fmt" + "strings" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +// JobController defines job control +type JobController struct { + configManager *config.ConfigManager + action string + filter models.JobControlFilter + jobIDs []string +} + +// NewJobController create job controller +func NewJobController(configManager *config.ConfigManager, action string) *JobController { + return &JobController{ + configManager: configManager, + action: action, + } +} + +// Run execute job control +func (c *JobController) RunJobsControl() error { + // Get server information + serverInfo, _ := c.configManager.GetServerInfo() + + // Create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + if len(c.jobIDs) > 0 { + // if job IDs are specified, use job ID interface + return c.controlJobsByIDs(apiClient, serverInfo.Token, serverInfo.CACert) + } else { + // otherwise use filter conditions + return c.controlJobsByFilter(apiClient, serverInfo.Token, serverInfo.CACert) + } +} + +// control jobs by job IDs +func (c *JobController) controlJobsByIDs(apiClient *request.APIClient, token string, certPath string) error { + jobIDsStr := strings.Join(c.jobIDs, ",") + url := fmt.Sprintf("/xce/v1/jobs/%s/control", jobIDsStr) + + req := models.JobControlJobIDRequest{ + Action: c.action, + } + // encode request data to JSON bytes + return c.printJobControlResponse(req, apiClient, token, certPath, url) +} + +// control jobs by filter conditions +func (c *JobController) controlJobsByFilter(apiClient *request.APIClient, token string, certPath string) error { + url := "/xce/v1/jobs/control" + + req := models.JobControlRequest{ + Action: c.action, + Filter: c.filter, + } + // encode request data to JSON bytes + return c.printJobControlResponse(req, apiClient, token, certPath, url) +} + +func (c *JobController) printJobControlResponse(req interface{}, apiClient *request.APIClient, token string, certPath string, url string) error { + reqData, err := json.Marshal(req) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + var resp models.JobControlResponse + err = apiClient.PostRequestToAPIServer( + token, + certPath, + url, + reqData, + &resp, + ) + + if err != nil { + return err + } + for _, data := range resp.Data.Accepted { + if data.ArrayIdx != 0 { + fmt.Printf("job <%d[%d]> %s\n", data.JobID, data.ArrayIdx, strings.TrimPrefix(data.Msg, "job ")) + } else { + fmt.Printf("job <%d> %s\n", data.JobID, strings.TrimPrefix(data.Msg, "job ")) + } + } + + for _, data := range resp.Data.Failed { + if data.ArrayIdx != 0 { + fmt.Printf("job <%d[%d]> %s\n", data.JobID, data.ArrayIdx, strings.TrimPrefix(data.Msg, "job ")) + } else { + fmt.Printf("job <%d> %s\n", data.JobID, strings.TrimPrefix(data.Msg, "job ")) + } + } + return nil +} + +// Parse parse command line arguments +// Parse command line arguments +func (c *JobController) ParseCmdArgs(cmd *cobra.Command, args []string) error { + if len(args) > 0 { + if len(args) == 1 && args[0] == "0" { + c.filter.AllJobs = true + } else { + // parse job ID parameters + c.jobIDs = strings.Split(args[0], ",") + return nil + } + } + // check if any filter conditions are set + if c.filter.JobName == "" && c.filter.QueueName == "" && c.filter.Users == "" && !c.filter.AllJobs { + // if no job IDs or filter conditions are specified, default to processing all jobs of the current user + return fmt.Errorf("no job IDs or filter conditions specified") + } + return nil +} + +// NewKillJobsCmd create kill job command +func NewKillJobsCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewJobController(configManager, "kill") + + cmd := &cobra.Command{ + Use: "kill", + Short: "Kill jobs", + Long: `Kill jobs by job IDs or filters. +Examples: + xce jobs kill jobid1,jobid2,... # kill jobs by job IDs + xce jobs kill -J jobname # kill jobs by job name + xce jobs kill -q queue_name # kill jobs in a specific queue + xce jobs kill -u username # kill jobs by user`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunJobsControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.StringVarP(&controller.filter.JobName, "job_name", "J", "", "filter by job name") + flags.StringVarP(&controller.filter.QueueName, "queue", "q", "", "filter by queue name") + flags.StringVarP(&controller.filter.Users, "user", "u", "", "filter by user") + flags.BoolVarP(&controller.filter.AllJobs, "all", "a", false, "select all jobs") + + return cmd +} + +// NewStopJobsCmd create stop job command +func NewStopJobsCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewJobController(configManager, "stop") + + cmd := &cobra.Command{ + Use: "stop", + Short: "Stop jobs", + Long: `Stop jobs by job IDs or filters. +Examples: + xce jobs stop jobid1,jobid2,... # stop jobs by job IDs + xce jobs stop -J jobname # stop jobs by job name + xce jobs stop -q queue_name # stop jobs in a specific queue + xce jobs stop -u username # stop jobs by user`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunJobsControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.StringVarP(&controller.filter.JobName, "job_name", "J", "", "filter by job name") + flags.StringVarP(&controller.filter.QueueName, "queue", "q", "", "filter by queue name") + flags.StringVarP(&controller.filter.Users, "user", "u", "", "filter by user") + flags.BoolVarP(&controller.filter.AllJobs, "all", "a", false, "select all jobs") + + return cmd +} + +// NewResumeJobsCmd create resume job command +func NewResumeJobsCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewJobController(configManager, "resume") + + cmd := &cobra.Command{ + Use: "resume", + Short: "Resume jobs", + Long: `Resume jobs by job IDs or filters. +Examples: + xce jobs resume jobid1,jobid2,... # resume jobs by job IDs + xce jobs resume -J jobname # resume jobs by job name + xce jobs resume -u username # resume jobs by user`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunJobsControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.StringVarP(&controller.filter.JobName, "job_name", "J", "", "filter by job name") + flags.StringVarP(&controller.filter.Users, "user", "u", "", "filter by user") + flags.BoolVarP(&controller.filter.AllJobs, "all", "a", false, "select all jobs") + + return cmd +} + +func NewCheckPointJobsCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewJobController(configManager, "checkpoint") + + cmd := &cobra.Command{ + Use: "checkpoint", + Short: "Checkpoint jobs", + Long: `Checkpoint jobs by job IDs or filters. +Examples: + xce jobs checkpoint jobid1,jobid2,... # checkpoint jobs by job IDs`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunJobsControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + return cmd +} + +func NewRestartJobsCmd(configManager *config.ConfigManager) *cobra.Command { + restarter := NewJobRestart(configManager) + + cmd := &cobra.Command{ + Use: "restart", + Short: "Restart jobs", + Long: `Restart jobs by job ID. +Examples: + xce jobs restart jobid # restart job by jobid + xce jobs restart -m hostname # restart jobs by hostname`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := restarter.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := restarter.RunJobsRestart(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.StringVarP(&restarter.askedHosts, "hostname", "m", "", "restart on the specific host name") + return cmd +} diff --git a/src/frontend/xcecli/cmd/jobs/files.go b/src/frontend/xcecli/cmd/jobs/files.go new file mode 100755 index 00000000..c5be903a --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/files.go @@ -0,0 +1,129 @@ +package jobs + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "xcecli/common/request" + "xcecli/models" +) + +type fileTransfer struct { + tranfers []*transferInfo + uploaded []models.UploadedFile +} + +func newFileTransfer(uploadFiles []string) *fileTransfer { + f := &fileTransfer{tranfers: make([]*transferInfo, 0), uploaded: make([]models.UploadedFile, 0)} + for _, uploadFile := range uploadFiles { + transfer, err := parseFileTransfer(uploadFile) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + fmt.Printf("parse file transfer config failed: [%s]", errMsg) + continue + } + f.tranfers = append(f.tranfers, transfer) + } + return f +} + +func (f *fileTransfer) uploadFiles(postFunc func([]string) (*models.UploadResponse, error)) error { + // upload files + files := make([]string, 0) + for _, transfer := range f.tranfers { + if transfer.isUpload { + files = append(files, transfer.sourcePath) + } + } + if len(files) > 0 { + resp, err := postFunc(files) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("upload files failed: [%s]", errMsg) + } + if resp.Data.Status == "accepted" { + f.uploaded = resp.Data.UploadedFiles + } else { + return fmt.Errorf("upload files failed: %v", resp.Data.Failed) + } + } + return nil +} + +// find the matching uploaded file +func (f *fileTransfer) findMatchingUploadedFile(transfer *transferInfo) *models.UploadedFile { + // get the source file name + srcFileName := filepath.Base(transfer.sourcePath) + // iterate the uploaded files + for _, uploadedFile := range f.uploaded { + // if the source file name matches + if uploadedFile.SrcFile == srcFileName { + return &uploadedFile + } + } + return nil +} + +func (f *fileTransfer) buildTransferFileList() []string { + var transferList []string + for _, transfer := range f.tranfers { + if transfer.isUpload { + // find the matching uploaded file + uploadedFile := f.findMatchingUploadedFile(transfer) + if uploadedFile == nil { + fmt.Printf("warning: no matching uploaded file (source file: %s, dest file: %s)\n", + transfer.sourcePath, transfer.destPath) + continue + } + transferList = append(transferList, fmt.Sprintf("%s %s %s", + uploadedFile.WebURL, + transfer.operator, + transfer.destPath)) + } else { + // for the files that do not need to be uploaded, add them to the transfer list + transferList = append(transferList, fmt.Sprintf("%s %s %s", + transfer.destPath, + transfer.operator, + transfer.sourcePath)) + } + } + return transferList +} + +// transferInfo upload file related information +type transferInfo struct { + sourcePath string // source file path + operator string // operator + destPath string // dest file path + isUpload bool // is upload + +} + +// parseFileTransfer parse the file transfer string. +// Accepts "src > dst" style specs or a plain path (treated as upload to basename). +func parseFileTransfer(transfer string) (*transferInfo, error) { + re := regexp.MustCompile(`^\s*([^<>]+?)\s*(>|><|<>|<<|<)\s*([^<>]+?)\s*$`) + matches := re.FindStringSubmatch(transfer) + if len(matches) == 4 { + operator := strings.TrimSpace(matches[2]) + isUpload := operator == ">" || operator == "><" || operator == "<>" + return &transferInfo{ + sourcePath: strings.TrimSpace(matches[1]), + operator: operator, + destPath: strings.TrimSpace(matches[3]), + isUpload: isUpload, + }, nil + } + // plain path — treat as upload to basename + plain := strings.TrimSpace(transfer) + if plain != "" { + return &transferInfo{ + sourcePath: plain, + operator: ">", + destPath: filepath.Base(plain), + isUpload: true, + }, nil + } + return nil, fmt.Errorf("cannot parse the file transfer config: [%s]", transfer) +} diff --git a/src/frontend/xcecli/cmd/jobs/files_test.go b/src/frontend/xcecli/cmd/jobs/files_test.go new file mode 100644 index 00000000..68174783 --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/files_test.go @@ -0,0 +1,43 @@ +package jobs + +import ( + "path/filepath" + "testing" +) + +func TestParseFileTransfer_ExplicitOperator(t *testing.T) { + ti, err := parseFileTransfer("/tmp/test.txt > remote.txt") + if err != nil { + t.Fatal(err) + } + if ti.sourcePath != "/tmp/test.txt" || ti.destPath != "remote.txt" || !ti.isUpload { + t.Errorf("unexpected: %+v", ti) + } +} + +func TestParseFileTransfer_PlainPath(t *testing.T) { + // plain path with no operator: treat as upload to basename + ti, err := parseFileTransfer("/tmp/test.txt") + if err != nil { + t.Fatal(err) + } + if ti.sourcePath != "/tmp/test.txt" { + t.Errorf("sourcePath: got %q want /tmp/test.txt", ti.sourcePath) + } + if ti.destPath != filepath.Base("/tmp/test.txt") { + t.Errorf("destPath: got %q want %q", ti.destPath, filepath.Base("/tmp/test.txt")) + } + if !ti.isUpload { + t.Error("expected isUpload=true for plain path") + } +} + +func TestParseFileTransfer_PlainFilename(t *testing.T) { + ti, err := parseFileTransfer("myfile.sh") + if err != nil { + t.Fatal(err) + } + if ti.sourcePath != "myfile.sh" || ti.destPath != "myfile.sh" || !ti.isUpload { + t.Errorf("unexpected: %+v", ti) + } +} diff --git a/src/frontend/xcecli/cmd/jobs/jobs.go b/src/frontend/xcecli/cmd/jobs/jobs.go new file mode 100755 index 00000000..95d28fe4 --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/jobs.go @@ -0,0 +1,36 @@ +package jobs + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +// NewJobsCmd creates a jobs query command +func NewJobsCmd(configManager *config.ConfigManager) *cobra.Command { + + // Create the parent jobs command + cmd := &cobra.Command{ + Use: "jobs", + GroupID: "query", + Short: "Manage jobs", + Long: `Manage jobs in the system. +Use 'xce jobs list' to query jobs information. +Use 'xce jobs ' to submit/kill/stop/resume/checkpoint/restart jobs.`, + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + cmd.AddCommand(NewListJobsCmd(configManager)) + // Add submit command to jobs command + cmd.AddCommand(NewSubmitJobsCmd(configManager)) + // Add job control sub-commands to jobs command + cmd.AddCommand(NewKillJobsCmd(configManager)) + cmd.AddCommand(NewStopJobsCmd(configManager)) + cmd.AddCommand(NewResumeJobsCmd(configManager)) + cmd.AddCommand(NewCheckPointJobsCmd(configManager)) + cmd.AddCommand(NewRestartJobsCmd(configManager)) + + return cmd +} diff --git a/src/frontend/xcecli/cmd/jobs/list.go b/src/frontend/xcecli/cmd/jobs/list.go new file mode 100755 index 00000000..36686ddc --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/list.go @@ -0,0 +1,369 @@ +package jobs + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + job_filter "xcecli/filters" + "xcecli/models" + + "github.com/spf13/cobra" +) + +var ( + resp models.JobsResponse + loopCount int + pageToken string + arrayJobsStatsResp models.ArrayJobsStatsResponse +) + +// JobsQuerier struct for querying jobs +type JobsQuerier struct { + configManager *config.ConfigManager + filter *job_filter.JobsFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool // wide format + arrayFlag bool // show array job statistics + arrayElementsFlag bool // show array job elements (sub-tasks of array jobs) + +} + +// NewJobsQuerier creates a new jobs querier +func NewJobsQuerier(configManager *config.ConfigManager) *JobsQuerier { + return &JobsQuerier{ + configManager: configManager, + filter: &job_filter.JobsFilter{}, + } + +} + +// Parses command line arguments, simplified version that only processes arguments not directly bound to filter +func (jq *JobsQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // Process JobIDs parameter, which needs special handling + // input multiple jobids, it will use , to connect + if len(args) > 0 { + jq.filter.JobIDs = strings.Join(args, ",") + } + + // Define mutual exclusions for verbose flag + if jq.verbose { + conflicts := make([]string, 0) + + if jq.wideFormat { + conflicts = append(conflicts, "--wided (-w)") + } + + if jq.fields != "" { + conflicts = append(conflicts, "--output (-o)") + } + + if len(conflicts) > 0 { + return fmt.Errorf("flag --verbose (-l) cannot be used with: %s\n\n%s", + strings.Join(conflicts, ", "), cmd.UsageString()) + } + } + return nil +} + +// Run executes the jobs query command +func (jq *JobsQuerier) RunJobsList() error { + // Get server information + serverInfo, _ := jq.configManager.GetServerInfo() + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // if array job statistics + if jq.arrayFlag { + if jq.verbose { + fmt.Println("not support -A and -l parameters together") + return nil + } + return jq.runArrayJobStats(serverInfo, apiClient) + } + + // if array job elements (sub-tasks of array jobs) + if jq.arrayElementsFlag { + return jq.runArrayJobElements(serverInfo, apiClient) + } + + // the following is the regular job query logic + // Build query parameters + params := make(map[string]string) + //fmt.Println(params) + + // Get filter string + filterStr, err := jq.filter.HandleJobFilter("") + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get parameter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + // Field mapping + fieldsMap := models.JobFieldsMap + if jq.verbose && jq.fields == "" { + for header := range fieldsMap { + jq.fields += header + "," + } + jq.fields = strings.TrimSuffix(jq.fields, ",") + } + + // Process field selection + if jq.fields != "" { + jq.fields = request.CleanFields(jq.fields) + mappedFields, err := request.CheckAndMapFields(jq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeJobs, output.WithFields(jq.fields), + output.WithFormatJson(jq.jsonOutput), output.WithNoHeader(jq.noHeader), + output.WithVerbose(jq.verbose), output.WithWideFormat(jq.wideFormat)) + + // record if the first page has been processed + isFirstPage := true + + // Loop to get all pages of data + loopCount = 0 + pageToken = "" + + for { + if loopCount > 0 && pageToken == "" { + break + } + + // Only add page_token to params if it's not empty + //if pageToken != "" { + //params["page_token"] = pageToken + //} + + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/jobs", &resp, params) + if err != nil { + return err + } + if len(resp.Data.Rows) == 0 { + if isFirstPage { + fmt.Println("no job found") + return nil + } + } + // process the first page + if isFirstPage { + // start printing process + pagePrinter.StartPrinting() + // + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + + // print current page + pagePrinter.PrintPage(&resp) + + // update page token and loop count + pageToken = resp.Data.Page_token + + loopCount++ + } + + // finish printing + pagePrinter.Finish() + + return nil +} + +// runArrayJobStats process array job statistics +func (jq *JobsQuerier) runArrayJobStats(serverInfo *config.APIServerInfo, apiClient *request.APIClient) error { + // set query parameters, request array job statistics + params := map[string]string{ + "type": "array_stats", + } + params["type"] = "array_stats" + + filterStr, err := jq.filter.HandleJobFilter("array_stats") + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get parameter: [%s]", errMsg) + } + + if filterStr != "" { + params["filter"] = filterStr + } + + // create response object + + // create array job page printer + pagePrinter := output.NewPagePrinter(output.DataTypeArrayJobs, output.WithFields(jq.fields), + output.WithFormatJson(jq.jsonOutput), output.WithNoHeader(jq.noHeader), + output.WithVerbose(jq.verbose), output.WithWideFormat(jq.wideFormat)) + + // record if the first page has been processed + isFirstPage := true + + // Loop to get all pages of data + loopCount = 0 + pageToken = "" + + for { + if loopCount > 0 && pageToken == "" { + break + } + + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/jobs", &arrayJobsStatsResp, params) + if err != nil { + return err + } + if len(arrayJobsStatsResp.Data.Rows) == 0 { + if isFirstPage { + fmt.Println("no job found") + return nil + } + } + // process the first page + if isFirstPage { + // start printing process + pagePrinter.StartPrinting() + // + pagePrinter.PrepareForPrinting(&arrayJobsStatsResp) + isFirstPage = false + } + + // print current page + pagePrinter.PrintPage(&arrayJobsStatsResp) + + // update page token and loop count + pageToken = arrayJobsStatsResp.Data.Page_token + + loopCount++ + } + // finish printing + pagePrinter.Finish() + + return nil +} + +// runArrayJobElements process array job elements +func (jq *JobsQuerier) runArrayJobElements(serverInfo *config.APIServerInfo, apiClient *request.APIClient) error { + // set query parameters, request array job elements + params := map[string]string{ + "filter": "array_idx:ne:0", // query array_idx not equal to 0, that is, the sub-tasks of array jobs + } + + // create response object + var jobsResp models.JobsResponse + + // request array job elements data + err := apiClient.GetRequestFromAPIServer(serverInfo.Token, "", serverInfo.CACert, "/xce/v1/jobs?limit=1000", &jobsResp, params) + if err != nil { + return err + } + + // create array job page printer + pagePrinter := output.NewPagePrinter(output.DataTypeArrayElements, output.WithFields(jq.fields), + output.WithFormatJson(jq.jsonOutput), output.WithNoHeader(jq.noHeader), + output.WithVerbose(jq.verbose), output.WithWideFormat(jq.wideFormat)) + // start printing process + pagePrinter.StartPrinting() + + // prepare and print data + pagePrinter.PrepareForPrinting(&jobsResp) + pagePrinter.PrintPage(&jobsResp) + + // finish printing + pagePrinter.Finish() + + return nil +} + +func NewListJobsCmd(configManager *config.ConfigManager) *cobra.Command { + jobsQuerier := NewJobsQuerier(configManager) + + // Create the list subcommand + listCmd := &cobra.Command{ + Use: "list", + Short: "Query jobs info", + Long: `Query jobs detail info. +Example: + xce jobs list # list all jobs + xce jobs list jobid0,jobid1,...jobidn # list jobs by job id + xce jobs list -a # list run,done,pending,suspended jobs + xce jobs list -d # list done jobs + xce jobs list -d -r # list done and running jobs + xce jobs list -d -r -s # list done and running and suspended jobs + xce jobs list -d -r -s -p # list done and running and suspended and pending jobs + xce jobs list -d -u all # list done jobs by all users + xce jobs list -d -r -s -p -u username # list done and running and suspended and pending jobs by user + xce jobs list -d -r -s -p -u username -q queuename # list done and running and suspended and pending jobs by user and queue + xce jobs list -j jobname # list jobs by job name + xce jobs list -Jd jobdescription # list jobs by job description + xce jobs list --Lp licenseprojectname # list license project jobs + xce jobs list -o "jobid submit_user ..." # list jobs by fields + xce jobs list -o "jobid submit_user ..." --json # list jobs by fields and output in json format + xce jobs list -p # list pending jobs + xce jobs list -p -u username # list pending jobs by user + xce jobs list -p -u all # list pending jobs by all users + xce jobs list -P projectname # show project jobs + xce jobs list -q queuename # list jobs by queue + xce jobs list -q queuename -u username # list jobs by user in queue + xce jobs list -r # list running jobs + xce jobs list -r -u username # list running jobs by user + xce jobs list -s # list suspended jobs + xce jobs list -s -u username # list suspended jobs by user + xce jobs list -s -u all # list suspended jobs by all users + xce jobs list --sla serviceclassname # list jobs by service class + xce jobs list -u all # list jobs by all users + xce jobs list -u username # list jobs by user + xce jobs list -u username all # list jobs by all user + xce jobs list -u username -q queuename # list jobs by user and queue + xce jobs list -u all -q queuename # list jobs by all users and queue + xce jobs list -q -u all # list jobs by all users and all queues + xce jobs list -A # list array job statistics + xce jobs list -W # list array job elements + xce jobs list -m host1 # list jobs executing on specific host + xce jobs list -m host1,host2 # list jobs executing on multiple hosts`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := jobsQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := jobsQuerier.RunJobsList(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + // Add command line parameters + flags := listCmd.Flags() + flags.BoolVarP(&jobsQuerier.filter.AllJobInfoFlag, "job", "a", false, "show all jobs status") + flags.StringVarP(&jobsQuerier.filter.User, "user", "u", "", "filter by user") + flags.StringVarP(&jobsQuerier.filter.Queue, "queue", "q", "", "filter by queue") + flags.BoolVarP(&jobsQuerier.filter.PendingFlag, "pending", "p", false, "show pending jobs") + flags.StringVarP(&jobsQuerier.filter.ProjectName, "project", "P", "", "show project jobs") + flags.BoolVarP(&jobsQuerier.filter.RunningFlag, "running", "r", false, "show running jobs") + flags.BoolVarP(&jobsQuerier.filter.DoneFlag, "done", "d", false, "show done jobs") + flags.BoolVarP(&jobsQuerier.filter.SSUSPFlag, "ssuspFlag", "s", false, "show suspended jobs") + flags.StringVarP(&jobsQuerier.filter.JobName, "job_name", "J", "", "filter by job name") + flags.StringVarP(&jobsQuerier.filter.LicenseProject, "Lp", "", "", "show license project jobs") + flags.StringVarP(&jobsQuerier.filter.JobDescription, "Jd", "", "", "show job description") + flags.StringVarP(&jobsQuerier.filter.ServiceClass, "sla", "", "", "show job by service class") + flags.BoolVarP(&jobsQuerier.arrayFlag, "array", "A", false, "show array job statistics") + //flags.BoolVarP(&jobsQuerier.arrayElementsFlag, "wide", "w", false, "show array job elements (sub-tasks of array jobs)") + flags.StringVarP(&jobsQuerier.filter.ExecHosts, "exechost", "m", "", "filter jobs by execution host, comma separated for multiple hosts") + flags.StringVarP(&jobsQuerier.fields, "output", "o", "", "output fields for jobs") + flags.BoolVarP(&jobsQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&jobsQuerier.verbose, "verbose", "l", false, "show job detail") + flags.BoolVarP(&jobsQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&jobsQuerier.noHeader, "noheader", "n", false, "output no header") + return listCmd +} diff --git a/src/frontend/xcecli/cmd/jobs/restart.go b/src/frontend/xcecli/cmd/jobs/restart.go new file mode 100644 index 00000000..f2bc5566 --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/restart.go @@ -0,0 +1,86 @@ +package jobs + +import ( + "encoding/json" + "fmt" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +func NewJobRestart(configManager *config.ConfigManager) *JobRestart { + return &JobRestart{ + configManager: configManager, + jobID: "", + askedHosts: "", + } +} + +type JobRestart struct { + configManager *config.ConfigManager + jobID string + askedHosts string +} + +func (c *JobRestart) ParseCmdArgs(cmd *cobra.Command, args []string) error { + for _, arg := range args { + if arg != "" && c.jobID == "" { + c.jobID = arg + } + } + // check if any filter conditions are set + if c.jobID == "" { + // if no job IDs or filter conditions are specified, default to processing all jobs of the current user + return fmt.Errorf("no job ID is specified") + } + return nil +} + +func (c *JobRestart) RunJobsRestart() error { + // Get server information + serverInfo, _ := c.configManager.GetServerInfo() + + // Create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + return c.restartJobsByID(apiClient, serverInfo.Token, serverInfo.CACert) +} + +func (c *JobRestart) restartJobsByID(apiClient *request.APIClient, token string, certPath string) error { + url := fmt.Sprintf("/xce/v1/jobs/%s/restart", c.jobID) + + req := &models.JobRestartRequest{ + AskedHosts: c.askedHosts, + } + + return c.printJobRestartResponse(req, apiClient, token, certPath, url) + +} + +func (c *JobRestart) printJobRestartResponse(req interface{}, apiClient *request.APIClient, token string, certPath string, url string) error { + reqData, err := json.Marshal(req) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + var resp models.JobRestartResponse + err = apiClient.PostRequestToAPIServer( + token, + certPath, + url, + reqData, + &resp, + ) + + if err != nil { + return err + } + row := resp.Data + if row != nil { + fmt.Printf("%s\n", row.Message) + } + return nil +} diff --git a/src/frontend/xcecli/cmd/jobs/submit.go b/src/frontend/xcecli/cmd/jobs/submit.go new file mode 100755 index 00000000..39bfb0f0 --- /dev/null +++ b/src/frontend/xcecli/cmd/jobs/submit.go @@ -0,0 +1,734 @@ +package jobs + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "runtime" + "strings" + "time" + "xcecli/common/request" + "xcecli/common/utils" + "xcecli/config" + logger "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "go.uber.org/zap" +) + +// JobSubmitter structure for managing job submission from the CLI parameters and operations +type JobSubmitter struct { + configManager *config.ConfigManager + queue string + resReq string + beginTime string + termTime string + jobName string + jobDescription string + cpuLimit string + command string + runLimit string + coreLimit string + fileLimit string + memLimit string + swapLimit string + dataLimit string + processLimit string + stackLimit string + threadLimit string + useShellLimit bool + useSystemEnv bool + subEnvVars []string + cwd string + uploadFiles []string + processorRange string + infile string + outfile string + errfile string + outdir string + configFile string + interactive bool + useTTY bool + useXJob bool + projectName string + askedHosts string + launchType string + preExecCmd string + postExecCmd string +} + +// NewJobSubmitter creates a new JobSubmitter instance +func NewJobSubmitter(configManager *config.ConfigManager) *JobSubmitter { + return &JobSubmitter{ + configManager: configManager, + } +} + +// Parse parses command line arguments into JobSubmitter struct +func (js *JobSubmitter) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // check if the -R flag is used more than once + if cmd.Flags().Changed("resreq") { + count := 0 + cmd.Flags().Visit(func(f *pflag.Flag) { + if f.Name == "resreq" { + count++ + } + }) + if count > 1 { + return fmt.Errorf("error: the -R/--resreq flag can only be used once") + } + } + + if js.interactive { + if js.useTTY && js.useXJob { + return fmt.Errorf("error: cannot use both --tty and --xjob") + } + } + + // check if there is standard input redirection + fileInfo, err := os.Stdin.Stat() + if err == nil && (fileInfo.Mode()&os.ModeCharDevice) == 0 { + if tmpFileName, err := js.handleStdin(); err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("error reading from stdin: [%s]", errMsg) + } else if tmpFileName != "" { + defer os.Remove(tmpFileName) + } + } + + // if the config file is specified, parse the config file first + if js.configFile != "" { + return js.parseConfigFile() + } + + // command line parameters will override the settings in the config file + if len(args) > 0 { + js.command = strings.Join(args, " ") + } + return nil +} + +func (js *JobSubmitter) handleStdin() (string, error) { + content, err := io.ReadAll(os.Stdin) + if err != nil { + return "", err + } + if len(content) == 0 { + return "", nil + } + tmpFile, err := os.CreateTemp("", "xcecli_stdin_*.xce") + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return "", fmt.Errorf("failed to create temporary file: [%s]", errMsg) + } + defer tmpFile.Close() + if _, err := tmpFile.Write(content); err != nil { + errMsg := request.ExtractErrorMessage(err) + return "", fmt.Errorf("failed to write to temporary file: [%s]", errMsg) + } + js.configFile = tmpFile.Name() + return js.configFile, nil +} + +// parseConfigFile parse the parameters in the config file +func (js *JobSubmitter) parseConfigFile() error { + // check if the file exists + fileInfo, err := os.Stat(js.configFile) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("config file %s does not exist or cannot be accessed: [%s]", js.configFile, errMsg) + } + + if fileInfo.IsDir() { + return fmt.Errorf("%s is a directory, not a valid config file", js.configFile) + } + + // if the file is a temporary file, delete it when the function ends + if strings.Contains(js.configFile, "xcecli_stdin_") { + defer os.Remove(js.configFile) + } + + // read the config file content + content, err := os.ReadFile(js.configFile) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to read the config file: %s", errMsg) + } + + lines := strings.Split(string(content), "\n") + fmt.Printf("config file %s contains %d lines of commands\n", js.configFile, len(lines)) + + // create the main command + rootCmd := NewSubmitJobsCmd(js.configManager) + + // collect all parameters and commands + var allParams []string + var allCmdParts []string + + // process each line + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue // skip empty lines + } + + if strings.HasPrefix(line, "#xce job submit") { + // process the parameters in the comment line + paramLine := strings.TrimPrefix(line, "#xce job submit") + paramLine = strings.TrimSpace(paramLine) + + if paramLine != "" { + parts := strings.Fields(paramLine) + + // separate the parameters and command parts + for j := 0; j < len(parts); j++ { + if strings.HasPrefix(parts[j], "-") { + allParams = append(allParams, parts[j]) + + // if the parameter has a value and is not a - + if j+1 < len(parts) && !strings.HasPrefix(parts[j+1], "-") { + allParams = append(allParams, parts[j+1]) + j++ + } + } else { + // non-parameter part as command + cmdPart := strings.Join(parts[j:], " ") + if cmdPart != "" { + allCmdParts = append(allCmdParts, cmdPart) + } + break + } + } + } + } else { + // non-comment line as command part + allCmdParts = append(allCmdParts, line) + } + } + + // merge all command parts + fullCommand := strings.Join(allCmdParts, " ") + + // merge all parameters + fullArgs := append(allParams, strings.Fields(fullCommand)...) + + fmt.Println("merged full command:") + fmt.Printf("xce jobs submit %s\n", strings.Join(fullArgs, " ")) + + // execute the merged command + rootCmd.SetArgs(fullArgs) + if err := rootCmd.Execute(); err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to execute the command: [%s]", errMsg) + } + os.Exit(0) + return nil + +} + +// BuildJobRequest +func (js *JobSubmitter) BuildJobRequest() (*models.JobSubmitRequest, error) { + // create the job request with only required fields + jobReq := &models.JobSubmitRequest{ + Command: js.command, + } + + if js.interactive { + if js.useTTY { + jobReq.Interactive = 1 // TTY mode + } else if js.useXJob { + jobReq.Interactive = 2 // XJob mode + } else { + jobReq.Interactive = 3 // only interactive, no special mode + } + } else { + jobReq.Interactive = 0 // default value + } + + // Helper function to format limit values with space + formatLimit := func(value string) string { + if value == "" { + return "" + } + re := regexp.MustCompile(`^(\d+)(MB|GB|TB|PB|[MGTP])$`) + matches := re.FindStringSubmatch(strings.ToUpper(value)) + if matches != nil { + return matches[1] + " " + matches[2] + } + return value + } + + // Set resource limits + jobReq.SwapLimit = formatLimit(js.swapLimit) + jobReq.DataLimit = formatLimit(js.dataLimit) + jobReq.MemLimit = formatLimit(js.memLimit) + jobReq.FileLimit = formatLimit(js.fileLimit) + jobReq.CoreLimit = formatLimit(js.coreLimit) + jobReq.ProcessLimit = formatLimit(js.processLimit) + jobReq.StackLimit = formatLimit(js.stackLimit) + jobReq.ThreadLimit = formatLimit(js.threadLimit) + jobReq.CPULimit = formatLimit(js.cpuLimit) + jobReq.RunLimit = formatLimit(js.runLimit) + + // Set other fields if explicitly set + jobReq.QueueName = js.queue + jobReq.Resreq = js.resReq + jobReq.JobName = js.jobName + jobReq.JobDescription = js.jobDescription + + // Parse processor range if specified + if js.processorRange != "" { + parts := strings.Split(js.processorRange, ",") + if len(parts) == 2 { + // Try to parse min and max processors + minProc, err := utils.ParseInt(parts[0]) + if err != nil { + return nil, errors.New("invalid min processor value") + } + maxProc, err := utils.ParseInt(parts[1]) + if err != nil { + return nil, errors.New("invalid max processor value") + } + jobReq.NumProcessors = minProc + jobReq.MaxNumProcessors = maxProc + } else { + // Single value case + numProc, err := utils.ParseInt(js.processorRange) + if err != nil { + return nil, errors.New("invalid processor value") + } + jobReq.NumProcessors = numProc + } + } + + // parse the begin time + if js.beginTime != "" { + // Parse the time in local timezone + parsedTime, err := utils.ParseTime(js.beginTime) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return nil, fmt.Errorf("invalid begin time value and %s", errMsg) + } + jobReq.BeginTime = parsedTime + } + // If the date is in the past, add appropriate time + + // parse the end time + if js.termTime != "" { + parsedTime, err := utils.ParseTime(js.termTime) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return nil, fmt.Errorf("invalid end time value and %s", errMsg) + } + jobReq.TermTime = parsedTime + } + + // if shell limit is enabled, get the system limits + if js.useShellLimit { + limits := utils.NewSystemLimits() + + // set the limits in the job request + if val, err := limits.GetCPULimit(); err == nil && val > 0 { + jobReq.CPULimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetFileLimit(); err == nil && val > 0 { + jobReq.FileLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetDataLimit(); err == nil && val > 0 { + jobReq.DataLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetStackLimit(); err == nil && val > 0 { + jobReq.StackLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetCoreLimit(); err == nil && val > 0 { + jobReq.CoreLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetProcessLimit(); err == nil && val > 0 { + jobReq.ProcessLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetThreadLimit(); err == nil && val > 0 { + jobReq.ThreadLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetMemLimit(); err == nil && val > 0 { + jobReq.MemLimit = fmt.Sprintf("%d", val) + } + if val, err := limits.GetSwapLimit(); err == nil && val > 0 { + jobReq.SwapLimit = fmt.Sprintf("%d", val) + } + } + + // if system environment variables are enabled, get and set them + if js.useSystemEnv { + systemEnv := utils.GetSystemEnv() + filteredEnv := []string{} + filteredEnv = append(filteredEnv, systemEnv...) + jobReq.Env = filteredEnv + } + + // Add submitted environment variables + if len(js.subEnvVars) > 0 { + jobReq.Env = append(jobReq.Env, js.getEnvironmentVariables()...) + } + + if js.infile != "" { + jobReq.InFile = js.infile + } + if js.outfile != "" { + jobReq.OutFile = js.outfile + } + if js.errfile != "" { + jobReq.ErrFile = js.errfile + } + if js.outdir != "" { + jobReq.OutDir = js.outdir + } + + if js.projectName != "" { + jobReq.ProjectName = js.projectName + } + if js.cwd != "" { + jobReq.CWD = js.cwd + } else { + cwd, _ := os.Getwd() + jobReq.CWD = cwd + } + + if js.askedHosts != "" { + jobReq.AskedHosts = js.askedHosts + } + if js.launchType != "" { + jobReq.LaunchType = js.launchType + } + if js.preExecCmd != "" { + jobReq.PreExecCmd = js.preExecCmd + } + if js.postExecCmd != "" { + jobReq.PostExecCmd = js.postExecCmd + } + return jobReq, nil +} + +// Run executes the job submission operation +func (js *JobSubmitter) RunJobsSubmit() error { + // Get server info and validate authentication + serverInfo, _ := js.configManager.GetServerInfo() + if js.command == "" { + return fmt.Errorf("please provide the command to execute") + } + + if js.infile != "" { + js.uploadFiles = append(js.uploadFiles, fmt.Sprintf("%s > %s", js.infile, js.infile)) + } + if js.outfile != "" { + js.uploadFiles = append(js.uploadFiles, fmt.Sprintf("%s < %s", js.outfile, js.outfile)) + } + if js.errfile != "" { + js.uploadFiles = append(js.uploadFiles, fmt.Sprintf("%s < %s", js.errfile, js.errfile)) + } + + fileExists := func(path string) error { + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("error: the file %s does not exist, please check the file name or path", path) + } + if info.IsDir() { + return fmt.Errorf("error: the file %s is a directory, please check the file name or path", path) + } + return nil + } + + if js.infile != "" { + if err := fileExists(js.infile); err != nil { + return err + } + } + + if js.outdir != "" { + info, err := os.Stat(js.outdir) + if os.IsNotExist(err) { + return fmt.Errorf("error: the output directory %s does not exist", js.outdir) + } + if !info.IsDir() { + return fmt.Errorf("error: the output directory %s is not a directory", js.outdir) + } + } + + // create the API client and submit the job + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + var transferFile []string + if len(js.uploadFiles) > 0 { + fileTrans := newFileTransfer(js.uploadFiles) + err := fileTrans.uploadFiles(func(s []string) (*models.UploadResponse, error) { + return apiClient.PostFile("Bearer "+serverInfo.Token, s) + }) + if err != nil { + return err + } + transferFile = fileTrans.buildTransferFileList() + } + // build the job request + jobReq, err := js.BuildJobRequest() + if err != nil { + return err + } + jobReq.TransferFiles = transferFile + jobResp, err := apiClient.SubmitJob("Bearer "+serverInfo.Token, jobReq) + logger.Debug("jobResp", zap.Any("jobResp", jobResp), zap.Error(err)) + logger.Info("jobResp", zap.Any("jobResp", jobResp), zap.Error(err)) + + if err != nil { + return err + } + + fmt.Printf("%s\n", jobResp.Data.Message) + logger.Debug("job submitted successfully", zap.String("message", jobResp.Data.Message)) + logger.Info("job submitted successfully", zap.String("message", jobResp.Data.Message)) + + if js.interactive { + fmt.Println("<>") + url, err := js.waitForInteractiveURL(apiClient, serverInfo.Token, fmt.Sprintf("%d", jobResp.Data.JobID)) + if err != nil { + return err + } + fmt.Printf("interactive session URL: %s\n", url) + + // automatically open the browser + if err := openBrowser(url); err != nil { + errMsg := request.ExtractErrorMessage(err) + fmt.Printf("warning: failed to open browser: [%s]\n", errMsg) + fmt.Println("Please manually open the URL in your browser.") + } + } + return nil +} + +func (js *JobSubmitter) checkJobStatus(apiClient *request.APIClient, token string, jobID string) (string, error) { + var resp models.JobsResponse + err := apiClient.GetRequestFromAPIServer( + "Bearer "+token, + "", + "", + fmt.Sprintf("/xce/v1/jobs/%s", jobID), + &resp, + map[string]string{ /*"fields": "interactive_job_url"*/ }, + ) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return "", fmt.Errorf("failed to get job details: [%s]", errMsg) + } + for _, row := range resp.Data.Rows { + switch row.JobStatus { + case "PEND": + return row.JobStatus, nil + case "RUN": + var execHosts models.ExecHosts + if err := json.Unmarshal([]byte(row.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 { + fmt.Println("<>") + } + return row.JobStatus, nil + default: + return "", fmt.Errorf("job %s is in state %s, please check the job status", jobID, row.JobStatus) + } + } + return "", fmt.Errorf("timeout waiting for job status") +} + +func (js *JobSubmitter) checkExtendData(apiClient *request.APIClient, token string, jobID string) (string, error) { + var resp models.JobsResponse + err := apiClient.GetRequestFromAPIServer( + "Bearer "+token, + "", + "", + fmt.Sprintf("/xce/v1/jobs/%s", jobID), + &resp, + map[string]string{"fields": "interactive_job_url"}, + ) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return "", fmt.Errorf("failed to get job details: [%s]", errMsg) + } + for _, row := range resp.Data.Rows { + //fmt.Println(row) + if row.InteractiveJobURL != "" { + return row.InteractiveJobURL, nil + } + } + return "", fmt.Errorf("no interactive job URL found") +} + +func (js *JobSubmitter) waitForInteractiveURL(apiClient *request.APIClient, token string, jobID string) (string, error) { + timeout := time.After(24 * time.Hour) + + timer := time.NewTimer(0) + defer timer.Stop() + + for { + select { + case <-timeout: + return "", fmt.Errorf("timeout waiting for interactive URL") + case <-timer.C: + status, err := js.checkJobStatus(apiClient, token, jobID) + if err != nil { + return "", err + } + if status == "PEND" { + timer.Reset(5 * time.Second) + continue + } + if status == "RUN" { + //time.Sleep(45 * time.Second) + return js.checkExtendData(apiClient, token, jobID) + } + return "", fmt.Errorf("job %s is in state %s, please check the job status", jobID, status) + } + } +} + +func openBrowser(url string) error { + var err error + switch runtime.GOOS { + case "linux": + err = exec.Command("xdg-open", url).Start() + case "windows": + err = exec.Command("cmd", "/c", "start", url).Start() + case "darwin": + err = exec.Command("open", url).Start() + default: + err = fmt.Errorf("unsupported platform") + } + return err +} + +// NewSubCmd creates a command for submitting jobs +func NewSubmitJobsCmd(configManager *config.ConfigManager) *cobra.Command { + jobSubmitter := NewJobSubmitter(configManager) + + cmd := &cobra.Command{ + Use: "submit", + Short: "Submit job to API Server", + Long: `Submit job to API Server. +Example: + xce jobs submit -q queue_name sleep 10 # submit job to queue queue_name with command "sleep 10" + xce jobs submit -R "select(!mg)" sleep 10 # submit job with resource requirement "select(!mg)" + xce jobs submit -c 10 sleep 10 # submit job with cpu limit 10 + xce jobs submit -C 10 sleep 10 # submit job with core limit 10 + xce jobs submit -b "2025-03-20 10:00" sleep 10 # submit job with begin time "2025-03-20 10:00" + xce jobs submit -t "2025-03-20 10:00" sleep 10 # submit job with termination time "2025-03-20 10:00" + xce jobs submit -J "test" sleep 10 # submit job with job name "test" + xce jobs submit --Jd "test" sleep 10 # submit job with job description "test" + xce jobs submit -F 1024 sleep 10 # submit job with file limit 1024 + xce jobs submit -W 10 sleep 10 # sets the runtime limit of the job + xce jobs submit -M 1024 sleep 10 # sets the memory limit of the job + xce jobs submit -v 1024 sleep 10 # sets the swap limit of the job + xce jobs submit -D 1024 sleep 10 # sets the data limit of the job + xce jobs submit -p 10 sleep 10 # sets the process limit of the job + xce jobs submit -S 1024 sleep 10 # sets the stack limit of the job + xce jobs submit -T 1024 sleep 10 # sets the thread limit of the job + xce jobs submit --ul sleep 10 # use current shell limits + xce jobs submit --env sleep 10 # use system environment variables + xce jobs submit --cwd "/tmp/" sleep 10 # use current working directory for job execution + xce jobs submit -f "t1.txt > t2.txt" sleep 10 # submit job with file "t1.txt" and redirect the output to "t2.txt" + xce jobs submit -q "normal" sleep 10 # submit job to queue "normal" and "priority" + xce jobs submit -n 4,6 sleep 10 # submit job with processor range 4-6 + xce jobs submit -P project_name sleep 10 # submit job with project name + xce jobs submit --configfile job.xce # submit job using parameters from a configuration file + xce jobs submit -m "host1 host2" sleep 10 # submit job into host1 host2 + xce jobs submit -I --tty sleep 10 # submit interactive job with tty + xce jobs submit -I --xjob sleep 10 # submit interactive job with xjob`, + + SilenceUsage: true, + Run: func(cmd *cobra.Command, args []string) { + if err := jobSubmitter.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + if err := jobSubmitter.RunJobsSubmit(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + }, + } + + // add the flags + flags := cmd.Flags() + + // modify the resreq flag + resReqFlag := &pflag.Flag{ + Name: "resreq", + Shorthand: "R", + Usage: "specify the resource requirement (can only be used once)", + Value: (*stringValue)(&jobSubmitter.resReq), + NoOptDefVal: "", + } + flags.AddFlag(resReqFlag) + + //modify the flags + flags.StringVarP(&jobSubmitter.beginTime, "begin_time", "b", "", "specify the begin time") + flags.StringVarP(&jobSubmitter.coreLimit, "core_limit", "C", "", "specify the core number") + flags.StringVarP(&jobSubmitter.jobName, "job_name", "J", "", "specify the job name") + flags.StringVarP(&jobSubmitter.jobDescription, "Jd", "", "", "specify the job description") + flags.StringVarP(&jobSubmitter.termTime, "term_time", "t", "", "specify the termination time") + flags.StringVarP(&jobSubmitter.cpuLimit, "cpu_limit", "c", "", "specify the cpu limit") + flags.StringVarP(&jobSubmitter.runLimit, "run_limit", "W", "", "specify the run time") + flags.StringVarP(&jobSubmitter.fileLimit, "file_limit", "F", "", "specify the file limit") + flags.StringVarP(&jobSubmitter.memLimit, "mem_limit", "M", "", "specify the memory limit") + flags.StringVarP(&jobSubmitter.swapLimit, "swap_limit", "v", "", "specify the swap limit") + flags.StringVarP(&jobSubmitter.dataLimit, "data_limit", "D", "", "specify the data limit") + flags.StringVarP(&jobSubmitter.processLimit, "process_limit", "p", "", "specify the process limit") + flags.StringVarP(&jobSubmitter.stackLimit, "stack_limit", "S", "", "specify the stack limit") + flags.StringVarP(&jobSubmitter.threadLimit, "thread_limit", "T", "", "specify the thread limit") + flags.BoolVarP(&jobSubmitter.useShellLimit, "ul", "", false, "use current shell limits") + flags.BoolVarP(&jobSubmitter.useSystemEnv, "env", "", false, "use system environment variables") + flags.StringVarP(&jobSubmitter.cwd, "cwd", "", "", "current working directory") + flags.StringArrayVarP(&jobSubmitter.uploadFiles, "file", "f", []string{}, "specify the files to upload") + flags.StringVarP(&jobSubmitter.queue, "queue_name", "q", "", "specify the job queue") + flags.StringVarP(&jobSubmitter.processorRange, "processor_range", "n", "", "specify processor range in format min,max (e.g. 4,6)") + flags.StringVarP(&jobSubmitter.infile, "infile", "i", "", "specify the input file") + flags.StringVarP(&jobSubmitter.outfile, "outfile", "o", "", "specify the output file") + flags.StringVarP(&jobSubmitter.errfile, "errfile", "e", "", "specify the error file") + flags.StringVarP(&jobSubmitter.outdir, "outdir", "d", "", "specify the output directory") + flags.StringVarP(&jobSubmitter.projectName, "project_name", "P", "", "specify the project name") + flags.StringVar(&jobSubmitter.configFile, "configfile", "", "specify the configuration file for job submission") + flags.BoolVarP(&jobSubmitter.interactive, "interactive", "I", false, "submit interactive job") + flags.BoolVar(&jobSubmitter.useTTY, "tty", false, "use TTY for interactive job") + flags.BoolVar(&jobSubmitter.useXJob, "xjob", false, "use XJob for interactive job") + flags.StringVarP(&jobSubmitter.askedHosts, "m", "m", "", "specify the running hosts, with space for multiple values") + flags.StringVarP(&jobSubmitter.launchType, "launch_type", "", "", "specify the launch type") + flags.StringVarP(&jobSubmitter.preExecCmd, "pre_exec", "", "", "specify the pre-execution command") + flags.StringVarP(&jobSubmitter.postExecCmd, "post_exec", "", "", "specify the post-execution command") + return cmd + // +} + +// modify the stringValue +type stringValue string + +func (s *stringValue) String() string { return string(*s) } +func (s *stringValue) Set(val string) error { + if *s != "" { + return fmt.Errorf("the -R/--resreq flag can only be used once") + } + *s = stringValue(val) + return nil +} +func (s *stringValue) Type() string { return "string" } + +func (js *JobSubmitter) getEnvironmentVariables() []string { + var envVars []string + for _, env := range js.subEnvVars { + // Skip environment variables with multiple equal signs + if strings.Count(env, "=") > 1 { + continue + } + envVars = append(envVars, env) + } + return envVars +} diff --git a/src/frontend/xcecli/cmd/main.go b/src/frontend/xcecli/cmd/main.go new file mode 100755 index 00000000..b6ad0a5b --- /dev/null +++ b/src/frontend/xcecli/cmd/main.go @@ -0,0 +1,66 @@ +/* +The MIT License (MIT) + +Copyright © 2025 xce + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +package main + +import ( + "fmt" + "os" + "strings" + "xcecli/config" + logger "xcecli/config" + cmd "xcecli/router" + + "go.uber.org/zap" +) + +func main() { + + // init logger + if err := logger.InitLogger(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + os.Exit(1) + } + logger.Info("********** Initialize Logger on client side done!") + + // initialize the config manager + configManager, err := config.NewConfigManager() + if err != nil { + fmt.Println("failed to initialize the config manager", zap.Error(err)) + os.Exit(1) + } + + // ensure the config manager is initialized correctly + if configManager == nil { + fmt.Println("failed to initialize the config manager") + os.Exit(1) + } + + // execute the command + if err := cmd.Execute(configManager); err != nil { + fmt.Println(strings.ToLower(err.Error())) + os.Exit(1) + } +} diff --git a/src/frontend/xcecli/cmd/permissions/list.go b/src/frontend/xcecli/cmd/permissions/list.go new file mode 100644 index 00000000..5b73f2ae --- /dev/null +++ b/src/frontend/xcecli/cmd/permissions/list.go @@ -0,0 +1,137 @@ +package permissions + +import ( + "fmt" + "strings" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +var ( + resp models.PermsResponse +) + +const ( + minIDWidth = 7 // Length of "PERM_ID" + minNameWidth = 9 // Length of "PERM_NAME" + padding = 2 + maxNameWidth = 60 +) + +type PermsQuerier struct { + configManager *config.ConfigManager + noHeader bool + fields []string +} + +func NewPermsQuerier(configManager *config.ConfigManager) *PermsQuerier { + return &PermsQuerier{ + configManager: configManager, + } +} + +func NewListPermsCmd(configManager *config.ConfigManager) *cobra.Command { + permsQuerier := NewPermsQuerier(configManager) + + listCmd := &cobra.Command{ + Use: "list", + Short: "Query permissions info", + Long: `Query permissions detail info. +Example: + xce permissions list # list all permissions`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := permsQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := permsQuerier.RunPermsList(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + flags := listCmd.Flags() + flags.BoolVarP(&permsQuerier.noHeader, "noheader", "n", false, "output no header") + return listCmd +} + +func (pq *PermsQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if len(args) > 0 { + return fmt.Errorf("too many arguments") + } + return nil +} +func (pq *PermsQuerier) RunPermsList() error { + // Get server information + serverInfo, err := pq.configManager.GetServerInfo() + if err != nil { + return fmt.Errorf("failed to get server info: %w", err) + } + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Build query parameters + params := make(map[string]string) + pq.fields = []string{} + + // Make API request + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, "", serverInfo.CACert, "/xce/v1/permissions", &resp, params) + if err != nil { + return err + } + data := resp.Data + + if len(data) == 0 { + fmt.Println("no permissions found") + return nil + } + + // Calculate column widths for proper alignment + permIDWidth := minIDWidth + padding + permNameWidth := minNameWidth + padding + + // Get the maximum ID width (last item) + if len(data) > 0 { + lastIDStr := fmt.Sprintf("%d", len(data)) + if len(lastIDStr) > permIDWidth-padding { + permIDWidth = len(lastIDStr) + padding + } + } + + // Get maximum permission name width + for _, permStr := range data { + if len(permStr) > permNameWidth-padding { + permNameWidth = len(permStr) + padding + } + } + + // Apply max width limit + if permNameWidth > maxNameWidth { + permNameWidth = maxNameWidth + } + + // Add headers with proper alignment + if !pq.noHeader { + fmt.Printf("%-*s %-*s\n", permIDWidth, "PERM_ID", permNameWidth, "PERM_NAME") + } + + // Print permissions with auto-incrementing IDs and aligned columns + for i, permStr := range data { + // Truncate long permission names + displayPermStr := permStr + if len(permStr) > permNameWidth-padding { + displayPermStr = permStr[:permNameWidth-padding] + ".." + } + fmt.Printf("%-*d %-*s\n", permIDWidth, i+1, permNameWidth, displayPermStr) + } + + return nil +} diff --git a/src/frontend/xcecli/cmd/permissions/permissions.go b/src/frontend/xcecli/cmd/permissions/permissions.go new file mode 100644 index 00000000..983b54d6 --- /dev/null +++ b/src/frontend/xcecli/cmd/permissions/permissions.go @@ -0,0 +1,26 @@ +package permissions + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +func NewPermsCmd(configManager *config.ConfigManager) *cobra.Command { + + // Create the parent permissions command + cmd := &cobra.Command{ + Use: "permissions", + GroupID: "query", + Short: "Manage permissions", + Long: `Manage permissions in the system. +Use 'xce permissions list' to query permissions information.`, + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + cmd.AddCommand(NewListPermsCmd(configManager)) + + return cmd +} diff --git a/src/frontend/xcecli/cmd/queues/controller.go b/src/frontend/xcecli/cmd/queues/controller.go new file mode 100755 index 00000000..9ee9d464 --- /dev/null +++ b/src/frontend/xcecli/cmd/queues/controller.go @@ -0,0 +1,239 @@ +package queues + +import ( + "encoding/json" + "fmt" + "strings" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" +) + +// QueueController queue controller +type QueueController struct { + configManager *config.ConfigManager + action string + queueNames []string + comment string + allQueues bool +} + +// NewQueueController create queue controller +func NewQueueController(configManager *config.ConfigManager, action string) *QueueController { + return &QueueController{ + configManager: configManager, + action: action, + queueNames: []string{}, + } +} + +// Run execute queue control +func (c *QueueController) RunQueuesControl() error { + // get server info and verify authentication + serverInfo, _ := c.configManager.GetServerInfo() + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // handle all queues case + if c.allQueues { + c.queueNames = []string{"all"} + } + + // if no queue names are specified, return an error + if len(c.queueNames) == 0 { + return fmt.Errorf("must specify queue names or use --all option") + } + + // create request + req := models.QueueControlRequest{ + ControlAction: c.action, + QueueName: c.queueNames, + Comment: c.comment, + } + + // convert request to JSON bytes + reqData, err := json.Marshal(req) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + + // send request + var resp models.QueueControlResponse + err = apiClient.PostRequestToAPIServer( + serverInfo.Token, + serverInfo.CACert, + "/xce/v1/queues/control", + reqData, + &resp, + ) + + if err != nil { + return err + } + + if len(c.queueNames) == 1 && c.queueNames[0] == "all" { + for _, data := range resp.Data.Accepted { + fmt.Printf("queue <%s> is being <%s>\n", data.QueueName, c.action) + } + for _, queue := range resp.Data.Failed { + fmt.Printf("queue <%s> cannot be <%s> for reason <%s>\n", queue.QueueName, c.action, queue.Msg) + } + } else { + for _, data := range resp.Data.Accepted { + fmt.Printf("queue <%s> is being <%s>\n", data.QueueName, c.action) + } + for _, queue := range resp.Data.Failed { + fmt.Printf("queue <%s> cannot be <%s> for reason <%s>\n", queue.QueueName, c.action, queue.Msg) + } + } + return nil +} + +// Parse parse command line arguments +func (c *QueueController) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // handle queue name parameters + if len(args) > 0 { + // Split comma-separated queue names + for _, arg := range args { + queues := strings.Split(arg, ",") + c.queueNames = append(c.queueNames, queues...) + } + } + + return nil +} + +// NewOpenQueuesCmd create open queues command +func NewOpenQueuesCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewQueueController(configManager, "open") + + cmd := &cobra.Command{ + Use: "open", + Short: "Open queues", + Long: `Open queues by queue name. +Opens queues to accept new job submissions. + +Examples: + xce queues open normal priority # open specific queues + xce queues open --all # open all queues`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunQueuesControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.BoolVarP(&controller.allQueues, "all", "a", false, "apply to all queues") + flags.StringVarP(&controller.comment, "comment", "c", "", "add comment for this operation") + + return cmd +} + +// NewCloseQueuesCmd create close queues command +func NewCloseQueuesCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewQueueController(configManager, "close") + + cmd := &cobra.Command{ + Use: "close", + Short: "Close queues", + Long: `Close queues by queue name. +Closes queues to prevent new job submissions. + +Examples: + xce queues close normal priority # close specific queues + xce queues close --all # close all queues`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunQueuesControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.BoolVarP(&controller.allQueues, "all", "a", false, "apply to all queues") + flags.StringVarP(&controller.comment, "comment", "c", "", "add comment for this operation") + + return cmd +} + +// NewActiveQueuesCmd create active queues command +func NewActiveQueuesCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewQueueController(configManager, "active") + + cmd := &cobra.Command{ + Use: "active", + Short: "Active queues", + Long: `Active queues by queue name. + +Examples: + xce queues active # queues_name status close/d`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunQueuesControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.BoolVarP(&controller.allQueues, "all", "a", false, "apply to all queues") + flags.StringVarP(&controller.comment, "comment", "c", "", "add comment for this operation") + + return cmd +} + +func NewDeactiveQueuesCmd(configManager *config.ConfigManager) *cobra.Command { + controller := NewQueueController(configManager, "deactive") + + cmd := &cobra.Command{ + Use: "deactive", + Short: "Deactive queues", + Long: `Deactive queues by queue name. + +Examples: + xce queues deactive # queues_name status close/d`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := controller.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := controller.RunQueuesControl(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // add control options + flags := cmd.Flags() + flags.BoolVarP(&controller.allQueues, "all", "a", false, "apply to all queues") + flags.StringVarP(&controller.comment, "comment", "c", "", "add comment for this operation") + + return cmd +} diff --git a/src/frontend/xcecli/cmd/queues/list.go b/src/frontend/xcecli/cmd/queues/list.go new file mode 100755 index 00000000..ac3786cb --- /dev/null +++ b/src/frontend/xcecli/cmd/queues/list.go @@ -0,0 +1,199 @@ +package queues + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + queue_filter "xcecli/filters" + + "xcecli/models" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +var ( + user string + queue string + fields string +) + +type QueuesQuerier struct { + configManager *config.ConfigManager + filter *queue_filter.QueuesFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool // wide format +} + +func NewQueuesQuerier(configManager *config.ConfigManager) *QueuesQuerier { + return &QueuesQuerier{ + configManager: configManager, + filter: &queue_filter.QueuesFilter{}, + } +} + +// Parse parses command line arguments, simplified version that doesn't need to process flags already bound to filter +func (qq *QueuesQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if len(args) > 0 { + qq.filter.QueueName = strings.Join(args, ",") + } + + // Define mutual exclusions for verbose flag + if qq.verbose { + conflicts := make([]string, 0) + + if qq.wideFormat { + conflicts = append(conflicts, "--wided (-w)") + } + + if qq.fields != "" { + conflicts = append(conflicts, "--output (-o)") + } + + if len(conflicts) > 0 { + return fmt.Errorf("flag --verbose (-l) cannot be used with: %s\n\n%s", + strings.Join(conflicts, ", "), cmd.UsageString()) + } + } + return nil +} + +func NewListQueuesCmd(configManager *config.ConfigManager) *cobra.Command { + queuesQuerier := NewQueuesQuerier(configManager) + + cmd := &cobra.Command{ + Use: "list", + Short: "Query queue info", + Long: `Query queue info, support filter by queue. +example: + xce queues list # query all queues + xce queues list -l # query all queues with detail + xce queues list -w # list queues with wide format`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := queuesQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := queuesQuerier.RunQueuesList(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&user, "user", "u", "", "filter by user") + flags.StringVarP(&queuesQuerier.filter.QueueName, "queue", "q", "", "filter by queue name") + flags.StringVarP(&queuesQuerier.fields, "output", "o", "", "output fields for queues") + flags.BoolVarP(&queuesQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&queuesQuerier.verbose, "verbose", "l", false, "show queue detail") + flags.BoolVarP(&queuesQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&queuesQuerier.noHeader, "noheader", "n", false, "output no header") + return cmd +} + +func (qq *QueuesQuerier) RunQueuesList() error { + // Get configuration + + // Get server information + serverInfo, _ := qq.configManager.GetServerInfo() + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Get filter string + filterStr, err := qq.filter.HandleQueueFilter() + + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get filter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + // Field mapping + fieldsMap := models.QueueFieldsMap + + if qq.verbose && qq.fields == "" { + for header := range fieldsMap { + qq.fields += header + "," + } + qq.fields = strings.TrimSuffix(qq.fields, ",") + } + // Process field selection + if qq.fields != "" { + qq.fields = request.CleanFields(qq.fields) + mappedFields, err := request.CheckAndMapFields(qq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeQueues, output.WithFields(qq.fields), + output.WithFormatJson(qq.jsonOutput), output.WithNoHeader(qq.noHeader), + output.WithVerbose(qq.verbose), output.WithWideFormat(qq.wideFormat)) + + // Record if the first page has been processed + isFirstPage := true + + var ( + resp models.QueuesResponse + loopCount int + pageToken string + ) + + // Loop to get all pages of data + for { + if loopCount > 0 && pageToken == "" { + break + } + + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/queues", &resp, params) + if err != nil { + return err + } + + if len(resp.Data.Rows) == 0 { + if isFirstPage { + fmt.Println("no queue found") + return nil + } + // + } + // Process the first page + if isFirstPage { + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + // Print the current page + pagePrinter.PrintPage(&resp) + + // Update page token and loop counter + pageToken = resp.Data.Page_token + loopCount++ + } + + // Finish printing + pagePrinter.Finish() + + logger.Debug("queues response completed", zap.Int("pages", loopCount)) + return nil +} diff --git a/src/frontend/xcecli/cmd/queues/queues.go b/src/frontend/xcecli/cmd/queues/queues.go new file mode 100644 index 00000000..4bc4db84 --- /dev/null +++ b/src/frontend/xcecli/cmd/queues/queues.go @@ -0,0 +1,34 @@ +package queues + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +// NewQueuesCmd creates a command for managing queues +func NewQueuesCmd(configManager *config.ConfigManager) *cobra.Command { + cmd := &cobra.Command{ + + Use: "queues", + GroupID: "query", + Short: "Manage queues", + Long: `Manage queues in the system. +Use 'xce queues list' to query queues information. +Use 'xce queues ' to open/close/active/deactive queues.`, + + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + + // Add list subcommand + cmd.AddCommand(NewListQueuesCmd(configManager)) + cmd.AddCommand(NewOpenQueuesCmd(configManager)) + cmd.AddCommand(NewCloseQueuesCmd(configManager)) + cmd.AddCommand(NewActiveQueuesCmd(configManager)) + cmd.AddCommand(NewDeactiveQueuesCmd(configManager)) + + return cmd +} diff --git a/src/frontend/xcecli/cmd/resources/list.go b/src/frontend/xcecli/cmd/resources/list.go new file mode 100644 index 00000000..6ebd386a --- /dev/null +++ b/src/frontend/xcecli/cmd/resources/list.go @@ -0,0 +1,359 @@ +package resources + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + "xcecli/models" + + logger "xcecli/config" + resource_filter "xcecli/filters" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +type ResourcesDistQuerier struct { + configManager *config.ConfigManager + filter *resource_filter.ResourceDistFilter + fields string + distType string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool // wide format +} + +func NewResourcesDistQuerier(configManager *config.ConfigManager) *ResourcesDistQuerier { + return &ResourcesDistQuerier{ + configManager: configManager, + filter: &resource_filter.ResourceDistFilter{}, + } +} + +func (rq *ResourcesDistQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if len(args) > 0 { + rq.filter.ResName = strings.Join(args, ",") + } + + // Define mutual exclusions for verbose flag + if rq.verbose { + conflicts := make([]string, 0) + + if rq.wideFormat { + conflicts = append(conflicts, "--wided (-w)") + } + + if len(conflicts) > 0 { + return fmt.Errorf("flag --verbose (-l) cannot be used with: %s\n\n%s", + strings.Join(conflicts, ", "), cmd.UsageString()) + } + } + return nil +} + +// ResourcesDefQuerier handles querying resource definition information +type ResourcesDefQuerier struct { + configManager *config.ConfigManager + filter *resource_filter.ResourceDefFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool // wide format +} + +func NewResourcesDefQuerier(configManager *config.ConfigManager) *ResourcesDefQuerier { + return &ResourcesDefQuerier{ + configManager: configManager, + filter: &resource_filter.ResourceDefFilter{}, + } +} + +func (rq *ResourcesDefQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if len(args) > 0 { + rq.filter.ResName = strings.Join(args, ",") + } + + // Define mutual exclusions for verbose flag + if rq.verbose { + conflicts := make([]string, 0) + + if rq.wideFormat { + conflicts = append(conflicts, "--wided (-w)") + } + + if len(conflicts) > 0 { + return fmt.Errorf("flag --verbose (-l) cannot be used with: %s\n\n%s", + strings.Join(conflicts, ", "), cmd.UsageString()) + } + } + return nil +} + +// NewListResourcesDefCmd creates a command for listing resource definition information +func NewListResourcesDefCmd(configManager *config.ConfigManager) *cobra.Command { + resourcesDefQuerier := NewResourcesDefQuerier(configManager) + + cmd := &cobra.Command{ + Use: "list", + Short: "Query the resources definition info", + Long: `Query the resources definition info. +Example: + xce resources definition list # query the definition info`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := resourcesDefQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := resourcesDefQuerier.RunResourcesDefList(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + + flags.BoolVarP(&resourcesDefQuerier.verbose, "verbose", "l", false, "show resource detail") + flags.BoolVarP(&resourcesDefQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&resourcesDefQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&resourcesDefQuerier.noHeader, "noheader", "n", false, "output no header") + return cmd +} + +func (rq *ResourcesDefQuerier) RunResourcesDefList() error { + // Get configuration + + // Get server information + serverInfo, _ := rq.configManager.GetServerInfo() + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Get filter string + filterStr, err := rq.filter.HandleResourceDefFilter() + + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get filter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + // Field mapping + fieldsMap := models.ResourceDefFieldsMap + + if rq.verbose && rq.fields == "" { + for header := range fieldsMap { + rq.fields += header + "," + } + rq.fields = strings.TrimSuffix(rq.fields, ",") + } + // Process field selection + if rq.fields != "" { + rq.fields = request.CleanFields(rq.fields) + mappedFields, err := request.CheckAndMapFields(rq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeResourcesDef, output.WithFields(rq.fields), + output.WithFormatJson(rq.jsonOutput), output.WithNoHeader(rq.noHeader), + output.WithVerbose(rq.verbose), output.WithWideFormat(rq.wideFormat)) + + // Record if the first page has been processed + isFirstPage := true + + var ( + resp models.ResourceDefResponse + loopCount int + pageToken string + ) + + // Loop to get all pages of data + for { + if loopCount > 0 && pageToken == "" { + break + } + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/resources/definition", &resp, params) + if err != nil { + return err + } + if len(resp.Data.Rows) == 0 { + if isFirstPage { + fmt.Println("no resource definition found") + return nil + } + } + + // Process the first page + if isFirstPage { + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + + // Print the current page + pagePrinter.PrintPage(&resp) + + // Update page token and loop counter + pageToken = resp.Data.Page_token + loopCount++ + } + + // Finish printing + pagePrinter.Finish() + + logger.Debug("resource definition response completed", zap.Int("pages", loopCount)) + return nil +} + +// NewListResourcesDistCmd creates a command for listing resource distribution information +func NewListResourcesDistCmd(configManager *config.ConfigManager) *cobra.Command { + resourcesDistQuerier := NewResourcesDistQuerier(configManager) + + cmd := &cobra.Command{ + Use: "list", + Short: "Query the resources distribution info", + Long: `Query the resources info, support static and dynamic type. +Example: + xce resources distribution list # query the distribution info`, + RunE: func(cmd *cobra.Command, args []string) error { + if err := resourcesDistQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := resourcesDistQuerier.RunResourcesDistList(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&resourcesDistQuerier.distType, "type", "t", "", "static or dynamic") + flags.BoolVarP(&resourcesDistQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&resourcesDistQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&resourcesDistQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} + +func (rq *ResourcesDistQuerier) RunResourcesDistList() error { + // Get configuration + + // Get server information + serverInfo, _ := rq.configManager.GetServerInfo() + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Get filter string + filterStr, err := rq.filter.HandleResourceDistFilter() + + if rq.distType != "" { + if rq.distType != "static" && rq.distType != "dynamic" { + return fmt.Errorf("invalid type, must be 'static' or 'dynamic'") + } else { + params["type"] = rq.distType + } + } + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get filter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + // Field mapping + fieldsMap := models.ResourceDistFieldsMap + + if rq.verbose && rq.fields == "" { + for header := range fieldsMap { + rq.fields += header + "," + } + rq.fields = strings.TrimSuffix(rq.fields, ",") + } + // Process field selection + if rq.fields != "" { + rq.fields = request.CleanFields(rq.fields) + mappedFields, err := request.CheckAndMapFields(rq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeResourcesDist, output.WithFields(rq.fields), + output.WithFormatJson(rq.jsonOutput), output.WithNoHeader(rq.noHeader), + output.WithVerbose(rq.verbose), output.WithWideFormat(rq.wideFormat)) + + // Record if the first page has been processed + isFirstPage := true + + var ( + resp models.ResourceDistResponse + loopCount int + pageToken string + ) + + // Loop to get all pages of data + for { + if loopCount > 0 && pageToken == "" { + break + } + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/resources/distribution", &resp, params) + if err != nil { + return err + } + if len(resp.Data.Rows) == 0 { + if isFirstPage { + fmt.Println("no resource found") + return nil + } + } + + // Process the first page + if isFirstPage { + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + + // Print the current page + pagePrinter.PrintPage(&resp) + + // Update page token and loop counter + pageToken = resp.Data.Page_token + loopCount++ + } + + // Finish printing + pagePrinter.Finish() + + logger.Debug("resource dirstribution response completed", zap.Int("pages", loopCount)) + return nil +} diff --git a/src/frontend/xcecli/cmd/resources/resources.go b/src/frontend/xcecli/cmd/resources/resources.go new file mode 100644 index 00000000..98917797 --- /dev/null +++ b/src/frontend/xcecli/cmd/resources/resources.go @@ -0,0 +1,56 @@ +package resources + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +// NewResourcesCmd creates a command for querying resource information +func NewResourcesCmd(configManager *config.ConfigManager) *cobra.Command { + cmd := &cobra.Command{ + Use: "resources", + GroupID: "query", + Short: "Manage resources", + Long: `Manage resources in the system. +Use 'xce resources distribution list' to query resources distribution information. +Use 'xce resources definition list' to query resources definition information.`, + + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + + // Add distribution subcommand + distributionCmd := &cobra.Command{ + Use: "distribution", + Short: "Manage resource distribution", + Long: `Manage resource distribution information. +Use 'xce resources distribution list' to query resources distribution information.`, + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + distributionCmd.AddCommand(NewListResourcesDistCmd(configManager)) + + // Add definition subcommand + definitionCmd := &cobra.Command{ + Use: "definition", + Short: "Manage resource definition", + Long: `Manage resource definition information. +Use 'xce resources definition list' to query resources definition information.`, + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + definitionCmd.AddCommand(NewListResourcesDefCmd(configManager)) + + // Add subcommands to main command + cmd.AddCommand(distributionCmd) + cmd.AddCommand(definitionCmd) + + return cmd +} diff --git a/src/frontend/xcecli/cmd/roles/addrole.go b/src/frontend/xcecli/cmd/roles/addrole.go new file mode 100644 index 00000000..3c5ffdba --- /dev/null +++ b/src/frontend/xcecli/cmd/roles/addrole.go @@ -0,0 +1,308 @@ +package roles + +import ( + "encoding/json" + "fmt" + "net/http" + "regexp" + "strings" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +func NewRoleSubmitter(configManager *config.ConfigManager) *RoleSubmitter { + return &RoleSubmitter{ + configManager: configManager, + } +} + +type RoleSubmitter struct { + configManager *config.ConfigManager + roleName string + rolePerms string + permissions []models.Permission +} + +func NewAddRolesCmd(configManager *config.ConfigManager) *cobra.Command { + roleSubmitter := NewRoleSubmitter(configManager) + + cmd := &cobra.Command{ + Use: "add", + Short: "Add role to API Server", + Long: `Add role to API Server. +Example: + xce roles add -r role_name -p perm_name=type[values,...] # add role with permission"`, + + SilenceUsage: true, + Run: func(cmd *cobra.Command, args []string) { + if err := roleSubmitter.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + if err := roleSubmitter.RunRolesAdd(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + }, + } + + // add the flags + flags := cmd.Flags() + + //modify the flags + flags.StringVarP(&roleSubmitter.roleName, "role_name", "r", "", "specify the role name") + flags.StringVarP(&roleSubmitter.rolePerms, "permissions", "p", "", "specify the permissions") + return cmd + // +} + +func NewAssignPermsCmd(configManager *config.ConfigManager) *cobra.Command { + roleSubmitter := NewRoleSubmitter(configManager) + + cmd := &cobra.Command{ + Use: "assignpermission", + Short: "Assign permissions to the existing role to API Server", + Long: `Assign permissions to the existing role to API Server. +Example: + xce roles assignpermission -r role_name -p perm_name=type[values,...] # assign permission to the role"`, + + SilenceUsage: true, + Run: func(cmd *cobra.Command, args []string) { + if err := roleSubmitter.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + if err := roleSubmitter.RunPermsAdd(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + }, + } + + // add the flags + flags := cmd.Flags() + + //modify the flags + flags.StringVarP(&roleSubmitter.roleName, "role_name", "r", "", "specify the role name") + flags.StringVarP(&roleSubmitter.rolePerms, "permissions", "p", "", "specify the permissions") + return cmd + // +} + +func (rs *RoleSubmitter) ParseCmdArgs(cmd *cobra.Command, args []string) error { + if rs.roleName == "" { + return fmt.Errorf("please provide the role name") + } + if len(rs.roleName) > 40 { + return fmt.Errorf("error: role name cannot be longer than 40 characters") + } + if rs.rolePerms == "" { + return fmt.Errorf("please provide the role permissions") + } else { + permissions, err := parsePermissions(rs.rolePerms) + if err != nil { + return err + } + rs.permissions = permissions + } + + return nil +} + +func parsePermissions(pStr string) ([]models.Permission, error) { + var permissions []models.Permission + + permItems := splitPermissions(pStr) + for _, item := range permItems { + item = strings.TrimSpace(item) + if item == "" { + continue + } + + perm, err := parseSinglePermission(item) + if err != nil { + return nil, fmt.Errorf("failed to parse the item <%s> for reason <%v>", item, err) + } + permissions = append(permissions, perm) + } + return permissions, nil +} + +func splitPermissions(pStr string) []string { + var result []string + var current strings.Builder + bracketDepth := 0 + + for _, char := range pStr { + switch char { + case ',': + if bracketDepth == 0 { + // Only split when not inside brackets + result = append(result, strings.TrimSpace(current.String())) + current.Reset() + } else { + current.WriteRune(char) + } + case '[': + bracketDepth++ + current.WriteRune(char) + case ']': + bracketDepth-- + current.WriteRune(char) + default: + current.WriteRune(char) + } + } + + // Add the last item + if current.Len() > 0 { + result = append(result, strings.TrimSpace(current.String())) + } + + return result +} +func parseSinglePermission(item string) (models.Permission, error) { + re := regexp.MustCompile(`^(.+?)=(.+?)(?:\[(.+?)\])?$`) + matches := re.FindStringSubmatch(item) + + if len(matches) != 4 { + return models.Permission{}, fmt.Errorf("incorrect format for permission, 'permission_name=domain_type' or 'permission_name=domain_type[value1,value2]' required") + } + + permName := strings.TrimSpace(matches[1]) + domainType := strings.TrimSpace(matches[2]) + valuesStr := strings.TrimSpace(matches[3]) + + // parse domain values + var domainValues []string + if valuesStr != "" { + values := strings.Split(valuesStr, ",") + for _, v := range values { + domainValues = append(domainValues, strings.TrimSpace(v)) + } + } + + if permName == "" { + return models.Permission{}, fmt.Errorf("permission_name cannot be empty") + } + if domainType == "" { + return models.Permission{}, fmt.Errorf("domain_type cannot be empty") + } + + return models.Permission{ + PermissionName: permName, + DomainType: domainType, + DomainValues: domainValues, + }, nil +} + +func (rs *RoleSubmitter) RunRolesAdd() error { + // Get server info and validate authentication + serverInfo, _ := rs.configManager.GetServerInfo() + // create the API client and submit the role + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // build the role request + roleReq, err := rs.BuildRoleRequest() + if err != nil { + return err + } + + // encode request to JSON bytes + reqData, err := json.Marshal(roleReq) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + var resp models.CreateRoleResponse + err = apiClient.PostRequestToAPIServer( + serverInfo.Token, + serverInfo.CACert, + "/xce/v1/roles/create", + reqData, + &resp, + ) + + if err != nil { + return err + } + + if resp.Code != http.StatusCreated { + logger.Info("create role failed", zap.Any("resp", resp), zap.Error(err)) + return fmt.Errorf("%s", resp.Data.Message) + } + logger.Debug("role created successfully", zap.String("message", resp.Data.Message)) + logger.Info("role created successfully", zap.String("message", resp.Data.Message)) + fmt.Printf("%s\n", resp.Data.Message) + return nil +} + +func (rs *RoleSubmitter) RunPermsAdd() error { + // Get server info and validate authentication + serverInfo, _ := rs.configManager.GetServerInfo() + // create the API client and submit the role + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // build the role request + roleReq, err := rs.BuildRoleRequest() + if err != nil { + return err + } + + // encode request to JSON bytes + reqData, err := json.Marshal(roleReq) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + + url := fmt.Sprintf("/xce/v1/roles/%s/permissions", rs.roleName) + var resp models.CreateRoleResponse + err = apiClient.PostRequestToAPIServer( + serverInfo.Token, + serverInfo.CACert, + url, + reqData, + &resp, + ) + + if err != nil { + return err + } + + if resp.Code != http.StatusCreated { + logger.Info("assign permission failed", zap.Any("resp", resp), zap.Error(err)) + return fmt.Errorf("%s", resp.Data.Message) + } + logger.Debug("assign permission successfully", zap.String("message", resp.Data.Message)) + logger.Info("assign permission successfully", zap.String("message", resp.Data.Message)) + fmt.Printf("%s\n", resp.Data.Message) + return nil +} + +func (rs *RoleSubmitter) BuildRoleRequest() (*models.CreateRoleRequest, error) { + // create the role request with only required fields + var perms []models.Permission + for _, perm := range rs.permissions { + rolePerm := models.Permission{ + PermissionName: perm.PermissionName, + DomainType: perm.DomainType, + } + if len(perm.DomainValues) > 0 { + rolePerm.DomainValues = perm.DomainValues + } + perms = append(perms, rolePerm) + } + roleReq := &models.CreateRoleRequest{ + RoleName: rs.roleName, + Permission: perms, + } + return roleReq, nil +} diff --git a/src/frontend/xcecli/cmd/roles/deleterole.go b/src/frontend/xcecli/cmd/roles/deleterole.go new file mode 100644 index 00000000..7ba4603e --- /dev/null +++ b/src/frontend/xcecli/cmd/roles/deleterole.go @@ -0,0 +1,92 @@ +package roles + +import ( + "fmt" + "net/http" + "strings" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +func NewDeleteRoleQuerier(configManager *config.ConfigManager) *DeleteRole { + return &DeleteRole{ + configManager: configManager, + } +} + +type DeleteRole struct { + configManager *config.ConfigManager + roleName string +} + +func NewDeleteRolesCmd(configManager *config.ConfigManager) *cobra.Command { + deleteRoleQuerier := NewDeleteRoleQuerier(configManager) + + cmd := &cobra.Command{ + Use: "delete", + Short: "Delete role", + Long: `Delete role, by specifying the role name. +example: + xce roles delete -r role_name # delete role`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := deleteRoleQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := deleteRoleQuerier.RunDeleteRoleInfo(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&deleteRoleQuerier.roleName, "role_name", "r", "", "filter by role name") + + return cmd +} + +func (dr *DeleteRole) ParseCmdArgs(cmd *cobra.Command, args []string) error { + if dr.roleName == "" { + return fmt.Errorf("please provide the role name") + } + return nil +} + +func (dr *DeleteRole) RunDeleteRoleInfo() error { + // Get server info and validate authentication + serverInfo, _ := dr.configManager.GetServerInfo() + // create the API client and submit the user + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + url := fmt.Sprintf("/xce/v1/roles/%s", dr.roleName) + var resp request.APIResponse + err := apiClient.DeleteRequestFromAPIServer( + serverInfo.Token, + serverInfo.CACert, + url, + []byte(nil), + &resp, + ) + + if err != nil { + return err + } + + if resp.Code != http.StatusOK { + logger.Info("delete role failed", zap.Any("resp", resp), zap.Error(err)) + return fmt.Errorf("%v", resp.Data) + } + logger.Debug("delete role successfully", zap.Any("data", resp.Data)) + logger.Info("delete role successfully", zap.Any("data", resp.Data)) + fmt.Printf("%v\n", resp.Data) + return nil +} diff --git a/src/frontend/xcecli/cmd/roles/list.go b/src/frontend/xcecli/cmd/roles/list.go new file mode 100644 index 00000000..7c037769 --- /dev/null +++ b/src/frontend/xcecli/cmd/roles/list.go @@ -0,0 +1,216 @@ +package roles + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + role_filter "xcecli/filters" + "xcecli/models" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +type RolesQuerier struct { + configManager *config.ConfigManager + filter *role_filter.RolesFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool // wide format +} + +func NewRolesQuerier(configManager *config.ConfigManager) *RolesQuerier { + return &RolesQuerier{ + configManager: configManager, + filter: &role_filter.RolesFilter{}, + } +} + +func NewListRolesCmd(configManager *config.ConfigManager) *cobra.Command { + rolesQuerier := NewRolesQuerier(configManager) + + cmd := &cobra.Command{ + Use: "list", + Short: "Query role info", + Long: `Query role info, support filter by role. +example: + xce roles list # query all roles + xce roles list -l # query all roles with detail + xce roles list -w # list roles with wide format`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := rolesQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := rolesQuerier.RunRolesInfo("list"); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&rolesQuerier.filter.RoleName, "role", "r", "", "filter by role name") + flags.StringVarP(&rolesQuerier.filter.Type, "type", "t", "", "output for all or builtin roles") + //flags.StringVarP(&rolesQuerier.fields, "output", "o", "", "output fields for roles") + flags.BoolVarP(&rolesQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&rolesQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&rolesQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} + +func (rq *RolesQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if rq.filter.Type != "" && rq.filter.Type != "all" && rq.filter.Type != "builtin" { + return fmt.Errorf("type must be either 'all' or 'builtin'") + } + if len(args) > 0 { + rq.filter.RoleName = strings.Join(args, ",") + } + return nil +} + +func (rq *RolesQuerier) RunRolesInfo(commandType string) error { + // Get server information + serverInfo, err := rq.configManager.GetServerInfo() + if err != nil { + return fmt.Errorf("failed to get server info: %w", err) + } + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Get filter string + filterStr, err := rq.filter.HandleRoleFilter() + + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get filter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + // Field mapping + + // Process field selection + // for list, only display role_name column; deduplication is done client-side + if commandType == "list" { + rq.fields = "ROLE_NAME" + params["fields"] = "role_name" + } else { + fieldsMap := models.RoleFieldsMap + if rq.verbose && rq.fields == "" { + for header := range fieldsMap { + rq.fields += header + "," + } + rq.fields = strings.TrimSuffix(rq.fields, ",") + } + + if rq.fields != "" { + rq.fields = request.CleanFields(rq.fields) + mappedFields, err := request.CheckAndMapFields(rq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + } + + if rq.filter.Type != "" { + params["type"] = rq.filter.Type + } + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeRoles, output.WithFields(rq.fields), + output.WithFormatJson(rq.jsonOutput), output.WithNoHeader(rq.noHeader), + output.WithVerbose(rq.verbose), output.WithWideFormat(rq.wideFormat)) + + // Record if the first page has been processed + isFirstPage := true + + var ( + resp models.RolesResponse + loopCount int + pageToken string + seen map[string]struct{} // cross-page dedup state for list mode + ) + if commandType == "list" { + seen = make(map[string]struct{}) + } + + // Loop to get all pages of data + for { + if loopCount > 0 && pageToken == "" { + break + } + resp = models.RolesResponse{} // reset to avoid stale rows from previous page + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/roles", &resp, params) + if err != nil { + return err + } + if resp.Data.Count == 0 { + if isFirstPage { + fmt.Println("no role found") + return nil + } + } + // For list mode, deduplicate role names client-side before printing + if commandType == "list" { + var unique []models.RoleInfo + for _, row := range resp.Data.Rows { + if _, ok := seen[row.RoleName]; !ok { + seen[row.RoleName] = struct{}{} + unique = append(unique, models.RoleInfo{RoleName: row.RoleName}) + } + } + resp.Data.Rows = unique + resp.Data.Count = len(unique) + } + // Process the first page + if isFirstPage { + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + // Print the current page + pagePrinter.PrintPage(&resp) + + // Update page token and loop counter + pageToken = resp.Data.PageToken + loopCount++ + } + + // Finish printing + pagePrinter.Finish() + + logger.Debug("roles response completed", zap.Int("pages", loopCount)) + return nil +} + +// uniqueRoleNames deduplicates role names from a RolesResponse, preserving order. +// The server returns one row per permission, so the same role_name appears multiple times. +func uniqueRoleNames(resp *models.RolesResponse) []string { + seen := make(map[string]struct{}) + var result []string + for _, row := range resp.Data.Rows { + if _, ok := seen[row.RoleName]; !ok { + seen[row.RoleName] = struct{}{} + result = append(result, row.RoleName) + } + } + return result +} diff --git a/src/frontend/xcecli/cmd/roles/list_test.go b/src/frontend/xcecli/cmd/roles/list_test.go new file mode 100644 index 00000000..d9432e2d --- /dev/null +++ b/src/frontend/xcecli/cmd/roles/list_test.go @@ -0,0 +1,82 @@ +package roles + +import ( + "testing" + "xcecli/models" +) + +// --- dedup (already tested, regression guard) --- + +func TestUniqueRoleNames_NoDuplicates(t *testing.T) { + resp := &models.RolesResponse{} + resp.Data.Rows = []models.RoleInfo{{RoleName: "admin"}, {RoleName: "basic_user"}} + if got := uniqueRoleNames(resp); len(got) != 2 { + t.Fatalf("expected 2, got %d", len(got)) + } +} + +func TestUniqueRoleNames_WithDuplicates(t *testing.T) { + resp := &models.RolesResponse{} + resp.Data.Rows = []models.RoleInfo{ + {RoleName: "admin", PermName: "job:view"}, + {RoleName: "admin", PermName: "host:view"}, + {RoleName: "basic_user", PermName: "job:view"}, + } + got := uniqueRoleNames(resp) + if len(got) != 2 || got[0] != "admin" || got[1] != "basic_user" { + t.Fatalf("unexpected: %v", got) + } +} + +func TestUniqueRoleNames_Empty(t *testing.T) { + resp := &models.RolesResponse{} + if got := uniqueRoleNames(resp); len(got) != 0 { + t.Fatalf("expected 0, got %d", len(got)) + } +} + +func TestUniqueRoleNames_CrossPage(t *testing.T) { + seen := make(map[string]struct{}) + dedup := func(rows []models.RoleInfo) []models.RoleInfo { + var unique []models.RoleInfo + for _, row := range rows { + if _, ok := seen[row.RoleName]; !ok { + seen[row.RoleName] = struct{}{} + unique = append(unique, models.RoleInfo{RoleName: row.RoleName}) + } + } + return unique + } + p1 := dedup([]models.RoleInfo{{RoleName: "admin"}, {RoleName: "admin"}, {RoleName: "basic_user"}}) + p2 := dedup([]models.RoleInfo{{RoleName: "basic_user"}, {RoleName: "operator"}}) + if len(p1) != 2 { + t.Fatalf("page1: expected 2, got %d", len(p1)) + } + if len(p2) != 1 || p2[0].RoleName != "operator" { + t.Fatalf("page2: expected [operator], got %v", p2) + } +} + +// --- column width calculation --- + +func TestUserRoleNameColumnWidth(t *testing.T) { + resp := &models.UserRolesResponse{} + resp.Data.Rows = []models.UserRoleInfo{ + {UserName: "alice", UserRoleName: "cluster_admin"}, + {UserName: "bob", UserRoleName: "basic_user"}, + } + width := calcUserRoleNameWidth(resp) + if width < len("cluster_admin") { + t.Errorf("ROLE_NAME width %d < len(%q)=%d", width, "cluster_admin", len("cluster_admin")) + } +} + +func calcUserRoleNameWidth(resp *models.UserRolesResponse) int { + w := len(models.HEADER_USERROLE_NAME) + for _, u := range resp.Data.Rows { + if n := len(u.UserRoleName); n > w { + w = n + } + } + return w +} diff --git a/src/frontend/xcecli/cmd/roles/roles.go b/src/frontend/xcecli/cmd/roles/roles.go new file mode 100644 index 00000000..fd2ad356 --- /dev/null +++ b/src/frontend/xcecli/cmd/roles/roles.go @@ -0,0 +1,31 @@ +package roles + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +func NewRolesCmd(configManager *config.ConfigManager) *cobra.Command { + + // Create the parent roles command + cmd := &cobra.Command{ + Use: "roles", + GroupID: "query", + Short: "Manage roles", + Long: `Manage roles in the system. +Use 'xce roles list' to query roles list only. +Use 'xce roles view' to query roles details. +Use 'xce roles ' to add roles.`, + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + cmd.AddCommand(NewListRolesCmd(configManager)) + cmd.AddCommand(NewViewRolesCmd(configManager)) + cmd.AddCommand(NewAddRolesCmd(configManager)) + cmd.AddCommand(NewAssignPermsCmd(configManager)) + cmd.AddCommand(NewDeleteRolesCmd(configManager)) + return cmd +} diff --git a/src/frontend/xcecli/cmd/roles/view.go b/src/frontend/xcecli/cmd/roles/view.go new file mode 100644 index 00000000..d6ec0c0b --- /dev/null +++ b/src/frontend/xcecli/cmd/roles/view.go @@ -0,0 +1,46 @@ +package roles + +import ( + "fmt" + "strings" + "xcecli/config" + + "github.com/spf13/cobra" +) + +func NewViewRolesCmd(configManager *config.ConfigManager) *cobra.Command { + rolesQuerier := NewRolesQuerier(configManager) + + cmd := &cobra.Command{ + Use: "view", + Short: "Query role info", + Long: `Query role info, support filter by role. +example: + xce roles view # query all roles + xce roles view -w # view roles with wide format`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := rolesQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := rolesQuerier.RunRolesInfo("view"); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&rolesQuerier.filter.RoleName, "role", "r", "", "filter by role name") + flags.StringVarP(&rolesQuerier.filter.Type, "type", "t", "", "output for all or builtin roles") + flags.StringVarP(&rolesQuerier.filter.PermName, "permission_name", "p", "", "filter by permission name") + flags.StringVarP(&rolesQuerier.fields, "output", "o", "", "output fields for roles") + flags.BoolVarP(&rolesQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&rolesQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&rolesQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} diff --git a/src/frontend/xcecli/cmd/users/assignrole.go b/src/frontend/xcecli/cmd/users/assignrole.go new file mode 100644 index 00000000..d0e664b4 --- /dev/null +++ b/src/frontend/xcecli/cmd/users/assignrole.go @@ -0,0 +1,127 @@ +package users + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + "xcecli/models" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +func NewUserSubmitter(configManager *config.ConfigManager) *UserSubmitter { + return &UserSubmitter{ + configManager: configManager, + } +} + +type UserSubmitter struct { + configManager *config.ConfigManager + userName string + roleName []string +} + +func NewAssignUserRolesCmd(configManager *config.ConfigManager) *cobra.Command { + userSubmitter := NewUserSubmitter(configManager) + + cmd := &cobra.Command{ + Use: "assignrole", + Short: "Assign role to API Server", + Long: `Assign role to API Server. +Example: + xce user assignrole -u user_name -r role_name1,role_name2 # assign role to a user"`, + + SilenceUsage: true, + Run: func(cmd *cobra.Command, args []string) { + if err := userSubmitter.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + if err := userSubmitter.RunUserAssignRole(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return + } + }, + } + + // add the flags + flags := cmd.Flags() + + //modify the flags + flags.StringVarP(&userSubmitter.userName, "user_name", "u", "", "specify the user name") + flags.StringSliceVarP(&userSubmitter.roleName, "role_name", "r", []string{}, "specify the role name") + return cmd + // +} + +func (rs *UserSubmitter) ParseCmdArgs(cmd *cobra.Command, args []string) error { + if rs.userName == "" { + return fmt.Errorf("please provide the user name") + } + if len(rs.userName) > 40 { + return fmt.Errorf("error: user name cannot be longer than 40 characters") + } + if len(rs.roleName) == 0 { + return fmt.Errorf("please provide the role name") + } + if len(args) > 0 { + return fmt.Errorf("too many arguments") + } + return nil +} + +func (us *UserSubmitter) RunUserAssignRole() error { + // Get server info and validate authentication + serverInfo, _ := us.configManager.GetServerInfo() + // create the API client and submit the user + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // build the user request + userReq, err := us.BuildUserRequest() + if err != nil { + return err + } + + // encode request to JSON bytes + reqData, err := json.Marshal(userReq) + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to encode request data: [%s]", errMsg) + } + url := fmt.Sprintf("/xce/v1/users/%s/roles", us.userName) + var resp models.UserAssignRoleResponse + err = apiClient.PostRequestToAPIServer( + serverInfo.Token, + serverInfo.CACert, + url, + reqData, + &resp, + ) + + if err != nil { + return err + } + + if resp.Code != http.StatusCreated { + logger.Info("user assign role failed", zap.Any("resp", resp), zap.Error(err)) + return fmt.Errorf("%s", resp.Data.Message) + } + logger.Debug("user assign role successfully", zap.String("message", resp.Data.Message)) + logger.Info("user assign role successfully", zap.String("message", resp.Data.Message)) + fmt.Printf("%s\n", resp.Data.Message) + return nil +} + +func (us *UserSubmitter) BuildUserRequest() (*models.AddRoleForUserRequest, error) { + roleReq := &models.AddRoleForUserRequest{ + UserName: us.userName, + RoleName: us.roleName, + } + return roleReq, nil +} diff --git a/src/frontend/xcecli/cmd/users/list.go b/src/frontend/xcecli/cmd/users/list.go new file mode 100644 index 00000000..3d743d73 --- /dev/null +++ b/src/frontend/xcecli/cmd/users/list.go @@ -0,0 +1,252 @@ +package users + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + userrole_filter "xcecli/filters" + "xcecli/models" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +type UserRolesQuerier struct { + configManager *config.ConfigManager + filter *userrole_filter.UserRolesFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool +} + +func NewUserRolesQuerier(configManager *config.ConfigManager) *UserRolesQuerier { + return &UserRolesQuerier{ + configManager: configManager, + filter: &userrole_filter.UserRolesFilter{}, + } +} + +func NewCurrentUserCmd(configManager *config.ConfigManager) *cobra.Command { + userrolesQuerier := NewUserRolesQuerier(configManager) + cmd := &cobra.Command{ + Use: "whoami", + Short: "Query the current loggin user", + Long: `Query the current loggin user. +example: + xce users whoami # query the current loggin user`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := userrolesQuerier.RunCurrentUserInfo(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + return cmd +} + +func NewListUserRolesCmd(configManager *config.ConfigManager) *cobra.Command { + userrolesQuerier := NewUserRolesQuerier(configManager) + + cmd := &cobra.Command{ + Use: "list", + Short: "Query user role info", + Long: `Query user role info, support filter by user. +example: + xce users list # query all users + xce users list -l # query all users with detail + xce users list -w # list users with wide format`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := userrolesQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := userrolesQuerier.RunUserRolesInfo("list"); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&userrolesQuerier.filter.UserName, "user_name", "u", "", "filter by user name, e.g. -u user1,user2") + + flags.BoolVarP(&userrolesQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&userrolesQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&userrolesQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} + +func (urq *UserRolesQuerier) ParseCmdArgs(cmd *cobra.Command, args []string) error { + // No additional processing needed, all parameters are bound in flags + if len(args) > 0 { + urq.filter.UserName = strings.Join(args, ",") + } + return nil +} + +func (urq *UserRolesQuerier) RunCurrentUserInfo() error { + // Get server information + serverInfo, err := urq.configManager.GetServerInfo() + if err != nil { + return fmt.Errorf("failed to get server info: %w", err) + } + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create a response structure for current user info + var currentUserResp models.UserResponse // You'll need to define this model + + // Make API request to get current user info using the token + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, "", serverInfo.CACert, "/xce/v1/users/current", ¤tUserResp, nil) + if err != nil { + return fmt.Errorf("failed to get current user info: %w", err) + } + + // Print the username + fmt.Println(currentUserResp.Data) // Adjust field name based on your model structure + + return nil +} +func (urq *UserRolesQuerier) RunUserRolesInfo(commandType string) error { + // Get server information + serverInfo, err := urq.configManager.GetServerInfo() + if err != nil { + return fmt.Errorf("failed to get server info: %w", err) + } + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Get filter string + filterStr, err := urq.filter.HandleUserRoleFilter() + + if err != nil { + errMsg := request.ExtractErrorMessage(err) + return fmt.Errorf("failed to get filter: [%s]", errMsg) + } else if filterStr != "" { + params["filter"] = filterStr + } + + if commandType == "list" { + urq.fields = "USER_NAME" + params["fields"] = "user_name" + } else { + // Field mapping + fieldsMap := models.UserRoleFieldsMap + + if urq.verbose && urq.fields == "" { + for header := range fieldsMap { + urq.fields += header + "," + } + urq.fields = strings.TrimSuffix(urq.fields, ",") + } + // Process field selection + if urq.fields != "" { + urq.fields = request.CleanFields(urq.fields) + mappedFields, err := request.CheckAndMapFields(urq.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + } + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeUsers, output.WithFields(urq.fields), + output.WithFormatJson(urq.jsonOutput), output.WithNoHeader(urq.noHeader), + output.WithVerbose(urq.verbose), output.WithWideFormat(urq.wideFormat)) + + // Record if the first page has been processed + isFirstPage := true + + var ( + resp models.UserRolesResponse + loopCount int + pageToken string + seen map[string]struct{} // cross-page dedup state for list mode + ) + if commandType == "list" { + seen = make(map[string]struct{}) + } + + // Loop to get all pages of data + for { + if loopCount > 0 && pageToken == "" { + break + } + resp = models.UserRolesResponse{} // reset to avoid stale rows from previous page + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, pageToken, serverInfo.CACert, "/xce/v1/users", &resp, params) + if err != nil { + return err + } + if resp.Data.Count == 0 { + if isFirstPage { + fmt.Println("no user found") + return nil + } + } + // For list mode, deduplicate user names client-side before printing + if commandType == "list" { + var unique []models.UserRoleInfo + for _, row := range resp.Data.Rows { + if _, ok := seen[row.UserName]; !ok { + seen[row.UserName] = struct{}{} + unique = append(unique, models.UserRoleInfo{UserName: row.UserName}) + } + } + resp.Data.Rows = unique + resp.Data.Count = len(unique) + } + // Process the first page + if isFirstPage { + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + isFirstPage = false + } + // Print the current page + pagePrinter.PrintPage(&resp) + + // Update page token and loop counter + pageToken = resp.Data.PageToken + loopCount++ + } + + // Finish printing + pagePrinter.Finish() + + logger.Debug("user role response completed", zap.Int("pages", loopCount)) + + return nil +} + +// uniqueUserNames deduplicates user names from a UserRolesResponse, preserving order. +// The server returns one row per role assignment, so the same user_name appears multiple times. +func uniqueUserNames(resp *models.UserRolesResponse) []string { + seen := make(map[string]struct{}) + var result []string + for _, row := range resp.Data.Rows { + if _, ok := seen[row.UserName]; !ok { + seen[row.UserName] = struct{}{} + result = append(result, row.UserName) + } + } + return result +} diff --git a/src/frontend/xcecli/cmd/users/list_test.go b/src/frontend/xcecli/cmd/users/list_test.go new file mode 100644 index 00000000..39eab6c3 --- /dev/null +++ b/src/frontend/xcecli/cmd/users/list_test.go @@ -0,0 +1,82 @@ +package users + +import ( + "testing" + "xcecli/models" +) + +// --- dedup (regression guard) --- + +func TestUniqueUserNames_NoDuplicates(t *testing.T) { + resp := &models.UserRolesResponse{} + resp.Data.Rows = []models.UserRoleInfo{{UserName: "alice"}, {UserName: "bob"}} + if got := uniqueUserNames(resp); len(got) != 2 { + t.Fatalf("expected 2, got %d", len(got)) + } +} + +func TestUniqueUserNames_WithDuplicates(t *testing.T) { + resp := &models.UserRolesResponse{} + resp.Data.Rows = []models.UserRoleInfo{ + {UserName: "alice", UserRoleName: "admin"}, + {UserName: "alice", UserRoleName: "basic_user"}, + {UserName: "bob", UserRoleName: "basic_user"}, + } + got := uniqueUserNames(resp) + if len(got) != 2 || got[0] != "alice" || got[1] != "bob" { + t.Fatalf("unexpected: %v", got) + } +} + +func TestUniqueUserNames_Empty(t *testing.T) { + resp := &models.UserRolesResponse{} + if got := uniqueUserNames(resp); len(got) != 0 { + t.Fatalf("expected 0, got %d", len(got)) + } +} + +func TestUniqueUserNames_CrossPage(t *testing.T) { + seen := make(map[string]struct{}) + dedup := func(rows []models.UserRoleInfo) []models.UserRoleInfo { + var unique []models.UserRoleInfo + for _, row := range rows { + if _, ok := seen[row.UserName]; !ok { + seen[row.UserName] = struct{}{} + unique = append(unique, models.UserRoleInfo{UserName: row.UserName}) + } + } + return unique + } + p1 := dedup([]models.UserRoleInfo{{UserName: "alice"}, {UserName: "alice"}, {UserName: "bob"}}) + p2 := dedup([]models.UserRoleInfo{{UserName: "bob"}, {UserName: "carol"}}) + if len(p1) != 2 { + t.Fatalf("page1: expected 2, got %d", len(p1)) + } + if len(p2) != 1 || p2[0].UserName != "carol" { + t.Fatalf("page2: expected [carol], got %v", p2) + } +} + +// --- column width --- + +func TestUserRoleNameColumnWidth(t *testing.T) { + resp := &models.UserRolesResponse{} + resp.Data.Rows = []models.UserRoleInfo{ + {UserName: "alice", UserRoleName: "cluster_admin"}, + {UserName: "bob", UserRoleName: "basic_user"}, + } + width := calcUserRoleNameWidth(resp) + if width < len("cluster_admin") { + t.Errorf("ROLE_NAME width %d < len(%q)=%d", width, "cluster_admin", len("cluster_admin")) + } +} + +func calcUserRoleNameWidth(resp *models.UserRolesResponse) int { + w := len(models.HEADER_USERROLE_NAME) + for _, u := range resp.Data.Rows { + if n := len(u.UserRoleName); n > w { + w = n + } + } + return w +} diff --git a/src/frontend/xcecli/cmd/users/perm4user.go b/src/frontend/xcecli/cmd/users/perm4user.go new file mode 100644 index 00000000..95b8494e --- /dev/null +++ b/src/frontend/xcecli/cmd/users/perm4user.go @@ -0,0 +1,141 @@ +package users + +import ( + "fmt" + "strings" + "xcecli/common/output" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + userrole_filter "xcecli/filters" + "xcecli/models" + + "github.com/spf13/cobra" +) + +func NewPerms4UserQuerier(configManager *config.ConfigManager) *Perms4User { + return &Perms4User{ + configManager: configManager, + } +} + +type Perms4User struct { + configManager *config.ConfigManager + userName string + filter *userrole_filter.UserRolesFilter + fields string + jsonOutput bool + noHeader bool + verbose bool + wideFormat bool +} + +func NewPermissions4UserCmd(configManager *config.ConfigManager) *cobra.Command { + perms4userQuerier := NewPerms4UserQuerier(configManager) + + cmd := &cobra.Command{ + Use: "permissions4user", + Short: "Query permissions for a user", + Long: `Query permissions for a user, by specifying the user name. +example: + xce users permissions4user -u user_name # query permissions for a user + xce users permissions4user -u user_name -w # view permissions with wide format`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := perms4userQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := perms4userQuerier.RunUserPermsInfo(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&perms4userQuerier.userName, "user_name", "u", "", "filter by user name, only one user name can be specified") + flags.StringVarP(&perms4userQuerier.fields, "output", "o", "", "output fields for user roles") + flags.BoolVarP(&perms4userQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&perms4userQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&perms4userQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} + +func (pu *Perms4User) ParseCmdArgs(cmd *cobra.Command, args []string) error { + if pu.userName == "" { + return fmt.Errorf("please provide the user name") + } + if len(args) > 0 { + return fmt.Errorf("too many arguments") + } + return nil +} + +func (up *Perms4User) RunUserPermsInfo() error { + // Get server information + serverInfo, err := up.configManager.GetServerInfo() + if err != nil { + return fmt.Errorf("failed to get server info: %w", err) + } + + // create API client + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + // Create query parameters + params := make(map[string]string) + + // Field mapping + fieldsMap := models.RoleFieldsMap + + if up.verbose && up.fields == "" { + for header := range fieldsMap { + up.fields += header + "," + } + up.fields = strings.TrimSuffix(up.fields, ",") + } + // Process field selection + if up.fields != "" { + up.fields = request.CleanFields(up.fields) + mappedFields, err := request.CheckAndMapFields(up.fields, fieldsMap) + if err != nil { + return err + } + params["fields"] = strings.Join(mappedFields, ",") + } + // create page printer + pagePrinter := output.NewPagePrinter(output.DataTypeRoles, output.WithFields(up.fields), + output.WithFormatJson(up.jsonOutput), output.WithNoHeader(up.noHeader), + output.WithVerbose(up.verbose), output.WithWideFormat(up.wideFormat)) + + var resp models.RolesResponse + + // Loop to get all pages of data + url := "/xce/v1/users/" + up.userName + "/permissions" + err = apiClient.GetRequestFromAPIServer(serverInfo.Token, "", serverInfo.CACert, url, &resp, nil) + if err != nil { + return err + } + if resp.Data.Count == 0 { + fmt.Println("no role found") + return nil + } + + // Start printing process + pagePrinter.StartPrinting() + // Prepare for printing, calculate column width and table header, etc. + pagePrinter.PrepareForPrinting(&resp) + + // Print the current page + pagePrinter.PrintPage(&resp) + + // Finish printing + pagePrinter.Finish() + + logger.Debug("roles response completed") + return nil +} diff --git a/src/frontend/xcecli/cmd/users/unassignrole.go b/src/frontend/xcecli/cmd/users/unassignrole.go new file mode 100644 index 00000000..09931dbd --- /dev/null +++ b/src/frontend/xcecli/cmd/users/unassignrole.go @@ -0,0 +1,100 @@ +package users + +import ( + "fmt" + "net/http" + "strings" + "xcecli/common/request" + "xcecli/config" + logger "xcecli/config" + + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +func NewUnassignUserQuerier(configManager *config.ConfigManager) *UnassignUser { + return &UnassignUser{ + configManager: configManager, + } +} + +type UnassignUser struct { + configManager *config.ConfigManager + userName string + roleName string +} + +func DeleteUserRolesCmd(configManager *config.ConfigManager) *cobra.Command { + unassignUserQuerier := NewUnassignUserQuerier(configManager) + + cmd := &cobra.Command{ + Use: "unassignrole", + Short: "Unassign permissions for a user", + Long: `Unassign permissions for a user, by specifying the user name. +example: + xce users unassignrole -u user_name -r role_name # unassign role for a user`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := unassignUserQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := unassignUserQuerier.RunDeleteUserRoleInfo(); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&unassignUserQuerier.userName, "user_name", "u", "", "filter by user name, only one user name can be specified") + flags.StringVarP(&unassignUserQuerier.roleName, "role_name", "r", "", "filter by role name, only one role name can be specified") + + return cmd +} + +func (uu *UnassignUser) ParseCmdArgs(cmd *cobra.Command, args []string) error { + if uu.userName == "" { + return fmt.Errorf("please provide the user name") + } + if uu.roleName == "" { + return fmt.Errorf("please provide the role name") + } + if len(args) > 0 { + return fmt.Errorf("too many arguments") + } + return nil +} + +func (uu *UnassignUser) RunDeleteUserRoleInfo() error { + // Get server info and validate authentication + serverInfo, _ := uu.configManager.GetServerInfo() + // create the API client and submit the user + apiClient := request.NewAPIClient(serverInfo.URL) + apiClient.SetRootCAs(serverInfo.CACert) + + url := fmt.Sprintf("/xce/v1/users/%s/%s", uu.userName, uu.roleName) + var resp request.APIResponse + err := apiClient.DeleteRequestFromAPIServer( + serverInfo.Token, + serverInfo.CACert, + url, + []byte(nil), + &resp, + ) + + if err != nil { + return err + } + + if resp.Code != http.StatusOK { + logger.Info("user unassign role failed", zap.Any("resp", resp), zap.Error(err)) + return fmt.Errorf("%v", resp.Data) + } + logger.Debug("user unassign role successfully", zap.Any("data", resp.Data)) + logger.Info("user unassign role successfully", zap.Any("data", resp.Data)) + fmt.Printf("%v\n", resp.Data) + return nil +} diff --git a/src/frontend/xcecli/cmd/users/users.go b/src/frontend/xcecli/cmd/users/users.go new file mode 100644 index 00000000..c3e88f56 --- /dev/null +++ b/src/frontend/xcecli/cmd/users/users.go @@ -0,0 +1,35 @@ +package users + +import ( + "xcecli/config" + + "github.com/spf13/cobra" +) + +func NewUsersCmd(configManager *config.ConfigManager) *cobra.Command { + + // Create the parent users command + cmd := &cobra.Command{ + Use: "users", + GroupID: "query", + Short: "Manage users", + Long: `Manage users in the system. +Use 'xce users whoami' to query current user name. +Use 'xce users list' to query users list only. +Use 'xce users view' to query users details. +Use 'xce users permissions4user' to query a user's permissions. +Use 'xce users unassignrole' to unassign a role for a user. +Use 'xce users ' to add users.`, + Run: func(cmd *cobra.Command, args []string) { + // Display help when no subcommand is specified + cmd.Help() + }, + } + cmd.AddCommand(NewCurrentUserCmd(configManager)) + cmd.AddCommand(NewListUserRolesCmd(configManager)) + cmd.AddCommand(NewViewUserRolesCmd(configManager)) + cmd.AddCommand(NewAssignUserRolesCmd(configManager)) + cmd.AddCommand(NewPermissions4UserCmd(configManager)) + cmd.AddCommand(DeleteUserRolesCmd(configManager)) + return cmd +} diff --git a/src/frontend/xcecli/cmd/users/view.go b/src/frontend/xcecli/cmd/users/view.go new file mode 100644 index 00000000..d4e67365 --- /dev/null +++ b/src/frontend/xcecli/cmd/users/view.go @@ -0,0 +1,45 @@ +package users + +import ( + "fmt" + "strings" + "xcecli/config" + + "github.com/spf13/cobra" +) + +func NewViewUserRolesCmd(configManager *config.ConfigManager) *cobra.Command { + userrolesQuerier := NewUserRolesQuerier(configManager) + + cmd := &cobra.Command{ + Use: "view", + Short: "Query user role info", + Long: `Query user role info, support filter by user. +example: + xce users view # query all users + xce users view -w # view users with wide format`, + + RunE: func(cmd *cobra.Command, args []string) error { + if err := userrolesQuerier.ParseCmdArgs(cmd, args); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + if err := userrolesQuerier.RunUserRolesInfo("view"); err != nil { + fmt.Println(strings.ToLower(err.Error())) + return nil + } + return nil + }, + } + + // Add command line parameters + flags := cmd.Flags() + flags.StringVarP(&userrolesQuerier.filter.UserName, "user_name", "u", "", "filter by user name, e.g. -u user1,user2") + flags.StringVarP(&userrolesQuerier.filter.RoleName, "role_name", "r", "", "filter by role name, e.g. -r role1,role2") + flags.StringVarP(&userrolesQuerier.fields, "output", "o", "", "output fields for user roles") + flags.BoolVarP(&userrolesQuerier.wideFormat, "wided", "w", false, "output in wide format") + flags.BoolVarP(&userrolesQuerier.jsonOutput, "json", "", false, "output in JSON format") + flags.BoolVarP(&userrolesQuerier.noHeader, "noheader", "n", false, "output no header") + + return cmd +} diff --git a/src/frontend/xcecli/common/output/output_clusters.go b/src/frontend/xcecli/common/output/output_clusters.go new file mode 100755 index 00000000..17cc9d86 --- /dev/null +++ b/src/frontend/xcecli/common/output/output_clusters.go @@ -0,0 +1,45 @@ +package output + +import ( + "fmt" + "os" + "text/tabwriter" + "xcecli/models" +) + +type Output struct { + fields string + formatJson bool + noheader bool + verbose bool + wideFormat bool + jobStatusFilter []string +} + +func NewOutput(fields string, formatJson bool, noheader bool) *Output { + return &Output{fields: fields, formatJson: formatJson, noheader: noheader} +} + +func (o *Output) SetVerbose(verbose bool) { + o.verbose = verbose +} + +func (o *Output) SetNoHeader(noheader bool) { + o.noheader = noheader +} + +func (o *Output) SetWideFormat(wideFormat bool) { + o.wideFormat = wideFormat +} + +func (o *Output) ClusterPrintData(cluster *models.ClusterInfo) { + printClustersVerboseOutput(cluster) +} + +func printClustersVerboseOutput(clusters *models.ClusterInfo) { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + defer w.Flush() + fmt.Fprintf(w, "XCE Standard Cluster\t%s\t%s\n", clusters.Version, "Jul 08 2025") + fmt.Fprintf(w, "My cluster name: <%s>\n", clusters.ClusterName) + fmt.Fprintf(w, "The cluster status: <%s>\n", clusters.Status) +} diff --git a/src/frontend/xcecli/common/output/page_print.go b/src/frontend/xcecli/common/output/page_print.go new file mode 100755 index 00000000..0402342b --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print.go @@ -0,0 +1,377 @@ +package output + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "sync" + "text/tabwriter" +) + +// PagePrinter +type PagePrinter interface { + // PrepareForPrinting + PrepareForPrinting(firstPage interface{}) + // PrintPage + PrintPage(page interface{}) + // Finish + Finish() + // StartPrinting + StartPrinting() +} + +const ( + DataTypeJobs = "jobs" + DataTypeHosts = "hosts" + DataTypeQueues = "queues" + DataTypeArrayJobs = "array_jobs" + DataTypeArrayElements = "array_elements" + DataTypeResourcesDist = "resources_dist" + DataTypeResourcesDef = "resource_def" + DataTypeRoles = "roles" + DataTypeUsers = "users" +) + +// NewPagePrinter +func NewPagePrinter(dataType string, opt ...BasePageOption) PagePrinter { + switch dataType { + case DataTypeJobs: + return NewJobsPagePrinter(opt...) + case DataTypeHosts: + return NewHostsPagePrinter(opt...) + case DataTypeQueues: + return NewQueuesPagePrinter(opt...) + case DataTypeArrayJobs: + return NewArrayJobsPagePrinter(opt...) + case DataTypeArrayElements: + return NewArrayElementsPagePrinter(opt...) + case DataTypeResourcesDist: + return NewResourcesDistPagePrinter(opt...) + case DataTypeResourcesDef: + return NewResourcesDefPagePrinter(opt...) + case DataTypeRoles: + return NewRolesPagePrinter(opt...) + case DataTypeUsers: + return NewUserRolesPagePrinter(opt...) + default: + fmt.Printf("unsupported data type: %s, using default Jobs printer\n", dataType) + return NewJobsPagePrinter(opt...) + } +} + +type BasePageOption interface { + apply(*BasePagePrinter) +} + +type basePageOptionFunc func(*BasePagePrinter) + +func (f basePageOptionFunc) apply(bpp *BasePagePrinter) { + f(bpp) +} + +func WithFields(fields string) BasePageOption { + return basePageOptionFunc(func(bpp *BasePagePrinter) { + bpp.fields = fields + }) +} + +func WithFormatJson(b bool) BasePageOption { + return basePageOptionFunc(func(bpp *BasePagePrinter) { + bpp.formatJson = b + }) +} + +func WithNoHeader(b bool) BasePageOption { + return basePageOptionFunc(func(bpp *BasePagePrinter) { + bpp.noheader = b + }) +} + +func WithVerbose(b bool) BasePageOption { + return basePageOptionFunc(func(bpp *BasePagePrinter) { + bpp.verbose = b + }) +} + +func WithWideFormat(b bool) BasePageOption { + return basePageOptionFunc(func(bpp *BasePagePrinter) { + bpp.wideFormat = b + }) +} + +func WithJobStatusFilter(filter []string) BasePageOption { + return basePageOptionFunc(func(bpp *BasePagePrinter) { + bpp.jobStatusFilter = filter + }) +} + +// BasePagePrinter +type BasePagePrinter struct { + /* external config */ + fields string + formatJson bool + noheader bool + verbose bool + wideFormat bool + jobStatusFilter []string + + /* internal variables */ + headerPrinted bool + columnWidths map[string]int // store column widths + fieldFormats []*fieldFormat // field formats + formattedHeaders []string // formatted headers + w *tabwriter.Writer + pageCount int // record the number of printed pages + startPrintOnce *sync.Once +} + +// newBasePagePrinter +func newBasePagePrinter(opts ...BasePageOption) *BasePagePrinter { + base := &BasePagePrinter{ + headerPrinted: false, + columnWidths: make(map[string]int), + w: tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0), + pageCount: 0, + startPrintOnce: &sync.Once{}, + } + for _, opt := range opts { + opt.apply(base) + } + return base +} + +// StartPrinting +func (p *BasePagePrinter) StartPrinting() { + p.startPrintOnce.Do(func() { + // if the format is JSON and need to print multiple pages, print the opening bracket + if p.formatJson && !p.verbose { + fmt.Print("[") + } + }) +} + +// prepareHeaders +func (p *BasePagePrinter) prepareHeaders() { + p.formattedHeaders = make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + p.formattedHeaders[i] = fmt.Sprintf("%"+format.align+"*s", p.columnWidths[format.name], format.name) + } +} + +// printHeader +func (p *BasePagePrinter) printHeader() { + if !p.headerPrinted && !p.noheader { + fmt.Fprintf(p.w, "%s\n", strings.Join(p.formattedHeaders, "\t")) + p.headerPrinted = true + } +} + +// Finish +func (p *BasePagePrinter) Finish() { + // if the format is JSON and printed data, add the closing bracket + if p.formatJson && !p.verbose && p.pageCount > 0 { + fmt.Println("]") + } + p.w.Flush() +} + +type fieldFormat struct { + name string + width int + align string // "-" for left align, "" for right align +} + +func parseFieldFormat(field string) *fieldFormat { + // parse the field format, support the following formats: + // "FIELD" - default format + // "FIELD:10" - specify the width to 10, right align + // "FIELD:-10" - specify the width to 10, left align + parts := strings.Split(field, ":") + if len(parts) == 1 { + return &fieldFormat{ + name: parts[0], + width: 0, + align: "-", + } + } + + width := 0 + align := "" + if strings.HasPrefix(parts[1], "-") { + align = "-" + width, _ = strconv.Atoi(parts[1][1:]) + } else { + width, _ = strconv.Atoi(parts[1]) + } + + return &fieldFormat{ + name: parts[0], + width: width, + align: align, + } +} + +func formatSize(value float64) (float64, string, error) { + if value <= 0 { + return 0, "MB", nil + } + switch { + case value >= 1024*1024*1024: // >= 1PB + return value / (1024 * 1024 * 1024), "PB", nil + case value >= 1024*1024: // >= 1TB + return value / (1024 * 1024), "TB", nil + case value >= 1024: // >= 1GB + return value / 1024, "GB", nil + default: // MB + return value, "MB", nil + } +} + +func formatSizeString(value float64) string { + size, unit, err := formatSize(value) + if err != nil { + return "-" + } + return fmt.Sprintf("%.2f%s", size, unit) +} + +func appendOrDefault(slice []string, value string, defaultValue string) []string { + if value != "" { + return append(slice, value) + } else { + return append(slice, defaultValue) + } +} + +func getFieldFormatIndex(fieldFormats []*fieldFormat, target string) int { + for i, f := range fieldFormats { + if f.name == target { + return i + } + } + return -1 +} + +func PrintWrappedLine(w *tabwriter.Writer, line string, width int, indentSpaces int) { + line = strings.TrimRight(line, " ") + indent := strings.Repeat(" ", indentSpaces) + first := true + for len(line) > 0 { + if line[len(line)-1] == ',' { + line = line[:len(line)-1] + ";" + } + if first { + // first line + if len(line) <= width { + fmt.Fprintf(w, "%s\n", line) + break + } else { + fmt.Fprintf(w, "%s\n", line[:width]) + line = line[width:] + } + first = false + } else { + avail := width - indentSpaces + if avail <= 0 { + avail = 1 + } + if len(line) <= avail { + fmt.Fprintf(w, "%s%s\n", indent, line) + break + } else { + fmt.Fprintf(w, "%s%s\n", indent, line[:avail]) + line = line[avail:] + } + } + } +} + +func (p *BasePagePrinter) PrintJsonPageGeneric(rows [][]string, headers []string) { + data := make([]map[string]string, len(rows)) + for i, row := range rows { + entry := make(map[string]string) + for j, header := range headers { + if j < len(row) { + entry[header] = strings.TrimSpace(row[j]) + } else { + entry[header] = "" + } + } + data[i] = entry + } + for i, item := range data { + jsonData, err := json.MarshalIndent(item, "", " ") + if err != nil { + fmt.Println("error marshalling to JSON:", err) + return + } + if p.pageCount > 1 || i > 0 { + fmt.Print(",\n") + } + fmt.Print(string(jsonData)) + } +} + +func FormatStringField(val string, width int, truncatFlag bool, prefixAtStart bool) string { + if len(val) > width && truncatFlag { + if prefixAtStart { + return "*" + val[len(val)-width+1:] + } else { + keepLen := width - 1 + if keepLen < 0 { + keepLen = 0 + } + return val[:keepLen] + "*" + } + } else { + if val == "" { + return fmt.Sprintf("%-*s", width, "-") + } else { + return fmt.Sprintf("%-*s", width, val) + } + } +} + +type TablePrinter struct { + Headers []string + Rows [][]string + ColWidths []int +} + +func (tp *TablePrinter) CalcColWidths() { + tp.ColWidths = make([]int, len(tp.Headers)) + for i, h := range tp.Headers { + tp.ColWidths[i] = len(h) + } + for _, row := range tp.Rows { + for i, cell := range row { + if len(cell) > tp.ColWidths[i] { + tp.ColWidths[i] = len(cell) + } + } + } +} + +func (tp *TablePrinter) Print() { + tp.CalcColWidths() + format := "" + for _, w := range tp.ColWidths { + format += fmt.Sprintf("%%-%ds ", w) + } + format += "\n" + + fmt.Printf(format, ToInterfaceSlice(tp.Headers)...) + for _, row := range tp.Rows { + fmt.Printf(format, ToInterfaceSlice(row)...) + } +} + +func ToInterfaceSlice(strs []string) []interface{} { + res := make([]interface{}, len(strs)) + for i, s := range strs { + res[i] = s + } + return res +} diff --git a/src/frontend/xcecli/common/output/page_print_array_elements_jobs.go b/src/frontend/xcecli/common/output/page_print_array_elements_jobs.go new file mode 100644 index 00000000..6945690e --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_array_elements_jobs.go @@ -0,0 +1,230 @@ +package output + +import ( + "encoding/json" + "fmt" + "strings" + "time" + "xcecli/models" +) + +// ArrayElementsPagePrinter +type ArrayElementsPagePrinter struct { + *BasePagePrinter +} + +// NewArrayElementsPagePrinter +func NewArrayElementsPagePrinter(opt ...BasePageOption) *ArrayElementsPagePrinter { + return &ArrayElementsPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +// PrepareForPrinting +func (p *ArrayElementsPagePrinter) PrepareForPrinting(firstPage interface{}) { + jobs, ok := firstPage.(*models.JobsResponse) + if !ok { + fmt.Println("error: wrong data type to ArrayElementsPagePrinter") + return + } + + // set field formats + p.fieldFormats = p.prepareFieldFormats() + + // init column widths + p.initColumnWidths(jobs) + + // prepare headers + p.prepareHeaders() +} + +// prepare field formats +func (p *ArrayElementsPagePrinter) prepareFieldFormats() []*fieldFormat { + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_JOB_JOBID, + models.HEADER_JOB_USER, + models.HEADER_JOB_STATUS, + models.HEADER_JOB_QUEUE, + models.HEADER_JOB_FROM_HOST, + models.HEADER_JOB_EXEC_HOST, + models.HEADER_JOB_NAME, + models.HEADER_JOB_SUBMIT_TIME, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths +func (p *ArrayElementsPagePrinter) initColumnWidths(jobs *models.JobsResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, job := range jobs.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_JOB_JOBID: + // format the job id as "jobID[idx]" + arrayJobIDStr := fmt.Sprintf("%d[%d]", job.JobID, job.ArrayIdx) + width = len(arrayJobIDStr) + case models.HEADER_JOB_USER: + width = len(job.SubmitUser) + case models.HEADER_JOB_STATUS: + width = len(job.JobStatus) + case models.HEADER_JOB_QUEUE: + width = len(job.QueueName) + case models.HEADER_JOB_FROM_HOST: + width = len(job.FromHost) + case models.HEADER_JOB_EXEC_HOST: + continue + case models.HEADER_JOB_NAME: + width = len(job.JobName) + case models.HEADER_JOB_SUBMIT_TIME: + width = len(job.SubmitTime) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage +func (p *ArrayElementsPagePrinter) PrintPage(page interface{}) { + jobs, ok := page.(*models.JobsResponse) + if !ok { + fmt.Println("error: wrong data type to ArrayElementsPagePrinter") + return + } + + p.pageCount++ + + if p.verbose { + printJobsVerboseOutput(jobs, p.jobStatusFilter) + return + } + + if p.formatJson { + p.printJsonPage(jobs) + return + } else { + // print header (if needed) + p.printHeader() + + // print data rows + rows := p.formatRows(jobs, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +// format data rows +func (p *ArrayElementsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, job := range jobs.Data.Rows { + mainRow := make([]string, 0, len(p.fieldFormats)) + var extraRows [][]string + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + align := format.align + switch format.name { + case models.HEADER_JOB_JOBID: + // format the job id as "jobID[idx]" + arrayJobIDStr := fmt.Sprintf("%d[%d]", job.JobID, job.ArrayIdx) + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", 20, arrayJobIDStr), "-") + case models.HEADER_JOB_USER: + mainRow = appendOrDefault(mainRow, FormatStringField(job.SubmitUser, width, truncatFlag, true), "-") + case models.HEADER_JOB_STATUS: + mainRow = appendOrDefault(mainRow, FormatStringField(job.JobStatus, width, truncatFlag, true), "-") + case models.HEADER_JOB_QUEUE: + mainRow = appendOrDefault(mainRow, FormatStringField(job.QueueName, width, truncatFlag, true), "-") + case models.HEADER_JOB_FROM_HOST: + mainRow = appendOrDefault(mainRow, FormatStringField(job.FromHost, width, truncatFlag, true), "-") + case models.HEADER_JOB_EXEC_HOST: + var execHosts models.ExecHosts + if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 { + firstHost := fmt.Sprintf("%d*%s", execHosts.HostSlots[0].NumSlots, execHosts.HostSlots[0].HostName) + mainRow = appendOrDefault(mainRow, FormatStringField(firstHost, width, truncatFlag, true), "-") + + if len(execHosts.HostSlots) > 1 { + for _, slot := range execHosts.HostSlots[1:] { + newLine := make([]string, len(p.fieldFormats)) + for i, f := range p.fieldFormats { + if f.name == "EXEC_HOST" { + continue + } + newLine[i] = strings.Repeat(" ", p.columnWidths[f.name]) + } + hostText := fmt.Sprintf("%d*%s", slot.NumSlots, slot.HostName) + newLine[getFieldFormatIndex(p.fieldFormats, "EXEC_HOST")] = FormatStringField(hostText, width, truncatFlag, true) + extraRows = append(extraRows, append([]string{}, newLine...)) + } + } + } else { + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-") + } + case models.HEADER_JOB_NAME: + mainRow = appendOrDefault(mainRow, FormatStringField(job.JobName, width, truncatFlag, true), "-") + case models.HEADER_JOB_SUBMIT_TIME: + // remove the timezone part and parse the time + timeStr := strings.Split(job.SubmitTime, " ")[0:2] + timeStrWithoutTZ := strings.Join(timeStr, " ") + + if submitTime, err := time.Parse("2006-01-02 15:04:05", timeStrWithoutTZ); err == nil { + month := submitTime.Format("Jan") + displayTime := fmt.Sprintf("%s %02d %02d:%02d", + month, + submitTime.Day(), + submitTime.Hour(), + submitTime.Minute()) + mainRow = appendOrDefault(mainRow, FormatStringField(displayTime, width, truncatFlag, true), "-") + } else { + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-") + } + } + } + rows = append(rows, mainRow) + rows = append(rows, extraRows...) + } + return rows +} + +// print JSON page +func (p *ArrayElementsPagePrinter) printJsonPage(jobs *models.JobsResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRows(jobs, false) + + p.PrintJsonPageGeneric(rows, headers) +} diff --git a/src/frontend/xcecli/common/output/page_print_array_jobs.go b/src/frontend/xcecli/common/output/page_print_array_jobs.go new file mode 100644 index 00000000..c87fbf0c --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_array_jobs.go @@ -0,0 +1,199 @@ +package output + +import ( + "fmt" + "strings" + "xcecli/models" +) + +// ArrayJobsPagePrinter +type ArrayJobsPagePrinter struct { + *BasePagePrinter +} + +// NewArrayJobsPagePrinter +func NewArrayJobsPagePrinter(opt ...BasePageOption) *ArrayJobsPagePrinter { + return &ArrayJobsPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +// PrepareForPrinting +func (p *ArrayJobsPagePrinter) PrepareForPrinting(firstPage interface{}) { + arrayJobsStats, ok := firstPage.(*models.ArrayJobsStatsResponse) + if !ok { + fmt.Println("error: wrong data type to ArrayJobsPagePrinter") + return + } + + // set field formats + p.fieldFormats = p.prepareFieldFormats() + + // init column widths + p.initColumnWidths(arrayJobsStats) + + // prepare headers + p.prepareHeaders() +} + +// prepare field formats +func (p *ArrayJobsPagePrinter) prepareFieldFormats() []*fieldFormat { + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_ARRAY_JOB_JOBID, + models.HEADER_ARRAY_JOB_SPEC, + models.HEADER_ARRAY_JOB_OWNER, + models.HEADER_ARRAY_JOB_NJOBS, + models.HEADER_ARRAY_JOB_PEND, + models.HEADER_ARRAY_JOB_DONE, + models.HEADER_ARRAY_JOB_RUN, + models.HEADER_ARRAY_JOB_EXIT, + models.HEADER_ARRAY_JOB_SSUSP, + models.HEADER_ARRAY_JOB_USUSP, + models.HEADER_ARRAY_JOB_PSUSP, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths +func (p *ArrayJobsPagePrinter) initColumnWidths(arrayJobsStats *models.ArrayJobsStatsResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, stat := range arrayJobsStats.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_ARRAY_JOB_JOBID: + width = len(fmt.Sprintf("%d", stat.JobID)) + case models.HEADER_ARRAY_JOB_SPEC: + width = len(stat.ArrayJobName) + case models.HEADER_ARRAY_JOB_OWNER: + width = len(stat.SubmitUser) + case models.HEADER_ARRAY_JOB_NJOBS: + width = len(fmt.Sprintf("%d", stat.NumJobs)) + case models.HEADER_ARRAY_JOB_PEND: + width = len(fmt.Sprintf("%d", stat.NumPend)) + case models.HEADER_ARRAY_JOB_DONE: + width = len(fmt.Sprintf("%d", stat.NumDone)) + case models.HEADER_ARRAY_JOB_RUN: + width = len(fmt.Sprintf("%d", stat.NumRun)) + case models.HEADER_ARRAY_JOB_EXIT: + width = len(fmt.Sprintf("%d", stat.NumExit)) + case models.HEADER_ARRAY_JOB_SSUSP: + width = len(fmt.Sprintf("%d", stat.NumSSusp)) + case models.HEADER_ARRAY_JOB_USUSP: + width = len(fmt.Sprintf("%d", stat.NumUSusp)) + case models.HEADER_ARRAY_JOB_PSUSP: + width = len(fmt.Sprintf("%d", stat.NumPSusp)) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage +func (p *ArrayJobsPagePrinter) PrintPage(page interface{}) { + arrayJobsStats, ok := page.(*models.ArrayJobsStatsResponse) + if !ok { + fmt.Println("error: wrong data type to ArrayJobsPagePrinter") + return + } + + p.pageCount++ + + if p.formatJson { + p.printJsonPage(arrayJobsStats) + return + } else { + // print header (if needed) + p.printHeader() + + // print data rows + rows := p.formatRows(arrayJobsStats, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } + +} + +// format data rows +// ./xce jobs list -A +func (p *ArrayJobsPagePrinter) formatRows(arrayJobsStats *models.ArrayJobsStatsResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, stat := range arrayJobsStats.Data.Rows { + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + align := format.align + switch format.name { + case models.HEADER_ARRAY_JOB_JOBID: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", 20, stat.JobID), "-") + case models.HEADER_ARRAY_JOB_SPEC: + row = appendOrDefault(row, FormatStringField(stat.ArrayJobName, width, truncatFlag, true), "-") + case models.HEADER_ARRAY_JOB_OWNER: + row = appendOrDefault(row, FormatStringField(stat.SubmitUser, width, truncatFlag, true), "-") + case models.HEADER_ARRAY_JOB_NJOBS: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumJobs), "0") + case models.HEADER_ARRAY_JOB_PEND: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumPend), "0") + case models.HEADER_ARRAY_JOB_DONE: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumDone), "0") + case models.HEADER_ARRAY_JOB_RUN: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumRun), "0") + case models.HEADER_ARRAY_JOB_EXIT: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumExit), "0") + case models.HEADER_ARRAY_JOB_SSUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumSSusp), "0") + case models.HEADER_ARRAY_JOB_USUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumUSusp), "0") + case models.HEADER_ARRAY_JOB_PSUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, stat.NumPSusp), "0") + } + } + rows = append(rows, row) + } + return rows +} + +// print JSON page +func (p *ArrayJobsPagePrinter) printJsonPage(arrayJobsStats *models.ArrayJobsStatsResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRows(arrayJobsStats, false) + + p.PrintJsonPageGeneric(rows, headers) +} diff --git a/src/frontend/xcecli/common/output/page_print_hosts.go b/src/frontend/xcecli/common/output/page_print_hosts.go new file mode 100644 index 00000000..969b86e3 --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_hosts.go @@ -0,0 +1,289 @@ +package output + +import ( + "fmt" + "strings" + "xcecli/models" +) + +// HostsPagePrinter +type HostsPagePrinter struct { + *BasePagePrinter +} + +// NewHostsPagePrinter +func NewHostsPagePrinter(opt ...BasePageOption) *HostsPagePrinter { + return &HostsPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +// PrepareForPrinting +func (p *HostsPagePrinter) PrepareForPrinting(firstPage interface{}) { + hosts, ok := firstPage.(*models.HostsResponse) + if !ok { + fmt.Println("error: wrong data type to HostsPagePrinter") + return + } + + // set field formats + p.fieldFormats = p.prepareFieldFormats() + + // init column widths + p.initColumnWidths(hosts) + + // prepare headers + p.prepareHeaders() +} + +// prepare field formats +func (p *HostsPagePrinter) prepareFieldFormats() []*fieldFormat { + // here define the default fields based on the host data + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_HOST_NAME, + models.HEADER_HOST_STATUS, + models.HEADER_HOST_NCPUS, + models.HEADER_HOST_MAX_SLOT, + models.HEADER_HOST_NUM_RUN, + models.HEADER_HOST_NUM_SSUSP, + models.HEADER_HOST_NUM_USUSP, + models.HEADER_HOST_HOST_TYPE, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths +func (p *HostsPagePrinter) initColumnWidths(hosts *models.HostsResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, host := range hosts.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_HOST_NAME: + width = len(host.HostName) + case models.HEADER_HOST_STATUS: + width = len(host.HostStatus) + case models.HEADER_HOST_HOST_TYPE: + width = len(host.HostType) + case models.HEADER_HOST_NCPUS: + width = len(fmt.Sprintf("%d", host.NumCPUs)) + case models.HEADER_HOST_MAX_SLOT: + width = len(fmt.Sprintf("%d", host.MaxSlot)) + case models.HEADER_HOST_NUM_RUN: + width = len(fmt.Sprintf("%d", host.NumRun)) + case models.HEADER_HOST_NUM_SSUSP: + width = len(fmt.Sprintf("%d", host.NumSSusp)) + case models.HEADER_HOST_NUM_USUSP: + width = len(fmt.Sprintf("%d", host.NumUSusp)) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage +func (p *HostsPagePrinter) PrintPage(page interface{}) { + hosts, ok := page.(*models.HostsResponse) + if !ok { + fmt.Println("error: wrong data type to HostsPagePrinter") + return + } + + p.pageCount++ + + if p.verbose { + printHostsVerboseOutput(hosts) + return + } + + if p.formatJson { + p.printJsonPage(hosts) + return + } else { + // print header (if needed) + p.printHeader() + + // print data rows + rows := p.formatRows(hosts, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +// format data rows +func (p *HostsPagePrinter) formatRows(hosts *models.HostsResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, host := range hosts.Data.Rows { + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + align := format.align + switch format.name { + case models.HEADER_HOST_NAME: + row = appendOrDefault(row, FormatStringField(host.HostName, width, truncatFlag, true), "-") + case models.HEADER_HOST_STATUS: + row = appendOrDefault(row, FormatStringField(host.HostStatus, width, truncatFlag, true), "-") + case models.HEADER_HOST_HOST_TYPE: + row = appendOrDefault(row, FormatStringField(host.HostType, width, truncatFlag, true), "-") + case models.HEADER_HOST_NCPUS: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumCPUs), "0") + case models.HEADER_HOST_MAX_SLOT: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.MaxSlot), "0") + case models.HEADER_HOST_NUM_RUN: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumRun), "0") + case models.HEADER_HOST_NUM_SSUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumSSusp), "0") + case models.HEADER_HOST_NUM_USUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, host.NumUSusp), "0") + } + } + rows = append(rows, row) + } + return rows +} + +// print JSON page +func (p *HostsPagePrinter) printJsonPage(hosts *models.HostsResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRows(hosts, false) + p.PrintJsonPageGeneric(rows, headers) +} + +func printHostsVerboseOutput(hosts *models.HostsResponse) { + for _, host := range hosts.Data.Rows { + fmt.Printf("HOST %s\n", host.HostName) + // table 1 + headers := models.HostDetailHeaders + row1 := make([]string, 0, len(headers)) + widths1 := make([]int, len(headers)) + for i, h := range headers { + var val string + switch h { + case models.HEADER_HOST_STATUS: + val = host.HostStatus + case models.HEADER_HOST_CPUF: + val = fmt.Sprintf("%.2f", host.CPUFactor) + case models.HEADER_HOST_JLU: + val = "-" + case models.HEADER_HOST_MAX: + val = fmt.Sprintf("%d", host.MaxSlot) + case models.HEADER_HOST_NJOBS: + val = fmt.Sprintf("%d", host.NumRun) + case models.HEADER_HOST_RUN: + val = fmt.Sprintf("%d", host.NumRun) + case models.HEADER_HOST_SSUSP: + val = fmt.Sprintf("%d", host.NumSSusp) + case models.HEADER_HOST_USUSP: + val = fmt.Sprintf("%d", host.NumUSusp) + case models.HEADER_HOST_RSV: + val = "-" + case models.HEADER_HOST_DISPT_WINDOW: + if str, ok := host.DisptWindow.(string); ok { + val = str + } else { + val = "-" + } + default: + val = "-" + } + widths1[i] = max(len(h), len(val)) + row1 = append(row1, FormatStringField(val, widths1[i], false, true)) + } + tp1 := TablePrinter{Headers: headers, Rows: [][]string{row1}, ColWidths: widths1} + tp1.Print() + fmt.Println("") + + fmt.Println("CURRENT LOAD USED FOR SCHEDULING:") + + // table2 + headers2 := append([]string{""}, models.HostLoadSchedulingHeaders...) + totalVals := []string{"Total", "0.0", "0.0", "0.0", "1%", "0.0", "33", "1", "1137", "51G", "3.9G", "6.8G", "8", "0.0"} + reserveVals := []string{"Reserved", "0.0", "0.0", "0.0", "0%", "0.0", "0", "0", "0", "0M", "0M", "0M", "-", "-"} + + widths2 := make([]int, len(headers2)) + for i := range headers2 { + widths2[i] = max(len(headers2[i]), len(totalVals[i])) + widths2[i] = max(widths2[i], len(reserveVals[i])) + } + + rowTotal := make([]string, len(headers2)) + rowReserve := make([]string, len(headers2)) + for i := range headers2 { + rowTotal[i] = FormatStringField(totalVals[i], widths2[i], false, true) + rowReserve[i] = FormatStringField(reserveVals[i], widths2[i], false, true) + } + + tp2 := TablePrinter{Headers: headers2, Rows: [][]string{rowTotal, rowReserve}, ColWidths: widths2} + tp2.Print() + fmt.Println("") + + // Table 3 + fmt.Println("LOAD THRESHOLD USED FOR SCHEDULING:") + + headers3 := append([]string{""}, models.HostLoadSchedulingHeaders...) + loadSched := []string{"loadSched", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"} + loadStop := []string{"loadStop", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"} + + widths3 := make([]int, len(headers3)) + for i := range headers3 { + widths3[i] = max(len(headers3[i]), len(loadSched[i])) + widths3[i] = max(widths3[i], len(loadStop[i])) + } + + rowSched := make([]string, len(headers3)) + rowStop := make([]string, len(headers3)) + for i := range headers3 { + rowSched[i] = FormatStringField(loadSched[i], widths2[i], false, true) + rowStop[i] = FormatStringField(loadStop[i], widths2[i], false, true) + } + + tp3 := TablePrinter{Headers: headers3, Rows: [][]string{rowSched, rowStop}, ColWidths: widths2} + tp3.Print() + fmt.Println("") + fmt.Println("CONFIGURED AFFINITY CPU LIST: all") + } +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/src/frontend/xcecli/common/output/page_print_jobs.go b/src/frontend/xcecli/common/output/page_print_jobs.go new file mode 100644 index 00000000..b58c7a1a --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_jobs.go @@ -0,0 +1,447 @@ +package output + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + "text/tabwriter" + "xcecli/models" +) + +// JobsPagePrinter is the printer for jobs +type JobsPagePrinter struct { + *BasePagePrinter +} + +// NewJobsPagePrinter creates a new jobs page printer +func NewJobsPagePrinter(opt ...BasePageOption) *JobsPagePrinter { + return &JobsPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +// PrepareForPrinting prepares for printing, initializes field formats and column widths +func (p *JobsPagePrinter) PrepareForPrinting(firstPage interface{}) { + jobs, ok := firstPage.(*models.JobsResponse) + if !ok { + fmt.Println("error: wrong type of data passed to JobsPagePrinter") + return + } + + // set the field formats + p.fieldFormats = p.prepareFieldFormats() + + // initialize the column widths + p.initColumnWidths(jobs) + + // prepare the headers + p.prepareHeaders() +} + +// prepare the field formats +func (p *JobsPagePrinter) prepareFieldFormats() []*fieldFormat { + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_JOB_JOBID, + models.HEADER_JOB_USER, + models.HEADER_JOB_STATUS, + models.HEADER_JOB_QUEUE, + models.HEADER_JOB_FROM_HOST, + models.HEADER_JOB_EXEC_HOST, + models.HEADER_JOB_NAME, + models.HEADER_JOB_SUBMIT_TIME, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user defined fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// initialize the column widths +func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) { + // initialize the column widths to the length of the headers + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the maximum width of the first page data + for _, job := range jobs.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_JOB_JOBID: + width = len(fmt.Sprintf("%d", job.JobID)) + case models.HEADER_JOB_USER: + width = len(job.SubmitUser) + case models.HEADER_JOB_STATUS: + width = len(job.JobStatus) + case models.HEADER_JOB_QUEUE: + width = len(job.QueueName) + case models.HEADER_JOB_FROM_HOST: + width = len(job.FromHost) + case models.HEADER_JOB_EXEC_HOST: + continue + case models.HEADER_JOB_NAME: + width = len(job.JobName) + case models.HEADER_JOB_SUBMIT_TIME: + width = len(job.SubmitTime) + case models.HEADER_JOB_CORE_LIMIT: + width = len(fmt.Sprintf("%f", job.CoreLimit)) + case models.HEADER_JOB_MEM_LIMIT: + width = len(fmt.Sprintf("%f", job.MemLimit)) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage prints a page of data +func (p *JobsPagePrinter) PrintPage(page interface{}) { + jobs, ok := page.(*models.JobsResponse) + if !ok { + fmt.Println("error: wrong type of data passed to JobsPagePrinter") + return + } + + p.pageCount++ + + if p.verbose { + printJobsVerboseOutput(jobs, p.jobStatusFilter) + return + } + + if p.formatJson { + p.printJsonPage(jobs) + return + } else { + // print the header (if needed) + p.printHeader() + + // print the data rows + rows := p.formatRows(jobs, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +func (p *JobsPagePrinter) printJsonPage(jobs *models.JobsResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + rows := p.formatRows(jobs, false) + p.PrintJsonPageGeneric(rows, headers) + +} + +// format the data rows +func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, job := range jobs.Data.Rows { + mainRow := make([]string, 0, len(p.fieldFormats)) + var extraRows [][]string + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + + align := format.align + + switch format.name { + case models.HEADER_JOB_JOBID: + var arrayJobIDStr string + if job.ArrayIdx != 0 { + arrayJobIDStr = fmt.Sprintf("%d[%d]", job.JobID, job.ArrayIdx) + } else { + arrayJobIDStr = fmt.Sprintf("%d", job.JobID) + } + mainRow = appendOrDefault(mainRow, + fmt.Sprintf("%"+align+"*s", 20, arrayJobIDStr), "-") + + case models.HEADER_JOB_EXEC_HOST: + var execHosts models.ExecHosts + if err := json.Unmarshal([]byte(job.ExecHosts.String()), &execHosts); err == nil && len(execHosts.HostSlots) > 0 { + firstHost := fmt.Sprintf("%d*%s", execHosts.HostSlots[0].NumSlots, execHosts.HostSlots[0].HostName) + mainRow = appendOrDefault(mainRow, + FormatStringField(firstHost, width, truncatFlag, false), "-") + if len(execHosts.HostSlots) > 1 { + for _, slot := range execHosts.HostSlots[1:] { + newLine := make([]string, len(p.fieldFormats)) + for i, f := range p.fieldFormats { + if f.name == "EXEC_HOST" { + continue + } + newLine[i] = strings.Repeat(" ", p.columnWidths[f.name]) + } + hostText := fmt.Sprintf("%d*%s", slot.NumSlots, slot.HostName) + newLine[getFieldFormatIndex(p.fieldFormats, "EXEC_HOST")] = FormatStringField(hostText, width, truncatFlag, false) + extraRows = append(extraRows, append([]string{}, newLine...)) + } + } + } else { + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-") + } + case models.HEADER_JOB_SUBMIT_TIME: + if submitTime, err := parseJobTime(job.SubmitTime); err == nil { + mainRow = appendOrDefault(mainRow, FormatStringField(formatJobTime(submitTime), width, truncatFlag, true), "-") + } else { + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, "-"), "-") + } + + // normal fields + case models.HEADER_JOB_NAME: + mainRow = appendOrDefault(mainRow, FormatStringField(job.JobName, width, truncatFlag, true), "-") + case models.HEADER_JOB_USER: + mainRow = appendOrDefault(mainRow, FormatStringField(job.SubmitUser, width, truncatFlag, true), "-") + case models.HEADER_JOB_STATUS: + mainRow = appendOrDefault(mainRow, FormatStringField(job.JobStatus, width, truncatFlag, true), "-") + case models.HEADER_JOB_QUEUE: + mainRow = appendOrDefault(mainRow, FormatStringField(job.QueueName, width, truncatFlag, true), "-") + case models.HEADER_JOB_FROM_HOST: + mainRow = appendOrDefault(mainRow, FormatStringField(job.FromHost, width, truncatFlag, true), "-") + case models.HEADER_JOB_COMMAND: + mainRow = appendOrDefault(mainRow, FormatStringField(job.Command, width, truncatFlag, true), "-") + case models.HEADER_JOB_PROJECT_NAME: + mainRow = appendOrDefault(mainRow, FormatStringField(job.ProjectName, width, truncatFlag, true), "-") + case models.HEADER_JOB_START_TIME: + mainRow = appendOrDefault(mainRow, FormatStringField(job.StartTime, width, truncatFlag, true), "-") + case models.HEADER_JOB_END_TIME: + mainRow = appendOrDefault(mainRow, FormatStringField(job.EndTime, width, truncatFlag, true), "-") + case models.HEADER_JOB_CWD: + mainRow = appendOrDefault(mainRow, FormatStringField(job.CWD, width, truncatFlag, true), "-") + case models.HEADER_JOB_LAUNCH_TYPE: + mainRow = appendOrDefault(mainRow, FormatStringField(job.LaunchType, width, truncatFlag, true), "-") + case models.HEADER_JOB_INTERACTIVE_URL: + mainRow = appendOrDefault(mainRow, FormatStringField(job.InteractiveJobURL, width, truncatFlag, true), "-") + + // not string field + case models.HEADER_JOB_NUM_PROCESSORS: + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*d", width, job.NumProcessors), "-") + case models.HEADER_JOB_CORE_LIMIT: + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.CoreLimit)), "-") + case models.HEADER_JOB_MEM_LIMIT: + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.MemLimit)), "-") + default: + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*v", width, "-"), "-") + } + } + rows = append(rows, mainRow) + rows = append(rows, extraRows...) + } + return rows +} + +func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + defer w.Flush() + for jobIndex, job := range jobs.Data.Rows { + // if the filter is not empty, only show the jobs that match the filter + if len(filter) > 0 { + found := false + for _, status := range filter { + if job.JobStatus == status { + found = true + break + } + } + if !found { + continue + } + } + // generate jobId for two format: 111 or 111[10] + jobId := fmt.Sprintf("%d", job.JobID) + index := strings.Index(job.JobName, "[") + if index != -1 { + jobId = fmt.Sprintf("%s%s", jobId, job.JobName[index:]) + } + + // (1) print the job baisc info when sending "jobs list -l" command + firstLine := "Job <" + jobId + ">, " + if job.JobName != "" { + firstLine = firstLine + "Job Name <" + job.JobName + ">, " + } + if job.SubmitUser != "" { + firstLine = firstLine + "User <" + job.SubmitUser + ">, " + } + if job.ProjectName != "" { + firstLine = firstLine + "Project <" + job.ProjectName + ">, " + } + if job.JobStatus != "" { + firstLine = firstLine + "Status <" + job.JobStatus + ">, " + } + + if job.QueueName != "" { + firstLine = firstLine + "Queue <" + job.QueueName + ">, " + } + if job.JobPriority != 0 { + firstLine = firstLine + "Priority <" + strconv.Itoa(job.JobPriority) + ">, " + } + if job.LaunchType != "" { + firstLine = firstLine + "Launch Type <" + job.LaunchType + ">, " + } + if job.Command != "" { + firstLine = firstLine + "Command <" + job.Command + ">, " + } + if job.JobDescription != "" { + firstLine = firstLine + "Job Description <" + job.JobDescription + "> " + } + PrintWrappedLine(w, firstLine, 80, 18) + + // (2) print the submit time info + submitLine := "" + if submitTime, err := parseJobTime(job.SubmitTime); err == nil { + submitLine = submitLine + fmt.Sprintf("%s %s %02d:%02d:%02d: ", + submitTime.Format("Jan"), submitTime.Format("02"), + submitTime.Hour(), submitTime.Minute(), submitTime.Second()) + if job.FromHost != "" { + submitLine = submitLine + "Submitted from host <" + job.FromHost + ">, " + } + if job.CWD != "" { + submitLine = submitLine + "CWD <" + job.CWD + ">, " + } + if job.NumProcessors != 0 { + submitLine = submitLine + strconv.Itoa(job.NumProcessors) + " Task(s), " + } + if job.Resreq != "" { + submitLine = submitLine + "Requested Resources <" + job.Resreq + ">, " + } + } + PrintWrappedLine(w, submitLine, 80, 18) + + // (3) print the start time info + startLine := "" + if startTime, err := parseJobTime(job.StartTime); err == nil { + month := startTime.Format("Jan") + day := startTime.Format("02") + exec_hostname := "" + started_slots := 0 + if len(job.ExecHosts.HostSlots) > 0 { + for _, host := range job.ExecHosts.HostSlots { + started_slots += host.NumSlots + if exec_hostname == "" { + exec_hostname = exec_hostname + fmt.Sprintf("<%d*%s>", host.NumSlots, host.HostName) + } else { + exec_hostname = exec_hostname + " " + fmt.Sprintf("<%d*%s>", host.NumSlots, host.HostName) + } + } + } + startLine = startLine + fmt.Sprintf("%s %s %02d:%02d:%02d: ", + month, day, startTime.Hour(), startTime.Minute(), startTime.Second()) + + if started_slots > 0 { + startSlotMsg := fmt.Sprintf("%d", started_slots) + startLine = startLine + "Started " + startSlotMsg + " Task(s) " + } + if exec_hostname != "" { + startLine = startLine + "on Host(s) " + exec_hostname + ", " + } + if job.ExecHome != "" { + startLine = startLine + "Execution Home <" + job.ExecHome + ">, " + } + if job.ExecCWD != "" { + startLine = startLine + "Execution CWD <" + job.ExecCWD + ">, " + } + if job.InteractiveJobURL != "" { + startLine = startLine + "Interactive Job URL <" + job.InteractiveJobURL + ">, " + } + + //fmt.Fprintf(w, "%s %s %02d:%02d:%02d: Resource usage collected.\n", + // month, day, submitTime.Hour(), submitTime.Minute(), submitTime.Second()) + //fmt.Fprintf(w, " MEM: Mbytes; SWAP: Mbytes; NTHREAD: \n") + pgids := make([]string, len(job.PGIDs)) + pids := make([]string, len(job.PID)) + for i, pgid := range job.PGIDs { + pgids[i] = fmt.Sprintf("%d", pgid) + } + for i, pid := range job.PID { + pids[i] = fmt.Sprintf("%d", pid) + } + + if len(pgids) > 0 { + startLine = startLine + "PGID: " + strings.Join(pgids, ",") + ", " + } else if len(pids) > 0 { + // only show pids + startLine = startLine + "PIDs: " + strings.Join(pids, ",") + ", " + } + } + PrintWrappedLine(w, startLine, 80, 18) + + // (4) print end line + endLine := "" + if job.JobStatus == "DONE" || job.JobStatus == "EXIT" { + if endTime, err := parseJobTime(job.EndTime); err == nil { + endLine = endLine + fmt.Sprintf("%s %s %02d:%02d:%02d: ", + endTime.Format("Jan"), endTime.Format("02"), + endTime.Hour(), endTime.Minute(), endTime.Second()) + } + if job.JobStatus == "EXIT" { + endLine = endLine + "Completed <" + strings.ToLower(job.JobStatus) + ">. " + } else { + endLine = endLine + "Done successfully. " + } + + if job.CPUTime != 0 { + endLine = endLine + "The CPU time used is <" + strconv.Itoa(job.CPUTime) + "> seconds." + + } + + } + PrintWrappedLine(w, endLine, 80, 18) + fmt.Println("") + // print the scheduling parameters table + fmt.Printf("SCHEDULING PARAMETERS\n") + + headers2 := append([]string{""}, models.JobLoadSchedulingHeaders...) + loadSched := []string{"loadSched", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"} + loadStop := []string{"loadStop", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"} + + widths2 := make([]int, len(headers2)) + for i := range headers2 { + widths2[i] = max(len(headers2[i]), len(loadSched[i])) + widths2[i] = max(widths2[i], len(loadStop[i])) + } + + rowSched := make([]string, len(headers2)) + rowStop := make([]string, len(headers2)) + for i := range headers2 { + rowSched[i] = FormatStringField(loadSched[i], widths2[i], false, true) + rowStop[i] = FormatStringField(loadStop[i], widths2[i], false, true) + } + + tp2 := TablePrinter{Headers: headers2, Rows: [][]string{rowSched, rowStop}, ColWidths: widths2} + tp2.Print() + fmt.Println("") + + // print the resource requirement details + fmt.Printf("RESOURCE REQUIREMENT DETAILS:\n") + fmt.Fprintln(w, " Combined: select[type == local] order[r15s:pg]") + fmt.Fprintln(w, " Effective: select[type == local] order[r15s:pg]") + if jobIndex != len(jobs.Data.Rows)-1 { + fmt.Fprintln(w, "------------------------------------------------------------------------------") + } + } +} diff --git a/src/frontend/xcecli/common/output/page_print_queues.go b/src/frontend/xcecli/common/output/page_print_queues.go new file mode 100644 index 00000000..5479e322 --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_queues.go @@ -0,0 +1,268 @@ +package output + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + "xcecli/models" +) + +// HostsPagePrinter +type QueuesPagePrinter struct { + *BasePagePrinter +} + +// NewHostsPagePrinter +func NewQueuesPagePrinter(opt ...BasePageOption) *QueuesPagePrinter { + return &QueuesPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +// PrepareForPrinting +func (p *QueuesPagePrinter) PrepareForPrinting(firstPage interface{}) { + queues, ok := firstPage.(*models.QueuesResponse) + if !ok { + fmt.Println("error: wrong data type to QueuesPagePrinter") + return + } + + // set field formats + p.fieldFormats = p.prepareFieldFormats() + + // init column widths + p.initColumnWidths(queues) + + // prepare headers + p.prepareHeaders() +} + +// prepare field formats +func (p *QueuesPagePrinter) prepareFieldFormats() []*fieldFormat { + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_QUEUE_NAME, + models.HEADER_QUEUE_PRIORITY, + models.HEADER_QUEUE_STATUS, + models.HEADER_QUEUE_NUM_PEND, + models.HEADER_QUEUE_NUM_RUN, + models.HEADER_QUEUE_NUM_SSUSP, + models.HEADER_QUEUE_NUM_USUSP, + models.HEADER_QUEUE_NUM_DONE, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths +func (p *QueuesPagePrinter) initColumnWidths(queues *models.QueuesResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, queue := range queues.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_QUEUE_NAME: + width = len(queue.QueueName) + case models.HEADER_QUEUE_PRIORITY: + width = len(fmt.Sprintf("%d", queue.Priority)) + case models.HEADER_QUEUE_STATUS: + width = len(queue.QueueStatus) + case models.HEADER_QUEUE_NUM_PEND: + width = len(fmt.Sprintf("%d", queue.NumPend)) + case models.HEADER_QUEUE_NUM_RUN: + width = len(fmt.Sprintf("%d", queue.NumRun)) + case models.HEADER_QUEUE_NUM_SSUSP: + width = len(fmt.Sprintf("%d", queue.NumSSusp)) + case models.HEADER_QUEUE_NUM_USUSP: + width = len(fmt.Sprintf("%d", queue.NumUSusp)) + case models.HEADER_QUEUE_NUM_DONE: + width = len(fmt.Sprintf("%d", queue.NumDone)) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage +func (p *QueuesPagePrinter) PrintPage(page interface{}) { + queues, ok := page.(*models.QueuesResponse) + if !ok { + fmt.Println("error: wrong data type to QueuesPagePrinter") + return + } + + p.pageCount++ + + if p.verbose { + printQueuesVerboseOutput(queues) + return + } + + if p.formatJson { + p.printJsonPage(queues) + return + } else { + // print header (if needed) + p.printHeader() + // print data rows + rows := p.formatRows(queues, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +// format data rows +func (p *QueuesPagePrinter) formatRows(queues *models.QueuesResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, queue := range queues.Data.Rows { + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + align := format.align + switch format.name { + case models.HEADER_QUEUE_NAME: + row = appendOrDefault(row, FormatStringField(queue.QueueName, width, truncatFlag, true), "-") + case models.HEADER_QUEUE_STATUS: + row = appendOrDefault(row, FormatStringField(queue.QueueStatus, width, truncatFlag, true), "-") + case models.HEADER_QUEUE_PRIORITY: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, queue.Priority), "0") + case models.HEADER_QUEUE_NUM_PEND: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, queue.NumPend), "0") + case models.HEADER_QUEUE_NUM_RUN: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, queue.NumRun), "0") + case models.HEADER_QUEUE_NUM_SSUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, queue.NumSSusp), "0") + case models.HEADER_QUEUE_NUM_USUSP: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, queue.NumUSusp), "0") + case models.HEADER_QUEUE_NUM_DONE: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, queue.NumDone), "0") + } + } + rows = append(rows, row) + } + return rows +} + +// print JSON page +func (p *QueuesPagePrinter) printJsonPage(queues *models.QueuesResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRows(queues, false) + p.PrintJsonPageGeneric(rows, headers) +} + +func printQueuesVerboseOutput(queues *models.QueuesResponse) { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + defer w.Flush() + + for _, queue := range queues.Data.Rows { + fmt.Printf("QUEUE: %s\n", queue.QueueName) + if queue.Description != "" { + fmt.Fprintf(w, "-- %s\n", queue.Description) + } + fmt.Println() + fmt.Printf("PARAMETERS/STATISTICS:\n") + // table 1 + headers := models.QueueDetailHeaders + row1 := make([]string, 0, len(headers)) + widths1 := make([]int, len(headers)) + + for i, q := range headers { + var val string + switch q { + case models.HEADER_QUEUE_PRIO: + val = fmt.Sprintf("%d", queue.Priority) + case models.HEADER_QUEUE_NICE: + val = "-" + case models.HEADER_QUEUE_STATUS: + val = queue.QueueStatus + case models.HEADER_QUEUE_MAX: + val = "-" + case models.HEADER_QUEUE_JLU: + val = "-" + case models.HEADER_QUEUE_JLP: + val = "-" + case models.HEADER_QUEUE_JLH: + val = "-" + case models.HEADER_QUEUE_NJOBS: + val = "-" + case models.HEADER_QUEUE_PEND: + val = fmt.Sprintf("%d", queue.NumPend) + case models.HEADER_QUEUE_RUN: + val = fmt.Sprintf("%d", queue.NumRun) + case models.HEADER_QUEUE_SSUSP: + val = fmt.Sprintf("%d", queue.NumSSusp) + case models.HEADER_QUEUE_USUSP: + val = fmt.Sprintf("%d", queue.NumUSusp) + case models.HEADER_HOST_RSV: + val = "-" + default: + val = "-" + } + widths1[i] = max(len(q), len(val)) + row1 = append(row1, FormatStringField(val, widths1[i], false, true)) + } + tp1 := TablePrinter{Headers: headers, Rows: [][]string{row1}, ColWidths: widths1} + tp1.Print() + fmt.Println() + + // table 2 + fmt.Printf("SCHEDULING PARAMETERS\n") + + headers2 := append([]string{""}, models.QueueLoadSchedulingHeaders...) + loadSched := []string{"loadSched", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"} + loadStop := []string{"loadStop", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"} + + widths2 := make([]int, len(headers2)) + for i := range headers2 { + widths2[i] = max(len(headers2[i]), len(loadSched[i])) + widths2[i] = max(widths2[i], len(loadStop[i])) + } + + rowSched := make([]string, len(headers2)) + rowStop := make([]string, len(headers2)) + for i := range headers2 { + rowSched[i] = FormatStringField(loadSched[i], widths2[i], false, true) + rowStop[i] = FormatStringField(loadStop[i], widths2[i], false, true) + } + + tp2 := TablePrinter{Headers: headers2, Rows: [][]string{rowSched, rowStop}, ColWidths: widths2} + tp2.Print() + fmt.Println("") + } +} diff --git a/src/frontend/xcecli/common/output/page_print_resources.go b/src/frontend/xcecli/common/output/page_print_resources.go new file mode 100644 index 00000000..32469cf4 --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_resources.go @@ -0,0 +1,365 @@ +package output + +import ( + "fmt" + "strings" + "xcecli/models" +) + +type ResourcesPagePrinter struct { + *BasePagePrinter + isDefinition bool // Flag to distinguish between distribution and definition +} + +func NewResourcesDistPagePrinter(opt ...BasePageOption) *ResourcesPagePrinter { + return &ResourcesPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + isDefinition: false, + } +} + +func NewResourcesDefPagePrinter(opt ...BasePageOption) *ResourcesPagePrinter { + return &ResourcesPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + isDefinition: true, + } +} + +// PrepareForPrinting supports both ResourceDistResponse and ResourceDefResponse +func (p *ResourcesPagePrinter) PrepareForPrinting(firstPage interface{}) { + // Check if it's ResourceDistResponse + if resourcesDist, ok := firstPage.(*models.ResourceDistResponse); ok { + p.isDefinition = false + // set field formats + p.fieldFormats = p.prepareFieldFormats() + // init column widths + p.initColumnWidthsForDist(resourcesDist) + // prepare headers + p.prepareHeaders() + return + } + + // Check if it's ResourceDefResponse + if resourcesDef, ok := firstPage.(*models.ResourceDefResponse); ok { + p.isDefinition = true + // set field formats + p.fieldFormats = p.prepareFieldFormats() + // init column widths + p.initColumnWidthsForDef(resourcesDef) + // prepare headers + p.prepareHeaders() + return + } + + fmt.Println("error: wrong data type to ResourcesPagePrinter") +} + +// prepare field formats +func (p *ResourcesPagePrinter) prepareFieldFormats() []*fieldFormat { + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields based on type + if p.isDefinition { + defaultFields := []string{ + models.HEADER_RESOURCE_NAME, + models.HEADER_RESOURCE_DEF_TYPE, + models.HEADER_RESOURCE_DEF_ORDER, + models.HEADER_RESOURCE_DEF_DESC, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, width: 20, align: "-"}) + } + } else { + defaultFields := []string{ + models.HEADER_RESOURCE_NAME, + models.HEADER_RESOURCE_VALUE, + models.HEADER_RESOURCE_LOC, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, width: 20, align: "-"}) + } + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths for ResourceDistResponse +func (p *ResourcesPagePrinter) initColumnWidthsForDist(resourcesDist *models.ResourceDistResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, resourceDist := range resourcesDist.Data.Rows { + for _, format := range p.fieldFormats { + width := 15 + switch format.name { + case models.HEADER_RESOURCE_NAME: + width = len(resourceDist.ResourceName) + case models.HEADER_RESOURCE_VALUE: + width = len(resourceDist.ResourceValue.String()) + case models.HEADER_RESOURCE_LOC: + width = len(resourceDist.ResourceLoc.String()) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] { + p.columnWidths[format.name] = width + } + } + } +} + +// init column widths for ResourceDefResponse +func (p *ResourcesPagePrinter) initColumnWidthsForDef(resourcesDef *models.ResourceDefResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, resourceDef := range resourcesDef.Data.Rows { + for _, format := range p.fieldFormats { + width := 15 + switch format.name { + case models.HEADER_RESOURCE_NAME: + width = len(resourceDef.ResourceName) + case models.HEADER_RESOURCE_DEF_TYPE: + width = len(resourceDef.ResourceType) + case models.HEADER_RESOURCE_DEF_DESC: + width = len(resourceDef.Description) + } + + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage supports both ResourceDistResponse and ResourceDefResponse +func (p *ResourcesPagePrinter) PrintPage(page interface{}) { + + p.pageCount++ + + if p.isDefinition { + resourcesDef, ok := page.(*models.ResourceDefResponse) + if !ok { + fmt.Println("error: wrong data type to ResourcesPagePrinter") + return + } + if p.verbose { + printResourcesDefVerboseOutput(resourcesDef) + return + } + } + if p.formatJson { + if p.isDefinition { + if resourcesDef, ok := page.(*models.ResourceDefResponse); ok { + p.printJsonPageForDef(resourcesDef) + return + } + } else { + if resourcesDist, ok := page.(*models.ResourceDistResponse); ok { + p.printJsonPageForDist(resourcesDist) + return + } + } + fmt.Println("error: wrong data type for JSON printing") + return + } else { + // print header (if needed) + p.printHeader() + + // print data rows + var rows [][]string + if p.isDefinition { + if resourcesDef, ok := page.(*models.ResourceDefResponse); ok { + rows = p.formatRowsForDef(resourcesDef, true) + } else { + fmt.Println("error: wrong data type to format rows") + return + } + } else { + if resourcesDist, ok := page.(*models.ResourceDistResponse); ok { + rows = p.formatRowsForDist(resourcesDist, true) + } else { + fmt.Println("error: wrong data type to format rows") + return + } + } + + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +// format data rows for ResourceDistResponse +func (p *ResourcesPagePrinter) formatRowsForDist(resourcesDist *models.ResourceDistResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, resourceDist := range resourcesDist.Data.Rows { + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + switch format.name { + case models.HEADER_RESOURCE_NAME: + row = appendOrDefault(row, FormatStringField(resourceDist.ResourceName, width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_VALUE: + row = appendOrDefault(row, FormatStringField(resourceDist.ResourceValue.String(), width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_LOC: + row = appendOrDefault(row, FormatStringField(resourceDist.ResourceLoc.String(), width, truncatFlag, true), "-") + } + } + rows = append(rows, row) + } + return rows +} + +// format data rows for ResourceDefResponse +func (p *ResourcesPagePrinter) formatRowsForDef(resourcesDef *models.ResourceDefResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + for _, resourceDef := range resourcesDef.Data.Rows { + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + align := format.align + switch format.name { + case models.HEADER_RESOURCE_NAME: + row = appendOrDefault(row, FormatStringField(resourceDef.ResourceName, width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_DEF_DESC: + row = appendOrDefault(row, FormatStringField(resourceDef.Description, width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_DEF_TYPE: + row = appendOrDefault(row, FormatStringField(resourceDef.ResourceType, width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_DEF_ORDER: + row = appendOrDefault(row, FormatStringField(resourceDef.OrderType, width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_DEF_BUILTIN: + // Extract builtin flag from the Flag object + builtinValue := "false" + if resourceDef.Flag != nil { + if builtin, ok := resourceDef.Flag["builtin"]; ok { + builtinValue = fmt.Sprintf("%v", builtin) + } + } + row = appendOrDefault(row, FormatStringField(builtinValue, width, truncatFlag, true), "-") + case models.HEADER_RESOURCE_DEF_INTERVAL: + row = appendOrDefault(row, fmt.Sprintf("%"+align+"*d", width, resourceDef.Interval), "0") + } + } + rows = append(rows, row) + } + return rows +} + +// print JSON page for ResourceDistResponse +func (p *ResourcesPagePrinter) printJsonPageForDist(resourcesDist *models.ResourceDistResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRowsForDist(resourcesDist, false) + p.PrintJsonPageGeneric(rows, headers) +} + +// print JSON page for ResourceDefResponse +func (p *ResourcesPagePrinter) printJsonPageForDef(resourcesDef *models.ResourceDefResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRowsForDef(resourcesDef, false) + p.PrintJsonPageGeneric(rows, headers) +} + +func printResourcesDefVerboseOutput(resourceDef *models.ResourceDefResponse) { + for _, resourceDef := range resourceDef.Data.Rows { + fmt.Printf("RESOURCE: %s\n", resourceDef.ResourceName) + fmt.Printf("DESCRIPTION: %s\n", resourceDef.Description) + // table 1 + headers := models.ResourceDefDetailHeaders + row1 := make([]string, 0, len(headers)) + widths1 := make([]int, len(headers)) + for i, h := range headers { + var val string + switch h { + case models.HEADER_RESOURCE_DEF_TYPE: + val = resourceDef.ResourceType + case models.HEADER_RESOURCE_DEF_ORDER: + val = resourceDef.OrderType + case models.HEADER_RESOURCE_DEF_BUILTIN: + // Extract builtin flag from the Flag object + builtinValue := "false" + if resourceDef.Flag != nil { + if builtin, ok := resourceDef.Flag["builtin"]; ok { + builtinValue = fmt.Sprintf("%v", builtin) + } + } + val = builtinValue + case models.HEADER_RESOURCE_DEF_CONSUMABLE: + // Extract consumable flag from the Flag object + consumableValue := "false" + if resourceDef.Flag != nil { + if consumable, ok := resourceDef.Flag["consumable"]; ok { + consumableValue = fmt.Sprintf("%v", consumable) + } + } + val = consumableValue + case models.HEADER_RESOURCE_DEF_DYNAMIC: + // Extract dynamic flag from the Flag object + dynamicValue := "false" + if resourceDef.Flag != nil { + if dynamic, ok := resourceDef.Flag["dynamic"]; ok { + dynamicValue = fmt.Sprintf("%v", dynamic) + } + } + val = dynamicValue + case models.HEADER_RESOURCE_DEF_RELEASE: + // Extract release flag from the Flag object + releaseValue := "false" + if resourceDef.Flag != nil { + if release, ok := resourceDef.Flag["release"]; ok { + releaseValue = fmt.Sprintf("%v", release) + } + } + val = releaseValue + case models.HEADER_RESOURCE_DEF_INTERVAL: + val = fmt.Sprintf("%d", resourceDef.Interval) + } + widths1[i] = max(len(h), len(val)) + row1 = append(row1, FormatStringField(val, widths1[i], false, true)) + } + tp1 := TablePrinter{Headers: headers, Rows: [][]string{row1}, ColWidths: widths1} + tp1.Print() + fmt.Println("") + } +} diff --git a/src/frontend/xcecli/common/output/page_print_roles.go b/src/frontend/xcecli/common/output/page_print_roles.go new file mode 100644 index 00000000..5910be29 --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_roles.go @@ -0,0 +1,191 @@ +package output + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + "xcecli/models" +) + +// HostsPagePrinter +type RolesPagePrinter struct { + *BasePagePrinter +} + +// NewHostsPagePrinter +func NewRolesPagePrinter(opt ...BasePageOption) *RolesPagePrinter { + return &RolesPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +// PrepareForPrinting +func (p *RolesPagePrinter) PrepareForPrinting(firstPage interface{}) { + roles, ok := firstPage.(*models.RolesResponse) + if !ok { + fmt.Println("error: wrong data type to RolesPagePrinter") + return + } + + // set field formats + p.fieldFormats = p.prepareFieldFormats() + + // init column widths + p.initColumnWidths(roles) + + // prepare headers + p.prepareHeaders() +} + +// prepare field formats +func (p *RolesPagePrinter) prepareFieldFormats() []*fieldFormat { + // here define the default fields based on the host data + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_ROLE_NAME, + models.HEADER_ROLE_PERM_NAME, + models.HEADER_ROLE_DOMAIN_TYPE, + models.HEADER_ROLE_DOMAIN_VALUES, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths +func (p *RolesPagePrinter) initColumnWidths(roles *models.RolesResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, role := range roles.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_ROLE_NAME: + width = len(role.RoleName) + case models.HEADER_ROLE_PERM_NAME: + width = len(role.PermName) + case models.HEADER_ROLE_DOMAIN_TYPE: + width = len(role.DomainType) + } + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage +func (p *RolesPagePrinter) PrintPage(page interface{}) { + roles, ok := page.(*models.RolesResponse) + if !ok { + fmt.Println("error: wrong data type to RolesPagePrinter") + return + } + + p.pageCount++ + + if p.verbose { + printRolesVerboseOutput(roles) + return + } + + if p.formatJson { + p.printJsonPage(roles) + return + } else { + // print header (if needed) + p.printHeader() + // print data rows + rows := p.formatRows(roles, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +// format data rows +func (p *RolesPagePrinter) formatRows(roles *models.RolesResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + + for _, role := range roles.Data.Rows { + // If role has no permissions, create one row with empty permission fields + + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + switch format.name { + case models.HEADER_ROLE_NAME: + row = appendOrDefault(row, FormatStringField(role.RoleName, width, truncatFlag, true), "-") + case models.HEADER_ROLE_PERM_NAME: + row = appendOrDefault(row, FormatStringField(role.PermName, width, truncatFlag, true), "-") + case models.HEADER_ROLE_DOMAIN_TYPE: + row = appendOrDefault(row, FormatStringField(role.DomainType, width, truncatFlag, true), "-") + case models.HEADER_ROLE_DOMAIN_VALUES: + domainValues := "-" + if role.DomainValues != nil { + // Handle as []string since that's what the API returns + if vals, ok := role.DomainValues.([]string); ok && len(vals) > 0 { + domainValues = strings.Join(vals, ", ") + } else if vals, ok := role.DomainValues.([]interface{}); ok && len(vals) > 0 { + // Alternative: handle as []interface{} and convert to strings + strVals := make([]string, len(vals)) + for i, v := range vals { + strVals[i] = fmt.Sprintf("%v", v) + } + domainValues = strings.Join(strVals, ", ") + } + } + row = appendOrDefault(row, FormatStringField(domainValues, width, truncatFlag, true), "-") + } + } + rows = append(rows, row) + } + return rows +} + +// print JSON page +func (p *RolesPagePrinter) printJsonPage(roles *models.RolesResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRows(roles, false) + p.PrintJsonPageGeneric(rows, headers) +} + +func printRolesVerboseOutput(roles *models.RolesResponse) { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) + defer w.Flush() + + for _, role := range roles.Data.Rows { + fmt.Printf("Role: %s\n", role.RoleName) + fmt.Println() + } +} diff --git a/src/frontend/xcecli/common/output/page_print_users.go b/src/frontend/xcecli/common/output/page_print_users.go new file mode 100644 index 00000000..7872862a --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_users.go @@ -0,0 +1,146 @@ +package output + +import ( + "fmt" + "strings" + "xcecli/models" +) + +type UserRolesPagePrinter struct { + *BasePagePrinter +} + +func NewUserRolesPagePrinter(opt ...BasePageOption) *UserRolesPagePrinter { + return &UserRolesPagePrinter{ + BasePagePrinter: newBasePagePrinter(opt...), + } +} + +func (p *UserRolesPagePrinter) PrepareForPrinting(firstPage interface{}) { + userroles, ok := firstPage.(*models.UserRolesResponse) + if !ok { + fmt.Println("error: wrong data type to UserRolesPagePrinter") + return + } + + // set field formats + p.fieldFormats = p.prepareFieldFormats() + + // init column widths + p.initColumnWidths(userroles) + + // prepare headers + p.prepareHeaders() +} + +// prepare field formats +func (p *UserRolesPagePrinter) prepareFieldFormats() []*fieldFormat { + var fieldFormats []*fieldFormat + if p.fields == "" { + // default fields + defaultFields := []string{ + models.HEADER_USER_NAME, + models.HEADER_USERROLE_NAME, + } + for _, field := range defaultFields { + fieldFormats = append(fieldFormats, &fieldFormat{name: field, align: "-"}) + } + } else { + // parse user specified fields + for _, field := range strings.Split(p.fields, ",") { + fieldFormats = append(fieldFormats, parseFieldFormat(field)) + } + } + return fieldFormats +} + +// init column widths +func (p *UserRolesPagePrinter) initColumnWidths(users *models.UserRolesResponse) { + // init column widths to header length + for _, format := range p.fieldFormats { + p.columnWidths[format.name] = len(format.name) + if format.width > 0 { + p.columnWidths[format.name] = format.width + } + } + + // calculate the max width of the first page data + for _, user := range users.Data.Rows { + for _, format := range p.fieldFormats { + width := 0 + switch format.name { + case models.HEADER_USER_NAME: + width = len(user.UserName) + case models.HEADER_USERROLE_NAME: + width = len(user.UserRoleName) + } + if !p.wideFormat && width > 15 { + width = 15 + } + + if width > p.columnWidths[format.name] && format.width == 0 { + p.columnWidths[format.name] = width + } + } + } +} + +// PrintPage +func (p *UserRolesPagePrinter) PrintPage(page interface{}) { + userroles, ok := page.(*models.UserRolesResponse) + if !ok { + fmt.Println("error: wrong data type to UserRolesPagePrinter") + return + } + + p.pageCount++ + + if p.formatJson { + p.printJsonPage(userroles) + return + } else { + // print header (if needed) + p.printHeader() + // print data rows + rows := p.formatRows(userroles, true) + for _, row := range rows { + fmt.Fprintln(p.w, strings.Join(row, "\t")) + } + p.w.Flush() + } +} + +// format data rows +func (p *UserRolesPagePrinter) formatRows(userroles *models.UserRolesResponse, truncatFlag bool) [][]string { + var rows [][]string + if p.wideFormat { + truncatFlag = false + } + + for _, user := range userroles.Data.Rows { + row := make([]string, 0, len(p.fieldFormats)) + for _, format := range p.fieldFormats { + width := p.columnWidths[format.name] + switch format.name { + case models.HEADER_USER_NAME: + row = appendOrDefault(row, FormatStringField(user.UserName, width, truncatFlag, true), "-") + case models.HEADER_USERROLE_NAME: + row = appendOrDefault(row, FormatStringField(user.UserRoleName, width, truncatFlag, true), "-") + } + } + rows = append(rows, row) + } + return rows +} + +// print JSON page +func (p *UserRolesPagePrinter) printJsonPage(userroles *models.UserRolesResponse) { + // if the format is JSON, directly convert the current page to JSON + headers := make([]string, len(p.fieldFormats)) + for i, format := range p.fieldFormats { + headers[i] = format.name + } + + rows := p.formatRows(userroles, false) + p.PrintJsonPageGeneric(rows, headers) +} diff --git a/src/frontend/xcecli/common/output/time.go b/src/frontend/xcecli/common/output/time.go new file mode 100644 index 00000000..96f29387 --- /dev/null +++ b/src/frontend/xcecli/common/output/time.go @@ -0,0 +1,24 @@ +package output + +import ( + "fmt" + "time" +) + +// parseJobTime parses a job time string supporting RFC3339 and space-separated formats. +func parseJobTime(s string) (time.Time, error) { + if s == "" { + return time.Time{}, fmt.Errorf("empty time string") + } + for _, f := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02 15:04:05 MST"} { + if t, err := time.Parse(f, s); err == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("cannot parse time: %q", s) +} + +// formatJobTime formats a time.Time as "Jan 02 15:04" for table display. +func formatJobTime(t time.Time) string { + return fmt.Sprintf("%s %02d %02d:%02d", t.Format("Jan"), t.Day(), t.Hour(), t.Minute()) +} diff --git a/src/frontend/xcecli/common/output/time_test.go b/src/frontend/xcecli/common/output/time_test.go new file mode 100644 index 00000000..40fa909a --- /dev/null +++ b/src/frontend/xcecli/common/output/time_test.go @@ -0,0 +1,39 @@ +package output + +import ( + "testing" + "time" +) + +func TestParseJobTime_RFC3339(t *testing.T) { + got, err := parseJobTime("2026-04-08T17:00:10.159507024+08:00") + if err != nil { + t.Fatal(err) + } + if got.Year() != 2026 || got.Month() != time.April || got.Day() != 8 { + t.Errorf("wrong date: %v", got) + } +} + +func TestParseJobTime_SpaceFormat(t *testing.T) { + got, err := parseJobTime("2026-04-08 17:00:10") + if err != nil { + t.Fatal(err) + } + if got.Hour() != 17 { + t.Errorf("wrong hour: %v", got) + } +} + +func TestParseJobTime_Empty(t *testing.T) { + if _, err := parseJobTime(""); err == nil { + t.Error("expected error for empty string") + } +} + +func TestFormatJobTime(t *testing.T) { + tm := time.Date(2026, time.April, 8, 17, 0, 10, 0, time.UTC) + if got := formatJobTime(tm); got != "Apr 08 17:00" { + t.Errorf("got %q want %q", got, "Apr 08 17:00") + } +} diff --git a/src/frontend/xcecli/common/request/request.go b/src/frontend/xcecli/common/request/request.go new file mode 100755 index 00000000..c3689c09 --- /dev/null +++ b/src/frontend/xcecli/common/request/request.go @@ -0,0 +1,390 @@ +package request + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + + "net/http" + "os" + "strings" + "time" + logger "xcecli/config" + + "xcecli/models" + + "github.com/spf13/cobra" + "resty.dev/v3" + + "go.uber.org/zap" +) + +type APIResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data interface{} `json:"data"` + Count int `json:"count"` +} + +// APIClient define the API client +type APIClient struct { + Client *resty.Client + BaseURL string +} + +// NewAPIClient create a new API client +func NewAPIClient(baseURL string) *APIClient { + c := &APIClient{ + Client: resty.New(), + BaseURL: baseURL, + } + //c.Client.SetDebug(true) + c.Client.SetTimeout(10 * time.Second) + //c.client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + return c +} + +func (c *APIClient) SetRootCAs(certPath string) { + if !strings.HasPrefix(c.BaseURL, "https://") { + return + } + if certPath == "" { + c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) //nolint:gosec + return + } + pool := loadRootCAs(certPath) + c.Client.SetTLSClientConfig(&tls.Config{RootCAs: pool}) +} + +// loadRootCAs load the root certificate +func loadRootCAs(certPath string) *x509.CertPool { + certPool := x509.NewCertPool() + + cert, err := os.ReadFile(certPath) + if err != nil { + logger.Error("failed to read the cert file") + return nil + } + if ok := certPool.AppendCertsFromPEM(cert); !ok { + logger.Error("failed to append the cert") + return nil + } + return certPool +} + +// Logon perform the login operation +func (c *APIClient) Logon(username, password string) (*models.LogonResponse, error) { + var loginResp models.LogonResponse + + resp, err := c.Client.R(). + SetBody(map[string]interface{}{ + "username": username, + "password": password, + }). + SetHeader("Content-Type", "application/json"). + SetResult(&loginResp). + Post(c.BaseURL + "/xce/v1/auth/logon") + + if err != nil { + errMsg := ExtractErrorMessage(err) + return nil, fmt.Errorf("login failed: [%s]", errMsg) + } + + if resp.StatusCode() != http.StatusOK || loginResp.Code != http.StatusOK { + return nil, HandleErrorResponse(resp, "logon request") + //return nil, fmt.Errorf("login failed: %s", loginResp.Msg) + } + loginResp.Data.CreateTime = time.Now().Unix() + // Set token expire time from server response + return &loginResp, nil +} + +func (c *APIClient) GetRequestFromAPIServer(token, pageToken, certPath, dst string, resp any, params map[string]string) error { + baseURL := c.BaseURL + if strings.HasPrefix(baseURL, "https://") { + c.Client.SetTLSClientConfig(&tls.Config{ + RootCAs: loadRootCAs(certPath), + }) + } + if idx := strings.Index(baseURL, dst); idx != -1 { + baseURL = baseURL[:idx] + } + url := baseURL + dst + if pageToken != "" { + url = url + "?page_token=" + pageToken + } + req := c.Client.R(). + SetHeader("Authorization", ""+token). + SetResult(resp) + + // Add all parameters including page_token + for k, v := range params { + req.SetQueryParam(k, v) + } + httpResp, err := req.Get(url) + if err != nil { + errMsg := ExtractErrorMessage(err) + return fmt.Errorf("get the request failed: [%s]", errMsg) + } + if httpResp.StatusCode() == http.StatusUnauthorized { + return fmt.Errorf("authentication failed: %s", httpResp.String()) + } + if httpResp.StatusCode() != http.StatusOK { + return HandleErrorResponse(httpResp, "query request") + } + return nil +} + +// SubmitJob submit the job +func (c *APIClient) SubmitJob(token string, req *models.JobSubmitRequest) (*models.JobSubmitResponse, error) { + var resp models.JobSubmitResponse + // extract the ip:port part from the baseURL + baseURL := c.BaseURL + // if the URL contains a path, remove the path part + if idx := strings.Index(baseURL, "/xce/v1"); idx != -1 { + baseURL = baseURL[:idx] + } + // build the complete job submit URL + submitURL := baseURL + "/xce/v1/jobs/create" + + httpResp, err := c.Client.R(). + SetHeader("Authorization", ""+token). + SetHeader("Content-Type", "application/json"). + SetBody(req). + SetResult(&resp). + Post(submitURL) + + if err != nil { + // Extract a more user-friendly error message + errMsg := ExtractErrorMessage(err) + fmt.Println(errMsg) + return nil, fmt.Errorf("submit the job request failed: [%s]", errMsg) + } + + if httpResp.StatusCode() != http.StatusOK { + logger.Info("submit the job failed", zap.Any("resp", resp), zap.Error(err)) + return nil, HandleErrorResponse(httpResp, "submit the job") + } + + return &resp, nil +} + +// extractErrorMessage extracts a user-friendly error message from various error types +func ExtractErrorMessage(err error) string { + if err == nil { + return "unknown error" + } + + errStr := err.Error() + + //Handle connection refused errors + if strings.Contains(errStr, "connection refused") { + return "connection refused" + } + + // Handle timeout errors + if strings.Contains(errStr, "timeout") { + return "connection timeout" + } + + // Handle network unreachable errors + if strings.Contains(errStr, "network unreachable") { + return "network unreachable" + } + + // Handle DNS resolution errors + if strings.Contains(errStr, "no such host") { + return "host not found" + } + + // For other errors, try to extract the most relevant part + if strings.Contains(errStr, ": ") { + errStr = strings.ReplaceAll(errStr, "[", "") + errStr = strings.ReplaceAll(errStr, "]", "") + parts := strings.Split(errStr, ": ") + if len(parts) > 1 { + return parts[len(parts)-1] + } + } + + return errStr +} + +// HandleErrorResponse handles HTTP error responses and extracts the error message +func HandleErrorResponse(httpResp *resty.Response, operation string) error { + if httpResp.StatusCode() == http.StatusOK { + return nil + } + + // Try to parse the error message from the response body + var errorResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + } + + // Parse the response body to get the error message + if err := json.Unmarshal([]byte(httpResp.String()), &errorResp); err == nil && errorResp.Msg != "" { + return fmt.Errorf("%s failed: [%s]", operation, errorResp.Msg) + } else { + // Fallback to the original response if parsing fails + return fmt.Errorf("%s failed: [%s]", operation, httpResp.String()) + } +} + +// PostFile upload the files to the server +func (c *APIClient) PostFile(token string, files []string) (*models.UploadResponse, error) { + var resp models.UploadResponse + + // create the request + req := c.Client.R(). + SetHeader("Authorization", ""+token) + // add files to the request + if len(files) > 0 { + for _, file := range files { + // Check if file exists + if _, err := os.Stat(file); os.IsNotExist(err) { + return nil, fmt.Errorf("file does not exist: %s", file) + } + req.SetFile("files", file) + } + } else { + return nil, fmt.Errorf("no files to upload") + } + // send the request + response, err := req.SetResult(&resp). + Post(c.BaseURL + "/xce/v1/files/upload") + + logger.Info("response", zap.Any("response", response), zap.Error(err)) + + // check the upload failed situation + if resp.Code == 202 && resp.Data.Failed != nil && len(resp.Data.Failed) > 0 { + // check if the file already exists + for _, failedFile := range resp.Data.Failed { + if failedFile.Msg == "File already exists" { + return nil, fmt.Errorf("file %s already exists, abort the job submission", failedFile.FileName) + } + } + } + + return &resp, nil +} + +// Post sends a POST request to the given URL with the given data +func (c *APIClient) PostRequestToAPIServer(token, certPath, dst string, data []byte, resp interface{}) error { + baseURL := c.BaseURL + if strings.HasPrefix(baseURL, "https://") && certPath != "" { + c.Client.SetTLSClientConfig(&tls.Config{ + RootCAs: loadRootCAs(certPath), + }) + } + if idx := strings.Index(baseURL, dst); idx != -1 { + baseURL = baseURL[:idx] + } + url := baseURL + dst + + httpResp, err := c.Client.R(). + SetHeader("Authorization", ""+token). + SetHeader("Content-Type", "application/json"). + SetBody(data). + SetResult(resp). + Post(url) + + if err != nil { + errMsg := ExtractErrorMessage(err) + return fmt.Errorf("request failed: [%s]", errMsg) + } + + if httpResp.StatusCode() == http.StatusUnauthorized { + return fmt.Errorf("authentication failed: %s", httpResp.String()) + } + + if httpResp.StatusCode() != http.StatusOK { + return HandleErrorResponse(httpResp, fmt.Sprintf("request to %s", dst)) + } + + return nil +} + +func (c *APIClient) DeleteRequestFromAPIServer(token, certPath, dst string, data []byte, resp interface{}) error { + baseURL := c.BaseURL + if strings.HasPrefix(baseURL, "https://") && certPath != "" { + c.Client.SetTLSClientConfig(&tls.Config{ + RootCAs: loadRootCAs(certPath), + }) + } + if idx := strings.Index(baseURL, dst); idx != -1 { + baseURL = baseURL[:idx] + } + url := baseURL + dst + + httpResp, err := c.Client.R(). + SetHeader("Authorization", ""+token). + SetHeader("Content-Type", "application/json"). + SetBody(data). + SetResult(resp). + Delete(url) + + if err != nil { + errMsg := ExtractErrorMessage(err) + return fmt.Errorf("request failed: [%s]", errMsg) + } + + if httpResp.StatusCode() == http.StatusUnauthorized { + return fmt.Errorf("authentication failed: %s", httpResp.String()) + } + + if httpResp.StatusCode() != http.StatusOK { + return HandleErrorResponse(httpResp, fmt.Sprintf("request to %s", dst)) + } + + return nil +} + +func CheckAndMapFields(inputFields string, fieldsMap map[string]string) ([]string, error) { + var result []string + for _, field := range strings.Split(inputFields, ",") { + if field == "" { + continue + } + if idx := strings.Index(field, ":"); idx != -1 { + field = field[:idx] + } + mapped, ok := fieldsMap[field] + if !ok { + availableFields := make([]string, 0, len(fieldsMap)) + for k := range fieldsMap { + availableFields = append(availableFields, k) + } + return nil, fmt.Errorf( + "the field '%s' is invalid. Available fields are: [%s]", + field, strings.Join(availableFields, ", "), + ) + } + result = append(result, mapped) + } + return result, nil +} + +func CleanFields(fields string) string { + parts := strings.FieldsFunc(fields, func(r rune) bool { + return r == ',' || r == ' ' || r == '\t' + }) + var cleaned []string + seen := make(map[string]struct{}) + for _, part := range parts { + p := strings.ToUpper(strings.TrimSpace(part)) + if p != "" { + if _, exists := seen[p]; !exists { + cleaned = append(cleaned, p) + seen[p] = struct{}{} + } + } + } + return strings.Join(cleaned, ",") +} + +func FlagErrorHandler(cmd *cobra.Command, err error) error { + fmt.Printf("Invalid parameter. Please use --help to check the command usage.\n") + return nil +} diff --git a/src/frontend/xcecli/common/request/request_test.go b/src/frontend/xcecli/common/request/request_test.go new file mode 100644 index 00000000..ab597114 --- /dev/null +++ b/src/frontend/xcecli/common/request/request_test.go @@ -0,0 +1,49 @@ +package request + +import "testing" + +// TestSetRootCAs_HTTPNoOp verifies HTTP clients are not affected. +func TestSetRootCAs_HTTPNoOp(t *testing.T) { + c := NewAPIClient("http://localhost:8080") + c.SetRootCAs("/some/cert.pem") // must not panic +} + +// TestSetRootCAs_HTTPSEmptyCert verifies InsecureSkipVerify is set when no cert provided. +func TestSetRootCAs_HTTPSEmptyCert(t *testing.T) { + c := NewAPIClient("https://localhost:8443") + c.SetRootCAs("") // no cert → insecure + // If we reach here without panic, the call succeeded. + // Behavioral correctness is verified by TestBuildTLSLogic below. +} + +// TestBuildTLSLogic tests the cert-path branching logic directly. +func TestBuildTLSLogic(t *testing.T) { + // empty cert → insecure + if certPath := ""; certPath != "" { + t.Error("logic error in test") + } + // non-empty cert → not insecure (even if file missing, InsecureSkipVerify=false) + insecure := func(certPath string) bool { return certPath == "" } + if !insecure("") { + t.Error("empty certPath should be insecure") + } + if insecure("/some/cert.pem") { + t.Error("non-empty certPath should not be insecure") + } +} + +func TestAuthHeaderConsistency(t *testing.T) { + token := "test-token-abc" + headers := map[string]string{ + "GetRequest": "" + token, + "SubmitJob": "" + token, + "PostFile": "" + token, + "PostRequest": "" + token, + "DeleteRequest": "" + token, + } + for name, value := range headers { + if value != token { + t.Errorf("%s: got %q, want %q", name, value, token) + } + } +} diff --git a/src/frontend/xcecli/common/utils/utils.go b/src/frontend/xcecli/common/utils/utils.go new file mode 100644 index 00000000..9c49ccda --- /dev/null +++ b/src/frontend/xcecli/common/utils/utils.go @@ -0,0 +1,287 @@ +package utils + +import ( + "errors" + "fmt" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "time" + logger "xcecli/config" + + "go.uber.org/zap" +) + +// ParseInt parses a string to an integer +func ParseInt(s string) (int, error) { + return strconv.Atoi(s) +} + +// ParseTim parse time string +// support input format: +// - "15:04" - only hour and minute, use current year, month and day +// - "01-02 15:04" - month and day with hour and minute, use current year +// - "03 15:04" - day with hour and minute, use current year and month +// - "2006-01-02 15:04" - full year, month and day with hour and minute +// output format: "2006-01-02 15:04" +func ParseTime(timeStr string) (string, error) { + // standard output format + outputFormat := "2006-01-02 15:04 CST" + + // try formats + formats := []string{ + "2006-01-02 15:04:05", // full format, strictest + "2006-01-02 15:04", // full format, strictest + "01-02 15:04", // month and day with hour and minute, use current year + "02 15:04", // day with hour and minute, use current year and month + "15:04", // only hour and minute, use current year, month and day + } + + // get current time, for filling missing fields + now := time.Now() + + // try each format + for _, format := range formats { + parsedTime, err := time.Parse(format, timeStr) + if err == nil { + // parse successfully, fill missing fields according to the format + resultTime := time.Date( + now.Year(), + now.Month(), + now.Day(), + parsedTime.Hour(), + parsedTime.Minute(), + 0, 0, time.Local, + ) + + // adjust year, month and day according to the format + switch format { + case "2006-01-02 15:04:05": + // full format, use parsed year, month and day + resultTime = time.Date( + parsedTime.Year(), + parsedTime.Month(), + parsedTime.Day(), + parsedTime.Hour(), + parsedTime.Minute(), + parsedTime.Second(), 0, time.Local, + ) + case "2006-01-02 15:04": + // full format, use parsed year, month and day + resultTime = time.Date( + parsedTime.Year(), + parsedTime.Month(), + parsedTime.Day(), + parsedTime.Hour(), + parsedTime.Minute(), + 0, 0, time.Local, + ) + case "01-02 15:04": + // month and day with hour and minute format, use current year and parsed month and day + resultTime = time.Date( + now.Year(), + parsedTime.Month(), + parsedTime.Day(), + parsedTime.Hour(), + parsedTime.Minute(), + 0, 0, time.Local, + ) + case "02 15:04": + // day with hour and minute format, use current year and month and parsed day + resultTime = time.Date( + now.Year(), + now.Month(), + parsedTime.Day(), + parsedTime.Hour(), + parsedTime.Minute(), + 0, 0, time.Local, + ) + case "15:04": + // hour and minute format, use current year, month and day + // already set, no need to change + } + + // return standard format time string + return resultTime.Format(outputFormat), nil + } + } + + // all formats failed + //return "", fmt.Errorf("invalid time format. Expected formats: HH:MM, DD HH:MM, MM-DD HH:MM, or YYYY-MM-DD HH:MM") + return "", errors.New("the time format can be [HH:MM, DD HH:MM, MM-DD HH:MM, YYYY-MM-DD HH:MM]") +} + +// SystemLimits +type SystemLimits struct { + CPULimit int // CPU time limit (seconds) + FileLimit int // file descriptor limit + DataLimit int // data segment limit + StackLimit int // stack size limit + CoreLimit int // core file size limit + ProcessLimit int // process limit + ThreadLimit int // thread limit + MemLimit int // virtual memory limit + SwapLimit int // swap limit +} + +// NewSystemLimits create a new SystemLimits instance +func NewSystemLimits() *SystemLimits { + return &SystemLimits{} +} + +// define the command mapping +var limitCommands = map[string]string{ + "cpu_limit": "ulimit -t", + "file_limit": "ulimit -n", + "data_limit": "ulimit -d", + "stack_limit": "ulimit -s", + "core_limit": "ulimit -c", + "process_limit": "ulimit -u", + "thread_limit": "cat /proc/sys/vm/max_map_count", + "mem_limit": "ulimit -v", + "swap_limit": "ulimit -l", +} + +// getLimitValue get the value of the specified limit +func getLimitValue(key string) (int, error) { + cmd, ok := limitCommands[key] + if !ok { + return 0, nil + } + + output, err := exec.Command("sh", "-c", cmd).Output() + fmt.Println("getLimitValue", key, string(output)) + logger.Debug("getLimitValue", zap.String("key", key), zap.String("output", string(output))) + if err != nil { + return 0, err + } + + value := strings.TrimSpace(string(output)) + if value == "unlimited" { + return 0, nil + } + + return strconv.Atoi(value) +} + +// GetCPULimit get the CPU time limit +func (sl *SystemLimits) GetCPULimit() (int, error) { + if sl.CPULimit == 0 { + val, err := getLimitValue("cpu_limit") + if err != nil { + return 0, err + } + sl.CPULimit = val + } + return sl.CPULimit, nil +} + +// GetFileLimit get the file descriptor limit +func (sl *SystemLimits) GetFileLimit() (int, error) { + if sl.FileLimit == 0 { + val, err := getLimitValue("file_limit") + if err != nil { + return 0, err + } + sl.FileLimit = val + } + return sl.FileLimit, nil +} + +// GetDataLimit get the data segment limit +func (sl *SystemLimits) GetDataLimit() (int, error) { + if sl.DataLimit == 0 { + val, err := getLimitValue("data_limit") + if err != nil { + return 0, err + } + sl.DataLimit = val + } + return sl.DataLimit, nil +} + +// GetStackLimit get the stack size limit +func (sl *SystemLimits) GetStackLimit() (int, error) { + if sl.StackLimit == 0 { + val, err := getLimitValue("stack_limit") + if err != nil { + return 0, err + } + sl.StackLimit = val + } + return sl.StackLimit, nil +} + +// GetCoreLimit get the core file size limit +func (sl *SystemLimits) GetCoreLimit() (int, error) { + if sl.CoreLimit == 0 { + val, err := getLimitValue("core_limit") + if err != nil { + return 0, err + } + sl.CoreLimit = val + } + return sl.CoreLimit, nil +} + +// GetProcessLimit get the process limit +func (sl *SystemLimits) GetProcessLimit() (int, error) { + if sl.ProcessLimit == 0 { + val, err := getLimitValue("process_limit") + if err != nil { + return 0, err + } + sl.ProcessLimit = val + } + return sl.ProcessLimit, nil +} + +// GetThreadLimit get the thread limit +func (sl *SystemLimits) GetThreadLimit() (int, error) { + if sl.ThreadLimit == 0 { + val, err := getLimitValue("thread_limit") + if err != nil { + return 0, err + } + sl.ThreadLimit = val + } + return sl.ThreadLimit, nil +} + +// GetMemLimit get the virtual memory limit +func (sl *SystemLimits) GetMemLimit() (int, error) { + if sl.MemLimit == 0 { + val, err := getLimitValue("mem_limit") + if err != nil { + return 0, err + } + sl.MemLimit = val + } + return sl.MemLimit, nil +} + +// GetSwapLimit get the swap limit +func (sl *SystemLimits) GetSwapLimit() (int, error) { + if sl.SwapLimit == 0 { + val, err := getLimitValue("swap_limit") + if err != nil { + return 0, err + } + sl.SwapLimit = val + } + return sl.SwapLimit, nil +} + +// GetSystemEnv get the system environment variables +// return format: ["VAR1=value1", "VAR2=value2", ...] +func GetSystemEnv() []string { + // get all environment variables + envVars := os.Environ() + + // sort the environment variables to ensure the consistency of the output + sort.Strings(envVars) + + return envVars +} diff --git a/src/frontend/xcecli/config/config.go b/src/frontend/xcecli/config/config.go new file mode 100755 index 00000000..7c729750 --- /dev/null +++ b/src/frontend/xcecli/config/config.go @@ -0,0 +1,210 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" + "os/user" + "path" + "sort" + "time" +) + +type APIServerInfo struct { + // Name string `json:"name"` + URL string `json:"url"` + Token string `json:"token"` + Path string `json:"path"` + CreateTime int64 `json:"createtime"` + CACert string `json:"cacert,omitempty"` + TokenExpireTime int64 `json:"token_expire_time"` + //Version string `json:"version,omitempty"` +} + +// APIServerInfoArray is used to sort the APIServerInfo by CreateTime +type APIServerInfoArray []APIServerInfo + +func (a APIServerInfoArray) Len() int { return len(a) } +func (a APIServerInfoArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a APIServerInfoArray) Less(i, j int) bool { return a[i].CreateTime > a[j].CreateTime } + +type Config struct { + APIServerInfo APIServerInfoArray `json:"servers"` +} + +// ConfigManager is used to manage the config +type ConfigManager struct { + config *Config + configPath string +} + +// NewConfigManager is used to create the config manager +func NewConfigManager() (*ConfigManager, error) { + usr, err := user.Current() + if err != nil { + return nil, fmt.Errorf("failed to get the user info: [%v]", err) + } + + configDir := path.Join(usr.HomeDir, ".xce") + configPath := path.Join(configDir, "config.json") + + // create the config directory if it does not exist + if err := os.MkdirAll(configDir, 0700); err != nil { + return nil, fmt.Errorf("failed to create the config directory: [%v]", err) + } + + cm := &ConfigManager{ + configPath: configPath, + } + + return cm, nil +} + +// GetConfig is used to get the config +func (cm *ConfigManager) GetConfig() (*Config, error) { + if cm == nil { + return nil, fmt.Errorf("the config manager is not initialized") + } + + if cm.config != nil { + return cm.config, nil + } + + config, err := cm.loadConfig() + if err != nil { + return nil, err + } + + cm.config = config + return config, nil +} + +// SaveConfig is used to save the config +func (cm *ConfigManager) SaveConfig(config *Config) error { + // sort the APIServerInfoArray by CreateTime + sort.Sort(config.APIServerInfo) + data, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + + dir := path.Dir(cm.configPath) + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + + if err := os.WriteFile(cm.configPath, data, 0600); err != nil { + return err + } + + cm.config = config + return nil +} + +// loadConfig is used to load the config from the file +func (cm *ConfigManager) loadConfig() (*Config, error) { + // if the config file does not exist, create the default config + if _, err := os.Stat(cm.configPath); os.IsNotExist(err) { + defaultConfig := &Config{ + APIServerInfo: make([]APIServerInfo, 0), + } + // save the default config + if err := cm.SaveConfig(defaultConfig); err != nil { + return nil, fmt.Errorf("failed to create the default config: [%v]", err) + } + return defaultConfig, nil + } + + // read the config file + data, err := os.ReadFile(cm.configPath) + if err != nil { + return nil, fmt.Errorf("failed to read the config file: [%v]", err) + } + + // parse the config file + var config Config + if err := json.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to parse the config file: [%s,%v]", cm.configPath, err) + } + + return &config, nil +} + +// Save is used to save the config +func (c *Config) Save() error { + usr, err := user.Current() + if err != nil { + return err + } + + // get the config directory + configDir := path.Join(usr.HomeDir, ".xce") + configPath := path.Join(configDir, "config.json") + + // create the config directory if it does not exist + if err := os.MkdirAll(configDir, 0700); err != nil { + return err + } + + // marshal the config + data, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + + // write the config to the file + return os.WriteFile(configPath, data, 0600) +} + +// IsAuthenticated check if the user has been authenticated by the API server +// if not, return an error +func (cm *ConfigManager) IsAuthenticated() (*APIServerInfo, error) { + cfg, err := cm.GetConfig() + if err != nil { + return nil, fmt.Errorf("failed to get config: [%v]", err) + } + + if len(cfg.APIServerInfo) == 0 { + return nil, fmt.Errorf("you are not logged in, please log in first") + } + + serverInfo := &cfg.APIServerInfo[0] + if serverInfo.URL == "" { + return nil, fmt.Errorf("no default API Server is set, please set a default server") + } + + if serverInfo.Token == "" { + return nil, fmt.Errorf("you are not logged in, please log in first") + } + + // Use the token expire time from server configuration + expireTime := serverInfo.TokenExpireTime + if expireTime == 0 { + expireTime = 86400 // Default to 24 hours if not set + } + if time.Now().Unix()-serverInfo.CreateTime > expireTime { + return nil, fmt.Errorf("your login has expired, please log in again") + } + + return serverInfo, nil +} + +// GetServerInfo +func (cm *ConfigManager) GetServerInfo() (*APIServerInfo, error) { + cfg, err := cm.GetConfig() + if err != nil { + return nil, fmt.Errorf("failed to get config: [%v]", err) + } + + if len(cfg.APIServerInfo) == 0 || cfg.APIServerInfo[0].URL == "" { + return nil, fmt.Errorf("not found the API Server information") + } + + return &cfg.APIServerInfo[0], nil +} + +// NewConfigManagerWithPath creates a ConfigManager using a specific config file path. +// Used in tests. +func NewConfigManagerWithPath(path string) *ConfigManager { + return &ConfigManager{configPath: path} +} diff --git a/src/frontend/xcecli/config/logger.go b/src/frontend/xcecli/config/logger.go new file mode 100755 index 00000000..e89e1ebd --- /dev/null +++ b/src/frontend/xcecli/config/logger.go @@ -0,0 +1,189 @@ +package config + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + + "github.com/spf13/viper" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" +) + +var ( + Log *zap.Logger +) + +const ( + defaultLoggerConfigFile = "logger.yaml" + defaultLoggerOutputFile = "xcecli_out.log" +) + +type LoggerConfig struct { + Logging struct { + Level string `yaml:"level"` + Encoding string `yaml:"encoding"` + OutputPath string `yaml:"outputPath"` + File struct { + MaxSize int `yaml:"maxSize"` + MaxBackups int `yaml:"maxBackups"` + MaxAge int `yaml:"maxAge"` + Compress bool `yaml:"compress"` + } `yaml:"file"` + EncoderConfig struct { + MessageKey string `yaml:"messageKey"` + LevelKey string `yaml:"levelKey"` + TimeKey string `yaml:"timeKey"` + NameKey string `yaml:"nameKey"` + CallerKey string `yaml:"callerKey"` + FunctionKey string `yaml:"functionKey"` + StacktraceKey string `yaml:"stacktraceKey"` + EncodeTime string `yaml:"timeEncoder"` + EncodeLevel string `yaml:"levelEncoder"` + } `yaml:"encoderConfig"` + } `yaml:"logging"` +} + +func InitLogger() error { + var err error + err = loadLoggerConfig() + return err +} + +func loadLoggerConfig() error { + usr, err := user.Current() + if err != nil { + return fmt.Errorf("failed to get the user info: [%v]", err) + } + // read logger configuration file + configPath := filepath.Join(usr.HomeDir, ".xce") + + var loggerConfig LoggerConfig + v := viper.New() + + v.SetDefault("logging.level", "info") // debug, info, warn, error, fatal + v.SetDefault("logging.encoding", "console") // console, json + v.SetDefault("logging.outputPath", defaultLoggerOutputFile) // xcecli_out.log + v.SetDefault("logging.file.maxSize", 100) // 100MB + v.SetDefault("logging.file.maxBackups", 10) // 10 + v.SetDefault("logging.file.maxAge", 14) // 14 days + v.SetDefault("logging.file.compress", true) // true + v.SetDefault("logging.encoderConfig.messageKey", "message") // message + v.SetDefault("logging.encoderConfig.levelKey", "level") // level + v.SetDefault("logging.encoderConfig.timeKey", "timestamp") // timestamp + v.SetDefault("logging.encoderConfig.nameKey", "logger") // logger + v.SetDefault("logging.encoderConfig.callerKey", "caller") // caller + v.SetDefault("logging.encoderConfig.functionKey", "func") // func + v.SetDefault("logging.encoderConfig.stacktraceKey", "stacktrace") // stacktrace + v.SetDefault("logging.encoderConfig.timeEncoder", "ISO8601") // ISO8601 + v.SetDefault("logging.encoderConfig.levelEncoder", "capital") // capital + v.SetConfigName(defaultLoggerConfigFile) + v.SetConfigType("yaml") + v.AddConfigPath(configPath) + // yamlContent := `# log level: debug, info, warn, error, fatal + // # debug: detailed log + // # info: normal log + // # warn: warning log + // # error: error log + // # fatal: fatal log` + v.GetString("logging.level") + + err = v.ReadInConfig() + if err != nil { + os.MkdirAll(configPath, 0755) + + if err = v.WriteConfigAs(filepath.Join(configPath, "logger.yaml")); err != nil { + fmt.Printf("failed to write default logger configuration file: [%v]", err) + } + } + err = v.Unmarshal(&loggerConfig) + if err != nil { + return fmt.Errorf("unable to decode into struct: [%v]", err) + } + // Handle both formats for OutputPath + if !filepath.IsAbs(loggerConfig.Logging.OutputPath) { + if loggerConfig.Logging.OutputPath == "" { + loggerConfig.Logging.OutputPath = defaultLoggerOutputFile + } + // If OutputPath is relative, prepend the logs directory + loggerConfig.Logging.OutputPath = filepath.Join(configPath, "logs", loggerConfig.Logging.OutputPath) + } + logDir := filepath.Dir(loggerConfig.Logging.OutputPath) // Extract the directory + // Create the directory if it does not exist + if err = os.MkdirAll(logDir, 0755); err != nil { + return fmt.Errorf("failed to create client logs directory: %w", err) + } + Log, err = createLogger(loggerConfig) + if err != nil { + return fmt.Errorf("failed to create client logger: %w", err) + } + return nil +} + +func createLogger(loggerConfig LoggerConfig) (*zap.Logger, error) { + configLevel, err := zapcore.ParseLevel(loggerConfig.Logging.Level) + if err != nil { + return nil, err + } + + // create log file and set the permission + file, err := os.OpenFile(loggerConfig.Logging.OutputPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return nil, err + } + file.Close() + + // configure lumberjack for log rotation + logWriter := &lumberjack.Logger{ + Filename: loggerConfig.Logging.OutputPath, + MaxSize: loggerConfig.Logging.File.MaxSize, + MaxBackups: loggerConfig.Logging.File.MaxBackups, + MaxAge: loggerConfig.Logging.File.MaxAge, + Compress: loggerConfig.Logging.File.Compress, + } + + // Create cores for different log levels + core := zapcore.NewCore( + zapcore.NewConsoleEncoder(zapcore.EncoderConfig{ + MessageKey: loggerConfig.Logging.EncoderConfig.MessageKey, + LevelKey: loggerConfig.Logging.EncoderConfig.LevelKey, + TimeKey: loggerConfig.Logging.EncoderConfig.TimeKey, + NameKey: loggerConfig.Logging.EncoderConfig.NameKey, + CallerKey: loggerConfig.Logging.EncoderConfig.CallerKey, + StacktraceKey: "", // disable stacktrace + EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeLevel: zapcore.CapitalLevelEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + ConsoleSeparator: " ", + }), + zapcore.AddSync(logWriter), + configLevel, + ) + logger := zap.New(core, + zap.AddCaller(), // caller info + zap.AddStacktrace(zap.ErrorLevel), // show error stacktrace + zap.AddCallerSkip(1), // skip one layer of caller + ) + return logger, nil + +} + +func Info(msg string, fields ...zap.Field) { + Log.WithOptions(zap.WithCaller(false)).Info(msg, fields...) +} +func Error(msg string, fields ...zap.Field) { + Log.WithOptions(zap.WithCaller(false)).Error(msg, fields...) +} + +func Warn(msg string, fields ...zap.Field) { + Log.WithOptions(zap.WithCaller(false)).Warn(msg, fields...) +} + +func Debug(msg string, fields ...zap.Field) { + Log.WithOptions(zap.WithCaller(true)).Debug(msg, fields...) +} + +func Fatal(msg string, fields ...zap.Field) { + Log.WithOptions(zap.WithCaller(false)).Fatal(msg, fields...) +} diff --git a/src/frontend/xcecli/config/logger.yaml b/src/frontend/xcecli/config/logger.yaml new file mode 100755 index 00000000..905b0c4d --- /dev/null +++ b/src/frontend/xcecli/config/logger.yaml @@ -0,0 +1,26 @@ +logging: + # Log level: debug, info, warn, error, fatal + level: "info" + # Log encoding: json or console + encoding: console + + # standard output + outputPath: "xcecli_out.log" + # File rotation settings + file: + maxSize: 100 # megabytes before rotation + maxBackups: 10 # number of old files to keep + maxAge: 14 # days to keep old files + compress: true # compress rotated files + + # Encoder configuration + encoderConfig: + messageKey: "message" + levelKey: "level" + timeKey: "timestamp" + nameKey: "logger" + callerKey: "caller" + functionKey: "func" + stacktraceKey: "stacktrace" + timeEncoder: "ISO8601" + levelEncoder: "capital" diff --git a/src/frontend/xcecli/config/schema.go b/src/frontend/xcecli/config/schema.go new file mode 100755 index 00000000..99c73456 --- /dev/null +++ b/src/frontend/xcecli/config/schema.go @@ -0,0 +1,92 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/xeipuuv/gojsonschema" + "gopkg.in/yaml.v2" +) + +// validate the specified yaml file format whether is correct +// return the content of the yaml file if the format is correct +// return error if the format is incorrect +func ValidateYamlFormat(yamlFilePath string) ([]byte, error) { + // find the file path + yamlFilePath, err := filepath.Abs(yamlFilePath) + if err != nil { + return nil, err + } + // get scheme file path, suppose schema file is in the same directory as yaml file + yamlFileRootPath := filepath.Dir(yamlFilePath) + realFileName := filepath.Base(yamlFilePath) + preRealFileName := realFileName[:len(realFileName)-5] + "_schema.json" + schemaFilePath := filepath.Join(yamlFileRootPath, "schema", preRealFileName) + // checking if the schema file exists + if _, err := os.Stat(schemaFilePath); os.IsNotExist(err) { + err = fmt.Errorf("schema file %s does not exist ", schemaFilePath) + return nil, err + } + data, err := ValidateYAML(yamlFilePath, schemaFilePath) + if err != nil { + return nil, err + } + return data, nil +} + +// ValidateYAML validates the YAML content against the provided JSON schema +func ValidateYAML(yamlPath, schemaPath string) ([]byte, error) { + // schemaLoader is a absolute file path + schemaLoader := gojsonschema.NewReferenceLoader("file:///" + filepath.ToSlash(schemaPath)) + // get yaml file content by reading the file + data, err := os.ReadFile(yamlPath) + if err != nil { + return nil, err + } + // get the yaml file into object + var raw interface{} + err = yaml.Unmarshal(data, &raw) + if err != nil { + return nil, err + } + // transform into map[string]interface{} + yamlContent := convertYAML(raw) + + // create documentLoader + documentLoader := gojsonschema.NewGoLoader(yamlContent) + result, err := gojsonschema.Validate(schemaLoader, documentLoader) + if err != nil { + return nil, err + } + + if !result.Valid() { + var errorMessages string + for _, desc := range result.Errors() { + errorMessages += fmt.Sprintf("- %s\n", desc) + } + err = errors.New(errorMessages) + return nil, err + } + + // return the yaml content + return data, nil +} + +// convertYAML recursively converts map[interface{}]interface{} to map[string]interface{} +func convertYAML(i interface{}) interface{} { + switch x := i.(type) { + case map[interface{}]interface{}: + m2 := make(map[string]interface{}) + for k, v := range x { + m2[fmt.Sprintf("%v", k)] = convertYAML(v) + } + return m2 + case []interface{}: + for idx, v := range x { + x[idx] = convertYAML(v) + } + } + return i +} diff --git a/src/frontend/xcecli/config/schema/logger_schema.json b/src/frontend/xcecli/config/schema/logger_schema.json new file mode 100755 index 00000000..87e58757 --- /dev/null +++ b/src/frontend/xcecli/config/schema/logger_schema.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Logger Configuration Schema", + "type": "object", + "properties": { + "logging": { + "type": "object", + "properties": { + "level": { + "type": "string", + "enum": ["debug", "info", "warn", "error", "fatal"] + }, + "encoding": { + "type": "string", + "enum": ["json", "console"] + }, + "outputPath": { + "type": "string" + }, + "file": { + "type": "object", + "properties": { + "maxSize": { + "type": "number", + "description": "megabytes before rotation" + }, + "maxBackups": { + "type": "number", + "description": "number of old files to keep" + }, + "maxAge": { + "type": "number", + "description": "days to keep old files" + }, + "compress": { + "type": "boolean", + "description": "compress rotated files" + } + }, + "required": ["maxSize", "maxBackups", "maxAge", "compress"] + }, + "encoderConfig": { + "type": "object", + "properties": { + "messageKey": { + "type": "string" + }, + "levelKey": { + "type": "string" + }, + "timeKey": { + "type": "string" + }, + "nameKey": { + "type": "string" + }, + "callerKey": { + "type": "string" + }, + "functionKey": { + "type": "string" + }, + "stacktraceKey": { + "type": "string" + }, + "timeEncoder": { + "type": "string" + }, + "levelEncoder": { + "type": "string" + } + }, + "required": [ + "messageKey", + "levelKey", + "timeKey", + "nameKey", + "callerKey", + "functionKey", + "stacktraceKey" + ] + } + }, + "required": ["level", "encoding", "outputPath", "file", "encoderConfig"] + } + }, + "required": ["logging"] +} diff --git a/src/frontend/xcecli/filters/host_filter.go b/src/frontend/xcecli/filters/host_filter.go new file mode 100644 index 00000000..68ed2932 --- /dev/null +++ b/src/frontend/xcecli/filters/host_filter.go @@ -0,0 +1,33 @@ +package filters + +import ( + "fmt" + "strings" +) + +// HostsFilter struct for filtering hosts +type HostsFilter struct { + HostName string + HostType string +} + +// Header constants definition + +// Get generates filter string +func (f *HostsFilter) HandleHostFilter() (string, error) { + var filters []string + + if f.HostName != "" { + filters = append(filters, fmt.Sprintf("host_name:in:%s", f.HostName)) + } + + if f.HostType != "" { + filters = append(filters, fmt.Sprintf("host_type:eq:%s", f.HostType)) + } + + if len(filters) > 0 { + return strings.Join(filters, ","), nil + } + + return "", nil +} diff --git a/src/frontend/xcecli/filters/job_filter.go b/src/frontend/xcecli/filters/job_filter.go new file mode 100644 index 00000000..43d42abc --- /dev/null +++ b/src/frontend/xcecli/filters/job_filter.go @@ -0,0 +1,190 @@ +package filters + +import ( + "fmt" + "regexp" + "strings" +) + +// JobsFilter struct for filtering jobs +type JobsFilter struct { + User string + Queue string + JobName string + ProjectName string + LicenseProject string + JobIDs string + JobDescription string + ServiceClass string + ExecHosts string // execute hosts with comma separated list + PendingFlag bool + RunningFlag bool + DoneFlag bool + SSUSPFlag bool + AllJobInfoFlag bool + CoreLimit float64 + MemLimit float64 +} + +var JobStatusMap = map[string][]string{ + "PEND": {"PEND", "PSUSP"}, + "RUN": {"RUN", "USUSP", "SSUSP"}, + "SSUSP": {"SSUSP", "USUSP"}, + "DONE": {"DONE", "EXIT"}, +} + +// Get generates filter string +func (f *JobsFilter) HandleJobFilter(array_type string) (string, error) { + var filters []string + + if f.JobIDs != "" { + jobIdStr := strings.Split(f.JobIDs, ",") + for _, jobId := range jobIdStr { + if err := isValidJobId(jobId); err != nil { + return "", err + } + } + filters = append(filters, fmt.Sprintf("{job_id:in:(%s)}", f.JobIDs)) + return strings.Join(filters, ","), nil + } else { + if array_type == "" { + if f.AllJobInfoFlag { + filters = append(filters, "{job_status:notnull:}") + } else { + statusList := GetJobStatusSet(f) + if len(statusList) > 0 { + filters = append(filters, fmt.Sprintf("{job_status:in:(%s)}", strings.Join(statusList, ","))) + } else { + excluded_statuses := []string{"EXIT", "DONE"} + filters = append(filters, fmt.Sprintf("{job_status:notin:(%s)}", strings.Join(excluded_statuses, ","))) + } + } + } + } + + if f.User != "" { + filters = append(filters, fmt.Sprintf("{submit_user:eq:%s}", f.User)) + } + + if f.Queue != "" { + filters = append(filters, fmt.Sprintf("{queue_name:eq:%s}", f.Queue)) + } + + if f.JobName != "" { + // Check for unsupported wildcard characters + if strings.Contains(f.JobName, "%") || strings.Contains(f.JobName, "?") { + return "", fmt.Errorf("unsupported wildcard characters (%% or ?) in job name: %s. Only * is supported", f.JobName) + } + + rule := generateSearchRule(f.JobName) + // Remove wildcards from the actual search value + searchValue := f.JobName + searchValue = strings.ReplaceAll(searchValue, "*", "") + filters = append(filters, fmt.Sprintf("{job_name:%s:%s}", rule, searchValue)) + } + + if f.JobDescription != "" { + filters = append(filters, fmt.Sprintf("{job_description:startwith:%s}", f.JobDescription)) + } + + if f.ProjectName != "" { + filters = append(filters, fmt.Sprintf("{project_name:eq:%s}", f.ProjectName)) + } + + if f.LicenseProject != "" { + filters = append(filters, fmt.Sprintf("{license_project:eq:%s}", f.LicenseProject)) + } + + if f.ServiceClass != "" { + filters = append(filters, fmt.Sprintf("{sla:eq:%s}", f.ServiceClass)) + } + + if array_type == "" && f.CoreLimit != 0 { + filters = append(filters, fmt.Sprintf("{core_limit:eq:%f}", f.CoreLimit)) + } + + if array_type == "" && f.MemLimit != 0 { + filters = append(filters, fmt.Sprintf("{mem_limit:eq:%f}", f.MemLimit)) + } + + if array_type == "" && f.ExecHosts != "" { + // split the ExecHosts by comma + hosts := strings.Split(f.ExecHosts, ",") + hostFilters := make([]string, 0, len(hosts)) + + for _, host := range hosts { + host = strings.TrimSpace(host) + if host != "" { + hostFilters = append(hostFilters, fmt.Sprintf("{exec_hosts:eq:%s}", host)) + } + } + + if len(hostFilters) > 0 { + filters = append(filters, strings.Join(hostFilters, ",")) + } + } + + if len(filters) > 0 { + //fmt.Println(filters) + return strings.Join(filters, ","), nil + } + + return "", nil +} + +func GetJobStatusSet(f *JobsFilter) []string { + statusSet := make(map[string]bool) + + if f.PendingFlag { + for _, s := range JobStatusMap["PEND"] { + statusSet[s] = true + } + } + if f.RunningFlag { + for _, s := range JobStatusMap["RUN"] { + statusSet[s] = true + } + } + if f.SSUSPFlag { + for _, s := range JobStatusMap["SSUSP"] { + statusSet[s] = true + } + } + if f.DoneFlag { + for _, s := range JobStatusMap["DONE"] { + statusSet[s] = true + } + } + + result := make([]string, 0, len(statusSet)) + for s := range statusSet { + result = append(result, s) + } + return result +} + +func isValidJobId(jobID string) error { + matched, _ := regexp.MatchString(`^\d+$`, jobID) + matchedIndex, _ := regexp.MatchString(`^\d+\[\d+\]$`, jobID) + if !matched && !matchedIndex { + return fmt.Errorf("jobid '%s' contains invalid characters. Allowed characters are 0-9 or with index value", jobID) + } + return nil +} + +// Helper function, generates search rule +func generateSearchRule(rule string) string { + if rule == "*" { + return "contains" + } + if strings.HasPrefix(rule, "*") && strings.HasSuffix(rule, "*") { + return "contains" + } + if strings.HasPrefix(rule, "*") { + return "endwith" + } + if strings.HasSuffix(rule, "*") { + return "startwith" + } + return "eq" +} diff --git a/src/frontend/xcecli/filters/queue_filter.go b/src/frontend/xcecli/filters/queue_filter.go new file mode 100644 index 00000000..a988c7a5 --- /dev/null +++ b/src/frontend/xcecli/filters/queue_filter.go @@ -0,0 +1,25 @@ +package filters + +import ( + "fmt" + "strings" +) + +type QueuesFilter struct { + QueueName string + QueueType string +} + +// Get generates filter string +func (f *QueuesFilter) HandleQueueFilter() (string, error) { + var filters []string + if f.QueueName != "" { + filters = append(filters, fmt.Sprintf("queue_name:in:%s", f.QueueName)) + } + + if len(filters) > 0 { + return strings.Join(filters, ","), nil + } + + return "", nil +} diff --git a/src/frontend/xcecli/filters/resource_filter.go b/src/frontend/xcecli/filters/resource_filter.go new file mode 100644 index 00000000..35c3691d --- /dev/null +++ b/src/frontend/xcecli/filters/resource_filter.go @@ -0,0 +1,49 @@ +package filters + +import ( + "fmt" + "strings" +) + +type ResourceDistFilter struct { + ResName string + Type string +} + +func (f *ResourceDistFilter) HandleResourceDistFilter() (string, error) { + var filters []string + + if f.ResName != "" { + filters = append(filters, fmt.Sprintf("res_name:eq:%s", f.ResName)) + } + + if f.Type != "" { + filters = append(filters, fmt.Sprintf("type:eq:%s", f.Type)) + } + + if len(filters) > 0 { + return strings.Join(filters, ","), nil + } + + return "", nil +} + +// ResourceDefFilter defines filters for resource definition queries +type ResourceDefFilter struct { + ResName string +} + +// HandleResourceDefFilter processes the resource definition filter +func (f *ResourceDefFilter) HandleResourceDefFilter() (string, error) { + var filters []string + + if f.ResName != "" { + filters = append(filters, fmt.Sprintf("res_name:eq:%s", f.ResName)) + } + + if len(filters) > 0 { + return strings.Join(filters, ","), nil + } + + return "", nil +} diff --git a/src/frontend/xcecli/filters/role_filter.go b/src/frontend/xcecli/filters/role_filter.go new file mode 100644 index 00000000..a37252a6 --- /dev/null +++ b/src/frontend/xcecli/filters/role_filter.go @@ -0,0 +1,29 @@ +package filters + +import ( + "fmt" + "strings" +) + +type RolesFilter struct { + RoleName string + Type string + PermName string +} + +// Get generates filter string +func (r *RolesFilter) HandleRoleFilter() (string, error) { + var filters []string + if r.RoleName != "" { + filters = append(filters, fmt.Sprintf("role_name:in:%s", r.RoleName)) + } + if r.PermName != "" { + filters = append(filters, fmt.Sprintf("action:in:%s", r.PermName)) + } + + if len(filters) > 0 { + return strings.Join(filters, ","), nil + } + + return "", nil +} diff --git a/src/frontend/xcecli/filters/user_filter.go b/src/frontend/xcecli/filters/user_filter.go new file mode 100644 index 00000000..da47093b --- /dev/null +++ b/src/frontend/xcecli/filters/user_filter.go @@ -0,0 +1,27 @@ +package filters + +import ( + "fmt" + "strings" +) + +type UserRolesFilter struct { + UserName string + RoleName string +} + +// Get generates filter string +func (ur *UserRolesFilter) HandleUserRoleFilter() (string, error) { + var filters []string + if ur.UserName != "" { + filters = append(filters, fmt.Sprintf("user_name:in:%s", ur.UserName)) + } + if ur.RoleName != "" { + filters = append(filters, fmt.Sprintf("role_name:in:%s", ur.RoleName)) + } + if len(filters) > 0 { + return strings.Join(filters, ","), nil + } + + return "", nil +} diff --git a/src/frontend/xcecli/go.mod b/src/frontend/xcecli/go.mod new file mode 100755 index 00000000..361c617c --- /dev/null +++ b/src/frontend/xcecli/go.mod @@ -0,0 +1,34 @@ +module xcecli + +go 1.23.4 + +require ( + github.com/spf13/cobra v1.9.1 + github.com/spf13/viper v1.20.1 + github.com/xeipuuv/gojsonschema v1.2.0 + go.uber.org/zap v1.27.0 + golang.org/x/term v0.27.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gopkg.in/yaml.v2 v2.4.0 + resty.dev/v3 v3.0.0-beta.2 +) + +require ( + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/src/frontend/xcecli/go.sum b/src/frontend/xcecli/go.sum new file mode 100755 index 00000000..16926d86 --- /dev/null +++ b/src/frontend/xcecli/go.sum @@ -0,0 +1,76 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= +resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= diff --git a/src/frontend/xcecli/meson.build b/src/frontend/xcecli/meson.build new file mode 100644 index 00000000..d95315ef --- /dev/null +++ b/src/frontend/xcecli/meson.build @@ -0,0 +1,19 @@ +# frontend/xcecli — Go CLI tool +# +# Pure Go binary: CGO_ENABLED=0 (static, no C dependencies). + +go_env = environment() +go_env.set('CGO_ENABLED', '0') +go_env.set('BUILDTYPE', get_option('buildtype')) + +custom_target('xcecli', + input: 'cmd/main.go', + output: 'xcecli', + command: [env_program, + 'CURRENT_SOURCE_DIR="@CURRENT_SOURCE_DIR@"', + 'BUILD_ROOT="@BUILD_ROOT@"', + go_build_program, + '@INPUT@', '@OUTPUT@'], + env: go_env, + install: true, + install_dir: get_option('prefix') / 'bin') diff --git a/src/frontend/xcecli/models/cluster.go b/src/frontend/xcecli/models/cluster.go new file mode 100755 index 00000000..5fb69dc2 --- /dev/null +++ b/src/frontend/xcecli/models/cluster.go @@ -0,0 +1,22 @@ +package models + +type ClusterResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []ClusterInfo `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +type ClusterInfo struct { + ClusterName string `json:"cluster_name"` + Version string `json:"version"` + Status string `json:"status"` + ControllerName string `json:"controller_name"` + Resources interface{} `json:"resources"` + HostTypes interface{} `json:"host_types"` + HostModels interface{} `json:"host_models"` + ClusterAdmins interface{} `json:"cluster_admins"` +} diff --git a/src/frontend/xcecli/models/exitstatus_test.go b/src/frontend/xcecli/models/exitstatus_test.go new file mode 100644 index 00000000..592b7ae3 --- /dev/null +++ b/src/frontend/xcecli/models/exitstatus_test.go @@ -0,0 +1,39 @@ +package models + +import ( + "encoding/json" + "testing" +) + +func TestQueryJob_ExitStatus_Null(t *testing.T) { + raw := `{"Fields":["job_id","exit_status"],"Data":{"job_id":1001,"exit_status":null}}` + var j QueryJob + if err := json.Unmarshal([]byte(raw), &j); err != nil { + t.Fatal(err) + } + if j.ExitStatus != "" { + t.Errorf("ExitStatus: got %q want empty", j.ExitStatus) + } +} + +func TestQueryJob_ExitStatus_String(t *testing.T) { + raw := `{"Fields":["job_id","exit_status"],"Data":{"job_id":1001,"exit_status":"{\"signal\":9}"}}` + var j QueryJob + if err := json.Unmarshal([]byte(raw), &j); err != nil { + t.Fatal(err) + } + if j.ExitStatus == "" { + t.Error("ExitStatus should not be empty for non-null value") + } +} + +func TestQueryJob_ExitStatus_Zero(t *testing.T) { + raw := `{"Fields":["job_id","exit_status"],"Data":{"job_id":1001,"exit_status":0}}` + var j QueryJob + if err := json.Unmarshal([]byte(raw), &j); err != nil { + t.Fatal(err) + } + if j.ExitStatus != "0" { + t.Errorf("ExitStatus: got %q want 0", j.ExitStatus) + } +} diff --git a/src/frontend/xcecli/models/hosts.go b/src/frontend/xcecli/models/hosts.go new file mode 100755 index 00000000..3e1563eb --- /dev/null +++ b/src/frontend/xcecli/models/hosts.go @@ -0,0 +1,154 @@ +package models + +const ( + HEADER_HOST_NAME = "HOST_NAME" + HEADER_HOST_STATUS = "STATUS" + HEADER_HOST_NCPUS = "NCPUS" + HEADER_HOST_MAX_SLOT = "MAX_SLOT" + HEADER_HOST_NUM_RUN = "NUM_RUN" + HEADER_HOST_NUM_SSUSP = "NUM_SSUSP" + HEADER_HOST_NUM_USUSP = "NUM_USUSP" + HEADER_HOST_HOST_TYPE = "HOST_TYPE" + // host details info + HEADER_HOST_CPUF = "CPUF" + HEADER_HOST_JLU = "JL/U" + HEADER_HOST_MAX = "MAX" + HEADER_HOST_NJOBS = "NJOBS" + HEADER_HOST_RUN = "RUN" + HEADER_HOST_SSUSP = "SSUSP" + HEADER_HOST_USUSP = "USUSP" + HEADER_HOST_RSV = "RSV" + HEADER_HOST_DISPT_WINDOW = "DISPATCH_WINDOW" + // second line + HEADER_HOST_R15S = "r15s" + HEADER_HOST_R1M = "r1m" + HEADER_HOST_R15M = "r15m" + HEADER_HOST_UT = "ut" + HEADER_HOST_PG = "pg" + HEADER_HOST_IO = "io" + HEADER_HOST_LS = "ls" + HEADER_HOST_IT = "it" + HEADER_HOST_TMP = "tmp" + HEADER_HOST_SWP = "swp" + HEADER_HOST_MEM = "mem" + HEADER_HOST_SLOTS = "slots" +) + +var HostDetailHeaders = []string{ + HEADER_HOST_STATUS, + HEADER_HOST_CPUF, + HEADER_HOST_JLU, + HEADER_HOST_MAX, + HEADER_HOST_NJOBS, + HEADER_HOST_RUN, + HEADER_HOST_SSUSP, + HEADER_HOST_USUSP, + HEADER_HOST_RSV, + HEADER_HOST_DISPT_WINDOW, +} + +var HostLoadSchedulingHeaders = []string{ + HEADER_HOST_R15S, + HEADER_HOST_R1M, + HEADER_HOST_R15M, + HEADER_HOST_UT, + HEADER_HOST_PG, + HEADER_HOST_IO, + HEADER_HOST_LS, + HEADER_HOST_IT, + HEADER_HOST_TMP, + HEADER_HOST_SWP, + HEADER_HOST_MEM, + HEADER_HOST_SLOTS, +} + +// HostsResponse define the data structure in the host query response +type HostsResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []XCEHost `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +// Host define the data structure in the host info\ +type XCEHost struct { + HostID int64 `json:"host_id"` + HostName string `json:"host_name"` + HostStatus string `json:"host_status"` + HostModel string `json:"host_model"` + HostType string `json:"host_type"` + HostGroup string `json:"host_group"` + Flag int `json:"flag"` + CPUFactor float64 `json:"cpu_factor"` + DisptWindow interface{} `json:"dispt_window"` + NumCPUs int `json:"num_cpus"` + MaxSlot int `json:"max_slot"` + MaxMem int `json:"max_mem"` + MaxSwap int `json:"max_swap"` + MaxTmp int `json:"max_tmp"` + NumDisk int `json:"num_disk"` + NumRun int `json:"num_run"` + NumSSusp int `json:"num_ssusp"` + NumUSusp int `json:"num_ususp"` + NumDone int `json:"num_done"` + AffTopo interface{} `json:"aff_topo"` // JSONB + GPUTopo interface{} `json:"gpu_topo"` // JSONB + SchedStop SchedStop `json:"sched_stop"` // JSONB + RealLoad interface{} `json:"real_load"` +} + +var HostFieldsMap = map[string]string{ + HEADER_HOST_NAME: "host_name", + HEADER_HOST_STATUS: "host_status", + HEADER_HOST_NCPUS: "num_cores", + HEADER_HOST_MAX_SLOT: "max_slot", + HEADER_HOST_NUM_RUN: "num_run", + HEADER_HOST_NUM_SSUSP: "num_ssusp", + HEADER_HOST_NUM_USUSP: "num_ususp", + HEADER_HOST_HOST_TYPE: "host_type", +} + +type SchedStop struct { + Ls string `json:"ls"` // Load Sharing + Pg string `json:"pg"` // Page + R1m string `json:"r1m"` // Recent 1 minute + Tmp string `json:"tmp"` // Temporary +} + +type HostControlRequest struct { + ControlAction string `json:"action" binding:"required"` //control action + Hostname []string `json:"host_name" binding:"required"` //host name + Comment string `json:"comment"` //comment +} + +type HostControlResponse struct { + Data struct { + Accepted []*HostControlResponseItem `json:"accepted"` + Failed []*HostControlResponseItem `json:"failed"` + } `json:"data"` +} + +type HostControlResponseItem struct { + HostName string `json:"host_name"` + Msg string `json:"msg"` +} + +func (h *XCEHost) UnmarshalJSON(data []byte) error { + type Alias XCEHost + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + *h = XCEHost(a) + return nil +} + +type ResReqSelectRequest struct { + Select string `json:"select"` + Order string `json:"order,omitempty"` + Limit int `json:"limit,omitempty"` + PageToken string `json:"page_token,omitempty"` +} diff --git a/src/frontend/xcecli/models/jobs.go b/src/frontend/xcecli/models/jobs.go new file mode 100755 index 00000000..57048bf8 --- /dev/null +++ b/src/frontend/xcecli/models/jobs.go @@ -0,0 +1,517 @@ +package models + +import ( + "encoding/json" + "fmt" + "time" + +) + +const ( + HEADER_JOB_JOBID = "JOBID" + HEADER_JOB_USER = "USER" + HEADER_JOB_STATUS = "STATUS" + HEADER_JOB_QUEUE = "QUEUE" + HEADER_JOB_FROM_HOST = "FROM_HOST" + HEADER_JOB_EXEC_HOST = "EXEC_HOST" + HEADER_JOB_COMMAND = "COMMAND" + HEADER_JOB_NAME = "JOB_NAME" + HEADER_JOB_SUBMIT_TIME = "SUBMIT_TIME" + HEADER_JOB_CORE_LIMIT = "CORE_LIMIT" + HEADER_JOB_MEM_LIMIT = "MEM_LIMIT" + HEADER_JOB_PID = "PID" + HEADER_JOB_PGID = "PGID" + HEADER_JOB_PROJECT_NAME = "PROJECT_NAME" + HEADER_JOB_START_TIME = "START_TIME" + HEADER_JOB_END_TIME = "END_TIME" + HEADER_JOB_CWD = "CWD" + HEADER_JOB_CPU_TIME = "CPU_TIME" + HEADER_JOB_RESREQ = "RESREQ" + //HEADER_JOB_PRIORITY = "PRIORITY" + HEADER_JOB_DESCRIPTION = "DESCRIPTION" + HEADER_JOB_LAUNCH_TYPE = "LAUNCH_TYPE" + HEADER_JOB_NUM_PROCESSORS = "NUM_PROCESSORS" + HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL" + + // second line + HEADER_JOB_R15S = "r15s" + HEADER_JOB_R1M = "r1m" + HEADER_JOB_R15M = "r15m" + HEADER_JOB_UT = "ut" + HEADER_JOB_PG = "pg" + HEADER_JOB_IO = "io" + HEADER_JOB_LS = "ls" + HEADER_JOB_IT = "it" + HEADER_JOB_TMP = "tmp" + HEADER_JOB_SWP = "swp" + HEADER_JOB_MEM = "mem" +) + +var JobLoadSchedulingHeaders = []string{ + HEADER_JOB_R15S, + HEADER_JOB_R1M, + HEADER_JOB_R15M, + HEADER_JOB_UT, + HEADER_JOB_PG, + HEADER_JOB_IO, + HEADER_JOB_LS, + HEADER_JOB_IT, + HEADER_JOB_TMP, + HEADER_JOB_SWP, + HEADER_JOB_MEM, +} + + + +// for array jobs +const ( + HEADER_ARRAY_JOB_JOBID = "JOBID" + HEADER_ARRAY_JOB_SPEC = "ARRAY_SPEC" + HEADER_ARRAY_JOB_OWNER = "OWNER" + HEADER_ARRAY_JOB_NJOBS = "NJOBS" + HEADER_ARRAY_JOB_PEND = "PEND" + HEADER_ARRAY_JOB_DONE = "DONE" + HEADER_ARRAY_JOB_RUN = "RUN" + HEADER_ARRAY_JOB_EXIT = "EXIT" + HEADER_ARRAY_JOB_SSUSP = "SSUSP" + HEADER_ARRAY_JOB_USUSP = "USUSP" + HEADER_ARRAY_JOB_PSUSP = "PSUSP" +) + +type JobsResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []QueryJob `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +type Jsonb struct { + String string +} + +// Job define the data structure in the job info +type QueryJob struct { + JobID int64 `json:"job_id"` + SubmitUser string `json:"submit_user"` + JobStatus string `json:"job_status"` + JobName string `json:"job_name"` + Command string `json:"command"` + QueueName string `json:"queue_name"` + ProjectName string `json:"project_name"` + UserGroup string `json:"user_group"` + LicenseProject string `json:"license_project"` + JobDescription string `json:"job_description"` + Resreq string `json:"resreq"` + RsvID string `json:"rsv_id"` + SLA string `json:"sla"` + MaxNumProcessors int `json:"maxnum_processors"` + NumProcessors int `json:"num_processors"` + AskedHosts string `json:"asked_hosts"` + BeginTime string `json:"begin_time"` + TermTime string `json:"term_time"` + UserPriority int `json:"user_priority"` + DependCond string `json:"depend_cond"` + SubmitTime string `json:"submit_time"` + ReserveTime time.Time `json:"reserve_time"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time"` + JobRusage string `json:"job_rusage"` + MaxMem float64 `json:"max_mem"` + AvgMem float64 `json:"avg_mem"` + UsedMem float64 `json:"used_mem"` + CPUTime int `json:"cpu_time"` + PeakCPUTime int `json:"peak_cpu_time"` + PeakCPUDuration float64 `json:"peak_cpu_duration"` + MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage + AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage + PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage + NumHosts int `json:"num_hosts"` // Number of hosts used by the job + FirstHost string `json:"first_host"` // First host where the job ran + AllocSlot string `json:"alloc_slots"` // Allocated slot information + Duration int `json:"duration"` + Umask int `json:"umask"` + CWD string `json:"cwd"` + SubmitHomeDir string `json:"submit_home_dir"` + FromHost string `json:"from_host"` + ExecHosts ExecHosts `json:"exec_hosts"` + ExitStatus ExitStatusValue `json:"exit_status"` + ExecUID int `json:"exec_uid"` + ExecHome string `json:"exec_home"` + ExecCWD string `json:"exec_cwd"` + ExecUserName string `json:"exec_user_name"` + JobPriority int `json:"job_priority"` + ChargedSAAP string `json:"charged_SAAP"` + PreExecCmd string `json:"pre_exec_cmd"` + PostExecCmd string `json:"post_exec_cmd"` + RequeueErrorValues string `json:"requeue_error_values"` + NumLicenses int `json:"num_licenses"` + LicenseNames []string `json:"license_names"` + RunTime int `json:"run_time"` + RunTimeEstimation int `json:"run_time_estimation"` + MigThreshold int `json:"mig_threshold"` + ChkpntPeriod int `json:"chkpnt_period"` + ChkpntDir string `json:"chkpnt_dir"` + OutDir string `json:"out_dir"` + InFile string `json:"in_file"` + OutFile string `json:"out_file"` + ErrFile string `json:"err_file"` + MailUser string `json:"mail_user"` + LoginShell string `json:"login_shell"` + ExceptList string `json:"except_list"` + RLimits64 []int64 `json:"rlimits64"` + SubEnvVars string `json:"sub_env_vars"` + PendingReason string `json:"pending_reason"` + JobStartInfo string `json:"job_start_info"` + JobExecInfo string `json:"job_exec_info"` + CoreLimit float64 `json:"core_limit"` + CPULimit string `json:"cpu_limit"` + RunLimit string `json:"run_limit"` + FileLimit float64 `json:"file_limit"` + DataLimit float64 `json:"data_limit"` + StackLimit float64 `json:"stack_limit"` + MemLimit float64 `json:"mem_limit"` + SwapLimit float64 `json:"swap_limit"` + ProcessLimit float64 `json:"process_limit"` + ThreadLimit float64 `json:"thread_limit"` + NumFiles int `json:"num_files"` + ArrayIdx int `json:"array_idx"` // + UniqueJobID int64 `json:"unique_job_id"` // + ExtendData JobExtendData `json:"extend_data"` + InteractiveJobURL string `json:"interactive_job_url"` + PGIDs IntSlice `json:"pgids"` + PID IntSlice `json:"pids"` + LaunchType string `json:"launch_type"` +} + +var JobFieldsMap = map[string]string{ + HEADER_JOB_JOBID: "job_id", + HEADER_JOB_USER: "submit_user", + HEADER_JOB_STATUS: "job_status", + HEADER_JOB_QUEUE: "queue_name", + HEADER_JOB_COMMAND: "command", + HEADER_JOB_NAME: "job_name", + HEADER_JOB_SUBMIT_TIME: "submit_time", + HEADER_JOB_FROM_HOST: "from_host", + HEADER_JOB_EXEC_HOST: "exec_hosts", + HEADER_JOB_CORE_LIMIT: "core_limit", + HEADER_JOB_MEM_LIMIT: "mem_limit", + HEADER_JOB_PID: "pids", + HEADER_JOB_PGID: "pgids", + HEADER_JOB_PROJECT_NAME: "project_name", + HEADER_JOB_START_TIME: "start_time", + HEADER_JOB_END_TIME: "end_time", + HEADER_JOB_CWD: "cwd", + HEADER_JOB_CPU_TIME: "cpu_time", + HEADER_JOB_RESREQ: "resreq", + HEADER_JOB_DESCRIPTION: "job_description", + HEADER_JOB_LAUNCH_TYPE: "launch_type", + HEADER_JOB_NUM_PROCESSORS: "num_processors", + HEADER_JOB_INTERACTIVE_URL: "interactive_job_url", + //job_filter.HEADER_JOB_PRIORITY: "job_priority", +} + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(data []byte) error { + // parse array + var arr []int + if err := json.Unmarshal(data, &arr); err == nil { + *s = arr + return nil + } + // parse {} + var obj map[string]interface{} + if err := json.Unmarshal(data, &obj); err == nil { + + if len(obj) == 0 { + *s = []int{} + return nil + } + } + return fmt.Errorf("cannot unmarshal IntSlice %s", string(data)) +} + +// ExecHosts represents the execution hosts for a job +type ExecHosts struct { + HostSlots []HostSlot `json:"host_slots"` + NumHosts int `json:"num_hosts"` +} + +// String returns the string representation of ExecHosts +// If HostSlots is empty, return "-" +// Otherwise, return the JSON formatted string +func (e ExecHosts) String() string { + if len(e.HostSlots) == 0 { + return "-" + } + + // Use json.Marshal to convert the structure to a JSON string + data, err := json.Marshal(e) + if err != nil { + return "-" + } + return string(data) +} + +// UnmarshalJSON customizes the JSON parsing method, compatible with both string "-" and structure formats +func (e *ExecHosts) UnmarshalJSON(data []byte) error { + // First try to parse as a string + var str string + if err := json.Unmarshal(data, &str); err == nil { + // If the string is "-", return an empty ExecHosts + if str == "-" { + e.HostSlots = []HostSlot{} + e.NumHosts = 0 + return nil + } + // If it's any other string, treat it as a single hostname + e.HostSlots = []HostSlot{{HostName: str, NumSlots: 1}} + e.NumHosts = 1 + return nil + } + + // If it's not a string, try to parse as an ExecHosts structure + type Alias ExecHosts + aux := &struct { + *Alias + }{ + Alias: (*Alias)(e), + } + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + return nil +} + +// HostSlot represents a single host slot +type HostSlot struct { + HostName string `json:"host_name"` + NumSlots int `json:"num_slots"` +} + +type JobSubmitRequest struct { + JobName string `json:"job_name,omitempty" binding:"required"` + Command string `json:"command" binding:"required"` + QueueName string `json:"queue_name,omitempty"` // set to DEFAULT_QUEUE in config + ProjectName string `json:"project_name,omitempty"` // default project: "default" + UserGroup string `json:"user_group,omitempty"` + LicenseProject string `json:"license_project,omitempty"` + JobDescription string `json:"job_description,omitempty"` + Resreq string `json:"resreq,omitempty"` // default: "type == local" + Gpureq string `json:"gpureq,omitempty"` + RsvID string `json:"rsv_id,omitempty"` + SLA string `json:"sla,omitempty"` + MaxNumProcessors int `json:"maxnum_processors,omitempty"` + NumProcessors int `json:"num_processors,omitempty"` + AskedHosts string `json:"asked_hosts,omitempty"` + BeginTime string `json:"begin_time,omitempty"` + TermTime string `json:"term_time,omitempty"` + UserPriority int `json:"user_priority,omitempty"` + DependCond string `json:"depend_cond,omitempty"` + JobGroup string `json:"job_group,omitempty"` + HostSpec string `json:"host_spec,omitempty"` + Env []string `json:"env,omitempty"` + Tags []string `json:"tags,omitempty"` + Exclusive bool `json:"exclusive,omitempty"` + Rerunable bool `json:"rerunable,omitempty"` + Interactive int `json:"interactive,omitempty"` + WarningTimePeriod int `json:"warning_time_period,omitempty"` + WarningAction string `json:"warning_action,omitempty"` + DataSpecFile string `json:"data_spec_file,omitempty"` + TransferFiles []string `json:"transfer_files,omitempty"` + PreExecCmd string `json:"pre_exec_cmd,omitempty"` + PostExecCmd string `json:"post_exec_cmd,omitempty"` + RequeueErrorValues string `json:"requeue_error_values,omitempty"` + RunTimeEstimation int `json:"run_time_estimation,omitempty"` + MigThreshold int `json:"mig_threshold,omitempty"` + ChkpntPeriod int `json:"chkpnt_period,omitempty"` + ChkpntDir string `json:"chkpnt_dir,omitempty"` + InitChkpntPeriod int `json:"init_chkpnt_period,omitempty"` + CWD string `json:"cwd,omitempty"` + OutDir string `json:"out_dir,omitempty"` + InFile string `json:"in_file,omitempty"` + OutFile string `json:"out_file,omitempty"` + ErrFile string `json:"err_file,omitempty"` + Mailuser string `json:"mailuser,omitempty"` + LoginShell string `json:"login_shell,omitempty"` + ExceptList string `json:"except_list,omitempty"` + CPULimit string `json:"cpu_limit,omitempty"` + RunLimit string `json:"run_limit,omitempty"` + MemLimit string `json:"mem_limit,omitempty"` + SwapLimit string `json:"swap_limit,omitempty"` + CoreLimit string `json:"core_limit,omitempty"` + DataLimit string `json:"data_limit,omitempty"` + FileLimit string `json:"file_limit,omitempty"` + ProcessLimit string `json:"process_limit,omitempty"` + StackLimit string `json:"stack_limit,omitempty"` + ThreadLimit string `json:"thread_limit,omitempty"` + ShellLimit string `json:"shell_limit,omitempty"` + NumFiles int `json:"num_files,omitempty"` + LaunchType string `json:"launch_type,omitempty"` +} + +// JobSubmitData define the data structure in the job submit response +type JobSubmitData struct { + JobID int64 `json:"jobid"` + Message string `json:"message"` +} + +// JobSubmitResponse define the data structure in the job submit response +type JobSubmitResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data JobSubmitData `json:"data"` + Count int `json:"count"` +} + +// UploadedFile represents the uploaded file +type UploadedFile struct { + DestFile string `json:"dest_file"` // dest file path + SrcFile string `json:"src_file"` // src file name + WebURL string `json:"web_url"` // file URL +} + +// UploadResponse represents the file upload response +type UploadResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + UploadedFiles []UploadedFile `json:"accepted"` + Failed []FailedFile `json:"failed"` + Status string `json:"status"` + } `json:"data"` +} + +type FailedFile struct { + FileName string `json:"file_name"` + Msg string `json:"msg"` +} + +// ------- job control ------------- +type JobControlFilter struct { + JobName string `json:"job_name,omitempty"` + JobGroupName string `json:"job_group_name,omitempty"` + QueueName string `json:"queue_name,omitempty"` + Users string `json:"users,omitempty"` + ExecHosts string `json:"exec_hosts,omitempty"` + SLA string `json:"sla,omitempty"` + AllJobs bool `json:"all_jobs,omitempty"` +} + +type JobControlRequest struct { + Filter JobControlFilter `json:"filter" binding:"required"` + Action string `json:"action" binding:"required"` + Signal string `json:"signal,omitempty"` + Comment string `json:"comment,omitempty"` + ForceKill bool `json:"force_kill,omitempty"` +} + +type JobControlJobIDRequest struct { + Action string `json:"action" binding:"required"` + Signal string `json:"signal,omitempty"` // signal value + Comment string `json:"comment,omitempty"` + ForceKill bool `json:"force_kill,omitempty"` +} + +type JobRestartRequest struct { + AskedHosts string `json:"asked_hosts,omitempty"` +} + +type JobRestartResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data *JobRestartResponseData `json:"data"` +} +type JobRestartResponseData struct { + JobID int64 `json:"jobid"` + QueueName string `json:"queue_name"` + Message string `json:"message"` +} + +type JobControlResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data *JobControlResponseData `json:"data"` +} + +type JobControlResponseData struct { + Accepted []*JobControlResponseItem `json:"accepted"` + Failed []*JobControlResponseItem `json:"failed"` + Status string `json:"status"` +} + +type JobControlResponseItem struct { + JobID int64 `json:"job_id"` + ArrayIdx int `json:"array_idx,omitempty"` + Msg string `json:"msg"` +} + +// job array statistics data structure +type ArrayJobStatsData struct { + JobID int64 `json:"job_id"` + ArrayJobName string `json:"array_job_name"` + SubmitUser string `json:"submit_user"` + NumJobs int `json:"num_jobs"` + NumDone int `json:"num_done"` + NumExit int `json:"num_exit"` + NumRun int `json:"num_run"` + NumPend int `json:"num_pend"` + NumSSusp int `json:"num_ssusp"` + NumUSusp int `json:"num_ususp"` + NumPSusp int `json:"num_psusp"` +} + +// job array statistics response data structure +type ArrayJobsStatsResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []ArrayJobStatsData `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +type JobExtendData struct { + Interactive int `json:"interactive"` + InteractiveJobURL string `json:"interactive_job_url"` + Limits struct { + CoreLimit int `json:"core_limit"` + } `json:"limits"` +} + +func (j *QueryJob) UnmarshalJSON(data []byte) error { + type Alias QueryJob + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + *j = QueryJob(a) + return nil +} + +// ExitStatusValue handles exit_status which can be null, a number, or a JSON string. +type ExitStatusValue string + +func (e *ExitStatusValue) UnmarshalJSON(data []byte) error { + s := string(data) + if s == "null" || s == "" { + *e = "" + return nil + } + // strip quotes if it's a JSON string + if len(s) >= 2 && s[0] == '"' { + var str string + if err := json.Unmarshal(data, &str); err != nil { + return err + } + *e = ExitStatusValue(str) + return nil + } + // bare number or other value — store as-is + *e = ExitStatusValue(s) + return nil +} diff --git a/src/frontend/xcecli/models/logon.go b/src/frontend/xcecli/models/logon.go new file mode 100755 index 00000000..e1fba09e --- /dev/null +++ b/src/frontend/xcecli/models/logon.go @@ -0,0 +1,19 @@ +package models + +// LogonData define the data structure in the logon response +type LogonData struct { + Token string `json:"token"` + Path string `json:"path"` + IP string `json:"ip"` + CreateTime int64 `json:"createtime"` + TokenExpireTime int64 `json:"token_expire_time"` +} + +// LogonResponse define the data structure in the logon response +type LogonResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data LogonData `json:"data"` + Count int `json:"count"` + CACert string `json:"cacert"` +} diff --git a/src/frontend/xcecli/models/ordermap.go b/src/frontend/xcecli/models/ordermap.go new file mode 100644 index 00000000..ce63733d --- /dev/null +++ b/src/frontend/xcecli/models/ordermap.go @@ -0,0 +1,19 @@ +package models + +import "encoding/json" + +// UnmarshalOrderMapRow extracts the Data sub-object from the server's OrderMap wire format +// {"Fields":[...],"Data":{...}} and unmarshals it into dst. +// If the input is not an OrderMap wrapper, it falls back to direct unmarshal. +func UnmarshalOrderMapRow(data []byte, dst interface{}) error { + var wrapper struct { + Data json.RawMessage `json:"Data"` + } + if err := json.Unmarshal(data, &wrapper); err != nil { + return err + } + if wrapper.Data == nil { + return json.Unmarshal(data, dst) + } + return json.Unmarshal(wrapper.Data, dst) +} diff --git a/src/frontend/xcecli/models/permissions.go b/src/frontend/xcecli/models/permissions.go new file mode 100644 index 00000000..76eba975 --- /dev/null +++ b/src/frontend/xcecli/models/permissions.go @@ -0,0 +1,13 @@ +package models + +type PermsResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data []string `json:"data"` +} + +type Permission struct { + PermissionName string `json:"permission_name" binding:"required"` + DomainType string `json:"domain_type" binding:"required"` + DomainValues []string `json:"domain_values"` +} diff --git a/src/frontend/xcecli/models/queues.go b/src/frontend/xcecli/models/queues.go new file mode 100755 index 00000000..4d792b5d --- /dev/null +++ b/src/frontend/xcecli/models/queues.go @@ -0,0 +1,131 @@ +package models + +const ( + HEADER_QUEUE_NAME = "QUEUE_NAME" + HEADER_QUEUE_PRIORITY = "PRIORITY" + HEADER_QUEUE_STATUS = "QUEUE_STATUS" + HEADER_QUEUE_NUM_PEND = "NUM_PEND" + HEADER_QUEUE_NUM_RUN = "NUM_RUN" + HEADER_QUEUE_NUM_SSUSP = "NUM_SSUSP" + HEADER_QUEUE_NUM_USUSP = "NUM_USUSP" + HEADER_QUEUE_NUM_DONE = "NUM_DONE" + // queue detail info + HEADER_QUEUE_PRIO = "PRIO" + HEADER_QUEUE_NICE = "NICE" + HEADER_QUEUE_MAX = "MAX" + HEADER_QUEUE_JLU = "JL/U" + HEADER_QUEUE_JLP = "JL/P" + HEADER_QUEUE_JLH = "JL/H" + HEADER_QUEUE_NJOBS = "NJOBS" + HEADER_QUEUE_PEND = "PEND" + HEADER_QUEUE_RUN = "RUN" + HEADER_QUEUE_SSUSP = "SSUSP" + HEADER_QUEUE_USUSP = "USUSP" + HEADER_QUEUE_RSV = "RSV" + // second line + HEADER_QUEUE_R15S = "r15s" + HEADER_QUEUE_R1M = "r1m" + HEADER_QUEUE_R15M = "r15m" + HEADER_QUEUE_UT = "ut" + HEADER_QUEUE_PG = "pg" + HEADER_QUEUE_IO = "io" + HEADER_QUEUE_LS = "ls" + HEADER_QUEUE_IT = "it" + HEADER_QUEUE_TMP = "tmp" + HEADER_QUEUE_SWP = "swp" + HEADER_QUEUE_MEM = "mem" +) + +var QueueDetailHeaders = []string{ + HEADER_QUEUE_PRIO, + HEADER_QUEUE_NICE, + HEADER_QUEUE_STATUS, + HEADER_QUEUE_MAX, + HEADER_QUEUE_JLU, + HEADER_QUEUE_JLP, + HEADER_QUEUE_JLH, + HEADER_QUEUE_NJOBS, + HEADER_QUEUE_PEND, + HEADER_QUEUE_RUN, + HEADER_QUEUE_SSUSP, + HEADER_QUEUE_USUSP, + HEADER_QUEUE_RSV, +} + +var QueueLoadSchedulingHeaders = []string{ + HEADER_QUEUE_R15S, + HEADER_QUEUE_R1M, + HEADER_QUEUE_R15M, + HEADER_QUEUE_UT, + HEADER_QUEUE_PG, + HEADER_QUEUE_IO, + HEADER_QUEUE_LS, + HEADER_QUEUE_IT, + HEADER_QUEUE_TMP, + HEADER_QUEUE_SWP, + HEADER_QUEUE_MEM, +} + +// QueuesResponse define the data structure in the queue query response +type QueuesResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []DBQueueInfo `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +var QueueFieldsMap = map[string]string{ + HEADER_QUEUE_NAME: "queue_name", + HEADER_QUEUE_PRIORITY: "priority", + HEADER_QUEUE_STATUS: "queue_status", + HEADER_QUEUE_NUM_PEND: "num_pend", + HEADER_QUEUE_NUM_RUN: "num_run", + HEADER_QUEUE_NUM_SSUSP: "num_ssusp", + HEADER_QUEUE_NUM_USUSP: "num_ususp", + HEADER_QUEUE_NUM_DONE: "num_done", +} + +// DBQueueInfo define the data structure in the queue info +type DBQueueInfo struct { + QueueName string `json:"queue_name"` + QueueStatus string `json:"queue_status"` + Priority int `json:"priority"` + Description string `json:"description"` + NumPend int `json:"num_pend"` + NumRun int `json:"num_run"` + NumSSusp int `json:"num_ssusp"` + NumUSusp int `json:"num_ususp"` + NumDone int `json:"num_done"` + FSInfo map[string]interface{} `json:"fs_info"` // JSONB +} + +type QueueControlRequest struct { + ControlAction string `json:"action" binding:"required"` //control action + QueueName []string `json:"queue_name" binding:"required"` //queue name + Comment string `json:"comment"` //comment +} + +type QueueControlResponse struct { + Data struct { + Accepted []*QueueControlResponseItem `json:"accepted"` + Failed []*QueueControlResponseItem `json:"failed"` + } `json:"data"` +} + +type QueueControlResponseItem struct { + QueueName string `json:"queue_name"` + Msg string `json:"msg"` +} + +func (q *DBQueueInfo) UnmarshalJSON(data []byte) error { + type Alias DBQueueInfo + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + *q = DBQueueInfo(a) + return nil +} diff --git a/src/frontend/xcecli/models/resource.go b/src/frontend/xcecli/models/resource.go new file mode 100644 index 00000000..cd72068f --- /dev/null +++ b/src/frontend/xcecli/models/resource.go @@ -0,0 +1,202 @@ +package models + +import ( + "encoding/json" + "fmt" + "strings" +) + +const ( + HEADER_RESOURCE_NAME = "RESOURCE" + HEADER_RESOURCE_VALUE = "VALUE" + HEADER_RESOURCE_LOC = "LOCATION" + HEADER_RESOURCE_DEF_TYPE = "TYPE" + HEADER_RESOURCE_DEF_ORDER = "ORDER" + HEADER_RESOURCE_DEF_BUILTIN = "BUILTIN" + HEADER_RESOURCE_DEF_CONSUMABLE = "CONSUMABLE" + HEADER_RESOURCE_DEF_DYNAMIC = "DYNAMIC" + HEADER_RESOURCE_DEF_RELEASE = "RELEASE" + HEADER_RESOURCE_DEF_INTERVAL = "INTERVAL" + HEADER_RESOURCE_DEF_DESC = "DESCRIPTION" +) + +var ResourceDistFieldsMap = map[string]string{ + HEADER_RESOURCE_NAME: "res_name", + HEADER_RESOURCE_VALUE: "value", + HEADER_RESOURCE_LOC: "location", +} + +// ResourceDefFieldsMap maps resource definition field headers to API field names +var ResourceDefFieldsMap = map[string]string{ + HEADER_RESOURCE_NAME: "res_name", + HEADER_RESOURCE_DEF_DESC: "description", + HEADER_RESOURCE_DEF_TYPE: "type", + HEADER_RESOURCE_DEF_ORDER: "order", + HEADER_RESOURCE_DEF_BUILTIN: "builtin", + HEADER_RESOURCE_DEF_CONSUMABLE: "consumable", + HEADER_RESOURCE_DEF_DYNAMIC: "dynamic", + HEADER_RESOURCE_DEF_RELEASE: "release", + HEADER_RESOURCE_DEF_INTERVAL: "interval", +} + +var ResourceDefDetailHeaders = []string{ + HEADER_RESOURCE_DEF_TYPE, + HEADER_RESOURCE_DEF_ORDER, + HEADER_RESOURCE_DEF_INTERVAL, + HEADER_RESOURCE_DEF_BUILTIN, + HEADER_RESOURCE_DEF_DYNAMIC, + HEADER_RESOURCE_DEF_RELEASE, + HEADER_RESOURCE_DEF_CONSUMABLE, +} + +// ResourceValue can handle both string and boolean values from the API +type ResourceValue struct { + Value string +} + +// ResourceLocation can handle both string and array values from the API +type ResourceLocation struct { + Value string +} + +// UnmarshalJSON customizes the JSON parsing method to handle string, boolean, and numeric values +func (rv *ResourceValue) UnmarshalJSON(data []byte) error { + // First try to parse as a string + var str string + if err := json.Unmarshal(data, &str); err == nil { + rv.Value = str + return nil + } + + // If it's not a string, try to parse as a boolean + var boolVal bool + if err := json.Unmarshal(data, &boolVal); err == nil { + if boolVal { + rv.Value = "true" + } else { + rv.Value = "false" + } + return nil + } + + // If it's not a boolean, try to parse as a number (int or float) + var num json.Number + if err := json.Unmarshal(data, &num); err == nil { + rv.Value = num.String() + return nil + } + + return fmt.Errorf("cannot unmarshal ResourceValue %s", string(data)) +} + +// String returns the string representation of ResourceValue +func (rv ResourceValue) String() string { + return rv.Value +} + +// UnmarshalJSON customizes the JSON parsing method to handle string and array values +func (rl *ResourceLocation) UnmarshalJSON(data []byte) error { + // First try to parse as a string + var str string + if err := json.Unmarshal(data, &str); err == nil { + rl.Value = str + return nil + } + + // If it's not a string, try to parse as an array + var arr []interface{} + if err := json.Unmarshal(data, &arr); err == nil { + // Convert array to comma-separated string + var strValues []string + for _, item := range arr { + if strItem, ok := item.(string); ok { + strValues = append(strValues, strItem) + } else { + // Convert non-string items to string + jsonBytes, err := json.Marshal(item) + if err != nil { + return fmt.Errorf("cannot marshal location array item: %v", err) + } + strValues = append(strValues, string(jsonBytes)) + } + } + rl.Value = strings.Join(strValues, ", ") + return nil + } + + return fmt.Errorf("cannot unmarshal ResourceLocation %s", string(data)) +} + +// String returns the string representation of ResourceLocation +func (rl ResourceLocation) String() string { + return rl.Value +} + +type ResourceDistResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []ResourceDist `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +type ResourceDist struct { + ResourceName string `json:"res_name"` + ResourceValue ResourceValue `json:"value"` + ResourceLoc ResourceLocation `json:"location"` +} + +func (r *ResourceDist) UnmarshalJSON(data []byte) error { + type Alias ResourceDist + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + *r = ResourceDist(a) + return nil +} + +// ResourceDefResponse represents the API response for resource definition queries +type ResourceDefResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []ResourceDef `json:"rows"` + Page_token string `json:"page_token"` + } `json:"data"` +} + +// ResourceDef represents a resource definition +type ResourceDef struct { + ResourceName string `json:"res_name"` + Description string `json:"description"` + ResourceType string `json:"-"` // populated from value_type int + OrderType string `json:"-"` // populated from order_type int + Flag map[string]interface{} `json:"flag"` + Interval int `json:"interval"` +} + +func (r *ResourceDef) UnmarshalJSON(data []byte) error { + type Alias struct { + ResourceName string `json:"res_name"` + Description string `json:"description"` + ResourceType string `json:"value_type"` + OrderType string `json:"order_type"` + Flag map[string]interface{} `json:"flag"` + Interval int `json:"interval"` + } + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + r.ResourceName = a.ResourceName + r.Description = a.Description + r.ResourceType = a.ResourceType + r.OrderType = a.OrderType + r.Flag = a.Flag + r.Interval = a.Interval + return nil +} diff --git a/src/frontend/xcecli/models/response_test.go b/src/frontend/xcecli/models/response_test.go new file mode 100644 index 00000000..0012f3df --- /dev/null +++ b/src/frontend/xcecli/models/response_test.go @@ -0,0 +1,150 @@ +package models + +import ( + "encoding/json" + "testing" +) + +// Wire format from actual server responses (OrderMap wrapper with string status fields) + +func TestHostsResponse_FullUnmarshal(t *testing.T) { + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["host_name","host_status","num_cpus","max_slot","num_run","num_ssusp","num_ususp","host_type"],"Data":{"host_name":"host1","host_status":"OK","num_cpus":16,"max_slot":16,"num_run":3,"num_ssusp":0,"num_ususp":0,"host_type":"linux"}}],"page_token":""}}` + var resp HostsResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + h := resp.Data.Rows[0] + if h.HostName != "host1" { + t.Errorf("HostName: got %q want host1", h.HostName) + } + if h.HostStatus != "OK" { + t.Errorf("HostStatus: got %q want OK", h.HostStatus) + } + if h.NumCPUs != 16 { + t.Errorf("NumCPUs: got %d want 16", h.NumCPUs) + } +} + +func TestQueuesResponse_FullUnmarshal(t *testing.T) { + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["queue_name","queue_status","priority","num_pend","num_run"],"Data":{"queue_name":"normal","queue_status":"open:active","priority":30,"num_pend":5,"num_run":3}}],"page_token":""}}` + var resp QueuesResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + q := resp.Data.Rows[0] + if q.QueueName != "normal" { + t.Errorf("QueueName: got %q want normal", q.QueueName) + } + if q.QueueStatus != "open:active" { + t.Errorf("QueueStatus: got %q want open:active", q.QueueStatus) + } + if q.NumPend != 5 { + t.Errorf("NumPend: got %d want 5", q.NumPend) + } +} + +func TestJobsResponse_FullUnmarshal(t *testing.T) { + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["job_id","submit_user","job_status","queue_name","job_name"],"Data":{"job_id":1001,"submit_user":"user1","job_status":"RUN","queue_name":"normal","job_name":"test_job_1"}}],"page_token":""}}` + var resp JobsResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + j := resp.Data.Rows[0] + if j.JobID != 1001 { + t.Errorf("JobID: got %d want 1001", j.JobID) + } + if j.JobStatus != "RUN" { + t.Errorf("JobStatus: got %q want RUN", j.JobStatus) + } + if j.SubmitUser != "user1" { + t.Errorf("SubmitUser: got %q want user1", j.SubmitUser) + } +} + +func TestRolesResponse_FullUnmarshal(t *testing.T) { + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["role_name","action","domain_type","domain_values"],"Data":{"role_name":"basic_user","action":"cluster:view","domain_type":"all","domain_values":null}}],"page_token":""}}` + var resp RolesResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + r := resp.Data.Rows[0] + if r.RoleName != "basic_user" { + t.Errorf("RoleName: got %q want basic_user", r.RoleName) + } + if r.PermName != "cluster:view" { + t.Errorf("PermName: got %q want cluster:view", r.PermName) + } +} + +func TestUserRolesResponse_FullUnmarshal(t *testing.T) { + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["user_name","role_name"],"Data":{"user_name":"admin","role_name":"cluster_admin"}}],"page_token":""}}` + var resp UserRolesResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + u := resp.Data.Rows[0] + if u.UserName != "admin" { + t.Errorf("UserName: got %q want admin", u.UserName) + } + if u.UserRoleName != "cluster_admin" { + t.Errorf("UserRoleName: got %q want cluster_admin", u.UserRoleName) + } +} + +func TestResourceDistResponse_FullUnmarshal(t *testing.T) { + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["res_name","value","location","is_dynamic"],"Data":{"res_name":"ncpus","value":16,"location":"host1","is_dynamic":false}}],"page_token":""}}` + var resp ResourceDistResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + r := resp.Data.Rows[0] + if r.ResourceName != "ncpus" { + t.Errorf("ResourceName: got %q want ncpus", r.ResourceName) + } + if r.ResourceValue.String() != "16" { + t.Errorf("ResourceValue: got %q want 16", r.ResourceValue.String()) + } + if r.ResourceLoc.String() != "host1" { + t.Errorf("ResourceLoc: got %q want host1", r.ResourceLoc.String()) + } +} + +func TestResourceDefResponse_FullUnmarshal(t *testing.T) { + // server sends value_type and order_type as strings, flag as object + raw := `{"code":200,"msg":"Success","data":{"count":1,"rows":[{"Fields":["res_name","description","value_type","order_type","flag","interval"],"Data":{"res_name":"gpu","description":"GPU devices","value_type":"Numeric","order_type":"Decr","flag":{"builtin":true,"consumable":true,"dynamic":true,"release":false},"interval":60}}],"page_token":""}}` + var resp ResourceDefResponse + if err := json.Unmarshal([]byte(raw), &resp); err != nil { + t.Fatal(err) + } + if len(resp.Data.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(resp.Data.Rows)) + } + r := resp.Data.Rows[0] + if r.ResourceName != "gpu" { + t.Errorf("ResourceName: got %q want gpu", r.ResourceName) + } + if r.ResourceType != "Numeric" { + t.Errorf("ResourceType: got %q want Numeric", r.ResourceType) + } + if r.OrderType != "Decr" { + t.Errorf("OrderType: got %q want Decr", r.OrderType) + } +} diff --git a/src/frontend/xcecli/models/roles.go b/src/frontend/xcecli/models/roles.go new file mode 100644 index 00000000..2a9331cf --- /dev/null +++ b/src/frontend/xcecli/models/roles.go @@ -0,0 +1,53 @@ +package models + +type CreateRoleRequest struct { + RoleName string `json:"role_name" binding:"required"` + Permission []Permission `json:"permissions" binding:"required"` +} + +type CreateRoleResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + RoleName string `json:"role_name"` + Message string `json:"message"` + } `json:"data"` +} + +type RoleInfo struct { + RoleName string `json:"role_name"` + PermName string `json:"action"` + DomainType string `json:"domain_type"` + DomainValues interface{} `json:"domain_values"` +} + +type RolesResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []RoleInfo `json:"rows"` + PageToken string `json:"page_token"` + } `json:"data"` +} + +const ( + HEADER_ROLE_NAME = "ROLE_NAME" + HEADER_ROLE_PERM_NAME = "PERM_NAME" + HEADER_ROLE_DOMAIN_TYPE = "DOMAIN_TYPE" + HEADER_ROLE_DOMAIN_VALUES = "DOMAIN_VALUES" +) + +var RoleFieldsMap = map[string]string{ + HEADER_ROLE_NAME: "role_name", +} + +func (r *RoleInfo) UnmarshalJSON(data []byte) error { + type Alias RoleInfo + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + *r = RoleInfo(a) + return nil +} diff --git a/src/frontend/xcecli/models/users.go b/src/frontend/xcecli/models/users.go new file mode 100644 index 00000000..6a0ea85e --- /dev/null +++ b/src/frontend/xcecli/models/users.go @@ -0,0 +1,54 @@ +package models + +const ( + HEADER_USER_NAME = "USER_NAME" + HEADER_USERROLE_NAME = "ROLE_NAME" +) + +var UserRoleFieldsMap = map[string]string{ + HEADER_USER_NAME: "user_name", +} + +type AddRoleForUserRequest struct { + UserName string `json:"user_name" binding:"required"` + RoleName []string `json:"role_name" binding:"required"` +} + +type UserAssignRoleResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + UserName string `json:"user_name"` + Message string `json:"message"` + } `json:"data"` +} +type UserRolesResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Count int `json:"count"` + Rows []UserRoleInfo `json:"rows"` + PageToken string `json:"page_token"` + } `json:"data"` +} + +type UserRoleInfo struct { + UserName string `json:"user_name"` + UserRoleName string `json:"role_name"` +} + +type UserResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data string `json:"data"` +} + +func (u *UserRoleInfo) UnmarshalJSON(data []byte) error { + type Alias UserRoleInfo + var a Alias + if err := UnmarshalOrderMapRow(data, &a); err != nil { + return err + } + *u = UserRoleInfo(a) + return nil +} diff --git a/src/frontend/xcecli/router/router.go b/src/frontend/xcecli/router/router.go new file mode 100755 index 00000000..6feece31 --- /dev/null +++ b/src/frontend/xcecli/router/router.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "fmt" + apiserver "xcecli/cmd/auth" + "xcecli/cmd/clusters" + "xcecli/cmd/hosts" + "xcecli/cmd/jobs" + "xcecli/cmd/permissions" + "xcecli/cmd/queues" + "xcecli/cmd/resources" + "xcecli/cmd/roles" + "xcecli/cmd/users" + "xcecli/common/request" + "xcecli/config" + + "github.com/spf13/cobra" +) + +var ( + rootCmd = &cobra.Command{ + Use: "xce", + Short: "XCE CLI tool for the API Server operations", + } + configManager *config.ConfigManager +) + +// execute the root command +func Execute(cm *config.ConfigManager) error { + if cm == nil { + return fmt.Errorf("the config manager is not initialized") + } + configManager = cm + rootCmd.CompletionOptions.DisableDefaultCmd = true + rootCmd.SilenceErrors = true + rootCmd.SilenceUsage = true + + // initialize the subcommands + initCommands() + + return rootCmd.Execute() +} + +// withAuth wraps a command with authentication check +// This function creates a middleware that checks authentication before executing any command +func withAuth(cmd *cobra.Command) *cobra.Command { + // Store original handlers + originalRunE := cmd.RunE + originalRun := cmd.Run + + // Create new RunE that checks authentication first + cmd.RunE = func(cmd *cobra.Command, args []string) error { + // Check authentication before executing the command + _, err := configManager.IsAuthenticated() + if err != nil { + return err + } + + // Execute the original command logic + if originalRunE != nil { + return originalRunE(cmd, args) + } + if originalRun != nil { + originalRun(cmd, args) + return nil + } + return nil + } + + return cmd +} + +// addCommandWithAuth adds a command and all its subcommands with authentication check +func addCommandWithAuth(parent *cobra.Command, cmd *cobra.Command) { + // Apply auth check to the command itself + withAuth(cmd) + + // Apply auth check to all subcommands recursively + for _, subCmd := range cmd.Commands() { + withAuth(subCmd) + } + + parent.AddCommand(cmd) +} + +// initialize all the subcommands +func initCommands() { + cobra.EnableCommandSorting = false + rootCmd.AddGroup( + &cobra.Group{ + ID: "api", + Title: "API Server Operations", + }, + &cobra.Group{ + ID: "query", + Title: "Query Operations", + }, + ) + + // Add authentication commands (no auth required) + rootCmd.AddCommand(apiserver.NewLogonCmd(configManager)) + rootCmd.AddCommand(apiserver.NewLogoutCmd(configManager)) + + // Add commands that require authentication + // These commands will automatically check authentication before execution + addCommandWithAuth(rootCmd, apiserver.NewListCmd(configManager)) + addCommandWithAuth(rootCmd, permissions.NewPermsCmd(configManager)) + addCommandWithAuth(rootCmd, roles.NewRolesCmd(configManager)) + addCommandWithAuth(rootCmd, users.NewUsersCmd(configManager)) + addCommandWithAuth(rootCmd, jobs.NewJobsCmd(configManager)) + addCommandWithAuth(rootCmd, hosts.NewHostsCmd(configManager)) + addCommandWithAuth(rootCmd, queues.NewQueuesCmd(configManager)) + addCommandWithAuth(rootCmd, clusters.NewClustersCmd(configManager)) + addCommandWithAuth(rootCmd, resources.NewResourcesCmd(configManager)) + rootCmd.SetFlagErrorFunc(request.FlagErrorHandler) +} diff --git a/src/frontend/xcecli/test/t.sh b/src/frontend/xcecli/test/t.sh new file mode 100644 index 00000000..5cc292ed --- /dev/null +++ b/src/frontend/xcecli/test/t.sh @@ -0,0 +1,37 @@ +Num=$1 + + + + + +script_dir=$(cd "$(dirnam + + +login_command="${XCECMD}/xce login -u user01 -p aaa123 --url http://172.31.1.61:8080" +list_jobs_nums="${XCECMD}/xce job list -u user01" +submit_command="${XCECMD}/xce job submit -R \"select(\!mg)\" -J test sleep 10" + + + + + + + + + + + + +for ((i=1; i<=Num; i++)); do + echo "Running command $i/" + $submit + + + + + + + + +echo "Total running time: $total_time seconds" +$list_jobs_nums diff --git a/src/lib/CLAUDE.md b/src/lib/CLAUDE.md new file mode 100644 index 00000000..7c69972c --- /dev/null +++ b/src/lib/CLAUDE.md @@ -0,0 +1,30 @@ +# XCE Shared Library (libxce) + +## Overview +Shared C library providing common utilities, data structures, and IPC primitives used by the controller, worker, and scheduler daemons. + +## Build +```bash +# From workspace root +nix develop +meson build && ninja -C build +``` + +## Architecture +- Produces `libxce.a` (static) and/or `libxce.so` (shared) +- Linked by all C/C++ system components: controller, worker, scheduler +- Public headers exposed via `src/include/xce/` +- Test suite in `src/lib/testsuite/` + +## Conventions +- C standard: C99 or later +- Format with `clang-format` (config at repo root if present) +- All public API headers go in `src/include/xce/` — not in `src/lib/` +- Implementation files stay in `src/lib/` + +## Key Files +- `src/include/xce/` — Public header files consumed by all components +- `src/lib/testsuite/` — Library unit tests + +## See Also +- Root guidelines: `/workspace/CLAUDE.md` diff --git a/src/lib/meson.build b/src/lib/meson.build index 7aa7fa58..19e548b0 100644 --- a/src/lib/meson.build +++ b/src/lib/meson.build @@ -1,11 +1,13 @@ capnp = find_program('capnp') packcc = find_program('packcc') +schema_dir = meson.current_source_dir() / '..' / 'schema' + gen_srcs = custom_target( 'controllercapnp.[ch]', output:['controller.capnp.c', 'controller.capnp.h'], input: '../schema/controller.capnp', - command: [capnp, 'compile','-o' ,'c:lib', '-I', '../schema','--src-prefix', '../schema', '@INPUT@'] + command: [capnp, 'compile','-o' ,'c:lib', '-I', schema_dir, '--src-prefix', schema_dir, '@INPUT@'] ) section_parser = custom_target( diff --git a/src/meson.build b/src/meson.build index 0d7e9272..fd77f71d 100644 --- a/src/meson.build +++ b/src/meson.build @@ -5,7 +5,8 @@ project('xce', default_options: [ 'c_std=gnu17']) -with_debug=false +component = get_option('component') +with_debug = false if get_option('buildtype') == 'debug' # add -DWITH_DEBUG=1 @@ -23,56 +24,63 @@ if get_option('buildtype') == 'debug' '-fvar-tracking-assignments', ], language: ['c', 'cpp']) add_project_link_arguments('-fno-lto', language: ['c', 'cpp']) - with_debug=true + with_debug = true endif add_project_arguments('-DVERSION_COMMIT_ID='+get_option('commit_id'), language: 'c') -c_compiler_id = meson.get_compiler('c').get_id() -if c_compiler_id == 'gcc' or c_compiler_id == 'clang' - wrap_link_args = ['-Wl,--wrap=printf', '-Wl,--wrap=perror'] -else - wrap_link_args = [] -endif - +# Programs needed by all components go_compiler = find_program('go') -pkgconfig = find_program('pkg-config') go_build = files('tools/go_build') go_build_program = find_program('go_build', go_build) env_program = find_program('env') -include_dir = include_directories('include', 'include/xce') +if component != 'xcecli' + c_compiler_id = meson.get_compiler('c').get_id() + if c_compiler_id == 'gcc' or c_compiler_id == 'clang' + wrap_link_args = ['-Wl,--wrap=printf', '-Wl,--wrap=perror'] + else + wrap_link_args = [] + endif -cc = meson.get_compiler('c') -math_dep = cc.find_library('m', required: false) -rt_dep = cc.find_library('rt', required: false) + pkgconfig = find_program('pkg-config') + include_dir = include_directories('include', 'include/xce') -struct2json_dep = dependency('struct2json') -libcyaml_dep = dependency('libcyaml') -libyaml_dep = dependency('yaml-0.1') -libtirpc_dep = dependency('libtirpc') -librtti_dep = dependency('librtti') + cc = meson.get_compiler('c') + math_dep = cc.find_library('m', required: false) + rt_dep = cc.find_library('rt', required: false) -yaml_deps = [ - libcyaml_dep, - libyaml_dep -] + struct2json_dep = dependency('struct2json') + libcyaml_dep = dependency('libcyaml') + libyaml_dep = dependency('yaml-0.1') + libtirpc_dep = dependency('libtirpc') + librtti_dep = dependency('librtti') -subdir('lib') -subdir('controller') -subdir('worker') -subdir('autoscaler') -subdir('scheduler') -subdir('apiserver') + yaml_deps = [ + libcyaml_dep, + libyaml_dep + ] -configure_file( - input: 'tools/appimage/entry.sh', - output: 'xce', - copy: true, - install: true, - install_dir: 'bin', - install_mode: 'rwxr-xr-x' -) + subdir('lib') + subdir('controller') + subdir('worker') + subdir('autoscaler') + subdir('scheduler') + subdir('frontend/apiserver') -install_data('docs/xce.md', - install_dir: 'share/docs') + configure_file( + input: 'tools/appimage/entry.sh', + output: 'xce', + copy: true, + install: true, + install_dir: 'bin', + install_mode: 'rwxr-xr-x' + ) + + install_data('docs/xce.md', + install_dir: 'share/docs') +endif + +if component != 'xcecore' + subdir('frontend/xcecli') +endif diff --git a/src/meson.options b/src/meson.options index 3152c579..abc77f6b 100644 --- a/src/meson.options +++ b/src/meson.options @@ -1,2 +1,6 @@ option('with_debug', type: 'boolean', value: false, yield: true) option('commit_id', type: 'string', value:'', yield: true) +option('component', type: 'combo', + choices: ['all', 'xcecore', 'xcecli'], + value: 'all', + description: 'Which component(s) to build: all (default, for dev), xcecore (C stack + apiserver), xcecli (CLI only)') diff --git a/src/nixSupport/default.nix b/src/nixSupport/default.nix index b434f325..5d3dd4d9 100644 --- a/src/nixSupport/default.nix +++ b/src/nixSupport/default.nix @@ -14,8 +14,12 @@ let withDebug = true; withAsan = true; }; + xcecli = pkgs.callPackage ./xcecli.nix { + inherit pkgs myCommitId; + }; in { xcecore = xcecore; xcecore_debug = xcecore_debug; + xcecli = xcecli; } diff --git a/src/nixSupport/xce.nix b/src/nixSupport/xce.nix index c4a06902..2203fe19 100644 --- a/src/nixSupport/xce.nix +++ b/src/nixSupport/xce.nix @@ -22,7 +22,10 @@ stdenv.mkDerivation { pkg-config cmake go - delve + go-swag # swagger doc generation: swag init + gotools # goimports, godoc + gopls # Go language server + delve # Go debugger (openssl.overrideAttrs (oldAttrs: { separateDebugInfo = true; })) @@ -57,6 +60,7 @@ stdenv.mkDerivation { separateDebugInfo = true; })) packcc + cacert ] ++ (if withDebug then [ (librtti_debug.overrideAttrs (oldattrs: { separateDebugInfo = true; @@ -76,6 +80,6 @@ stdenv.mkDerivation { (lib.optional withDebug "--buildtype=debug") (lib.optional withAsan "-Db_sanitize=address") ("-Dcommit_id=" + myCommitId) - # (lib.optional (!with_debug) "-Dbuildtype=release") + "-Dcomponent=xcecore" ]; } diff --git a/src/nixSupport/xcecli.nix b/src/nixSupport/xcecli.nix new file mode 100644 index 00000000..981b3f2f --- /dev/null +++ b/src/nixSupport/xcecli.nix @@ -0,0 +1,32 @@ +{ + lib, + stdenv, + pkgs, + myCommitId, + ... +}: + +stdenv.mkDerivation { + pname = "xcecli"; + version = "0.0.1"; + + src = ./..; + + nativeBuildInputs = with pkgs; [ + meson + ninja + go + cacert + which + findutils + ]; + + build-system = [ pkgs.meson ]; + + impureEnvVars = [ "HOME" ]; + + mesonFlags = [ + "-Dcomponent=xcecli" + ("-Dcommit_id=" + myCommitId) + ]; +} diff --git a/src/scheduler/CLAUDE.md b/src/scheduler/CLAUDE.md new file mode 100644 index 00000000..1f859a0e --- /dev/null +++ b/src/scheduler/CLAUDE.md @@ -0,0 +1,28 @@ +# XCE Scheduler + +## Overview +C-based job scheduling engine with a plugin architecture. Determines job-to-node assignments based on resource requirements and scheduling policies. + +## Build +```bash +# From workspace root +nix develop +meson build && ninja -C build +``` + +## Architecture +- Plugin-based: scheduling policies live under `src/plugins/` +- Each plugin implements a standard scheduling interface +- Core engine selects the active plugin at runtime + +Links against `libxce` from `src/lib/`. + +## Conventions +- C standard: C99 or later +- Format with `clang-format` +- New scheduling policies → add as a plugin under `src/plugins/`, do not modify the core engine +- Shared types defined in `src/include/xce/` (libxce) + +## See Also +- Shared library: `src/lib/CLAUDE.md` +- Root guidelines: `/workspace/CLAUDE.md` diff --git a/src/worker/CLAUDE.md b/src/worker/CLAUDE.md new file mode 100644 index 00000000..34c1a5fa --- /dev/null +++ b/src/worker/CLAUDE.md @@ -0,0 +1,28 @@ +# XCE Worker Daemon (xcecd) + +## Overview +C daemon running on each compute node. Manages job execution, hardware monitoring, and process lifecycle on worker nodes. + +## Build +```bash +# From workspace root +nix develop +meson build && ninja -C build +``` + +## Architecture +- `src/xcecd.c` — Main daemon entry point +- `src/hardware_monitor.c` — CPU/memory/GPU resource monitoring +- `src/process_manager.c` — Job process lifecycle (spawn, track, reap) + +Links against `libxce` from `src/lib/`. + +## Conventions +- C standard: C99 or later +- Format with `clang-format` +- Headers in `include/` +- Shared types defined in `src/include/xce/` (libxce) + +## See Also +- Shared library: `src/lib/CLAUDE.md` +- Root guidelines: `/workspace/CLAUDE.md`