forked from lei.chen/myxce
795 lines
26 KiB
Bash
Executable file
795 lines
26 KiB
Bash
Executable file
#!/bin/bash
|
||
|
||
# prepare env begin
|
||
# sudo chmod 775 /nix/var/nix/daemon-socket
|
||
# sudo systemctl restart nix-daemon
|
||
# prepare env end
|
||
|
||
# customzie begin
|
||
#g_code_base="/home/jsxing/allwork/git/xce"
|
||
g_code_base="/home/jsxing/allwork/git/myxce"
|
||
|
||
# services registry: name|binary path, managed by start/stop/restart/status
|
||
g_services=(
|
||
"controller|${g_code_base}/src/build/controller/xce_controller"
|
||
"scheduler|${g_code_base}/src/build/scheduler/xce_scheduler"
|
||
"worker|${g_code_base}/src/build/worker/xcecd"
|
||
"apiserver|${g_code_base}/src/build/frontend/apiserver/xce_apiserver"
|
||
)
|
||
|
||
# Unified deployment root: ALL configs, logs and runtime state live under one
|
||
# directory next to all.sh, so nothing references system dirs (/opt/xce) or
|
||
# stale repo paths. Layout:
|
||
# deploy/conf/ worker.yaml + other generated configs
|
||
# deploy/run/ pid files, certs, controller workdir, apiserver topdir
|
||
# deploy/logs/ controller/scheduler/worker log files
|
||
g_deploy_dir="${g_code_base}/deploy"
|
||
g_conf_dir="${g_deploy_dir}/conf" # generated config files (worker.yaml, ...)
|
||
g_run_dir="${g_deploy_dir}/run" # pid files, certs, controller/apiserver data
|
||
g_log_dir="${g_deploy_dir}/logs" # log files
|
||
|
||
# Log verbosity for ALL components. One of: info | debug | debug2 | debug3.
|
||
# controller/scheduler: mapped to -v / -vv / -vvv (argp --verbose)
|
||
# worker (xcecd): written into worker.yaml log_level field
|
||
# apiserver: written into logger.yaml level (zap; debug2/3 -> debug)
|
||
g_loglevel="debug3"
|
||
|
||
# Per-job worker binary install dir. xcecd_worker is exec'd by systemd AS the
|
||
# job's exec user (e.g. admin), so it must live somewhere every user can reach.
|
||
# The repo path under /home/<user> is usually 750 and blocks other users ->
|
||
# exec fails with "Permission denied" (sh exit 126) and every job dies instantly.
|
||
# /opt/xce/bin is the canonical XCE layout. svc_prepare_worker installs it there
|
||
# via sudo and svc_env_worker points XCE_SERVER_DIR at it. Can be overridden.
|
||
g_worker_bin_dir="/opt/xce/bin"
|
||
# customize end
|
||
|
||
# usage begin
|
||
|
||
# ./all.sh build -c # 清理
|
||
# ./all.sh build -d # 开发构建（带耗时统计）
|
||
# ./all.sh build -p # 生产构建（带耗时统计）
|
||
# ./all.sh config deploy # 构建+部署集群配置（cluster.bin）并重启 controller
|
||
# ./all.sh config backup # 备份所有服务配置到 ~/all_configs
|
||
# ./all.sh start [svc] # 启动服务（默认全部）：controller scheduler worker apiserver
|
||
# ./all.sh stop [svc] # 停止服务（默认全部）
|
||
# ./all.sh restart [svc] # 重启服务（默认全部）
|
||
# ./all.sh status [svc] # 查看服务状态（默认全部）
|
||
|
||
# -d/-p：增量编译，只有 xceadmin 每次固定重跑（~11s）+ 首次/源码变更时全量。初次全量约 60s。
|
||
# -c：彻底，重建一次即可得到干净状态。
|
||
# svc 取值为：controller / scheduler / worker，可传多个，如 ./all.sh start controller scheduler
|
||
# worker (xcecd) 需要 root，通过 sudo 启动/停止；sudo 需免密或已有缓存凭证（sudo -n）
|
||
# 无 cluster.bin 时 controller 会拒绝 worker join（EINVAL）——用 ./all.sh config deploy 部署
|
||
# 所有配置/logs/run 统一在 ${g_deploy_dir}（deploy/）下：
|
||
# worker.yaml 生成在 ${g_conf_dir}/worker.yaml（不再用 /opt/xce/conf/worker.yaml）
|
||
# controller 配置在 ${g_run_dir}/controller/ctrl-1/conf
|
||
# apiserver 配置在 ${g_run_dir}/apiserver/config
|
||
# 日志在 ${g_log_dir}/，pid 在 ${g_run_dir}/
|
||
|
||
# usage end
|
||
|
||
# log in a function, please print the calling function name
|
||
function _in() {
|
||
echo "[$(date +'%Y-%m-%d %H:%M:%S')] [${FUNCNAME[1]}] $1"
|
||
}
|
||
|
||
# Map g_loglevel -> argp --verbose flag count (-v/-vv/-vvv) for controller and
|
||
# scheduler (they call verbose_to_loglevel(); see src/lib/utils.c).
|
||
function _verbose_flags() {
|
||
case "${g_loglevel:-info}" in
|
||
debug3) echo "-vvv" ;;
|
||
debug2) echo "-vv" ;;
|
||
debug) echo "-v" ;;
|
||
*) echo "" ;;
|
||
esac
|
||
}
|
||
|
||
# Map g_loglevel -> worker.yaml log_level value (xcecd level_to_int()).
|
||
function _worker_loglevel() {
|
||
case "${g_loglevel:-info}" in
|
||
debug|debug2|debug3|notice|warn|error|fatal|off) echo "${g_loglevel}" ;;
|
||
*) echo "info" ;;
|
||
esac
|
||
}
|
||
|
||
# Map g_loglevel -> apiserver logger.yaml level (zap; only debug/info/warn/...).
|
||
function _apiserver_loglevel() {
|
||
case "${g_loglevel:-info}" in
|
||
debug|debug2|debug3) echo "debug" ;;
|
||
*) echo "info" ;;
|
||
esac
|
||
}
|
||
|
||
# run a command and log the command and its output
|
||
function run_cmd() {
|
||
local datestr=$(date "+%Y-%m-%d %H:%M:%S")
|
||
local cmd="$1 2>&1"
|
||
echo "[${datestr}]executing: ${1}"
|
||
|
||
echo "${cmd}" | awk '
|
||
function cmd( c )
|
||
{
|
||
while( (c|getline foo) > 0 )
|
||
printf( "%s\n", foo )
|
||
close( c );
|
||
}
|
||
|
||
{
|
||
cmd( $0 );
|
||
}
|
||
'
|
||
}
|
||
|
||
# ---- service management begin ----
|
||
|
||
# Return the startup arguments for a service.
|
||
# Override these in the customize section to fit your deployment.
|
||
function svc_args_controller() {
|
||
echo "--myname ctrl-1 --peers ctrl-1=localhost:9990 \
|
||
--workdir ${g_run_dir}/controller --certdir ${g_run_dir}/certs \
|
||
--logfile ${g_log_dir}/controller.log --vcpus 4 $(_verbose_flags)"
|
||
}
|
||
function svc_args_scheduler() {
|
||
echo "-l ${g_log_dir}/scheduler.log $(_verbose_flags)"
|
||
}
|
||
function svc_args_worker() {
|
||
# NOTE: when -c is given xcecd IGNORES -l/-p/-m (config file wins), so the
|
||
# log dir is fully controlled by ${g_conf_dir}/worker.yaml (log_dir field).
|
||
echo "-c ${g_conf_dir}/worker.yaml -n localhost01"
|
||
}
|
||
|
||
# Extra environment for the worker (xcecd) process. XCE_SERVER_DIR tells xcecd
|
||
# where to find xcecd_worker for per-job workers; it must point at a directory
|
||
# every job exec user can reach (see g_worker_bin_dir). Without it xcecd infers
|
||
# the dir from its own binary path (under /home/<user>) which job users can't
|
||
# traverse -> exec fails "Permission denied" / exit 126. svc_start prepends
|
||
# these via `env` for sudo-launched services.
|
||
function svc_env_worker() {
|
||
echo "XCE_SERVER_DIR=${g_worker_bin_dir}"
|
||
}
|
||
function svc_args_apiserver() {
|
||
echo "-d ${g_run_dir}/apiserver"
|
||
}
|
||
|
||
# services that must run as root (launched/stopped via sudo)
|
||
function svc_needs_sudo() {
|
||
case "$1" in
|
||
worker) return 0 ;;
|
||
*) return 1 ;;
|
||
esac
|
||
}
|
||
|
||
# Optional per-service prepare hook, called by svc_start before launch.
|
||
# controller: ensure the self-signed TLS cert/key exist (like deploy-ha.sh).
|
||
function svc_prepare_controller() {
|
||
local certdir="${g_run_dir}/certs"
|
||
mkdir -p "${certdir}"
|
||
if [ ! -f "${certdir}/xce.crt" ] || [ ! -f "${certdir}/xce.key" ]; then
|
||
_in "Generating self-signed TLS cert/key in ${certdir}"
|
||
openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
|
||
-keyout "${certdir}/xce.key" -out "${certdir}/xce.crt" \
|
||
-subj "/CN=xce-dev-test" 2>/dev/null
|
||
fi
|
||
}
|
||
|
||
# apiserver: ensure <topdir>/config/app.yaml + logger.yaml exist
|
||
# (app.yaml schema-valid, backend -> controller on 9990). Without them the
|
||
# Go binary refuses to start.
|
||
function svc_prepare_apiserver() {
|
||
local topdir="${g_run_dir}/apiserver"
|
||
local confdir="${topdir}/config"
|
||
mkdir -p "${confdir}"
|
||
if [ ! -f "${confdir}/logger.yaml" ]; then
|
||
_in "Generating default ${confdir}/logger.yaml (level=$(_apiserver_loglevel))"
|
||
cat > "${confdir}/logger.yaml" <<EOF
|
||
logging:
|
||
level: "$(_apiserver_loglevel)"
|
||
encoding: "console"
|
||
outputPath: "apiserver.log"
|
||
file:
|
||
maxSize: 64
|
||
maxBackups: 10
|
||
maxAge: 30
|
||
compress: true
|
||
encoderConfig:
|
||
messageKey: "msg"
|
||
levelKey: "level"
|
||
timeKey: "ts"
|
||
nameKey: "logger"
|
||
callerKey: "caller"
|
||
functionKey: "func"
|
||
stacktraceKey: "stacktrace"
|
||
timeEncoder: "iso8601"
|
||
levelEncoder: "capital"
|
||
EOF
|
||
else
|
||
# keep existing logger.yaml but sync level to the configured g_loglevel
|
||
local want="$(_apiserver_loglevel)"
|
||
if ! grep -q '^ level: "'"${want}"'"' "${confdir}/logger.yaml"; then
|
||
_in "updating ${confdir}/logger.yaml level -> ${want}"
|
||
sed -i 's/^ level: ".*"/ level: "'"${want}"'"/' "${confdir}/logger.yaml"
|
||
fi
|
||
fi
|
||
if [ ! -f "${confdir}/rbac.yaml" ]; then
|
||
if [ -f "${g_code_base}/src/frontend/apiserver/config/rbac.yaml" ]; then
|
||
_in "Copying sample rbac.yaml"
|
||
cp "${g_code_base}/src/frontend/apiserver/config/rbac.yaml" \
|
||
"${confdir}/rbac.yaml"
|
||
else
|
||
_in "Generating minimal rbac.yaml"
|
||
cat > "${confdir}/rbac.yaml" <<'EOF'
|
||
permissions: []
|
||
roles: {}
|
||
users: {}
|
||
EOF
|
||
fi
|
||
fi
|
||
if [ ! -f "${confdir}/auth.yaml" ]; then
|
||
if [ -f "${g_code_base}/src/frontend/apiserver/config/auth.yaml" ]; then
|
||
_in "Copying sample auth.yaml"
|
||
cp "${g_code_base}/src/frontend/apiserver/config/auth.yaml" \
|
||
"${confdir}/auth.yaml"
|
||
else
|
||
_in "Generating minimal auth.yaml"
|
||
cat > "${confdir}/auth.yaml" <<'EOF'
|
||
provider: config_file
|
||
config_file:
|
||
users:
|
||
- username: admin
|
||
password: admin
|
||
roles: [cluster_admin]
|
||
EOF
|
||
fi
|
||
fi
|
||
if [ ! -f "${confdir}/app.yaml" ]; then
|
||
_in "Generating default ${confdir}/app.yaml"
|
||
cat > "${confdir}/app.yaml" <<'EOF'
|
||
app:
|
||
name: "xce_apiserver"
|
||
runmode: "release"
|
||
http_port: 8080
|
||
version: "1.0.0"
|
||
|
||
security:
|
||
https:
|
||
enabled: false
|
||
port: 8443
|
||
cert_file: "config/ssl/server.crt"
|
||
key_file: "config/ssl/server.key"
|
||
|
||
authentication:
|
||
token:
|
||
type: "jwt"
|
||
secretKey: "xce-api-key"
|
||
token_expire_time: 86400
|
||
|
||
rate_limit:
|
||
global:
|
||
rate: 1000
|
||
burst: 2000
|
||
window: second
|
||
endpoints:
|
||
/xce/v1/auth/logon:
|
||
rate: 100
|
||
burst: 200
|
||
window: second
|
||
|
||
whitelist:
|
||
- "0.0.0.0/0"
|
||
- "::/0"
|
||
|
||
file_upload:
|
||
max_size: 100 MB
|
||
allowed_types:
|
||
- "application/x-tar"
|
||
- "application/zip"
|
||
|
||
job:
|
||
max_array_size: 1000
|
||
|
||
cache:
|
||
refresh_interval: 30
|
||
|
||
cors:
|
||
enabled: true
|
||
allowed_origins:
|
||
- "*"
|
||
|
||
backend:
|
||
address: "localhost"
|
||
port: 9990
|
||
from: "xce_apiserver"
|
||
key: "keepsecret$"
|
||
EOF
|
||
fi
|
||
}
|
||
|
||
# worker: generate worker.yaml under ${g_conf_dir} so the worker never reads a
|
||
# system path (/opt/xce/conf/worker.yaml) or a stale repo's log_dir. The
|
||
# log_dir field must point at our ${g_log_dir}; when -c is given xcecd ignores
|
||
# -l so this file is the single source of truth for the worker's log location.
|
||
function svc_prepare_worker() {
|
||
mkdir -p "${g_conf_dir}"
|
||
local wf="${g_conf_dir}/worker.yaml"
|
||
local lvl="$(_worker_loglevel)"
|
||
if [ -f "$wf" ]; then
|
||
# keep existing file but sync log_level to the configured g_loglevel
|
||
if ! grep -q "log_level: \"${lvl}\"" "$wf"; then
|
||
_in "updating worker.yaml log_level -> ${lvl}"
|
||
sed -i "s/log_level:.*/log_level: \"${lvl}\"/" "$wf"
|
||
fi
|
||
_in "worker config: ${wf} (log_level=${lvl})"
|
||
else
|
||
_in "Generating ${wf}"
|
||
cat > "$wf" <<EOF
|
||
---
|
||
workerconf:
|
||
management_list: ["localhost"]
|
||
controller_port: 9990
|
||
log_level: "${lvl}"
|
||
log_dir: "${g_log_dir}"
|
||
EOF
|
||
fi
|
||
|
||
# Install xcecd_worker somewhere every job exec user can run it. The job's
|
||
# worker is launched by systemd with User=<exec user>; the repo path under
|
||
# /home/<user> is usually 750, so other users get "Permission denied"
|
||
# (exit 126) and jobs die instantly. xcecd finds the binary via
|
||
# XCE_SERVER_DIR (svc_env_worker), so it must match this install dir.
|
||
local worker_bin="${g_code_base}/src/build/worker/xcecd_worker"
|
||
local worker_dst="${g_worker_bin_dir}/xcecd_worker"
|
||
if [ -x "${worker_bin}" ]; then
|
||
if [ ! -f "${worker_dst}" ] || [ "${worker_bin}" -nt "${worker_dst}" ]; then
|
||
_in "installing ${worker_bin} -> ${worker_dst}"
|
||
sudo -n mkdir -p "${g_worker_bin_dir}"
|
||
sudo -n install -m 0755 "${worker_bin}" "${worker_dst}"
|
||
fi
|
||
else
|
||
_in "WARN: ${worker_bin} not found; jobs will fail to exec xcecd_worker" >&2
|
||
fi
|
||
}
|
||
|
||
# Optional per-service precheck hook, called by svc_start BEFORE launching.
|
||
# controller: refuse to start without a deployed cluster.bin — without one the
|
||
# controller never becomes primary and apiserver's backend init fails with
|
||
# -10013 (bug-1514). Prints the active config when present.
|
||
function svc_check_controller() {
|
||
local confdir="${g_run_dir}/controller/ctrl-1/conf"
|
||
local cur ref bin
|
||
if [ ! -f "${confdir}/current" ]; then
|
||
_in "controller 未部署集群配置（缺少 ${confdir}/current）" >&2
|
||
_in "请先运行: ./all.sh config deploy # 从 src/configs 生成 cluster.bin 并部署" >&2
|
||
_in "然后再: ./all.sh start" >&2
|
||
return 1
|
||
fi
|
||
ref="$(cat "${confdir}/current" 2>/dev/null)"
|
||
bin="${confdir}/${ref}"
|
||
if [ -n "$ref" ] && [ -f "$bin" ]; then
|
||
_in "controller 使用集群配置: ${bin} (current -> ${ref})"
|
||
return 0
|
||
fi
|
||
_in "controller 配置指针 current -> ${ref} 指向的文件不存在（${bin}）" >&2
|
||
_in "请重新运行: ./all.sh config deploy" >&2
|
||
return 1
|
||
}
|
||
|
||
# list registered service names
|
||
function _svc_names() {
|
||
local entry n
|
||
for entry in "${g_services[@]}"; do
|
||
n="${entry%%|*}"
|
||
echo "$n"
|
||
done
|
||
}
|
||
|
||
# resolve binary path for a service name (empty if unknown)
|
||
function _svc_bin() {
|
||
local name="$1" entry n b
|
||
for entry in "${g_services[@]}"; do
|
||
n="${entry%%|*}"; b="${entry#*|}"
|
||
[ "$n" = "$name" ] && { echo "$b"; return 0; }
|
||
done
|
||
return 1
|
||
}
|
||
|
||
function _svc_pidfile() { echo "${g_run_dir}/$1.pid"; }
|
||
|
||
# check liveness of a service pid; uses sudo for root (sudo) services
|
||
function _svc_alive() {
|
||
local name="$1" pid="$2"
|
||
if svc_needs_sudo "$name"; then
|
||
sudo -n kill -0 "$pid" 2>/dev/null
|
||
else
|
||
kill -0 "$pid" 2>/dev/null
|
||
fi
|
||
}
|
||
|
||
# send a signal to a service pid; uses sudo for root (sudo) services
|
||
function _svc_kill() {
|
||
local name="$1" sig="$2" pid="$3"
|
||
if svc_needs_sudo "$name"; then
|
||
if [ -n "$sig" ]; then
|
||
sudo -n kill -"$sig" "$pid" 2>/dev/null
|
||
else
|
||
sudo -n kill "$pid" 2>/dev/null
|
||
fi
|
||
else
|
||
if [ -n "$sig" ]; then
|
||
kill -"$sig" "$pid" 2>/dev/null
|
||
else
|
||
kill "$pid" 2>/dev/null
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# print the live pid of a service (or empty); cleanup stale pidfile
|
||
function _svc_pid() {
|
||
local name="$1" pidfile pid
|
||
pidfile="$(_svc_pidfile "$name")"
|
||
if [ -f "$pidfile" ]; then
|
||
pid="$(cat "$pidfile" 2>/dev/null)"
|
||
if [ -n "$pid" ] && _svc_alive "$name" "$pid"; then
|
||
echo "$pid"
|
||
return 0
|
||
fi
|
||
rm -f "$pidfile"
|
||
fi
|
||
return 1
|
||
}
|
||
|
||
# resolve target services from args; default to all registered
|
||
function _svc_resolve() {
|
||
if [ $# -eq 0 ]; then
|
||
_svc_names
|
||
return 0
|
||
fi
|
||
local name bin
|
||
for name in "$@"; do
|
||
bin="$(_svc_bin "$name")" || {
|
||
_in "unknown service: $name (valid: $(_svc_names | tr '\n' ' '))" >&2
|
||
return 1
|
||
}
|
||
echo "$name"
|
||
done
|
||
}
|
||
|
||
# start a single service (background, pidfile + log)
|
||
function svc_start() {
|
||
local name="$1" bin args pidfile log
|
||
bin="$(_svc_bin "$name")" || return 1
|
||
args="$(svc_args_${name} 2>/dev/null)" || args=""
|
||
|
||
[ -x "$bin" ] || { _in "binary not found: $bin (run ./all.sh build first)"; return 1; }
|
||
if [ -n "$(_svc_pid "$name")" ]; then
|
||
# already running: still report the active config (best-effort).
|
||
# stdout carries the "using config" line; stderr hints are muted here.
|
||
if declare -F "svc_check_${name}" >/dev/null 2>&1; then
|
||
"svc_check_${name}" 2>/dev/null || true
|
||
fi
|
||
_in "$name already running (pid $(_svc_pid "$name"))"
|
||
return 0
|
||
fi
|
||
|
||
mkdir -p "${g_run_dir}" "${g_log_dir}"
|
||
pidfile="$(_svc_pidfile "$name")"
|
||
log="${g_log_dir}/${name}.log"
|
||
|
||
# optional per-service prepare hook (e.g. controller TLS certs)
|
||
if declare -F "svc_prepare_${name}" >/dev/null 2>&1; then
|
||
"svc_prepare_${name}" || return 1
|
||
fi
|
||
|
||
# optional per-service precheck hook (e.g. controller requires cluster.bin)
|
||
if declare -F "svc_check_${name}" >/dev/null 2>&1; then
|
||
"svc_check_${name}" || return 1
|
||
fi
|
||
|
||
if svc_needs_sudo "$name"; then
|
||
_in "Starting ${name} as root via sudo: ${bin} ${args}"
|
||
# optional per-service env (e.g. worker: XCE_SERVER_DIR so job workers
|
||
# can exec xcecd_worker from a world-accessible dir)
|
||
local envp=""
|
||
if declare -F "svc_env_${name}" >/dev/null 2>&1; then
|
||
envp="$(svc_env_${name})"
|
||
fi
|
||
# inner bash writes its own pid, then exec's into the binary so the
|
||
# pidfile holds the real (root) process pid; nohup guards SIGHUP.
|
||
# sudo -n = non-interactive; requires NOPASSWD or a cached credential.
|
||
sudo -n env ${envp} bash -c "echo \$\$ > '${pidfile}'; exec nohup '${bin}' ${args}" \
|
||
>>"${log}" 2>&1 < /dev/null &
|
||
else
|
||
_in "Starting ${name}: ${bin} ${args}"
|
||
setsid nohup ${bin} ${args} >>"${log}" 2>&1 < /dev/null &
|
||
echo $! > "${pidfile}"
|
||
fi
|
||
disown 2>/dev/null || true
|
||
|
||
sleep 1
|
||
local pid
|
||
pid="$(_svc_pid "$name")"
|
||
if [ -n "$pid" ]; then
|
||
_in "${name} started (pid $pid, log ${log})"
|
||
else
|
||
_in "${name} FAILED to start -- check ${log}"
|
||
rm -f "${pidfile}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# stop a single service (pidfile, then pgrep fallback)
|
||
function svc_stop() {
|
||
local name="$1" bin pid pidfile
|
||
bin="$(_svc_bin "$name")" || return 1
|
||
pidfile="$(_svc_pidfile "$name")"
|
||
|
||
pid="$(_svc_pid "$name")"
|
||
if [ -z "$pid" ]; then
|
||
# fallback: match by binary name
|
||
pid="$(pgrep -f "$(basename "$bin")" 2>/dev/null | head -n1)"
|
||
fi
|
||
|
||
if [ -z "$pid" ]; then
|
||
_in "${name} is not running"
|
||
rm -f "${pidfile}"
|
||
return 0
|
||
fi
|
||
|
||
_in "Stopping ${name} (pid $pid)"
|
||
_svc_kill "$name" "" "$pid"
|
||
local i
|
||
for i in $(seq 1 10); do
|
||
_svc_alive "$name" "$pid" || break
|
||
sleep 0.3
|
||
done
|
||
if _svc_alive "$name" "$pid"; then
|
||
_in "${name} did not exit, sending SIGKILL"
|
||
_svc_kill "$name" 9 "$pid"
|
||
sleep 0.3
|
||
fi
|
||
rm -f "${pidfile}"
|
||
_in "${name} stopped"
|
||
}
|
||
|
||
# restart a single service
|
||
function svc_restart() {
|
||
local name="$1"
|
||
svc_stop "$name"
|
||
svc_start "$name"
|
||
}
|
||
|
||
# print status of a single service
|
||
function svc_status() {
|
||
local name="$1" bin pid pidfile log state
|
||
bin="$(_svc_bin "$name")" || return 1
|
||
pidfile="$(_svc_pidfile "$name")"
|
||
log="${g_log_dir}/${name}.log"
|
||
|
||
pid="$(_svc_pid "$name")"
|
||
if [ -n "$pid" ]; then
|
||
state="running"
|
||
else
|
||
state="stopped"
|
||
fi
|
||
printf "%-12s %-8s %-8s\n" "$name" "$state" "${pid:--}"
|
||
printf " binary: %s\n" "$bin"
|
||
if [ -f "$log" ]; then
|
||
printf " log: %s\n" "$log"
|
||
printf " last: %s\n" "$(tail -n1 "$log" 2>/dev/null)"
|
||
fi
|
||
}
|
||
|
||
# ---- service management end ----
|
||
|
||
# clean
|
||
function build_clean() {
|
||
_in "Cleaning build artifacts..."
|
||
local src_dir=${g_code_base}/src
|
||
for dir in build build-prod; do
|
||
if [ -d "${src_dir}/${dir}" ]; then
|
||
run_cmd "rm -rf ${src_dir}/${dir}"
|
||
_in "Removed ${dir}/"
|
||
fi
|
||
done
|
||
}
|
||
|
||
function build_dev() {
|
||
_in "Building development version..."
|
||
local src_dir=${g_code_base}/src
|
||
cd ${src_dir}
|
||
local start_ts=$(date +%s)
|
||
run_cmd "nix develop --extra-experimental-features \"nix-command flakes\" --command bash -c \"meson setup build --buildtype=debug -Db_sanitize=address && meson compile -C build\""
|
||
local end_ts=$(date +%s)
|
||
local elapsed=$((end_ts - start_ts))
|
||
_in "Build finished in $((elapsed / 60))m $((elapsed % 60))s (${elapsed}s total)"
|
||
echo "output is in ${src_dir}/build"
|
||
}
|
||
|
||
function build_prod() {
|
||
_in "Building production version..."
|
||
local src_dir=${g_code_base}/src
|
||
cd ${src_dir}
|
||
local start_ts=$(date +%s)
|
||
run_cmd "nix develop --extra-experimental-features \"nix-command flakes\" --command bash -c \"meson setup build-prod --buildtype=release && meson compile -C build-prod\""
|
||
local end_ts=$(date +%s)
|
||
local elapsed=$((end_ts - start_ts))
|
||
_in "Build finished in $((elapsed / 60))m $((elapsed % 60))s (${elapsed}s total)"
|
||
echo "output is in ${src_dir}/build-prod"
|
||
}
|
||
|
||
# deploy cluster config: build cluster.bin from src/configs via xceadmin
|
||
# (in nix develop), place it in ctrl-1/conf, set conf/current, then restart
|
||
# the controller so it loads the config. Without a deployed cluster.bin the
|
||
# controller rejects worker joins with EINVAL (no hostconfig matched).
|
||
function config_deploy() {
|
||
_in "Deploying cluster config..."
|
||
local src_dir=${g_code_base}/src
|
||
local xceadmin=""
|
||
local candidate
|
||
|
||
# xceadmin lives under whichever meson build dir; try each in turn
|
||
for candidate in \
|
||
"${src_dir}/build/xceadmin/.venv/bin/xceadmin" \
|
||
"${src_dir}/build-prod/xceadmin/.venv/bin/xceadmin" \
|
||
"$(command -v xceadmin 2>/dev/null || true)"
|
||
do
|
||
if [ -n "$candidate" ] && [ -x "$candidate" ]; then
|
||
xceadmin="$candidate"
|
||
break
|
||
fi
|
||
done
|
||
if [ -z "$xceadmin" ]; then
|
||
_in "xceadmin not found (run ./all.sh build first)"
|
||
return 1
|
||
fi
|
||
_in "Using xceadmin: ${xceadmin}"
|
||
|
||
local tmpbin="${g_run_dir}/.cluster.bin.tmp"
|
||
local confdir="${g_run_dir}/controller/ctrl-1/conf"
|
||
|
||
_in "Building cluster.bin from ${src_dir}/configs (in nix develop)"
|
||
cd ${src_dir}
|
||
run_cmd "nix develop --extra-experimental-features \"nix-command flakes\" --command bash -c \"${xceadmin} config build ${src_dir}/configs -o ${tmpbin}\""
|
||
|
||
if [ ! -s "${tmpbin}" ]; then
|
||
_in "cluster.bin build failed -- check output above"
|
||
rm -f "${tmpbin}"
|
||
return 1
|
||
fi
|
||
|
||
mkdir -p "${confdir}"
|
||
local tag
|
||
tag="$(sha256sum "${tmpbin}" | cut -c1-12)"
|
||
mv "${tmpbin}" "${confdir}/cluster.bin.${tag}"
|
||
echo "cluster.bin.${tag}" > "${confdir}/current"
|
||
_in "Deployed cluster.bin.${tag} (current -> cluster.bin.${tag}) in ${confdir}"
|
||
|
||
_in "Restarting controller to load config"
|
||
svc_restart controller
|
||
}
|
||
|
||
# back up all services' runtime configs (controller certs/conf, worker.yaml,
|
||
# apiserver configs) to ~/all_configs, preserving the directory structure.
|
||
function config_backup() {
|
||
_in "Backing up all service configs to ~/all_configs..."
|
||
local dest=~/all_configs
|
||
local n=0
|
||
|
||
# controller: TLS certs
|
||
if [ -d "${g_run_dir}/certs" ]; then
|
||
mkdir -p "${dest}/controller/certs"
|
||
cp -f ${g_run_dir}/certs/* "${dest}/controller/certs/" 2>/dev/null
|
||
_in "controller certs -> ${dest}/controller/certs/"
|
||
n=$((n + 1))
|
||
fi
|
||
|
||
# controller: cluster.bin + current
|
||
if [ -d "${g_run_dir}/controller/ctrl-1/conf" ]; then
|
||
mkdir -p "${dest}/controller/conf"
|
||
cp -f ${g_run_dir}/controller/ctrl-1/conf/cluster.bin.* \
|
||
"${g_run_dir}/controller/ctrl-1/conf/current" \
|
||
"${dest}/controller/conf/" 2>/dev/null
|
||
_in "controller conf -> ${dest}/controller/conf/"
|
||
n=$((n + 1))
|
||
fi
|
||
|
||
# worker: ${g_conf_dir}/worker.yaml (generated by svc_prepare_worker)
|
||
mkdir -p "${dest}/worker"
|
||
if [ -f "${g_conf_dir}/worker.yaml" ]; then
|
||
cp -f "${g_conf_dir}/worker.yaml" "${dest}/worker/"
|
||
_in "worker.yaml -> ${dest}/worker/"
|
||
n=$((n + 1))
|
||
else
|
||
_in "worker.yaml (${g_conf_dir}/worker.yaml) not found, skipped"
|
||
fi
|
||
|
||
# apiserver: 4 config files
|
||
if [ -d "${g_run_dir}/apiserver/config" ]; then
|
||
mkdir -p "${dest}/apiserver"
|
||
cp -f ${g_run_dir}/apiserver/config/*.yaml "${dest}/apiserver/" 2>/dev/null
|
||
_in "apiserver config -> ${dest}/apiserver/"
|
||
n=$((n + 1))
|
||
fi
|
||
|
||
# scheduler: no standalone config file
|
||
mkdir -p "${dest}/scheduler"
|
||
printf 'scheduler 无独立配置文件（启动参数 -l <logfile> 已由 all.sh svc_args_scheduler 管理）\n' \
|
||
> "${dest}/scheduler/README.txt"
|
||
|
||
_in "Backup complete (${n} groups) in ${dest}"
|
||
}
|
||
|
||
# main sub-command dispatch
|
||
function cmd_build() {
|
||
local mode="dev"
|
||
while getopts "cdp" opt; do
|
||
case $opt in
|
||
c) build_clean; exit 0 ;;
|
||
d) mode="dev" ;;
|
||
p) mode="prod" ;;
|
||
?) echo "Usage: $0 build [-c] [-d|-p]" && exit 1 ;;
|
||
esac
|
||
done
|
||
|
||
if [ "$mode" = "dev" ]; then
|
||
build_dev
|
||
else
|
||
build_prod
|
||
fi
|
||
}
|
||
|
||
function _cmd_svc_foreach() {
|
||
# $1 = action (start|stop|restart|status), rest = services
|
||
local action="$1"; shift
|
||
local targets rc=0
|
||
targets="$(_svc_resolve "$@")" || exit 1
|
||
local name
|
||
for name in $targets; do
|
||
if ! "svc_${action}" "$name"; then
|
||
rc=1
|
||
# fail-fast for start/restart: one bad service (e.g. controller
|
||
# without cluster.bin) should stop the whole launch so the user
|
||
# sees the precheck hint (./all.sh config deploy) immediately.
|
||
case "$action" in
|
||
start|restart) break ;;
|
||
esac
|
||
fi
|
||
done
|
||
return "$rc"
|
||
}
|
||
|
||
function cmd_start() { _cmd_svc_foreach start "$@"; }
|
||
function cmd_stop() { _cmd_svc_foreach stop "$@"; }
|
||
function cmd_restart() { _cmd_svc_foreach restart "$@"; }
|
||
|
||
function cmd_status() {
|
||
local targets
|
||
targets="$(_svc_resolve "$@")" || return 1
|
||
printf "%-12s %-8s %-8s\n" "SERVICE" "STATE" "PID"
|
||
local name
|
||
for name in $targets; do
|
||
svc_status "$name"
|
||
done
|
||
}
|
||
|
||
function cmd_config() {
|
||
case "${1:-deploy}" in
|
||
deploy) config_deploy ;;
|
||
backup) config_backup ;;
|
||
*) echo "Usage: $0 config {deploy|backup}" && exit 1 ;;
|
||
esac
|
||
}
|
||
|
||
# dispatch
|
||
case "${1:-}" in
|
||
build) shift; cmd_build "$@" ;;
|
||
config) shift; cmd_config "$@" ;;
|
||
start) shift; cmd_start "$@" ;;
|
||
stop) shift; cmd_stop "$@" ;;
|
||
restart) shift; cmd_restart "$@" ;;
|
||
status) shift; cmd_status "$@" ;;
|
||
*) echo "Usage: $0 {build|config|start|stop|restart|status} [args]" && exit 1 ;;
|
||
esac
|
||
|
||
|
||
|
||
|