#!/bin/bash

# prepare env begin
# sudo chmod 775 /nix/var/nix/daemon-socket
# sudo systemctl restart nix-daemon
# prepare env end

# customzie begin
#g_code_base="/home/jsxing/allwork/git/xce"
g_code_base="/home/jsxing/allwork/git/myxce"

# services registry: name|binary path, managed by start/stop/restart/status
g_services=(
    "controller|${g_code_base}/src/build/controller/xce_controller"
    "scheduler|${g_code_base}/src/build/scheduler/xce_scheduler"
    "worker|${g_code_base}/src/build/worker/xcecd"
    "apiserver|${g_code_base}/src/build/frontend/apiserver/xce_apiserver"
)

# Unified deployment root: ALL configs, logs and runtime state live under one
# directory next to all.sh, so nothing references system dirs (/opt/xce) or
# stale repo paths. Layout:
#   deploy/conf/          worker.yaml + other generated configs
#   deploy/run/           pid files, certs, controller workdir, apiserver topdir
#   deploy/logs/          controller/scheduler/worker log files
g_deploy_dir="${g_code_base}/deploy"
g_conf_dir="${g_deploy_dir}/conf" # generated config files (worker.yaml, ...)
g_run_dir="${g_deploy_dir}/run"   # pid files, certs, controller/apiserver data
g_log_dir="${g_deploy_dir}/logs"  # log files

# Log verbosity for ALL components. One of: info | debug | debug2 | debug3.
#   controller/scheduler: mapped to -v / -vv / -vvv (argp --verbose)
#   worker (xcecd):       written into worker.yaml log_level field
#   apiserver:            written into logger.yaml level (zap; debug2/3 -> debug)
g_loglevel="debug3"

# Per-job worker binary install dir. xcecd_worker is exec'd by systemd AS the
# job's exec user (e.g. admin), so it must live somewhere every user can reach.
# The repo path under /home/<user> is usually 750 and blocks other users ->
# exec fails with "Permission denied" (sh exit 126) and every job dies instantly.
# /opt/xce/bin is the canonical XCE layout. svc_prepare_worker installs it there
# via sudo and svc_env_worker points XCE_SERVER_DIR at it. Can be overridden.
g_worker_bin_dir="/opt/xce/bin"
# customize end

# usage begin

# ./all.sh build -c       # 清理
# ./all.sh build -d       # 开发构建（带耗时统计）
# ./all.sh build -p       # 生产构建（带耗时统计）
# ./all.sh config deploy  # 构建+部署集群配置（cluster.bin）并重启 controller
# ./all.sh config backup  # 备份所有服务配置到 ~/all_configs
# ./all.sh start [svc]   # 启动服务（默认全部）：controller scheduler worker apiserver
# ./all.sh stop  [svc]   # 停止服务（默认全部）
# ./all.sh restart [svc] # 重启服务（默认全部）
# ./all.sh status [svc]  # 查看服务状态（默认全部）

# -d/-p：增量编译，只有 xceadmin 每次固定重跑（~11s）+ 首次/源码变更时全量。初次全量约 60s。
# -c：彻底，重建一次即可得到干净状态。
# svc 取值为：controller / scheduler / worker，可传多个，如 ./all.sh start controller scheduler
# worker (xcecd) 需要 root，通过 sudo 启动/停止；sudo 需免密或已有缓存凭证（sudo -n）
# 无 cluster.bin 时 controller 会拒绝 worker join（EINVAL）——用 ./all.sh config deploy 部署
# 所有配置/logs/run 统一在 ${g_deploy_dir}（deploy/）下：
#   worker.yaml 生成在 ${g_conf_dir}/worker.yaml（不再用 /opt/xce/conf/worker.yaml）
#   controller 配置在 ${g_run_dir}/controller/ctrl-1/conf
#   apiserver 配置在 ${g_run_dir}/apiserver/config
#   日志在 ${g_log_dir}/，pid 在 ${g_run_dir}/

# usage end

# log in a function, please print the calling function name
function _in() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] [${FUNCNAME[1]}] $1"
}

# Map g_loglevel -> argp --verbose flag count (-v/-vv/-vvv) for controller and
# scheduler (they call verbose_to_loglevel(); see src/lib/utils.c).
function _verbose_flags() {
    case "${g_loglevel:-info}" in
        debug3) echo "-vvv" ;;
        debug2) echo "-vv" ;;
        debug)  echo "-v" ;;
        *)      echo "" ;;
    esac
}

# Map g_loglevel -> worker.yaml log_level value (xcecd level_to_int()).
function _worker_loglevel() {
    case "${g_loglevel:-info}" in
        debug|debug2|debug3|notice|warn|error|fatal|off) echo "${g_loglevel}" ;;
        *) echo "info" ;;
    esac
}

# Map g_loglevel -> apiserver logger.yaml level (zap; only debug/info/warn/...).
function _apiserver_loglevel() {
    case "${g_loglevel:-info}" in
        debug|debug2|debug3) echo "debug" ;;
        *) echo "info" ;;
    esac
}

# run a command and log the command and its output
function run_cmd() {
    local datestr=$(date "+%Y-%m-%d %H:%M:%S")
    local cmd="$1 2>&1"
    echo "[${datestr}]executing: ${1}"

    echo "${cmd}" | awk '
        function cmd( c )
        {
            while( (c|getline foo) > 0 )
                 printf( "%s\n", foo )
            close( c );
        }

        {
            cmd( $0 );
        }
    '
}

# ---- service management begin ----

# Return the startup arguments for a service.
# Override these in the customize section to fit your deployment.
function svc_args_controller() {
    echo "--myname ctrl-1 --peers ctrl-1=localhost:9990 \
--workdir ${g_run_dir}/controller --certdir ${g_run_dir}/certs \
--logfile ${g_log_dir}/controller.log --vcpus 4 $(_verbose_flags)"
}
function svc_args_scheduler() {
    echo "-l ${g_log_dir}/scheduler.log $(_verbose_flags)"
}
function svc_args_worker() {
    # NOTE: when -c is given xcecd IGNORES -l/-p/-m (config file wins), so the
    # log dir is fully controlled by ${g_conf_dir}/worker.yaml (log_dir field).
    echo "-c ${g_conf_dir}/worker.yaml -n localhost01"
}

# Extra environment for the worker (xcecd) process. XCE_SERVER_DIR tells xcecd
# where to find xcecd_worker for per-job workers; it must point at a directory
# every job exec user can reach (see g_worker_bin_dir). Without it xcecd infers
# the dir from its own binary path (under /home/<user>) which job users can't
# traverse -> exec fails "Permission denied" / exit 126. svc_start prepends
# these via `env` for sudo-launched services.
function svc_env_worker() {
    echo "XCE_SERVER_DIR=${g_worker_bin_dir}"
}
function svc_args_apiserver() {
    echo "-d ${g_run_dir}/apiserver"
}

# services that must run as root (launched/stopped via sudo)
function svc_needs_sudo() {
    case "$1" in
        worker) return 0 ;;
        *)      return 1 ;;
    esac
}

# Optional per-service prepare hook, called by svc_start before launch.
# controller: ensure the self-signed TLS cert/key exist (like deploy-ha.sh).
function svc_prepare_controller() {
    local certdir="${g_run_dir}/certs"
    mkdir -p "${certdir}"
    if [ ! -f "${certdir}/xce.crt" ] || [ ! -f "${certdir}/xce.key" ]; then
        _in "Generating self-signed TLS cert/key in ${certdir}"
        openssl req -x509 -newkey rsa:2048 -nodes -days 3650 \
            -keyout "${certdir}/xce.key" -out "${certdir}/xce.crt" \
            -subj "/CN=xce-dev-test" 2>/dev/null
    fi
}

# apiserver: ensure <topdir>/config/app.yaml + logger.yaml exist
# (app.yaml schema-valid, backend -> controller on 9990). Without them the
# Go binary refuses to start.
function svc_prepare_apiserver() {
    local topdir="${g_run_dir}/apiserver"
    local confdir="${topdir}/config"
    mkdir -p "${confdir}"
    if [ ! -f "${confdir}/logger.yaml" ]; then
        _in "Generating default ${confdir}/logger.yaml (level=$(_apiserver_loglevel))"
        cat > "${confdir}/logger.yaml" <<EOF
logging:
  level: "$(_apiserver_loglevel)"
  encoding: "console"
  outputPath: "apiserver.log"
  file:
    maxSize: 64
    maxBackups: 10
    maxAge: 30
    compress: true
  encoderConfig:
    messageKey: "msg"
    levelKey: "level"
    timeKey: "ts"
    nameKey: "logger"
    callerKey: "caller"
    functionKey: "func"
    stacktraceKey: "stacktrace"
    timeEncoder: "iso8601"
    levelEncoder: "capital"
EOF
    else
        # keep existing logger.yaml but sync level to the configured g_loglevel
        local want="$(_apiserver_loglevel)"
        if ! grep -q '^  level: "'"${want}"'"' "${confdir}/logger.yaml"; then
            _in "updating ${confdir}/logger.yaml level -> ${want}"
            sed -i 's/^  level: ".*"/  level: "'"${want}"'"/' "${confdir}/logger.yaml"
        fi
    fi
    if [ ! -f "${confdir}/rbac.yaml" ]; then
        if [ -f "${g_code_base}/src/frontend/apiserver/config/rbac.yaml" ]; then
            _in "Copying sample rbac.yaml"
            cp "${g_code_base}/src/frontend/apiserver/config/rbac.yaml" \
                "${confdir}/rbac.yaml"
        else
            _in "Generating minimal rbac.yaml"
            cat > "${confdir}/rbac.yaml" <<'EOF'
permissions: []
roles: {}
users: {}
EOF
        fi
    fi
    if [ ! -f "${confdir}/auth.yaml" ]; then
        if [ -f "${g_code_base}/src/frontend/apiserver/config/auth.yaml" ]; then
            _in "Copying sample auth.yaml"
            cp "${g_code_base}/src/frontend/apiserver/config/auth.yaml" \
                "${confdir}/auth.yaml"
        else
            _in "Generating minimal auth.yaml"
            cat > "${confdir}/auth.yaml" <<'EOF'
provider: config_file
config_file:
  users:
    - username: admin
      password: admin
      roles: [cluster_admin]
EOF
        fi
    fi
    if [ ! -f "${confdir}/app.yaml" ]; then
        _in "Generating default ${confdir}/app.yaml"
        cat > "${confdir}/app.yaml" <<'EOF'
app:
  name: "xce_apiserver"
  runmode: "release"
  http_port: 8080
  version: "1.0.0"

security:
  https:
    enabled: false
    port: 8443
    cert_file: "config/ssl/server.crt"
    key_file: "config/ssl/server.key"

authentication:
  token:
    type: "jwt"
    secretKey: "xce-api-key"
    token_expire_time: 86400

rate_limit:
  global:
    rate: 1000
    burst: 2000
    window: second
  endpoints:
    /xce/v1/auth/logon:
      rate: 100
      burst: 200
      window: second

whitelist:
  - "0.0.0.0/0"
  - "::/0"

file_upload:
  max_size: 100 MB
  allowed_types:
    - "application/x-tar"
    - "application/zip"

job:
  max_array_size: 1000

cache:
  refresh_interval: 30

cors:
  enabled: true
  allowed_origins:
    - "*"

backend:
  address: "localhost"
  port: 9990
  from: "xce_apiserver"
  key: "keepsecret$"
EOF
    fi
}

# worker: generate worker.yaml under ${g_conf_dir} so the worker never reads a
# system path (/opt/xce/conf/worker.yaml) or a stale repo's log_dir. The
# log_dir field must point at our ${g_log_dir}; when -c is given xcecd ignores
# -l so this file is the single source of truth for the worker's log location.
function svc_prepare_worker() {
    mkdir -p "${g_conf_dir}"
    local wf="${g_conf_dir}/worker.yaml"
    local lvl="$(_worker_loglevel)"
    if [ -f "$wf" ]; then
        # keep existing file but sync log_level to the configured g_loglevel
        if ! grep -q "log_level: \"${lvl}\"" "$wf"; then
            _in "updating worker.yaml log_level -> ${lvl}"
            sed -i "s/log_level:.*/log_level: \"${lvl}\"/" "$wf"
        fi
        _in "worker config: ${wf} (log_level=${lvl})"
    else
        _in "Generating ${wf}"
        cat > "$wf" <<EOF
---
workerconf:
  management_list: ["localhost"]
  controller_port: 9990
  log_level: "${lvl}"
  log_dir: "${g_log_dir}"
EOF
    fi

    # Install xcecd_worker somewhere every job exec user can run it. The job's
    # worker is launched by systemd with User=<exec user>; the repo path under
    # /home/<user> is usually 750, so other users get "Permission denied"
    # (exit 126) and jobs die instantly. xcecd finds the binary via
    # XCE_SERVER_DIR (svc_env_worker), so it must match this install dir.
    local worker_bin="${g_code_base}/src/build/worker/xcecd_worker"
    local worker_dst="${g_worker_bin_dir}/xcecd_worker"
    if [ -x "${worker_bin}" ]; then
        if [ ! -f "${worker_dst}" ] || [ "${worker_bin}" -nt "${worker_dst}" ]; then
            _in "installing ${worker_bin} -> ${worker_dst}"
            sudo -n mkdir -p "${g_worker_bin_dir}"
            sudo -n install -m 0755 "${worker_bin}" "${worker_dst}"
        fi
    else
        _in "WARN: ${worker_bin} not found; jobs will fail to exec xcecd_worker" >&2
    fi
}

# Optional per-service precheck hook, called by svc_start BEFORE launching.
# controller: refuse to start without a deployed cluster.bin — without one the
# controller never becomes primary and apiserver's backend init fails with
# -10013 (bug-1514). Prints the active config when present.
function svc_check_controller() {
    local confdir="${g_run_dir}/controller/ctrl-1/conf"
    local cur ref bin
    if [ ! -f "${confdir}/current" ]; then
        _in "controller 未部署集群配置（缺少 ${confdir}/current）" >&2
        _in "请先运行: ./all.sh config deploy   # 从 src/configs 生成 cluster.bin 并部署" >&2
        _in "然后再:   ./all.sh start" >&2
        return 1
    fi
    ref="$(cat "${confdir}/current" 2>/dev/null)"
    bin="${confdir}/${ref}"
    if [ -n "$ref" ] && [ -f "$bin" ]; then
        _in "controller 使用集群配置: ${bin} (current -> ${ref})"
        return 0
    fi
    _in "controller 配置指针 current -> ${ref} 指向的文件不存在（${bin}）" >&2
    _in "请重新运行: ./all.sh config deploy" >&2
    return 1
}

# list registered service names
function _svc_names() {
    local entry n
    for entry in "${g_services[@]}"; do
        n="${entry%%|*}"
        echo "$n"
    done
}

# resolve binary path for a service name (empty if unknown)
function _svc_bin() {
    local name="$1" entry n b
    for entry in "${g_services[@]}"; do
        n="${entry%%|*}"; b="${entry#*|}"
        [ "$n" = "$name" ] && { echo "$b"; return 0; }
    done
    return 1
}

function _svc_pidfile() { echo "${g_run_dir}/$1.pid"; }

# check liveness of a service pid; uses sudo for root (sudo) services
function _svc_alive() {
    local name="$1" pid="$2"
    if svc_needs_sudo "$name"; then
        sudo -n kill -0 "$pid" 2>/dev/null
    else
        kill -0 "$pid" 2>/dev/null
    fi
}

# send a signal to a service pid; uses sudo for root (sudo) services
function _svc_kill() {
    local name="$1" sig="$2" pid="$3"
    if svc_needs_sudo "$name"; then
        if [ -n "$sig" ]; then
            sudo -n kill -"$sig" "$pid" 2>/dev/null
        else
            sudo -n kill "$pid" 2>/dev/null
        fi
    else
        if [ -n "$sig" ]; then
            kill -"$sig" "$pid" 2>/dev/null
        else
            kill "$pid" 2>/dev/null
        fi
    fi
}

# print the live pid of a service (or empty); cleanup stale pidfile
function _svc_pid() {
    local name="$1" pidfile pid
    pidfile="$(_svc_pidfile "$name")"
    if [ -f "$pidfile" ]; then
        pid="$(cat "$pidfile" 2>/dev/null)"
        if [ -n "$pid" ] && _svc_alive "$name" "$pid"; then
            echo "$pid"
            return 0
        fi
        rm -f "$pidfile"
    fi
    return 1
}

# resolve target services from args; default to all registered
function _svc_resolve() {
    if [ $# -eq 0 ]; then
        _svc_names
        return 0
    fi
    local name bin
    for name in "$@"; do
        bin="$(_svc_bin "$name")" || {
            _in "unknown service: $name (valid: $(_svc_names | tr '\n' ' '))" >&2
            return 1
        }
        echo "$name"
    done
}

# start a single service (background, pidfile + log)
function svc_start() {
    local name="$1" bin args pidfile log
    bin="$(_svc_bin "$name")" || return 1
    args="$(svc_args_${name} 2>/dev/null)" || args=""

    [ -x "$bin" ] || { _in "binary not found: $bin (run ./all.sh build first)"; return 1; }
    if [ -n "$(_svc_pid "$name")" ]; then
        # already running: still report the active config (best-effort).
        # stdout carries the "using config" line; stderr hints are muted here.
        if declare -F "svc_check_${name}" >/dev/null 2>&1; then
            "svc_check_${name}" 2>/dev/null || true
        fi
        _in "$name already running (pid $(_svc_pid "$name"))"
        return 0
    fi

    mkdir -p "${g_run_dir}" "${g_log_dir}"
    pidfile="$(_svc_pidfile "$name")"
    log="${g_log_dir}/${name}.log"

    # optional per-service prepare hook (e.g. controller TLS certs)
    if declare -F "svc_prepare_${name}" >/dev/null 2>&1; then
        "svc_prepare_${name}" || return 1
    fi

    # optional per-service precheck hook (e.g. controller requires cluster.bin)
    if declare -F "svc_check_${name}" >/dev/null 2>&1; then
        "svc_check_${name}" || return 1
    fi

    if svc_needs_sudo "$name"; then
        _in "Starting ${name} as root via sudo: ${bin} ${args}"
        # optional per-service env (e.g. worker: XCE_SERVER_DIR so job workers
        # can exec xcecd_worker from a world-accessible dir)
        local envp=""
        if declare -F "svc_env_${name}" >/dev/null 2>&1; then
            envp="$(svc_env_${name})"
        fi
        # inner bash writes its own pid, then exec's into the binary so the
        # pidfile holds the real (root) process pid; nohup guards SIGHUP.
        # sudo -n = non-interactive; requires NOPASSWD or a cached credential.
        sudo -n env ${envp} bash -c "echo \$\$ > '${pidfile}'; exec nohup '${bin}' ${args}" \
            >>"${log}" 2>&1 < /dev/null &
    else
        _in "Starting ${name}: ${bin} ${args}"
        setsid nohup ${bin} ${args} >>"${log}" 2>&1 < /dev/null &
        echo $! > "${pidfile}"
    fi
    disown 2>/dev/null || true

    sleep 1
    local pid
    pid="$(_svc_pid "$name")"
    if [ -n "$pid" ]; then
        _in "${name} started (pid $pid, log ${log})"
    else
        _in "${name} FAILED to start -- check ${log}"
        rm -f "${pidfile}"
        return 1
    fi
}

# stop a single service (pidfile, then pgrep fallback)
function svc_stop() {
    local name="$1" bin pid pidfile
    bin="$(_svc_bin "$name")" || return 1
    pidfile="$(_svc_pidfile "$name")"

    pid="$(_svc_pid "$name")"
    if [ -z "$pid" ]; then
        # fallback: match by binary name
        pid="$(pgrep -f "$(basename "$bin")" 2>/dev/null | head -n1)"
    fi

    if [ -z "$pid" ]; then
        _in "${name} is not running"
        rm -f "${pidfile}"
        return 0
    fi

    _in "Stopping ${name} (pid $pid)"
    _svc_kill "$name" "" "$pid"
    local i
    for i in $(seq 1 10); do
        _svc_alive "$name" "$pid" || break
        sleep 0.3
    done
    if _svc_alive "$name" "$pid"; then
        _in "${name} did not exit, sending SIGKILL"
        _svc_kill "$name" 9 "$pid"
        sleep 0.3
    fi
    rm -f "${pidfile}"
    _in "${name} stopped"
}

# restart a single service
function svc_restart() {
    local name="$1"
    svc_stop "$name"
    svc_start "$name"
}

# print status of a single service
function svc_status() {
    local name="$1" bin pid pidfile log state
    bin="$(_svc_bin "$name")" || return 1
    pidfile="$(_svc_pidfile "$name")"
    log="${g_log_dir}/${name}.log"

    pid="$(_svc_pid "$name")"
    if [ -n "$pid" ]; then
        state="running"
    else
        state="stopped"
    fi
    printf "%-12s %-8s %-8s\n" "$name" "$state" "${pid:--}"
    printf "  binary: %s\n" "$bin"
    if [ -f "$log" ]; then
        printf "  log:    %s\n" "$log"
        printf "  last:   %s\n" "$(tail -n1 "$log" 2>/dev/null)"
    fi
}

# ---- service management end ----

# clean
function build_clean() {
    _in "Cleaning build artifacts..."
    local src_dir=${g_code_base}/src
    for dir in build build-prod; do
        if [ -d "${src_dir}/${dir}" ]; then
            run_cmd "rm -rf ${src_dir}/${dir}"
            _in "Removed ${dir}/"
        fi
    done
}

function build_dev() {
    _in "Building development version..."
    local src_dir=${g_code_base}/src
    cd ${src_dir}
    local start_ts=$(date +%s)
    run_cmd "nix develop --extra-experimental-features \"nix-command flakes\" --command bash -c \"meson setup build --buildtype=debug -Db_sanitize=address && meson compile -C build\""
    local end_ts=$(date +%s)
    local elapsed=$((end_ts - start_ts))
    _in "Build finished in $((elapsed / 60))m $((elapsed % 60))s (${elapsed}s total)"
    echo "output is in ${src_dir}/build"
}

function build_prod() {
    _in "Building production version..."
    local src_dir=${g_code_base}/src
    cd ${src_dir}
    local start_ts=$(date +%s)
    run_cmd "nix develop --extra-experimental-features \"nix-command flakes\" --command bash -c \"meson setup build-prod --buildtype=release && meson compile -C build-prod\""
    local end_ts=$(date +%s)
    local elapsed=$((end_ts - start_ts))
    _in "Build finished in $((elapsed / 60))m $((elapsed % 60))s (${elapsed}s total)"
    echo "output is in ${src_dir}/build-prod"
}

# deploy cluster config: build cluster.bin from src/configs via xceadmin
# (in nix develop), place it in ctrl-1/conf, set conf/current, then restart
# the controller so it loads the config. Without a deployed cluster.bin the
# controller rejects worker joins with EINVAL (no hostconfig matched).
function config_deploy() {
    _in "Deploying cluster config..."
    local src_dir=${g_code_base}/src
    local xceadmin=""
    local candidate

    # xceadmin lives under whichever meson build dir; try each in turn
    for candidate in \
        "${src_dir}/build/xceadmin/.venv/bin/xceadmin" \
        "${src_dir}/build-prod/xceadmin/.venv/bin/xceadmin" \
        "$(command -v xceadmin 2>/dev/null || true)"
    do
        if [ -n "$candidate" ] && [ -x "$candidate" ]; then
            xceadmin="$candidate"
            break
        fi
    done
    if [ -z "$xceadmin" ]; then
        _in "xceadmin not found (run ./all.sh build first)"
        return 1
    fi
    _in "Using xceadmin: ${xceadmin}"

    local tmpbin="${g_run_dir}/.cluster.bin.tmp"
    local confdir="${g_run_dir}/controller/ctrl-1/conf"

    _in "Building cluster.bin from ${src_dir}/configs (in nix develop)"
    cd ${src_dir}
    run_cmd "nix develop --extra-experimental-features \"nix-command flakes\" --command bash -c \"${xceadmin} config build ${src_dir}/configs -o ${tmpbin}\""

    if [ ! -s "${tmpbin}" ]; then
        _in "cluster.bin build failed -- check output above"
        rm -f "${tmpbin}"
        return 1
    fi

    mkdir -p "${confdir}"
    local tag
    tag="$(sha256sum "${tmpbin}" | cut -c1-12)"
    mv "${tmpbin}" "${confdir}/cluster.bin.${tag}"
    echo "cluster.bin.${tag}" > "${confdir}/current"
    _in "Deployed cluster.bin.${tag} (current -> cluster.bin.${tag}) in ${confdir}"

    _in "Restarting controller to load config"
    svc_restart controller
}

# back up all services' runtime configs (controller certs/conf, worker.yaml,
# apiserver configs) to ~/all_configs, preserving the directory structure.
function config_backup() {
    _in "Backing up all service configs to ~/all_configs..."
    local dest=~/all_configs
    local n=0

    # controller: TLS certs
    if [ -d "${g_run_dir}/certs" ]; then
        mkdir -p "${dest}/controller/certs"
        cp -f ${g_run_dir}/certs/* "${dest}/controller/certs/" 2>/dev/null
        _in "controller certs -> ${dest}/controller/certs/"
        n=$((n + 1))
    fi

    # controller: cluster.bin + current
    if [ -d "${g_run_dir}/controller/ctrl-1/conf" ]; then
        mkdir -p "${dest}/controller/conf"
        cp -f ${g_run_dir}/controller/ctrl-1/conf/cluster.bin.* \
              "${g_run_dir}/controller/ctrl-1/conf/current" \
              "${dest}/controller/conf/" 2>/dev/null
        _in "controller conf -> ${dest}/controller/conf/"
        n=$((n + 1))
    fi

    # worker: ${g_conf_dir}/worker.yaml (generated by svc_prepare_worker)
    mkdir -p "${dest}/worker"
    if [ -f "${g_conf_dir}/worker.yaml" ]; then
        cp -f "${g_conf_dir}/worker.yaml" "${dest}/worker/"
        _in "worker.yaml -> ${dest}/worker/"
        n=$((n + 1))
    else
        _in "worker.yaml (${g_conf_dir}/worker.yaml) not found, skipped"
    fi

    # apiserver: 4 config files
    if [ -d "${g_run_dir}/apiserver/config" ]; then
        mkdir -p "${dest}/apiserver"
        cp -f ${g_run_dir}/apiserver/config/*.yaml "${dest}/apiserver/" 2>/dev/null
        _in "apiserver config -> ${dest}/apiserver/"
        n=$((n + 1))
    fi

    # scheduler: no standalone config file
    mkdir -p "${dest}/scheduler"
    printf 'scheduler 无独立配置文件（启动参数 -l <logfile> 已由 all.sh svc_args_scheduler 管理）\n' \
        > "${dest}/scheduler/README.txt"

    _in "Backup complete (${n} groups) in ${dest}"
}

# main sub-command dispatch
function cmd_build() {
    local mode="dev"
    while getopts "cdp" opt; do
        case $opt in
            c) build_clean; exit 0 ;;
            d) mode="dev" ;;
            p) mode="prod" ;;
            ?) echo "Usage: $0 build [-c] [-d|-p]" && exit 1 ;;
        esac
    done

    if [ "$mode" = "dev" ]; then
        build_dev
    else
        build_prod
    fi
}

function _cmd_svc_foreach() {
    # $1 = action (start|stop|restart|status), rest = services
    local action="$1"; shift
    local targets rc=0
    targets="$(_svc_resolve "$@")" || exit 1
    local name
    for name in $targets; do
        if ! "svc_${action}" "$name"; then
            rc=1
            # fail-fast for start/restart: one bad service (e.g. controller
            # without cluster.bin) should stop the whole launch so the user
            # sees the precheck hint (./all.sh config deploy) immediately.
            case "$action" in
                start|restart) break ;;
            esac
        fi
    done
    return "$rc"
}

function cmd_start()   { _cmd_svc_foreach start   "$@"; }
function cmd_stop()    { _cmd_svc_foreach stop    "$@"; }
function cmd_restart() { _cmd_svc_foreach restart "$@"; }

function cmd_status() {
    local targets
    targets="$(_svc_resolve "$@")" || return 1
    printf "%-12s %-8s %-8s\n" "SERVICE" "STATE" "PID"
    local name
    for name in $targets; do
        svc_status "$name"
    done
}

function cmd_config() {
    case "${1:-deploy}" in
        deploy) config_deploy ;;
        backup) config_backup ;;
        *) echo "Usage: $0 config {deploy|backup}" && exit 1 ;;
    esac
}

# dispatch
case "${1:-}" in
    build)   shift; cmd_build "$@" ;;
    config)  shift; cmd_config "$@" ;;
    start)   shift; cmd_start "$@" ;;
    stop)    shift; cmd_stop "$@" ;;
    restart) shift; cmd_restart "$@" ;;
    status)  shift; cmd_status "$@" ;;
    *) echo "Usage: $0 {build|config|start|stop|restart|status} [args]" && exit 1 ;;
esac