lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(xce-deploy-ha): support real xcecd worker deployment alongside virtual

Browse source 
Adds auto-detection of real vs virtual worker based on systemd
availability (/run/systemd/system), plus an XCE_REAL_WORKER override.
Real xcecd requires root (systemd D-Bus transient units, hardcoded
/var/run paths) and is used opportunistically now that passwordless sudo
is available in the sandbox; defaults to xcecd_virtual when systemd/D-Bus
isn't available.
This commit is contained in:
Rongsong Shen 2026-08-19 09:48:53 +00:00
parent c9acd502c0
commit 83e61eecc8
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
2 changed files with 150 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

74
src/tools/deploy-ha.sh
View file

@ -85,9 +85,48 @@ fi
# only: xce-test-ha's soak driver also calls this script and never builds
# either, so a missing worker/scheduler binary must not fail the deploy --
# it just skips generating that one start script, with a warning.
WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd_virtual"
#
# Which of the two sibling worker binaries this deploy's start-worker.sh
# targets is auto-detected by default, overridable with XCE_REAL_WORKER=0/1:
# xcecd_virtual: skips the root-only sqlite db-recovery and domain-socket
# subsystems, runs as an ordinary user, no sudo needed.
# xcecd (the production daemon): worker/src/main.c's check_user()
# unconditionally requires geteuid()==0, so the generated
# start-worker.sh below wraps its exec in `sudo` and preflights
# `sudo mkdir -p /var/run` (xcecd hardcodes db_path=/var/run/xcecd.db
# and domain_sock_file=/var/run/.xcecd.sock -- neither is a flag).
# Also, actually dispatching a job goes through sd_bus_open_system() ->
# org.freedesktop.systemd1.Manager.StartTransientUnit
# (worker/src/xcecd_cgroups.c) -- it needs a real systemd manager
# answering on the system dbus, not just root.
#
# Auto-detection: /run/systemd/system existing is systemd's own sd_booted(3)
# signal for "a real systemd instance is running as this host's manager" --
# true on an actual host/VM, false in sandboxes that boot under something
# else (verified 2026-08-19: this dev sandbox has no /run at all, PID 1 is
# s6-svscan, no dbus-daemon/systemctl even under `nix develop`). Real job
# dispatch needs that systemd manager reachable regardless of root, so
# picking xcecd_virtual when it's absent -- and xcecd when it's present --
# means this one deploy-ha.sh works unmodified on both a throwaway sandbox
# and a real HA test host. XCE_REAL_WORKER=0/1 overrides the auto-detect in
# either direction (e.g. force xcecd_virtual on a real host, or force xcecd
# on a sandbox to exercise its root-gated startup path even though job
# dispatch there will still fail).
if [ -n "${XCE_REAL_WORKER:-}" ]; then
USE_REAL_WORKER="$XCE_REAL_WORKER"
elif [ -d /run/systemd/system ]; then
USE_REAL_WORKER=1
else
USE_REAL_WORKER=0
fi
if [ "$USE_REAL_WORKER" = "1" ]; then
WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd"
else
WORKER_BIN="$BINARY_BUILD_DIR/worker/xcecd_virtual"
fi
echo "Worker binary: $([ "$USE_REAL_WORKER" = "1" ] && echo "xcecd (real, root via sudo)" || echo "xcecd_virtual (no root)") -- override with XCE_REAL_WORKER=0/1"
if [ ! -x "$WORKER_BIN" ]; then
echo "Warning: no xcecd_virtual found at $WORKER_BIN -- skipping scripts/start-worker.sh generation (run '/xce-build worker' first if you need it)"
echo "Warning: no $(basename "$WORKER_BIN") found at $WORKER_BIN -- skipping scripts/start-worker.sh generation (run '/xce-build worker' first if you need it)"
WORKER_BIN=""
fi
SCHED_BIN="$BINARY_BUILD_DIR/scheduler/xce_scheduler"
@ -229,15 +268,29 @@ chmod +x "$TOP/scripts/cleanup.sh"
# Generate worker start script (component-failover-adoption's --peers path):
# same $PEERS_STR the controllers themselves were just given, so a worker
# started via this script fails over across the identical candidate list.
# xcecd_virtual (not production xcecd) is the default -- it skips the
# root-only sqlite/db-recovery and domain-socket subsystems, so it runs as
# an ordinary user in this same dev/CI environment with no sudo, which is
# what this skill's "local testing" scope calls for. Override with
# XCE_WORKER_BIN=/path/to/xcecd for a real worker if you specifically need
# job execution / cgroups (requires root -- run it yourself, outside this
# script, since this skill/session cannot sudo).
# Which binary WORKER_BIN points at (xcecd_virtual or xcecd -- auto-detected
# or XCE_REAL_WORKER-overridden above) was decided once, at deploy time;
# XCE_WORKER_BIN can still override the path at run time (e.g. to point at a
# differently-built xcecd), but does NOT flip the sudo/preflight branch below
# -- that follows USE_REAL_WORKER as resolved for THIS deploy, since a
# run-time path swap doesn't change whether the binary needs root.
if [ -n "$WORKER_BIN" ]; then
cat > "$TOP/scripts/start-worker.sh" << EOF
if [ "$USE_REAL_WORKER" = "1" ]; then
cat > "$TOP/scripts/start-worker.sh" << EOF
#!$GEN_BASH
set -e
NAME=\${1:-real-worker}
[ \$# -gt 0 ] && shift
WORKER_BIN="\${XCE_WORKER_BIN:-$WORKER_BIN}"
mkdir -p $TOP/logs
# xcecd (production) requires root (worker/src/main.c:check_user()) and
# hardcodes db_path=/var/run/xcecd.db + domain_sock_file=/var/run/.xcecd.sock
# -- neither is a flag, so /var/run must exist before it will start.
sudo mkdir -p /var/run
sudo "\$WORKER_BIN" --peers "$PEERS_STR" -n "\$NAME" -l $TOP/logs \$*
EOF
else
cat > "$TOP/scripts/start-worker.sh" << EOF
#!$GEN_BASH
set -e
NAME=\${1:-worker-1}
@ -246,6 +299,7 @@ WORKER_BIN="\${XCE_WORKER_BIN:-$WORKER_BIN}"
mkdir -p $TOP/logs
"\$WORKER_BIN" --peers "$PEERS_STR" -n "\$NAME" -l $TOP/logs \$*
EOF
fi
chmod +x "$TOP/scripts/start-worker.sh"
fi