fix(deploy-ha): stop-all.sh/cleanup.sh must find the real worker regardless of sudo process-tree depth (bug-1823)
bug-1820's own fix assumed sudo always forks exactly one direct child that IS the real target, and looked for it via `pgrep -P $PID` (direct children only). On a real host, sudo commonly interposes an additional "monitor" process between the invoked sudo and the actual target (sudo -> monitor -> target, three processes, not two -- standard sudo 1.8.7+ architecture for signal/exit-status relay) -- a shape this sandbox's own simpler sudo build doesn't reproduce, which is why bug-1820 verified clean here but still left the real worker orphaned on the reporting user's box. `pgrep -P` found and killed the monitor (a direct child) while the actual worker (the monitor's own child, not a direct child of the pidfile'd PID) survived untouched. Replaces the parent/child tree-walking approach entirely with a depth-independent identity-based re-scan: both generated scripts now search system-wide (pgrep -f) for any still-running process whose own cmdline matches the worker's distinguishing identity (-n $NAME plus this deploy's -l $TOP/logs -- the same two args start-worker.sh's own "already running" guard checks) and kill every match directly, however many hops deep it landed. No longer needs to know or guess sudo's internal process-tree depth. Live-verified: regression-checked against this sandbox's own real (2-level) sudo deploy (still clean), and built + killed a synthetic 3-level process tree modeled on the user's exact live report, confirming the deepest process is found directly and the intermediate hop then exits on its own once its child is gone. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
parent
bfb0db4af9
commit
7b274ec0df
2 changed files with 92 additions and 55 deletions
|
|
@ -325,33 +325,40 @@ for PF in $TOP/run/*.pid; do
|
|||
NAME=\$(basename "\$PF" .pid)
|
||||
if [ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null; then
|
||||
echo "cleanup.sh: stopping \$NAME (PID \$PID)"
|
||||
# bug-1820: the real (sudo-launched) worker's pidfile records sudo's
|
||||
# OWN pid -- start-worker.sh's "exec sudo \$WORKER_BIN ..." replaces
|
||||
# the pidfile-writing shell with sudo in place (same pid), but sudo
|
||||
# then forks a SEPARATE, root-owned child that actually runs the
|
||||
# worker. kill -9 on sudo's pid kills only sudo -- SIGKILL cannot be
|
||||
# forwarded to a child -- silently orphaning the real worker, and the
|
||||
# "still alive" check below never catches it either (sudo's own pid,
|
||||
# owned by the invoking user, dies cleanly). Detect a sudo wrapper by
|
||||
# its cmdline and kill its child(ren) first, via sudo (root-owned).
|
||||
CHILDREN=""
|
||||
if tr '\0' ' ' < /proc/\$PID/cmdline 2>/dev/null | grep -q '^sudo '; then
|
||||
CHILDREN=\$(pgrep -P "\$PID" 2>/dev/null || true)
|
||||
for CHILD in \$CHILDREN; do
|
||||
echo " \$NAME (PID \$PID) is a sudo wrapper -- also stopping its child (PID \$CHILD)"
|
||||
sudo kill -9 "\$CHILD" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
kill -9 "\$PID" 2>/dev/null || true
|
||||
sleep 0.2
|
||||
STILL_ALIVE=0
|
||||
kill -0 "\$PID" 2>/dev/null && STILL_ALIVE=1
|
||||
for CHILD in \$CHILDREN; do
|
||||
[ -d "/proc/\$CHILD" ] && STILL_ALIVE=1
|
||||
done
|
||||
if [ "\$STILL_ALIVE" -eq 1 ]; then
|
||||
echo "Warning: \$NAME (PID \$PID \$CHILDREN) still alive after kill -9 -- likely the real worker running as root; try: sudo kill -9 \$PID \$CHILDREN"
|
||||
fi
|
||||
fi
|
||||
# bug-1820/1823: a real (sudo-launched) worker's pidfile records sudo's
|
||||
# OWN pid (start-worker.sh's "exec sudo \$WORKER_BIN ..." replaces the
|
||||
# pidfile-writing shell with sudo in place, same pid) -- but the ACTUAL
|
||||
# worker process can be several sudo-internal hops away: modern sudo
|
||||
# (1.8.7+) commonly interposes an additional "monitor" process before
|
||||
# the real target (sudo -> monitor -> target, three processes, not
|
||||
# two), so a parent/child walk of just \$PID's DIRECT children (this
|
||||
# fix's own first attempt, bug-1820) finds and kills the monitor while
|
||||
# the real worker -- the monitor's OWN child, not a direct child of
|
||||
# the pidfile'd PID -- survives untouched. Live-reproduced: process-
|
||||
# tree depth varies by sudo version/config, so rather than guess how
|
||||
# many hops deep it goes on any given host, independently re-scan for
|
||||
# ANY still-running process whose own cmdline matches this worker's
|
||||
# distinguishing identity (its -n \$NAME flag plus this deploy's -l
|
||||
# $TOP/logs flag -- the same two args start-worker.sh's own "already
|
||||
# running" guard already checks) and kill it directly, wherever it
|
||||
# landed in the process tree. A no-op for ctrl-N/scheduler/apiserver
|
||||
# pidfiles, which use neither flag.
|
||||
SURVIVORS=\$(pgrep -f -- "-n \$NAME .*-l $TOP/logs" 2>/dev/null || true)
|
||||
for SURVIVOR in \$SURVIVORS; do
|
||||
echo " \$NAME: process \$SURVIVOR still running -- stopping it directly (sudo process-tree workaround)"
|
||||
kill -9 "\$SURVIVOR" 2>/dev/null || true
|
||||
sudo kill -9 "\$SURVIVOR" 2>/dev/null || true
|
||||
done
|
||||
sleep 0.2
|
||||
STILL_ALIVE=0
|
||||
[ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null && STILL_ALIVE=1
|
||||
for SURVIVOR in \$SURVIVORS; do
|
||||
[ -d "/proc/\$SURVIVOR" ] && STILL_ALIVE=1
|
||||
done
|
||||
if [ "\$STILL_ALIVE" -eq 1 ]; then
|
||||
echo "Warning: \$NAME (PID \$PID \$SURVIVORS) still alive after kill -9 -- likely the real worker running as root; try: sudo kill -9 \$PID \$SURVIVORS"
|
||||
fi
|
||||
done
|
||||
rm -f $TOP/logs/*.log
|
||||
|
|
@ -378,38 +385,41 @@ for PF in $TOP/run/*.pid; do
|
|||
[ -f "\$PF" ] || continue
|
||||
PID=\$(cat "\$PF" 2>/dev/null || true)
|
||||
NAME=\$(basename "\$PF" .pid)
|
||||
FOUND_SOMETHING=0
|
||||
if [ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null; then
|
||||
echo "Stopping \$NAME (PID \$PID)"
|
||||
# bug-1820: see cleanup.sh's identical comment above -- the real
|
||||
# (sudo-launched) worker's pidfile records sudo's OWN pid (its
|
||||
# start-worker.sh does "exec sudo \$WORKER_BIN ...", so the
|
||||
# pidfile-writing shell becomes sudo in place, same pid), but sudo
|
||||
# forks a SEPARATE, root-owned child that actually runs the worker.
|
||||
# kill -9 on sudo's pid only kills sudo -- SIGKILL cannot be
|
||||
# forwarded to a child -- silently orphaning the real worker, and
|
||||
# the "still alive" check below never caught it either (sudo's own
|
||||
# pid, owned by the invoking user, dies cleanly). Detect a sudo
|
||||
# wrapper by its cmdline and kill its child(ren) first, via sudo.
|
||||
CHILDREN=""
|
||||
if tr '\0' ' ' < /proc/\$PID/cmdline 2>/dev/null | grep -q '^sudo '; then
|
||||
CHILDREN=\$(pgrep -P "\$PID" 2>/dev/null || true)
|
||||
for CHILD in \$CHILDREN; do
|
||||
echo " \$NAME (PID \$PID) is a sudo wrapper -- also stopping its child (PID \$CHILD)"
|
||||
sudo kill -9 "\$CHILD" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
kill -9 "\$PID" 2>/dev/null || true
|
||||
sleep 0.2
|
||||
STILL_ALIVE=0
|
||||
kill -0 "\$PID" 2>/dev/null && STILL_ALIVE=1
|
||||
for CHILD in \$CHILDREN; do
|
||||
[ -d "/proc/\$CHILD" ] && STILL_ALIVE=1
|
||||
done
|
||||
if [ "\$STILL_ALIVE" -eq 1 ]; then
|
||||
echo "Warning: \$NAME (PID \$PID \$CHILDREN) still alive after kill -9 -- likely the real worker running as root; try: sudo kill -9 \$PID \$CHILDREN"
|
||||
else
|
||||
STOPPED=1
|
||||
fi
|
||||
FOUND_SOMETHING=1
|
||||
fi
|
||||
# bug-1820/1823: see cleanup.sh's identical comment above -- a real
|
||||
# (sudo-launched) worker's pidfile records sudo's OWN pid, but the
|
||||
# ACTUAL worker process can be several sudo-internal hops away (modern
|
||||
# sudo commonly interposes an additional "monitor" process before the
|
||||
# real target, so a parent/child walk of just \$PID's DIRECT children
|
||||
# -- this fix's own first attempt, bug-1820 -- finds and kills the
|
||||
# monitor while the real worker, the monitor's OWN child, survives
|
||||
# untouched). Rather than guess the process-tree depth, independently
|
||||
# re-scan for ANY still-running process whose own cmdline matches this
|
||||
# worker's distinguishing identity (-n \$NAME plus this deploy's -l
|
||||
# $TOP/logs) and kill it directly. A no-op for ctrl-N/scheduler/
|
||||
# apiserver pidfiles, which use neither flag.
|
||||
SURVIVORS=\$(pgrep -f -- "-n \$NAME .*-l $TOP/logs" 2>/dev/null || true)
|
||||
[ -n "\$SURVIVORS" ] && FOUND_SOMETHING=1
|
||||
for SURVIVOR in \$SURVIVORS; do
|
||||
echo " \$NAME: process \$SURVIVOR still running -- stopping it directly (sudo process-tree workaround)"
|
||||
kill -9 "\$SURVIVOR" 2>/dev/null || true
|
||||
sudo kill -9 "\$SURVIVOR" 2>/dev/null || true
|
||||
done
|
||||
sleep 0.2
|
||||
STILL_ALIVE=0
|
||||
[ -n "\$PID" ] && kill -0 "\$PID" 2>/dev/null && STILL_ALIVE=1
|
||||
for SURVIVOR in \$SURVIVORS; do
|
||||
[ -d "/proc/\$SURVIVOR" ] && STILL_ALIVE=1
|
||||
done
|
||||
if [ "\$STILL_ALIVE" -eq 1 ]; then
|
||||
echo "Warning: \$NAME (PID \$PID \$SURVIVORS) still alive after kill -9 -- likely the real worker running as root; try: sudo kill -9 \$PID \$SURVIVORS"
|
||||
elif [ "\$FOUND_SOMETHING" -eq 1 ]; then
|
||||
STOPPED=1
|
||||
fi
|
||||
rm -f "\$PF"
|
||||
done
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue