lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(worker): VNC start failure reports JOB_FAILED instead of killing the worker; fix generate_url snprintf overflow

Browse source 
xjob_start_vnc's four failure points (XCE_SERVER_DIR unset, pre-exec spawn
failure, DISPLAY export failure, port-wait timeout) called exit(1), taking
down the whole per-job worker with no user-visible reason and leaving the
job RUNNING forever (the master never learned of the death until its
instance reap path). It now returns int; the worker_job_callback call site
reports the standard failed-start pattern -- JOB_FAILED + status_message
"Failed to start VNC server (check worker log)" + worker_exit_flag so the
master reaps the instance and starts a fresh one for the next job.

xjob_stop_vnc is cleanup-only: its failures now log and return instead of
exit(1) -- a failed stop must not kill the worker or clobber the job's
exit code.

generate_url sized its snprintf with CMD_MAX_LENGTH (4096) while the url
buffer is URL_MAX_LENGTH (256). Latent since import: the old build's
non-fortified snprintf silently truncated, but any _FORTIFY_SOURCE build
aborts with SIGABRT (__chk_fail) on the FIRST interactive job -- found via
core dump after t1 bare -I jobs died ~1s after start ("*** buffer overflow
detected ***" in generate_url). Size both snprintfs with URL_MAX_LENGTH.

Verified on t1: -I --xjob on a host without vncserver fails in ~1s with
the reason in worker/master logs, worker exits cleanly, next batch job
completes; bare -I submits, dispatches, attaches via the relay and runs
until killed; daemons healthy.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-10-15 03:40:29 +00:00
parent a8c3e91b59
commit 6d5281daf1
2 changed files with 44 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/worker/src/inter_job_utils.c
View file

@ -242,10 +242,13 @@ char* generate_url(port_info_t port, int interactive_mode)
return NULL;
}
/* url is a URL_MAX_LENGTH buffer -- sizing the snprintf with
* CMD_MAX_LENGTH here is a latent overflow that glibc's _FORTIFY_SOURCE
* build turns into a SIGABRT (__chk_fail) on the first interactive job. */
if (interactive_mode == TTY || interactive_mode == DISPLAY) {
snprintf(url, CMD_MAX_LENGTH, "http://%s:%d", local_ip, port.tty_port);
snprintf(url, URL_MAX_LENGTH, "http://%s:%d", local_ip, port.tty_port);
} else if (interactive_mode == XJOB) {
snprintf(url, CMD_MAX_LENGTH, "%s::%d", local_ip, port.xjob_port);
snprintf(url, URL_MAX_LENGTH, "%s::%d", local_ip, port.xjob_port);
} else {
free(url);
free(local_ip);

50
src/worker/src/xcecd_worker.c
View file

@ -58,7 +58,7 @@ uv_loop_t* uv_loop = NULL;
static void check_and_reconnect();
static void worker_reconnect_callback(bool connected);
static void xjob_start_vnc();
static int xjob_start_vnc();
static void xjob_stop_vnc();
static bool check_process_status(pid_t target_pid, int expected_signal,
int timeout_seconds);
@ -599,7 +599,7 @@ static void check_and_reconnect()
reconnect_retry_limit = 5;
}
static void xjob_start_vnc()
static int xjob_start_vnc()
{
char jobidStr[64];
fmt_jobid(jobidStr, sizeof(jobidStr), jobid);
@ -618,7 +618,7 @@ static void xjob_start_vnc()
XCECD_WORKER_LOG(
XCE_LOG_ERROR,
"XCE_SERVER_DIR not set, cannot find VNC pre-exec command");
exit(1);
return -1;
}
} else {
snprintf(vnc_start_cmd, sizeof(vnc_start_cmd), "%s %d", XJOB_PRE_EXEC,
@ -630,14 +630,14 @@ static void xjob_start_vnc()
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Failed to start VNC server: %s",
vnc_start_cmd);
exit(1);
return -1;
}
if (export_display_env(port_info.xjob_port) == -1) {
XCECD_WORKER_LOG(
XCE_LOG_ERROR,
"Failed to export DISPLAY environment variable for VNC server");
exit(1);
return -1;
} else {
XCECD_WORKER_LOG(
XCE_LOG_DEBUG3,
@ -655,7 +655,7 @@ static void xjob_start_vnc()
"Timeout waiting for VNC server to start on port "
"%d after %d seconds",
port_info.xjob_port, timeout_seconds);
exit(1);
return -1;
}
XCECD_WORKER_LOG(
@ -668,6 +668,7 @@ static void xjob_start_vnc()
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "VNC server started for job %s",
jobidStr);
return 0;
}
static void xjob_stop_vnc()
@ -689,18 +690,20 @@ static void xjob_stop_vnc()
XCECD_WORKER_LOG(
XCE_LOG_ERROR,
"XCE_SERVER_DIR not set, cannot find VNC post-exec command");
exit(1);
return;
}
} else {
snprintf(vnc_stop_cmd, sizeof(vnc_stop_cmd), "%s %d", XJOB_POST_EXEC,
port_info.xjob_port);
}
/* Cleanup-only path: a failed stop must not kill the worker or clobber
* the job's exit code -- the VNC server will be reaped when its job's
* user session goes away or by the next startvnc on this port. */
if (uv_proc_post_exec(&uv_proc_manager, vnc_stop_cmd, NULL) != 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "Failed to stop VNC server: %s",
vnc_stop_cmd);
exit(1);
return;
}
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "VNC server stopped for job %s",
@ -834,9 +837,34 @@ static void worker_job_callback(job_assignment_t* info)
/* export XCE environment variables here */
apply_env_vars(&mydata.env_vars, jobid);
/* X server start here (blocked) */
/* X server start here (blocked). Failure must not kill the worker
* process: report JOB_FAILED with a reason and exit the worker loop
* normally, so the master can reap this instance and start a fresh one
* for the next job (previously exit(1) here took down the whole
* per-job worker with no user-visible reason). */
if (XJOB == mydata.transfer_info->interactive_mode) {
xjob_start_vnc();
if (xjob_start_vnc() != 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"Failed to start VNC server for job %s",
jobidStr);
xjob_stop_vnc(); /* best-effort cleanup of a partial start */
check_and_reconnect();
job_runtime_info.job_pid = 0;
job_runtime_info.job_status = JOB_FAILED;
job_runtime_info.worker_status = WORKER_EXITED;
job_runtime_info.exit_code = 1;
ipc_worker_send_status(
&job_runtime_info,
"Failed to start VNC server (check worker log)");
worker_exit_flag = 1;
return_code = 1;
FREEUP(job_cmd);
ipc_worker_run(UV_RUN_NOWAIT);
return;
}
}
/* Queue pre-exec */