fix(worker): run the job in its cwd with HOME = exec home; apply nofile_limit (bug-1936)

Cause: the start info's execcwd/exechome reached xcecd and its worker-list
entry, but the MSG_JOB_ASSIGNED reply to the per-job xcecd_worker
(job_assignment_t) carried neither, and uv_proc_exec() spawned the job
with cwd NULL, so every job ran in the xcecd_worker unit's directory
whatever --cwd said, and exec home was never applied. parse_limits()
ignored the apiserver's "nofile_limit", so num_files never became an
rlimit.

Fix: job_assignment_t gets cwd/home, filled by the new
ipc_fill_job_assignment() (uv_ipc.c, which also NUL-ends every copied
string). xcecd_worker's job callback calls the new
uv_proc_enter_job_dirs() (uv_proc.c) before pre-exec and the job: it
exports HOME and chdir()s into cwd, falling back to /tmp with a warning
when the directory cannot be entered (the xcecd_worker runs as the job
user, so this is the user's own access). parse_limits() reads
nofile_limit into res_limits_t.nofile_limit and start_systemd_job() sets
LimitNOFILE like LimitNPROC. xcecd_virtual spawns no job and is
unaffected.

Tests: testsuite/test_job_dirs.c (assignment carries cwd/home, enter
cwd/home, fallback, a spawned job's pwd is the cwd) and
test_apiserver_nofile_limit_parsed in test_exdata_parse.c.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-10 11:48:32 +00:00
parent 81e7187ddc
commit 725b1d23fd
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
11 changed files with 406 additions and 44 deletions

View file

@ -73,6 +73,10 @@ typedef struct {
char job_pre_exec[COMMAND_LENGTH];
char job_post_exec[COMMAND_LENGTH];
char exdata[EXDATA_LENGTH];
/* bug-1936: the start info's execcwd/exechome; the job runs in cwd with
* HOME set to home (uv_proc_enter_job_dirs()). Empty when not given. */
char cwd[PATH_LENGTH];
char home[PATH_LENGTH];
} job_assignment_t;
typedef struct {
@ -183,6 +187,12 @@ bool ipc_worker_send_rusages(xce_job_rusages_t* rusages);
void on_write_complete(uv_write_t* req, int status);
struct xcecd_worker;
/* Fill the MSG_JOB_ASSIGNED payload xcecd sends xcecd_worker from the
* worker-list entry of its job (strings cut to their buffers, NUL ended). */
void ipc_fill_job_assignment(job_assignment_t* assignment,
const struct xcecd_worker* worker);
uv_loop_t* ipc_get_event_loop();
#ifdef __cplusplus

View file

@ -175,4 +175,19 @@ int uv_proc_exec_single(uv_proc_manager_t* manager, const char* command,
char* const args[], uv_proc_exit_cb exit_cb,
void* user_data);
/**
* bug-1936: enter the job's working directory and export its home, in the
* per-job xcecd_worker process before pre-exec and the job are started, so
* every process of the job inherits both.
*
* @param cwd job working directory (execcwd); NULL or "" keeps the current
* one. A directory that cannot be entered falls back to
* UV_PROC_FALLBACK_CWD.
* @param home job home (exechome); NULL or "" leaves HOME unchanged.
* @return 0 when cwd (if any) was entered, 1 when it fell back to
* UV_PROC_FALLBACK_CWD, -errno when not even that worked.
*/
#define UV_PROC_FALLBACK_CWD "/tmp"
int uv_proc_enter_job_dirs(const char* cwd, const char* home);
#endif /* UV_PROC_H */

View file

@ -38,6 +38,7 @@ typedef struct {
uint64_t fsize_limit; // Max file size (in bytes)
uint64_t stack_limit; // Max stack size (in bytes)
uint64_t process_limit; // Max number of processes
uint64_t nofile_limit; // Max open files (count, LimitNOFILE)
// OOM score
int32_t oom_score_adjust; // OOM (-1000 to 1000)

View file

@ -16,6 +16,39 @@ int exit_flag = 0;
uv_loop_t* loop = NULL;
static char socket_path[256];
/* Copy src (NULL = empty) into a fixed buffer, always NUL ended. */
static void assignment_copy(char* dst, size_t size, const char* src)
{
if (!src) {
dst[0] = '\0';
return;
}
strncpy(dst, src, size - 1);
dst[size - 1] = '\0';
}
void ipc_fill_job_assignment(job_assignment_t* assignment,
const struct xcecd_worker* worker)
{
assignment->job_id = worker->job_info.jobid;
assignment->ntasks = worker->ntasks;
assignment_copy(assignment->cmd, sizeof(assignment->cmd), worker->job_cmd);
assignment_copy(assignment->exdata, sizeof(assignment->exdata),
worker->job_exdata);
assignment_copy(assignment->q_pre_exec, sizeof(assignment->q_pre_exec),
worker->q_pre_exec);
assignment_copy(assignment->q_post_exec, sizeof(assignment->q_post_exec),
worker->q_post_exec);
assignment_copy(assignment->job_pre_exec, sizeof(assignment->job_pre_exec),
worker->job_pre_exec);
assignment_copy(assignment->job_post_exec,
sizeof(assignment->job_post_exec), worker->job_post_exec);
/* bug-1936: the job's working directory and home */
assignment_copy(assignment->cwd, sizeof(assignment->cwd), worker->execcwd);
assignment_copy(assignment->home, sizeof(assignment->home),
worker->exechome);
}
static uv_pipe_t master_pipe;
static job_status_callback_t master_status_callback = NULL;
static client_disconnect_callback_t master_disconnect_callback = NULL;
@ -433,48 +466,8 @@ static void on_master_read(uv_stream_t* client, ssize_t nread,
}
memset(&response, 0, sizeof(response));
response.type = MSG_JOB_ASSIGNED;
response.data.assignment.job_id = worker->job_info.jobid;
response.data.assignment.ntasks = worker->ntasks;
strncpy(response.data.assignment.cmd, worker->job_cmd,
sizeof(response.data.assignment.cmd));
if (worker->job_exdata) {
strncpy(response.data.assignment.exdata, worker->job_exdata,
sizeof(response.data.assignment.exdata));
} else {
response.data.assignment.exdata[0] = '\0';
}
if (worker->q_pre_exec) {
strncpy(response.data.assignment.q_pre_exec, worker->q_pre_exec,
sizeof(response.data.assignment.q_pre_exec));
} else {
response.data.assignment.q_pre_exec[0] = '\0';
}
if (worker->q_post_exec) {
strncpy(response.data.assignment.q_post_exec,
worker->q_post_exec,
sizeof(response.data.assignment.q_post_exec));
} else {
response.data.assignment.q_post_exec[0] = '\0';
}
if (worker->job_pre_exec) {
strncpy(response.data.assignment.job_pre_exec,
worker->job_pre_exec,
sizeof(response.data.assignment.job_pre_exec));
} else {
response.data.assignment.job_pre_exec[0] = '\0';
}
if (worker->job_post_exec) {
strncpy(response.data.assignment.job_post_exec,
worker->job_post_exec,
sizeof(response.data.assignment.job_post_exec));
} else {
response.data.assignment.job_post_exec[0] = '\0';
}
response.type = MSG_JOB_ASSIGNED;
ipc_fill_job_assignment(&response.data.assignment, worker);
if (requesting_client) {
add_job_client_mapping(worker->job_info.jobid,

View file

@ -491,3 +491,40 @@ int uv_proc_exec_single(uv_proc_manager_t* manager, const char* command,
{
return uv_proc_exec(manager, command, args, exit_cb, user_data, 1);
}
int uv_proc_enter_job_dirs(const char* cwd, const char* home)
{
int rc = 0;
if (home && home[0] != '\0') {
if (setenv("HOME", home, 1) != 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR, "set HOME=%s failed: %s", home,
strerror(errno));
return -errno;
}
}
if (!cwd || cwd[0] == '\0') {
return 0;
}
/* xcecd_worker runs as the job's user, so this is the user's own
* access to the directory. Like a traditional batch system, a job
* whose cwd does not exist (or is not accessible) on this host runs in
* /tmp instead of failing; the warning names both directories. */
if (chdir(cwd) != 0) {
int err = errno;
XCECD_WORKER_LOG(XCE_LOG_WARN,
"cannot enter job cwd %s (%s), running the job in %s",
cwd, strerror(err), UV_PROC_FALLBACK_CWD);
if (chdir(UV_PROC_FALLBACK_CWD) != 0) {
err = errno;
XCECD_WORKER_LOG(XCE_LOG_ERROR, "cannot enter %s either: %s",
UV_PROC_FALLBACK_CWD, strerror(err));
return -err;
}
rc = 1;
}
return rc;
}

View file

@ -930,6 +930,33 @@ char* start_systemd_job(const char* command, const res_limits_t* limits,
goto finish;
}
// Open-file limit (bug-1936), a count like LimitNPROC
if (limits->nofile_limit > 0) {
r = sd_bus_message_open_container(m, 'r', "sv");
if (r < 0)
goto finish;
r = sd_bus_message_append(m, "s", "LimitNOFILE");
if (r < 0)
goto finish;
r = sd_bus_message_open_container(m, 'v', "t");
if (r < 0)
goto finish;
r = sd_bus_message_append(m, "t", limits->nofile_limit);
if (r < 0)
goto finish;
r = sd_bus_message_close_container(m);
if (r < 0)
goto finish;
r = sd_bus_message_close_container(m);
if (r < 0)
goto finish;
}
// === OOM设置 ===
// OOMScoreAdjust

View file

@ -353,14 +353,22 @@ bool parse_limits(const char* exdata, res_limits_t* limits)
limits->process_limit = (uint64_t)process_limit->valuedouble;
}
/* bug-1936: the apiserver's num_files, a count of open files */
cJSON* nofile_limit
= cJSON_GetObjectItemCaseSensitive(limits_obj, "nofile_limit");
if (cJSON_IsNumber(nofile_limit)) {
limits->nofile_limit = (uint64_t)nofile_limit->valuedouble;
}
XCECD_WORKER_LOG(
XCE_LOG_DEBUG3,
"Parsed limits: cputime=%lu, coresize=%lu, rss=%lu, runlimit=%lu, "
"data=%lu, swap=%lu, fsize=%lu, stack=%lu, thread=%lu, process=%lu",
"data=%lu, swap=%lu, fsize=%lu, stack=%lu, thread=%lu, process=%lu, "
"nofile=%lu",
limits->cpu_limit, limits->core_size_limit, limits->memory_max,
limits->run_limit, limits->data_limit, limits->memory_swap_max,
limits->fsize_limit, limits->stack_limit, limits->tasks_max,
limits->process_limit);
limits->process_limit, limits->nofile_limit);
success = true;

View file

@ -818,6 +818,19 @@ static void worker_job_callback(job_assignment_t* info)
job_cmd = generate_cmd((char*)info->cmd, port_info,
mydata.transfer_info->interactive_mode);
/* bug-1936: run pre-exec and the job in the submission's cwd, with HOME
* = its exec home; before apply_env_vars() so an explicit --env HOME
* still wins. */
XCECD_WORKER_LOG(XCE_LOG_DEBUG3, "Job %s cwd: %s, home: %s", jobidStr,
info->cwd[0] ? info->cwd : "(none)",
info->home[0] ? info->home : "(none)");
if (uv_proc_enter_job_dirs(info->cwd, info->home) < 0) {
XCECD_WORKER_LOG(XCE_LOG_ERROR,
"Failed to enter a working directory for job %s",
jobidStr);
exit(1);
}
/* export XCE environment variables here */
apply_env_vars(&mydata.env_vars, jobid);

View file

@ -247,6 +247,21 @@ test('worker exdata_parse test', exdata_parse_test,
timeout: 60,
env: ['ASAN_OPTIONS=detect_leaks=0'])
# bug-1936: the job runs in its submission's cwd with HOME = exec home --
# job_assignment_t carries both (uv_ipc.c), uv_proc_enter_job_dirs()
# (uv_proc.c) applies them, and a spawned job inherits them. White-box
# #include of uv_ipc.c and uv_proc.c.
job_dirs_test = executable('job_dirs_test',
'test_job_dirs.c',
include_directories: [worker_test_incdir],
dependencies: worker_test_deps,
install: false)
test('worker job_dirs test', job_dirs_test,
suite: 'worker',
timeout: 60,
env: ['ASAN_OPTIONS=detect_leaks=0'])
# bug-1908: pending-start signal table (pending_signal.c).
pending_signal_test = executable('pending_signal_test',
['test_pending_signal.c',

View file

@ -134,6 +134,21 @@ static void test_apiserver_limits_parsed(void)
TEST_ASSERT_EQUAL_UINT64(1024, limits.memory_max); /* MB */
}
/* bug-1936: the apiserver writes num_files as "nofile_limit" (a count of
* open files); parse_limits() must read it so the job unit gets
* LimitNOFILE like the other rlimits. */
static void test_apiserver_nofile_limit_parsed(void)
{
res_limits_t limits;
memset(&limits, 0, sizeof(limits));
TEST_ASSERT_TRUE(parse_limits(
"{\"limits\":{\"nofile_limit\":2048,\"run_limit\":60}}", &limits));
TEST_ASSERT_EQUAL_UINT64(2048, limits.nofile_limit);
TEST_ASSERT_EQUAL_UINT64(60, limits.run_limit);
}
int main(void)
{
UNITY_BEGIN();
@ -142,5 +157,6 @@ int main(void)
RUN_TEST(test_legacy_transfer_file_key_still_parsed);
RUN_TEST(test_both_keys_keep_first_list);
RUN_TEST(test_apiserver_limits_parsed);
RUN_TEST(test_apiserver_nofile_limit_parsed);
return UNITY_END();
}

View file

@ -0,0 +1,227 @@
/*
* test_job_dirs.c -- bug-1936: a job runs in the working directory and with
* the home its submission asked for.
*
* The start info's execcwd/exechome reach xcecd (controller_connector.c) and
* its worker-list entry (xcecd_worker_utils.c add_worker), but xcecd's
* MSG_JOB_ASSIGNED reply to the per-job xcecd_worker carried neither, and
* uv_proc_exec() spawned the job with cwd NULL: every job ran in the
* xcecd_worker unit's own directory whatever --cwd said.
*
* Checks the three seams: ipc_fill_job_assignment() (uv_ipc.c) puts cwd and
* home into the assignment; uv_proc_enter_job_dirs() (uv_proc.c) enters the
* directory and exports HOME (falling back to /tmp for a directory that
* cannot be entered); and a job spawned by uv_proc_exec_single() afterwards
* really runs there.
*
* White-box #include of ../src/uv_ipc.c and ../src/uv_proc.c (same
* convention as test_uv_ipc_job_exit_severity.c).
*/
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include <unity/unity.h>
#include "xcecd.h"
xcecd_context_t xcecd_ctx;
#include "../src/uv_ipc.c"
#include "../src/uv_proc.c"
/* Link-only stubs, as in test_uv_ipc_job_exit_severity.c. */
xcecd_worker_t* find_worker_by_pid(void* worker_list, int pid)
{
(void)worker_list;
(void)pid;
return NULL;
}
int xcecd_job_remote_start(uint64_t jobid, const char* hostname,
const char* command)
{
(void)jobid;
(void)hostname;
(void)command;
return 0;
}
#define JD_JOBID ((uint64_t)936190001ULL)
static char g_saved_cwd[PATH_MAX];
static char* g_saved_home;
static char g_dir[64];
void setUp(void)
{
const char* home = getenv("HOME");
TEST_ASSERT_NOT_NULL(getcwd(g_saved_cwd, sizeof(g_saved_cwd)));
g_saved_home = home ? strdup(home) : NULL;
strcpy(g_dir, "/tmp/xce_jobdirs_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(g_dir));
}
void tearDown(void)
{
char path[PATH_MAX];
char out[256];
TEST_ASSERT_EQUAL_INT(0, chdir(g_saved_cwd));
if (g_saved_home) {
setenv("HOME", g_saved_home, 1);
free(g_saved_home);
} else {
unsetenv("HOME");
}
snprintf(path, sizeof(path), "%s/pwd.out", g_dir);
unlink(path);
rmdir(g_dir);
xcecd_job_output_path(out, sizeof(out), JD_JOBID, 0);
unlink(out);
}
static void current_dir(char* buf, size_t size)
{
TEST_ASSERT_NOT_NULL(getcwd(buf, size));
}
static void test_assignment_carries_cwd_and_home(void)
{
xcecd_worker_t w;
job_assignment_t a;
memset(&w, 0, sizeof(w));
memset(&a, 0, sizeof(a));
w.job_info.jobid = JD_JOBID;
w.ntasks = 2;
w.job_cmd = "sleep 1";
w.execcwd = "/data/run";
w.exechome = "/home/alice";
ipc_fill_job_assignment(&a, &w);
TEST_ASSERT_EQUAL_UINT64(JD_JOBID, a.job_id);
TEST_ASSERT_EQUAL_INT(2, a.ntasks);
TEST_ASSERT_EQUAL_STRING("sleep 1", a.cmd);
TEST_ASSERT_EQUAL_STRING("/data/run", a.cwd);
TEST_ASSERT_EQUAL_STRING("/home/alice", a.home);
}
static void test_assignment_without_cwd_and_home_is_empty(void)
{
xcecd_worker_t w;
job_assignment_t a;
memset(&w, 0, sizeof(w));
memset(&a, 'x', sizeof(a));
w.job_cmd = "true";
ipc_fill_job_assignment(&a, &w);
TEST_ASSERT_EQUAL_STRING("", a.cwd);
TEST_ASSERT_EQUAL_STRING("", a.home);
TEST_ASSERT_EQUAL_STRING("", a.exdata);
}
static void test_enter_job_dirs_enters_cwd_and_sets_home(void)
{
char want[PATH_MAX];
char got[PATH_MAX];
TEST_ASSERT_NOT_NULL(realpath(g_dir, want));
TEST_ASSERT_EQUAL_INT(0, uv_proc_enter_job_dirs(g_dir, "/home/alice"));
current_dir(got, sizeof(got));
TEST_ASSERT_EQUAL_STRING(want, got);
TEST_ASSERT_EQUAL_STRING("/home/alice", getenv("HOME"));
}
static void test_enter_job_dirs_without_values_changes_nothing(void)
{
char got[PATH_MAX];
setenv("HOME", "/home/unchanged", 1);
TEST_ASSERT_EQUAL_INT(0, uv_proc_enter_job_dirs(NULL, ""));
current_dir(got, sizeof(got));
TEST_ASSERT_EQUAL_STRING(g_saved_cwd, got);
TEST_ASSERT_EQUAL_STRING("/home/unchanged", getenv("HOME"));
}
static void test_enter_job_dirs_unusable_cwd_falls_back(void)
{
char want[PATH_MAX];
char got[PATH_MAX];
char missing[128];
snprintf(missing, sizeof(missing), "%s/no/such/dir", g_dir);
TEST_ASSERT_NOT_NULL(realpath(UV_PROC_FALLBACK_CWD, want));
TEST_ASSERT_EQUAL_INT(1, uv_proc_enter_job_dirs(missing, NULL));
current_dir(got, sizeof(got));
TEST_ASSERT_EQUAL_STRING(want, got);
}
static int g_exit_status = -1;
static void on_job_exit(int pid, int exit_status, int term_signal,
void* user_data)
{
(void)pid;
(void)term_signal;
(void)user_data;
g_exit_status = exit_status;
}
/* The spawned job inherits the directory: a relative output path lands in
* the job's cwd, and the job's own `pwd` is that directory. */
static void test_job_spawned_after_enter_runs_in_cwd(void)
{
uv_loop_t uvloop;
uv_proc_manager_t m;
char want[PATH_MAX];
char path[PATH_MAX + 16];
char got[PATH_MAX] = { 0 };
FILE* f;
TEST_ASSERT_NOT_NULL(realpath(g_dir, want));
TEST_ASSERT_EQUAL_INT(0, uv_loop_init(&uvloop));
TEST_ASSERT_EQUAL_INT(0, uv_proc_init(&m, &uvloop));
m.job_id = JD_JOBID;
TEST_ASSERT_EQUAL_INT(0, uv_proc_enter_job_dirs(g_dir, NULL));
TEST_ASSERT_EQUAL_INT(0, uv_proc_exec_single(&m, "pwd -P > pwd.out", NULL,
on_job_exit, NULL));
uv_run(&uvloop, UV_RUN_DEFAULT);
TEST_ASSERT_EQUAL_INT(0, g_exit_status);
snprintf(path, sizeof(path), "%s/pwd.out", want);
f = fopen(path, "r");
TEST_ASSERT_NOT_NULL_MESSAGE(f, "the job did not write pwd.out in cwd");
TEST_ASSERT_NOT_NULL(fgets(got, sizeof(got), f));
fclose(f);
got[strcspn(got, "\n")] = '\0';
TEST_ASSERT_EQUAL_STRING(want, got);
uv_proc_cleanup(&m);
TEST_ASSERT_EQUAL_INT(0, uv_loop_close(&uvloop));
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_assignment_carries_cwd_and_home);
RUN_TEST(test_assignment_without_cwd_and_home_is_empty);
RUN_TEST(test_enter_job_dirs_enters_cwd_and_sets_home);
RUN_TEST(test_enter_job_dirs_without_values_changes_nothing);
RUN_TEST(test_enter_job_dirs_unusable_cwd_falls_back);
RUN_TEST(test_job_spawned_after_enter_runs_in_cwd);
return UNITY_END();
}