lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): send job control to every host of a multi-host job (bug-1942)

Browse source 
Cause: job_ctrl (job.c) sent kill/stop/resume only to allocs[0]->host.
The tasks on the other hosts are started through runtasks (by the head
worker for a parallel job, or by xrun as the MPI launcher's rsh agent)
and run under their own worker entry on each host; the head worker
signals only its local process group and forwards nothing, so they kept
running.

Fix: job_ctrl sends the request to every allocated host through
job_ctrl_to_host. The head decides the result as before; an unconnected
remote host is logged and skipped (its tasks belong to host failure
handling). Sends are non-blocking enqueues. Test:
testsuite/test_job_ctrl_hosts.c.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-10 08:36:05 +00:00
parent f244661b14
commit 8568c8ab80
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
2 changed files with 106 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

20
src/controller/job.c
View file

@ -2175,11 +2175,25 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
lock(j, RTTI_LOCK_WRITE);
if (j->job.allocs != NULL) {
xce_host_alloc_t** allocs = j->job.allocs->allocs;
int i;
/* job control command will be sent to first node
* for parallel jobs
*/
/* bug-1942: the control goes to every host of the job, not only
* the head (allocs[0]). The tasks on the other hosts are started
* through runtasks (task.c) -- by the head worker for a plain
* parallel job, or by xrun as the MPI launcher's rsh agent -- and
* each runs under that host's own worker entry for this jobid; the
* head worker only signals its local process group and does not
* forward. The head decides the result, as before; an unconnected
* remote host is logged by job_ctrl_to_host and skipped. Each send
* only enqueues on the worker endpoint. */
rc = job_ctrl_to_host(c, allocs[0]->host, ctrinfo);
for (i = 1; i < j->job.allocs->nhosts; i++) {
int rc1 = job_ctrl_to_host(c, allocs[i]->host, ctrinfo);
if ((rc1 < 0) && (rc1 != -E_XCE_BAD_WORKER) && (rc == 0)) {
rc = rc1;
}
}
} else {
/* job has not been started yet */
job_failinfo_t failinfo = {

90
src/controller/testsuite/test_job_ctrl_hosts.c
View file

@ -276,11 +276,99 @@ static void test_jc_unconnected_host_fails_with_bad_worker(void)
jch_teardown();
}
/* bug-1942: the tasks of a multi-host job on hosts[1..] run under their own
* worker entry on that host (runtasks), and the head's worker signals only
* its local process group -- so every allocated host must get the request,
* with the signal and force unchanged. */
static void test_jc_multi_host_job_reaches_every_host(void)
{
const char* hosts[] = { "jch-h0", "jch-h1", "jch-h2" };
int i;
int rc;
jch_setup();
jch_seed_worker("jch-h0", 0, &g_jch_ep[0]);
jch_seed_worker("jch-h1", XCE_WORKER_CLOSED, &g_jch_ep[1]);
jch_seed_worker("jch-h2", 0, &g_jch_ep[2]);
jch_seed_running_job(93300, 3, hosts);
rc = jch_kill(93300, SIGUSR1, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
for (i = 0; i < 3; i++) {
jch_obs_t obs = jch_scan(&g_jch_ep[i]);
char msg[96];
snprintf(msg, sizeof(msg),
"bug-1942: host %s of a 3-host job must get the control",
hosts[i]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, msg);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(93300, obs.jobid, msg);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(SIGUSR1, obs.signo, msg);
TEST_ASSERT_EQUAL_INT32_MESSAGE(1, obs.force, msg);
}
jch_teardown();
}
/* bug-1942: an unconnected non-head host does not fail the request (its
* tasks are lost with the host; host failure handling owns them) -- the
* head and the other hosts still get it. */
static void test_jc_multi_host_remote_unconnected_still_controls_rest(void)
{
const char* hosts[] = { "jch-h0", "jch-h1", "jch-h2" };
jch_obs_t obs;
int rc;
jch_setup();
jch_seed_worker("jch-h0", 0, &g_jch_ep[0]);
jch_seed_worker("jch-h1", XCE_WORKER_UNAVAIL, &g_jch_ep[1]);
jch_seed_worker("jch-h2", 0, &g_jch_ep[2]);
jch_seed_running_job(93301, 3, hosts);
rc = jch_kill(93301, SIGTERM, 0);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"an unconnected non-head host must not fail the request");
obs = jch_scan(&g_jch_ep[0]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, "head host must get the control");
obs = jch_scan(&g_jch_ep[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, obs.n,
"nothing may be sent to an UNAVAIL host");
obs = jch_scan(&g_jch_ep[2]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n,
"bug-1942: connected remote host must get the control");
jch_teardown();
}
/* bug-1942: an unconnected head fails the request (as for a single-host
* job), but the connected remote hosts still get it. */
static void test_jc_multi_host_head_unconnected_fails_but_reaches_rest(void)
{
const char* hosts[] = { "jch-h0", "jch-h1" };
jch_obs_t obs;
int rc;
jch_setup();
jch_seed_worker("jch-h0", XCE_WORKER_UNAVAIL, &g_jch_ep[0]);
jch_seed_worker("jch-h1", 0, &g_jch_ep[1]);
jch_seed_running_job(93302, 2, hosts);
rc = jch_kill(93302, SIGKILL, 1);
TEST_ASSERT_EQUAL_INT_MESSAGE(-E_XCE_BAD_WORKER, rc,
"an unconnected head host must fail the request");
obs = jch_scan(&g_jch_ep[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n,
"bug-1942: connected remote host must still get the control");
jch_teardown();
}
void test_job_ctrl_hosts(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> job_ctrl() host selection (bug-1941)\n\n");
printf("\n>> job_ctrl() host selection (bug-1941, bug-1942)\n\n");
RUN_TEST(test_jc_connected_flagged_host_is_controllable);
RUN_TEST(test_jc_unconnected_host_fails_with_bad_worker);
RUN_TEST(test_jc_multi_host_job_reaches_every_host);
RUN_TEST(test_jc_multi_host_remote_unconnected_still_controls_rest);
RUN_TEST(test_jc_multi_host_head_unconnected_fails_but_reaches_rest);
}