lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): send job control to every host of a multi-host job (bug-1942)

Browse source 
Cause: job_ctrl (job.c) sent kill/stop/resume only to allocs[0]->host.
The tasks on the other hosts are started through runtasks (by the head
worker for a parallel job, or by xrun as the MPI launcher's rsh agent)
and run under their own worker entry on each host; the head worker
signals only its local process group and forwards nothing, so they kept
running.

Fix: job_ctrl sends the request to every allocated host through
job_ctrl_to_host. The head decides the result as before; an unconnected
remote host is logged and skipped (its tasks belong to host failure
handling). Sends are non-blocking enqueues. Test:
testsuite/test_job_ctrl_hosts.c.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-10 08:36:05 +00:00
parent f244661b14
commit 8568c8ab80
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
2 changed files with 106 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

90
src/controller/testsuite/test_job_ctrl_hosts.c
View file

@ -276,11 +276,99 @@ static void test_jc_unconnected_host_fails_with_bad_worker(void)
jch_teardown();
}
/* bug-1942: the tasks of a multi-host job on hosts[1..] run under their own
* worker entry on that host (runtasks), and the head's worker signals only
* its local process group -- so every allocated host must get the request,
* with the signal and force unchanged. */
static void test_jc_multi_host_job_reaches_every_host(void)
{
const char* hosts[] = { "jch-h0", "jch-h1", "jch-h2" };
int i;
int rc;
jch_setup();
jch_seed_worker("jch-h0", 0, &g_jch_ep[0]);
jch_seed_worker("jch-h1", XCE_WORKER_CLOSED, &g_jch_ep[1]);
jch_seed_worker("jch-h2", 0, &g_jch_ep[2]);
jch_seed_running_job(93300, 3, hosts);
rc = jch_kill(93300, SIGUSR1, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
for (i = 0; i < 3; i++) {
jch_obs_t obs = jch_scan(&g_jch_ep[i]);
char msg[96];
snprintf(msg, sizeof(msg),
"bug-1942: host %s of a 3-host job must get the control",
hosts[i]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, msg);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(93300, obs.jobid, msg);
TEST_ASSERT_EQUAL_UINT32_MESSAGE(SIGUSR1, obs.signo, msg);
TEST_ASSERT_EQUAL_INT32_MESSAGE(1, obs.force, msg);
}
jch_teardown();
}
/* bug-1942: an unconnected non-head host does not fail the request (its
* tasks are lost with the host; host failure handling owns them) -- the
* head and the other hosts still get it. */
static void test_jc_multi_host_remote_unconnected_still_controls_rest(void)
{
const char* hosts[] = { "jch-h0", "jch-h1", "jch-h2" };
jch_obs_t obs;
int rc;
jch_setup();
jch_seed_worker("jch-h0", 0, &g_jch_ep[0]);
jch_seed_worker("jch-h1", XCE_WORKER_UNAVAIL, &g_jch_ep[1]);
jch_seed_worker("jch-h2", 0, &g_jch_ep[2]);
jch_seed_running_job(93301, 3, hosts);
rc = jch_kill(93301, SIGTERM, 0);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"an unconnected non-head host must not fail the request");
obs = jch_scan(&g_jch_ep[0]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, "head host must get the control");
obs = jch_scan(&g_jch_ep[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, obs.n,
"nothing may be sent to an UNAVAIL host");
obs = jch_scan(&g_jch_ep[2]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n,
"bug-1942: connected remote host must get the control");
jch_teardown();
}
/* bug-1942: an unconnected head fails the request (as for a single-host
* job), but the connected remote hosts still get it. */
static void test_jc_multi_host_head_unconnected_fails_but_reaches_rest(void)
{
const char* hosts[] = { "jch-h0", "jch-h1" };
jch_obs_t obs;
int rc;
jch_setup();
jch_seed_worker("jch-h0", XCE_WORKER_UNAVAIL, &g_jch_ep[0]);
jch_seed_worker("jch-h1", 0, &g_jch_ep[1]);
jch_seed_running_job(93302, 2, hosts);
rc = jch_kill(93302, SIGKILL, 1);
TEST_ASSERT_EQUAL_INT_MESSAGE(-E_XCE_BAD_WORKER, rc,
"an unconnected head host must fail the request");
obs = jch_scan(&g_jch_ep[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n,
"bug-1942: connected remote host must still get the control");
jch_teardown();
}
void test_job_ctrl_hosts(void)
{
UnitySetTestFile(__FILE__);
printf("\n>> job_ctrl() host selection (bug-1941)\n\n");
printf("\n>> job_ctrl() host selection (bug-1941, bug-1942)\n\n");
RUN_TEST(test_jc_connected_flagged_host_is_controllable);
RUN_TEST(test_jc_unconnected_host_fails_with_bad_worker);
RUN_TEST(test_jc_multi_host_job_reaches_every_host);
RUN_TEST(test_jc_multi_host_remote_unconnected_still_controls_rest);
RUN_TEST(test_jc_multi_host_head_unconnected_fails_but_reaches_rest);
}