diff --git a/src/controller/job.c b/src/controller/job.c index e32bfa5e..2c4291a2 100644 --- a/src/controller/job.c +++ b/src/controller/job.c @@ -2175,11 +2175,25 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo) lock(j, RTTI_LOCK_WRITE); if (j->job.allocs != NULL) { xce_host_alloc_t** allocs = j->job.allocs->allocs; + int i; - /* job control command will be sent to first node - * for parallel jobs - */ + /* bug-1942: the control goes to every host of the job, not only + * the head (allocs[0]). The tasks on the other hosts are started + * through runtasks (task.c) -- by the head worker for a plain + * parallel job, or by xrun as the MPI launcher's rsh agent -- and + * each runs under that host's own worker entry for this jobid; the + * head worker only signals its local process group and does not + * forward. The head decides the result, as before; an unconnected + * remote host is logged by job_ctrl_to_host and skipped. Each send + * only enqueues on the worker endpoint. */ rc = job_ctrl_to_host(c, allocs[0]->host, ctrinfo); + for (i = 1; i < j->job.allocs->nhosts; i++) { + int rc1 = job_ctrl_to_host(c, allocs[i]->host, ctrinfo); + + if ((rc1 < 0) && (rc1 != -E_XCE_BAD_WORKER) && (rc == 0)) { + rc = rc1; + } + } } else { /* job has not been started yet */ job_failinfo_t failinfo = { diff --git a/src/controller/testsuite/test_job_ctrl_hosts.c b/src/controller/testsuite/test_job_ctrl_hosts.c index 251959d5..f33da3f5 100644 --- a/src/controller/testsuite/test_job_ctrl_hosts.c +++ b/src/controller/testsuite/test_job_ctrl_hosts.c @@ -276,11 +276,99 @@ static void test_jc_unconnected_host_fails_with_bad_worker(void) jch_teardown(); } +/* bug-1942: the tasks of a multi-host job on hosts[1..] run under their own + * worker entry on that host (runtasks), and the head's worker signals only + * its local process group -- so every allocated host must get the request, + * with the signal and force unchanged. */ +static void test_jc_multi_host_job_reaches_every_host(void) +{ + const char* hosts[] = { "jch-h0", "jch-h1", "jch-h2" }; + int i; + int rc; + + jch_setup(); + jch_seed_worker("jch-h0", 0, &g_jch_ep[0]); + jch_seed_worker("jch-h1", XCE_WORKER_CLOSED, &g_jch_ep[1]); + jch_seed_worker("jch-h2", 0, &g_jch_ep[2]); + jch_seed_running_job(93300, 3, hosts); + + rc = jch_kill(93300, SIGUSR1, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + for (i = 0; i < 3; i++) { + jch_obs_t obs = jch_scan(&g_jch_ep[i]); + char msg[96]; + + snprintf(msg, sizeof(msg), + "bug-1942: host %s of a 3-host job must get the control", + hosts[i]); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, msg); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(93300, obs.jobid, msg); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(SIGUSR1, obs.signo, msg); + TEST_ASSERT_EQUAL_INT32_MESSAGE(1, obs.force, msg); + } + jch_teardown(); +} + +/* bug-1942: an unconnected non-head host does not fail the request (its + * tasks are lost with the host; host failure handling owns them) -- the + * head and the other hosts still get it. */ +static void test_jc_multi_host_remote_unconnected_still_controls_rest(void) +{ + const char* hosts[] = { "jch-h0", "jch-h1", "jch-h2" }; + jch_obs_t obs; + int rc; + + jch_setup(); + jch_seed_worker("jch-h0", 0, &g_jch_ep[0]); + jch_seed_worker("jch-h1", XCE_WORKER_UNAVAIL, &g_jch_ep[1]); + jch_seed_worker("jch-h2", 0, &g_jch_ep[2]); + jch_seed_running_job(93301, 3, hosts); + + rc = jch_kill(93301, SIGTERM, 0); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, + "an unconnected non-head host must not fail the request"); + obs = jch_scan(&g_jch_ep[0]); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, "head host must get the control"); + obs = jch_scan(&g_jch_ep[1]); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, obs.n, + "nothing may be sent to an UNAVAIL host"); + obs = jch_scan(&g_jch_ep[2]); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, + "bug-1942: connected remote host must get the control"); + jch_teardown(); +} + +/* bug-1942: an unconnected head fails the request (as for a single-host + * job), but the connected remote hosts still get it. */ +static void test_jc_multi_host_head_unconnected_fails_but_reaches_rest(void) +{ + const char* hosts[] = { "jch-h0", "jch-h1" }; + jch_obs_t obs; + int rc; + + jch_setup(); + jch_seed_worker("jch-h0", XCE_WORKER_UNAVAIL, &g_jch_ep[0]); + jch_seed_worker("jch-h1", 0, &g_jch_ep[1]); + jch_seed_running_job(93302, 2, hosts); + + rc = jch_kill(93302, SIGKILL, 1); + TEST_ASSERT_EQUAL_INT_MESSAGE(-E_XCE_BAD_WORKER, rc, + "an unconnected head host must fail the request"); + obs = jch_scan(&g_jch_ep[1]); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, obs.n, + "bug-1942: connected remote host must still get the control"); + jch_teardown(); +} + void test_job_ctrl_hosts(void) { UnitySetTestFile(__FILE__); - printf("\n>> job_ctrl() host selection (bug-1941)\n\n"); + printf("\n>> job_ctrl() host selection (bug-1941, bug-1942)\n\n"); RUN_TEST(test_jc_connected_flagged_host_is_controllable); RUN_TEST(test_jc_unconnected_host_fails_with_bad_worker); + RUN_TEST(test_jc_multi_host_job_reaches_every_host); + RUN_TEST(test_jc_multi_host_remote_unconnected_still_controls_rest); + RUN_TEST(test_jc_multi_host_head_unconnected_fails_but_reaches_rest); }