lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): allow control of running jobs on connected hosts of any status (bug-1941)

Browse source 
Cause: job_ctrl (job.c) sent kill/stop/resume of a running job only when
the host's status was exactly 0, so any status bit (CLOSED,
CLOSED_BY_ADMIN, BUSY, LOCKED...) refused the request, and the refusal
returned -errno, i.e. whatever errno happened to hold (often 0 or
-ENOENT).

Fix: new job_ctrl_to_host sends the request whenever the worker is
connected (endpoint present, no XCE_WORKER_UNAVAIL); an unknown or
unconnected worker fails with -E_XCE_BAD_WORKER and an ERROR naming the
host. Nothing else changes for jobs on dead hosts. Test:
testsuite/test_job_ctrl_hosts.c.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-10 08:34:59 +00:00
parent 4d2e80c4bb
commit f244661b14
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
4 changed files with 344 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

90
src/controller/job.c
View file

@ -2078,6 +2078,60 @@ int dispatch_a_job_test(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
}
#endif /* XCE_TESTING */
/* Sends a job control request to the worker on `host`. Returns 0 once it is
* queued (xce_jobctrl_req() only enqueues, it never blocks), or a negative
* error.
*
* bug-1941: what matters is whether the worker is connected -- an endpoint
* and no XCE_WORKER_UNAVAIL. Other status bits (CLOSED, CLOSED_BY_ADMIN,
* BUSY, LOCKED...) only stop new dispatches; the jobs already running there
* must stay controllable. A worker that is unknown or not connected gives
* -E_XCE_BAD_WORKER (it used to be -errno, whatever errno held). */
static int job_ctrl_to_host(cluster_t* c, const char* host,
xce_jobctrl_t* ctrinfo)
{
worker_t* w = NULL;
int rc;
lock(c->workermgr, RTTI_LOCK_READ);
rc = key_find(c->workermgr, (void*)host, (void**)&w);
if (w != NULL) {
reference(w);
}
unlock(c->workermgr);
if (rc < 0) {
return rc;
}
if (w == NULL) {
ERROR("job ctrl %ld: host %s is not a known worker", ctrinfo->jobid,
host);
SET_ERROR(E_XCE_BAD_WORKER);
return -E_XCE_BAD_WORKER;
}
lock(w, RTTI_LOCK_READ);
if ((w->ep == NULL) || IS_WORKER_UNAVAIL(w->host.status)) {
ERROR("job ctrl %ld: worker %s is not connected, status %x ep %p",
ctrinfo->jobid, w->host.name, w->host.status, w->ep);
SET_ERROR(E_XCE_BAD_WORKER);
rc = -E_XCE_BAD_WORKER;
} else {
rc = xce_jobctrl_req(w->ep, ctrinfo);
if (rc < 0) {
ERROR("job ctrl %ld on host %s returns %d", ctrinfo->jobid,
w->host.name, rc);
} else {
rc = 0;
}
}
unlock(w);
dereference(w);
return rc;
}
int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
{
uint64_t jobid = ctrinfo->jobid;
@ -2121,45 +2175,11 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
lock(j, RTTI_LOCK_WRITE);
if (j->job.allocs != NULL) {
xce_host_alloc_t** allocs = j->job.allocs->allocs;
worker_t* w = NULL;
/* job control command will be sent to first node
* for parallel jobs
*/
lock(c->workermgr, RTTI_LOCK_READ);
rc = key_find(c->workermgr, (void*)allocs[0]->host, (void**)&w);
if (w != NULL) {
reference(w);
}
unlock(c->workermgr);
if (rc < 0) {
unlock(j);
dereference(j);
return rc;
}
if (w != NULL) {
lock(w, RTTI_LOCK_READ);
/* FIXME: better method to check host status */
if ((w->ep != NULL) && (w->host.status == 0)) {
rc = xce_jobctrl_req(w->ep, ctrinfo);
if (rc < 0) {
ERROR("job ctrl %ld on host %s returns %d", jobid,
w->host.name, rc);
}
} else {
ERROR("worker %s is not in correct state, status %x ep %p",
w->host.name, w->host.status, w->ep);
rc = -errno;
}
unlock(w);
dereference(w);
} else {
rc = -errno;
}
rc = job_ctrl_to_host(c, allocs[0]->host, ctrinfo);
} else {
/* job has not been started yet */
job_failinfo_t failinfo = {