forked from lei.chen/myxce
save works
This commit is contained in:
parent
acfbec875b
commit
3f74f14687
38 changed files with 751 additions and 448 deletions
|
|
@ -157,7 +157,13 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
TRACE("%s jobid %ld", __func__, alloc->jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)alloc->jobid, (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
|
@ -170,6 +176,8 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
nc = alloc->nhosts;
|
||||
if (nc <= 0) {
|
||||
dereference(j);
|
||||
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
|
|
@ -180,7 +188,12 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
allocs = (xce_host_alloc_t**)alloc->allocs;
|
||||
/* using head node of job */
|
||||
lock(c->workermgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->workermgr, (void*)allocs[0]->host, (void**)&w);
|
||||
if (w != NULL) {
|
||||
reference(w);
|
||||
}
|
||||
unlock(c->workermgr);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
|
@ -190,12 +203,18 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
return -errno;
|
||||
}
|
||||
|
||||
lock(w, RTTI_LOCK_READ);
|
||||
if ((w->ep == NULL) || (w->host.status != 0)) {
|
||||
ERROR("worker %s is not in correct state, status %x ep %p",
|
||||
w->host.name, w->host.status, w->ep);
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
SET_ERROR(E_XCE_BAD_WORKER);
|
||||
|
||||
unlock(w);
|
||||
|
||||
dereference(w);
|
||||
|
||||
return -errno;
|
||||
}
|
||||
|
||||
|
|
@ -209,11 +228,9 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
return rc;
|
||||
}
|
||||
|
||||
if (j->job.submit->start_info->q_pre_exec != NULL) {
|
||||
}
|
||||
if (j->job.submit->start_info->q_pre_exec != NULL) { }
|
||||
|
||||
if (j->job.submit->start_info->q_post_exec != NULL) {
|
||||
}
|
||||
if (j->job.submit->start_info->q_post_exec != NULL) { }
|
||||
|
||||
j->job.submit->start_info->q_pre_exec = q_pre_exec;
|
||||
j->job.submit->start_info->q_post_exec = q_post_exec;
|
||||
|
|
@ -225,6 +242,10 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
rc = xce_startjob_req(w->ep, &einfo);
|
||||
|
||||
unlock(w);
|
||||
|
||||
dereference(w);
|
||||
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
if (j->job.allocs != NULL) {
|
||||
|
|
@ -236,7 +257,6 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
MOVE_PTR(j->job.allocs->allocs, allocs);
|
||||
alloc->allocs = NULL;
|
||||
|
||||
|
||||
if (rc < 0) {
|
||||
SET_ERROR(-rc);
|
||||
PERROR("xce_startjob_req");
|
||||
|
|
@ -254,7 +274,13 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
|
||||
TRACE("%s job_ctrl job %ld", __func__, jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
|
@ -264,6 +290,8 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* FIXME: eliminate held multiple locks */
|
||||
lock(j, RTTI_LOCK_READ);
|
||||
if (j->job.allocs != NULL) {
|
||||
xce_host_alloc_t** allocs = j->job.allocs->allocs;
|
||||
worker_t* w = NULL;
|
||||
|
|
@ -271,11 +299,19 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
/* job control command will be sent to first node
|
||||
* for parallel jobs
|
||||
*/
|
||||
|
||||
lock(c->workermgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->workermgr, (void*)allocs[0]->host, (void**)&w);
|
||||
if (w != NULL) {
|
||||
reference(w);
|
||||
}
|
||||
unlock(c->workermgr);
|
||||
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
if (w != NULL) {
|
||||
lock(w, RTTI_LOCK_READ);
|
||||
/* FIXME: better method to check host status */
|
||||
if ((w->ep != NULL) && (w->host.status == 0)) {
|
||||
rc = xce_jobctrl_req(w->ep, ctrinfo);
|
||||
|
|
@ -288,6 +324,9 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
w->host.name, w->host.status, w->ep);
|
||||
rc = -errno;
|
||||
}
|
||||
unlock(w);
|
||||
|
||||
dereference(w);
|
||||
} else {
|
||||
rc = -errno;
|
||||
}
|
||||
|
|
@ -353,6 +392,8 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
}
|
||||
}
|
||||
|
||||
unlock(j);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
|
@ -409,10 +450,7 @@ int handle_jobdispatch(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
return rc;
|
||||
}
|
||||
|
||||
static int xlist_deref_job_item_(job_t* j, void* arg)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
static int xlist_deref_job_item_(job_t* j, void* arg) { return 0; }
|
||||
|
||||
int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
||||
{
|
||||
|
|
@ -457,7 +495,13 @@ int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
for (i = 0; i < submit->array_size; i++) {
|
||||
/* Run hooks to tell scheduler */
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)(jobid + i), (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if ((rc <= 0) || (j == NULL)) {
|
||||
char idstring[128];
|
||||
|
||||
|
|
@ -514,14 +558,23 @@ int on_job_start(cluster_t* c, job_pidinfo_t* info)
|
|||
|
||||
TRACE("%s jobid %ld", __func__, info->jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->jobmgr, (void*)info->jobid, (void**)&job);
|
||||
if (job != NULL) {
|
||||
reference(job);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if ((rc <= 0) || (job == NULL)) {
|
||||
ERROR("can not find job %ld", info->jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
mark_job_start(c, job, info);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -563,14 +616,23 @@ int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
|
|||
TRACE("%s job finish jobid %ld exit code %d", __func__, finish->jobid,
|
||||
finish->exit_code);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)finish->jobid, (void**)&job);
|
||||
if (job != NULL) {
|
||||
reference(job);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", finish->jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
mark_job_finish(c, job, finish);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -625,13 +687,23 @@ int on_job_fail(cluster_t* c, job_failinfo_t* failinfo)
|
|||
TRACE("%s job fail jobid %ld code %d", __func__, failinfo->jobid,
|
||||
failinfo->code);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)failinfo->jobid, (void**)&job);
|
||||
if (job != NULL) {
|
||||
reference(job);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", failinfo->jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
mark_job_fail(c, job, failinfo);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -643,16 +715,26 @@ int on_job_stop(cluster_t* c, uint64_t jobid)
|
|||
|
||||
TRACE("%s job stop jobid %ld", __func__, jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
if (job != NULL) {
|
||||
reference(job);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
state = job->job.state;
|
||||
|
||||
state |= JOB_STAT_USUSP;
|
||||
mark_job_update(c, job, state);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -664,15 +746,25 @@ int on_job_resume(cluster_t* c, uint64_t jobid)
|
|||
|
||||
TRACE("%s job resume jobid %ld", __func__, jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
if (job != NULL) {
|
||||
reference(job);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
state = job->job.state;
|
||||
state &= ~JOB_STAT_USUSP;
|
||||
mark_job_update(c, job, state);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -684,12 +776,19 @@ int on_job_postdone(cluster_t* c, endpoint_t* ep, uint64_t jobid)
|
|||
|
||||
TRACE("%s job post done jobid %ld", __func__, jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
if (job != NULL) {
|
||||
reference(job);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(job, RTTI_LOCK_WRITE);
|
||||
state = job->job.state;
|
||||
if ((state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
|
||||
WARN("job %ld got post exec fail notification but job's exit status "
|
||||
|
|
@ -702,6 +801,9 @@ int on_job_postdone(cluster_t* c, endpoint_t* ep, uint64_t jobid)
|
|||
|
||||
state |= JOB_STAT_PDONE;
|
||||
mark_job_update(c, job, state);
|
||||
unlock(job);
|
||||
|
||||
dereference(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -722,22 +824,32 @@ int on_job_ckpt(cluster_t* c, endpoint_t* ep, xce_ckptinfo_t* ckptinfo)
|
|||
|
||||
TRACE("%s job %ld ckptinfo %s", __func__, jobid, info);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (j == NULL) {
|
||||
ERROR("%s can not find job %ld", __func__, jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_CKPT, c, j);
|
||||
if (j->job.ckptinfo != NULL) {
|
||||
dereference(j->job.ckptinfo);
|
||||
}
|
||||
|
||||
j->job.ckptinfo = makeString(info);
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_CKPT, c, j);
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -769,7 +881,13 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_RUSAGE_UPDATE, c, rusages);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)rusages->jobid, (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
return -errno;
|
||||
|
|
@ -782,12 +900,15 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
host = makeString(w->host.name);
|
||||
if (host == NULL) {
|
||||
dereference(j);
|
||||
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
TRACE("%s update job (%ld) rusage for host %s", __func__, rusages->jobid,
|
||||
host);
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||||
update_host_runusage(&(j->job.run_usage), host, rusages->count,
|
||||
|
|
@ -805,6 +926,9 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_RUSAGE_UPDATE, c, rusages);
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -816,12 +940,19 @@ static int set_preason(cluster_t* c, xce_pendreason_t* reasons)
|
|||
|
||||
TRACE("%s jobid %ld", __func__, reasons->jobid);
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)reasons->jobid, (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_SETPREASON, c, j);
|
||||
job = &(j->job);
|
||||
|
|
@ -833,6 +964,9 @@ static int set_preason(cluster_t* c, xce_pendreason_t* reasons)
|
|||
MOVE_PTR(job->pend_reasons, reasons->reasons);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_SETPREASON, c, j);
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -882,12 +1016,19 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
job_t* j;
|
||||
xce_job_t* job;
|
||||
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j != NULL) {
|
||||
reference(j);
|
||||
}
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
continue;
|
||||
}
|
||||
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_CLRPREASON, c, j);
|
||||
|
||||
|
|
@ -899,7 +1040,9 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_CLRPREASON, c, j);
|
||||
unlock(j);
|
||||
|
||||
dereference(j);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue