forked from lei.chen/myxce
save works
This commit is contained in:
parent
b1faeb94b6
commit
b748e30d20
35 changed files with 1247 additions and 2149 deletions
|
|
@ -3,10 +3,12 @@
|
|||
#include <time.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "jobj.h"
|
||||
#include "preason.h"
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/errnox.h"
|
||||
#include "xce/types.h"
|
||||
#include "xstring.h"
|
||||
|
||||
static int mark_job_start(cluster_t*, job_t*, job_pidinfo_t*);
|
||||
static int mark_job_finish(cluster_t*, job_t*, xce_jobfinish_t*);
|
||||
|
|
@ -19,20 +21,15 @@ uint64_t jobid = 100;
|
|||
|
||||
static job_t* create_job(cluster_t* c, xce_job_t* j)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
static int typeid = -1;
|
||||
job_t* job = NULL;
|
||||
|
||||
if (typeid < 0) {
|
||||
typeid = typeid_of_name(&(c->objmgr), "xjob");
|
||||
TRACE("%s type %s id %d", __func__, "xjob", typeid);
|
||||
}
|
||||
|
||||
if (typeid < 0) {
|
||||
SET_ERROR(-typeid);
|
||||
job = calloc(1, sizeof(*job));
|
||||
if (job == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
job = create_obj_by_typeid(&(c->objmgr), typeid, j);
|
||||
/* FIXME: */
|
||||
job->job = makeJob(j);
|
||||
|
||||
return job;
|
||||
}
|
||||
|
|
@ -72,26 +69,26 @@ uint64_t job_submit(void* c, xce_job_submit_t* submit)
|
|||
int load_jobs(void* c)
|
||||
{
|
||||
#if 0
|
||||
job_t* job;
|
||||
int i;
|
||||
char cmdline[1024];
|
||||
job_t* job;
|
||||
int i;
|
||||
char cmdline[1024];
|
||||
|
||||
for (i = 1; i < NJOBS; i++) {
|
||||
sprintf(cmdline, "echo normal; sleep %d", i);
|
||||
job = create_job(c, jobid++, "normal", "rshen", "/home/rshen", "/tmp",
|
||||
"rshen", (char*)cmdline, NULL, 1, 1);
|
||||
if (job == NULL) {
|
||||
return -errno;
|
||||
for (i = 1; i < NJOBS; i++) {
|
||||
sprintf(cmdline, "echo normal; sleep %d", i);
|
||||
job = create_job(c, jobid++, "normal", "rshen", "/home/rshen", "/tmp",
|
||||
"rshen", (char*)cmdline, NULL, 1, 1);
|
||||
if (job == NULL) {
|
||||
return -errno;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 1; i < NJOBS; i++) {
|
||||
job = create_job(c, jobid++, "xce", "rshen", "/home/rshen", "/tmp",
|
||||
"rshen", "echo xce; sleep 30", NULL, 1, 1);
|
||||
if (job == NULL) {
|
||||
return -errno;
|
||||
for (i = 1; i < NJOBS; i++) {
|
||||
job = create_job(c, jobid++, "xce", "rshen", "/home/rshen", "/tmp",
|
||||
"rshen", "echo xce; sleep 30", NULL, 1, 1);
|
||||
if (job == NULL) {
|
||||
return -errno;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
|
|
@ -157,8 +154,8 @@ static int mk_launchinfo(xce_launch_t* launch, xce_job_alloc_t* alloc)
|
|||
static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
||||
{
|
||||
int rc = 0;
|
||||
job_t* j;
|
||||
worker_t* w;
|
||||
job_t* j = NULL;
|
||||
worker_t* w = NULL;
|
||||
xce_host_alloc_t** allocs;
|
||||
int nc;
|
||||
xce_job_exec_t einfo;
|
||||
|
|
@ -166,7 +163,10 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
TRACE("%s jobid %ld", __func__, alloc->jobid);
|
||||
|
||||
j = jobmgr_lockjob(&(c->jobmgr), alloc->jobid, OBJREF_UPDATE);
|
||||
rc = key_find(c->jobmgr, (void*)alloc->jobid, (void**)&j);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
|
|
@ -187,17 +187,20 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
|
||||
allocs = (xce_host_alloc_t**)alloc->allocs;
|
||||
/* using head node of job */
|
||||
w = workermgr_lock(&(c->workermgr), allocs[0]->host, OBJREF_READ);
|
||||
rc = key_find(c->workermgr, (void*)allocs[0]->host, (void**)&w);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (w == NULL) {
|
||||
PERROR("workermgr_lock");
|
||||
free_launchinfo(&launchinfo);
|
||||
obj_deref(j);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
if ((w->ep == NULL) || (w->host.status != 0)) {
|
||||
if ((w->ep == NULL) || (w->host->status != 0)) {
|
||||
ERROR("worker %s is not in correct state, status %x ep %p",
|
||||
w->host.name, w->host.status, w->ep);
|
||||
w->host->name, w->host->status, w->ep);
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
obj_deref(w);
|
||||
|
|
@ -212,40 +215,41 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
char* q_pre_exec;
|
||||
char* q_post_exec;
|
||||
|
||||
rc = get_queue_exec(c, j->job.submit->queue, &q_pre_exec, &q_post_exec);
|
||||
rc = get_queue_exec(c, j->job->submit->queue, &q_pre_exec,
|
||||
&q_post_exec);
|
||||
if (rc < 0) {
|
||||
obj_deref(j);
|
||||
obj_deref(w);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (j->job.submit->start_info->q_pre_exec != NULL) {
|
||||
obj_deref(j->job.submit->start_info->q_pre_exec);
|
||||
if (j->job->submit->start_info->q_pre_exec != NULL) {
|
||||
obj_deref(j->job->submit->start_info->q_pre_exec);
|
||||
}
|
||||
|
||||
if (j->job.submit->start_info->q_post_exec != NULL) {
|
||||
obj_deref(j->job.submit->start_info->q_post_exec);
|
||||
if (j->job->submit->start_info->q_post_exec != NULL) {
|
||||
obj_deref(j->job->submit->start_info->q_post_exec);
|
||||
}
|
||||
|
||||
j->job.submit->start_info->q_pre_exec = q_pre_exec;
|
||||
j->job.submit->start_info->q_post_exec = q_post_exec;
|
||||
j->job->submit->start_info->q_pre_exec = q_pre_exec;
|
||||
j->job->submit->start_info->q_post_exec = q_post_exec;
|
||||
}
|
||||
|
||||
einfo.jobid = alloc->jobid;
|
||||
einfo.start_info = j->job.submit->start_info;
|
||||
einfo.start_info = j->job->submit->start_info;
|
||||
einfo.launchinfo = &launchinfo;
|
||||
|
||||
rc = xce_startjob_req(w->ep, &einfo);
|
||||
|
||||
free_launchinfo(&launchinfo);
|
||||
|
||||
if (j->job.allocs != NULL) {
|
||||
free_JobAlloc_ptr(&(j->job.allocs));
|
||||
if (j->job->allocs != NULL) {
|
||||
free_JobAlloc_ptr(&(j->job->allocs));
|
||||
}
|
||||
j->job.allocs = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t));
|
||||
j->job.allocs->jobid = alloc->jobid;
|
||||
j->job.allocs->nhosts = nc;
|
||||
MOVE_PTR(j->job.allocs->allocs, allocs);
|
||||
j->job->allocs = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t));
|
||||
j->job->allocs->jobid = alloc->jobid;
|
||||
j->job->allocs->nhosts = nc;
|
||||
MOVE_PTR(j->job->allocs->allocs, allocs);
|
||||
alloc->allocs = NULL;
|
||||
|
||||
DEREF_OBJ(w);
|
||||
|
|
@ -263,40 +267,47 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc)
|
|||
int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
||||
{
|
||||
uint64_t jobid = ctrinfo->jobid;
|
||||
job_t* j;
|
||||
int rc = 0;
|
||||
job_t* j = NULL;
|
||||
int rc = 0;
|
||||
|
||||
TRACE("%s job_ctrl job %ld", __func__, jobid);
|
||||
j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
|
||||
|
||||
rc = key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
if (j->job.allocs != NULL) {
|
||||
xce_host_alloc_t** allocs = j->job.allocs->allocs;
|
||||
worker_t* w;
|
||||
if (j->job->allocs != NULL) {
|
||||
xce_host_alloc_t** allocs = j->job->allocs->allocs;
|
||||
worker_t* w = NULL;
|
||||
|
||||
/* job control command will be sent to first node
|
||||
* for parallel jobs
|
||||
*/
|
||||
w = workermgr_lock(&(c->workermgr), allocs[0]->host, OBJREF_READ);
|
||||
rc = key_find(c->workermgr, (void*)allocs[0]->host, (void**)&w);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
if (w != NULL) {
|
||||
/* FIXME: better method to check host status */
|
||||
if ((w->ep != NULL) && (w->host.status == 0)) {
|
||||
if ((w->ep != NULL) && (w->host->status == 0)) {
|
||||
rc = xce_jobctrl_req(w->ep, ctrinfo);
|
||||
if (rc < 0) {
|
||||
ERROR("job ctrl %ld on host %s returns %d", jobid,
|
||||
w->host.name, rc);
|
||||
w->host->name, rc);
|
||||
}
|
||||
} else {
|
||||
ERROR("worker %s is not in correct state, status %x ep %p",
|
||||
w->host.name, w->host.status, w->ep);
|
||||
w->host->name, w->host->status, w->ep);
|
||||
rc = -errno;
|
||||
}
|
||||
obj_deref(w);
|
||||
} else {
|
||||
PERROR("workermgr_lock");
|
||||
rc = -errno;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -311,9 +322,9 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
case JobControl_u_siginfo:
|
||||
switch (ctrinfo->u.siginfo->signo) {
|
||||
case SIGCONT:
|
||||
if (j->job.state
|
||||
if (j->job->state
|
||||
& (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP)) {
|
||||
int state = j->job.state;
|
||||
int state = j->job->state;
|
||||
|
||||
/* resume suspend job? */
|
||||
state &= ~(JOB_STAT_PSUSP | JOB_STAT_USUSP);
|
||||
|
|
@ -322,9 +333,9 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
break;
|
||||
case SIGSTOP:
|
||||
case SIGTSTP:
|
||||
if (!(j->job.state
|
||||
if (!(j->job->state
|
||||
& (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP))) {
|
||||
int state = j->job.state;
|
||||
int state = j->job->state;
|
||||
|
||||
state |= JOB_STAT_PSUSP;
|
||||
mark_job_update(c, j, state);
|
||||
|
|
@ -333,7 +344,7 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
case SIGKILL:
|
||||
case SIGTERM:
|
||||
default:
|
||||
if (j->job.state
|
||||
if (j->job->state
|
||||
& (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -346,7 +357,8 @@ int job_ctrl(cluster_t* c, xce_jobctrl_t* ctrinfo)
|
|||
}
|
||||
break;
|
||||
case JobControl_u_ckptoption:
|
||||
if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
|
||||
if (j->job->state
|
||||
& (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
|
||||
ERROR("job <%ld> has been finished, no checkpoint can be done.",
|
||||
jobid);
|
||||
} else {
|
||||
|
|
@ -433,6 +445,7 @@ int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
xlist_t jlist;
|
||||
job_t* j = NULL;
|
||||
int i;
|
||||
int rc;
|
||||
|
||||
submit = request->body.jobsubmsg;
|
||||
|
||||
|
|
@ -467,8 +480,8 @@ int handle_jobsubmit(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
for (i = 0; i < submit->array_size; i++) {
|
||||
/* Run hooks to tell scheduler */
|
||||
j = jobmgr_lockjob(&(c->jobmgr), jobid + i, OBJREF_READ);
|
||||
if (j == NULL) {
|
||||
rc = key_find(c->jobmgr, (void*)(jobid + i), (void**)&j);
|
||||
if ((rc <= 0) || (j == NULL)) {
|
||||
char idstring[128];
|
||||
|
||||
fmt_jobid(idstring, sizeof(idstring) - 1, jobid);
|
||||
|
|
@ -496,21 +509,20 @@ static int mark_job_start(cluster_t* c, job_t* job, job_pidinfo_t* info)
|
|||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_START, c, info, job);
|
||||
/* update job information */
|
||||
job->job.state = JOB_STAT_RUN;
|
||||
job->job.start_time = time(NULL);
|
||||
if (job->job.pidinfo == NULL) {
|
||||
job->job.pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t));
|
||||
job->job->state = JOB_STAT_RUN;
|
||||
job->job->start_time = time(NULL);
|
||||
if (job->job->pidinfo == NULL) {
|
||||
job->job->pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t));
|
||||
}
|
||||
|
||||
job->job.pidinfo->npids = copy_int_list((int**)&(job->job.pidinfo->pids),
|
||||
(int*)info->pids, info->npids);
|
||||
job->job.pidinfo->npgids = copy_int_list((int**)&(job->job.pidinfo->pgids),
|
||||
(int*)info->pgids, info->npgids);
|
||||
job->job->pidinfo->npids = copy_int_list((int**)&(job->job->pidinfo->pids),
|
||||
(int*)info->pids, info->npids);
|
||||
job->job->pidinfo->npgids = copy_int_list(
|
||||
(int**)&(job->job->pidinfo->pgids), (int*)info->pgids, info->npgids);
|
||||
if (info->exdata != NULL) {
|
||||
job->job.pidinfo->exdata
|
||||
= create_obj_by_typename(&(c->objmgr), "xstring", info->exdata);
|
||||
job->job->pidinfo->exdata = makeString(info->exdata);
|
||||
} else {
|
||||
job->job.pidinfo->exdata = NULL;
|
||||
job->job->pidinfo->exdata = NULL;
|
||||
}
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
|
|
@ -522,20 +534,19 @@ static int mark_job_start(cluster_t* c, job_t* job, job_pidinfo_t* info)
|
|||
int on_job_start(cluster_t* c, job_pidinfo_t* info)
|
||||
{
|
||||
job_t* job = NULL;
|
||||
int rc;
|
||||
|
||||
TRACE("%s jobid %ld", __func__, info->jobid);
|
||||
|
||||
job = jobmgr_lockjob(&(c->jobmgr), info->jobid, OBJREF_UPDATE);
|
||||
if (job == NULL) {
|
||||
rc = key_find(c->jobmgr, (void*)info->jobid, (void**)&job);
|
||||
|
||||
if ((rc <= 0) || (job == NULL)) {
|
||||
ERROR("can not find job %ld", info->jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
mark_job_start(c, job, info);
|
||||
|
||||
/* unlock the job */
|
||||
DEREF_OBJ(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -545,23 +556,24 @@ static int mark_job_finish(cluster_t* c, job_t* job, xce_jobfinish_t* finish)
|
|||
HOOK_DETAIL_JOB_EXIT, c, finish, job);
|
||||
/* update job information */
|
||||
if (finish->exit_code == 0) {
|
||||
job->job.state = JOB_STAT_DONE;
|
||||
job->job->state = JOB_STAT_DONE;
|
||||
} else {
|
||||
job->job.state = JOB_STAT_EXIT;
|
||||
job->job->state = JOB_STAT_EXIT;
|
||||
}
|
||||
job->job.finish_time = time(NULL);
|
||||
job->job->finish_time = time(NULL);
|
||||
|
||||
/* fill job run time/rusage information */
|
||||
|
||||
if (job->job.exit_info == NULL) {
|
||||
job->job.exit_info = (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
|
||||
if (job->job->exit_info == NULL) {
|
||||
job->job->exit_info
|
||||
= (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
|
||||
}
|
||||
|
||||
if (job->job.exit_info != NULL) {
|
||||
job->job.exit_info->exitcode = finish->exit_code;
|
||||
if (job->job->exit_info != NULL) {
|
||||
job->job->exit_info->exitcode = finish->exit_code;
|
||||
}
|
||||
|
||||
job->job.run_time = job->job.finish_time - job->job.start_time;
|
||||
job->job->run_time = job->job->finish_time - job->job->start_time;
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_EXIT, c, finish, job);
|
||||
|
|
@ -576,7 +588,8 @@ int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
|
|||
TRACE("%s job finish jobid %ld exit code %d", __func__, finish->jobid,
|
||||
finish->exit_code);
|
||||
|
||||
job = jobmgr_lockjob(&(c->jobmgr), finish->jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)finish->jobid, (void**)&job);
|
||||
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", finish->jobid);
|
||||
return -errno;
|
||||
|
|
@ -584,9 +597,6 @@ int on_job_finish(cluster_t* c, xce_jobfinish_t* finish)
|
|||
|
||||
mark_job_finish(c, job, finish);
|
||||
|
||||
/* unlock the job */
|
||||
DEREF_OBJ(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -599,20 +609,20 @@ static int mark_job_fail(cluster_t* c, job_t* job, job_failinfo_t* failinfo)
|
|||
switch (failinfo->stage) {
|
||||
case JOB_EXEC_STAGE_Q_POST_EXEC:
|
||||
case JOB_EXEC_STAGE_POST_EXEC:
|
||||
if ((job->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
|
||||
if ((job->job->state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
|
||||
WARN("job %ld got post exec fail notification but job's exit "
|
||||
"status has not been set",
|
||||
job->job.jobid);
|
||||
job->job.state |= JOB_STAT_EXIT;
|
||||
job->job->jobid);
|
||||
job->job->state |= JOB_STAT_EXIT;
|
||||
}
|
||||
job->job.state |= JOB_STAT_PERR;
|
||||
job->job->state |= JOB_STAT_PERR;
|
||||
break;
|
||||
default:
|
||||
job->job.state = JOB_STAT_EXIT;
|
||||
job->job->state = JOB_STAT_EXIT;
|
||||
break;
|
||||
}
|
||||
|
||||
job->job.finish_time = time(NULL);
|
||||
job->job->finish_time = time(NULL);
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_FAIL, c, failinfo, job);
|
||||
|
|
@ -625,7 +635,7 @@ static int mark_job_update(cluster_t* c, job_t* job, int state)
|
|||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_UPDATE, c, job);
|
||||
/* update job state */
|
||||
job->job.state = state;
|
||||
job->job->state = state;
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_UPDATE, c, job);
|
||||
|
|
@ -640,7 +650,7 @@ int on_job_fail(cluster_t* c, job_failinfo_t* failinfo)
|
|||
TRACE("%s job fail jobid %ld code %d", __func__, failinfo->jobid,
|
||||
failinfo->code);
|
||||
|
||||
job = jobmgr_lockjob(&(c->jobmgr), failinfo->jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)failinfo->jobid, (void**)&job);
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", failinfo->jobid);
|
||||
return -errno;
|
||||
|
|
@ -648,9 +658,6 @@ int on_job_fail(cluster_t* c, job_failinfo_t* failinfo)
|
|||
|
||||
mark_job_fail(c, job, failinfo);
|
||||
|
||||
/* unlock the job */
|
||||
DEREF_OBJ(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -661,20 +668,17 @@ int on_job_stop(cluster_t* c, uint64_t jobid)
|
|||
|
||||
TRACE("%s job stop jobid %ld", __func__, jobid);
|
||||
|
||||
job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
state = job->job.state;
|
||||
state = job->job->state;
|
||||
|
||||
state |= JOB_STAT_USUSP;
|
||||
mark_job_update(c, job, state);
|
||||
|
||||
/* unlock the job */
|
||||
DEREF_OBJ(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -685,19 +689,16 @@ int on_job_resume(cluster_t* c, uint64_t jobid)
|
|||
|
||||
TRACE("%s job resume jobid %ld", __func__, jobid);
|
||||
|
||||
job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
state = job->job.state;
|
||||
state = job->job->state;
|
||||
state &= ~JOB_STAT_USUSP;
|
||||
mark_job_update(c, job, state);
|
||||
|
||||
/* unlock the job */
|
||||
DEREF_OBJ(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -708,28 +709,25 @@ int on_job_postdone(cluster_t* c, endpoint_t* ep, uint64_t jobid)
|
|||
|
||||
TRACE("%s job post done jobid %ld", __func__, jobid);
|
||||
|
||||
job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&job);
|
||||
if (job == NULL) {
|
||||
ERROR("can not find job %ld", jobid);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
state = job->job.state;
|
||||
state = job->job->state;
|
||||
if ((state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
|
||||
WARN("job %ld got post exec fail notification but job's exit status "
|
||||
"has not been set",
|
||||
job->job.jobid);
|
||||
job->job->jobid);
|
||||
state |= JOB_STAT_EXIT;
|
||||
/* ensure that job finish time has been set */
|
||||
job->job.finish_time = time(NULL);
|
||||
job->job->finish_time = time(NULL);
|
||||
}
|
||||
|
||||
state |= JOB_STAT_PDONE;
|
||||
mark_job_update(c, job, state);
|
||||
|
||||
/* unlock the job */
|
||||
DEREF_OBJ(job);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -737,7 +735,7 @@ int on_job_ckpt(cluster_t* c, endpoint_t* ep, xce_ckptinfo_t* ckptinfo)
|
|||
{
|
||||
uint64_t jobid;
|
||||
char* info = NULL;
|
||||
job_t* j;
|
||||
job_t* j = NULL;
|
||||
|
||||
if (ckptinfo == NULL) {
|
||||
ERROR("%s ckptinfo should not be NULL", __func__);
|
||||
|
|
@ -749,7 +747,8 @@ int on_job_ckpt(cluster_t* c, endpoint_t* ep, xce_ckptinfo_t* ckptinfo)
|
|||
|
||||
TRACE("%s job %ld ckptinfo %s", __func__, jobid, info);
|
||||
|
||||
j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
|
||||
if (j == NULL) {
|
||||
ERROR("%s can not find job %ld", __func__, jobid);
|
||||
return -errno;
|
||||
|
|
@ -757,11 +756,12 @@ int on_job_ckpt(cluster_t* c, endpoint_t* ep, xce_ckptinfo_t* ckptinfo)
|
|||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_CKPT, c, j);
|
||||
if (j->job.ckptinfo != NULL) {
|
||||
obj_deref(j->job.ckptinfo);
|
||||
if (j->job->ckptinfo != NULL) {
|
||||
obj_deref(j->job->ckptinfo);
|
||||
}
|
||||
|
||||
j->job.ckptinfo = create_obj_by_typename(&(c->objmgr), "xstring", info);
|
||||
j->job->ckptinfo = makeString(info);
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_CKPT, c, j);
|
||||
|
||||
|
|
@ -788,28 +788,28 @@ int do_jobcontrol_op(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
||||
{
|
||||
xce_job_rusages_t* rusages = request->body.updatemsg->data.jobrusages;
|
||||
job_t* j;
|
||||
worker_t* w;
|
||||
char* host = NULL;
|
||||
job_t* j = NULL;
|
||||
worker_t* w = NULL;
|
||||
char* host = NULL;
|
||||
|
||||
TRACE("%s trying to update job(%ld) rusage", __func__, rusages->jobid);
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_RUSAGE_UPDATE, c, rusages);
|
||||
|
||||
j = jobmgr_lockjob(&(c->jobmgr), rusages->jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)rusages->jobid, (void**)&j);
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
return -errno;
|
||||
}
|
||||
|
||||
w = get_ep_worker(c, ep, OBJREF_READ);
|
||||
w = get_ep_worker(c, ep);
|
||||
if (w == NULL) {
|
||||
obj_deref(j);
|
||||
return errno;
|
||||
}
|
||||
|
||||
host = create_obj_by_typename(&(c->objmgr), "xstring", w->host.name);
|
||||
host = makeString(w->host->name);
|
||||
obj_deref(w);
|
||||
|
||||
if (host == NULL) {
|
||||
|
|
@ -821,12 +821,12 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||||
update_host_runusage(&(j->job.run_usage), host, rusages->count,
|
||||
update_host_runusage(&(j->job->run_usage), host, rusages->count,
|
||||
rusages->usages);
|
||||
|
||||
update_pids(&(j->job.pidinfo), rusages->npids, rusages->pids);
|
||||
update_pids(&(j->job->pidinfo), rusages->npids, rusages->pids);
|
||||
|
||||
update_pgids(&(j->job.pidinfo), rusages->npgids, rusages->pgids);
|
||||
update_pgids(&(j->job->pidinfo), rusages->npgids, rusages->pgids);
|
||||
|
||||
/* FIXME
|
||||
* update other fields of job rusage later
|
||||
|
|
@ -843,12 +843,12 @@ int update_job_rusages(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
|
||||
static int set_preason(cluster_t* c, xce_pendreason_t* reasons)
|
||||
{
|
||||
job_t* j;
|
||||
xce_job_t* job;
|
||||
job_t* j = NULL;
|
||||
xce_job_t* job = NULL;
|
||||
|
||||
TRACE("%s jobid %ld", __func__, reasons->jobid);
|
||||
|
||||
j = jobmgr_lockjob(&(c->jobmgr), reasons->jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)reasons->jobid, (void**)&j);
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
return -ENOENT;
|
||||
|
|
@ -856,7 +856,7 @@ static int set_preason(cluster_t* c, xce_pendreason_t* reasons)
|
|||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_SETPREASON, c, j);
|
||||
job = &(j->job);
|
||||
job = j->job;
|
||||
if (job->pend_reasons != NULL) {
|
||||
erase_job_preason(job->pend_reasons);
|
||||
FREEUP(job->pend_reasons);
|
||||
|
|
@ -865,7 +865,6 @@ static int set_preason(cluster_t* c, xce_pendreason_t* reasons)
|
|||
MOVE_PTR(job->pend_reasons, reasons->reasons);
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_SETPREASON, c, j);
|
||||
obj_deref(j);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -915,7 +914,7 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
job_t* j;
|
||||
xce_job_t* job;
|
||||
|
||||
j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
|
||||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||||
if (j == NULL) {
|
||||
PERROR("jobmgr_lockjob");
|
||||
continue;
|
||||
|
|
@ -924,7 +923,7 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, HOOK_BEFORE_EVENT,
|
||||
HOOK_DETAIL_CLRPREASON, c, j);
|
||||
|
||||
job = &(j->job);
|
||||
job = j->job;
|
||||
if (job->pend_reasons != NULL) {
|
||||
erase_job_preason(job->pend_reasons);
|
||||
FREEUP(job->pend_reasons);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue