update controller code

src/controller/job.c

@@ -0,0 +1,922 @@
+#include <errno.h>
+#include <signal.h>
+#include <time.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+#include "xce/types.h"
+
+static int mark_job_start(cluster_t*, job_t*, job_pidinfo_t*);
+static int mark_job_finish(cluster_t*, job_t*, xce_jobfinish_t*);
+static int mark_job_fail(cluster_t*, job_t*, job_failinfo_t*);
+static int mark_job_update(cluster_t*, job_t*, int);
+
+#define NJOBS (100)
+
+uint64_t jobid = 100;
+
+static job_t* create_job(cluster_t* c,
+                         uint64_t   jobid,
+                         char*      queue,
+                         char*      user,
+                         char*      exechome,
+                         char*      execcwd,
+                         char*      execuser,
+                         char*      cmdline,
+                         char*      exdata,
+                         uint32_t   mintasks,
+                         uint32_t   maxtasks)
+{
+    job_t* job = NULL;
+    return job;
+}
+
+uint64_t job_submit(void*    c,
+                    char*    user,
+                    char*    queue,
+                    char*    resreq,
+                    char*    cmdline,
+                    char*    exdata,
+                    uint32_t mintasks,
+                    uint32_t maxtasks)
+{
+    job_t* job = NULL;
+
+    job = create_job(c, jobid++, queue,
+                     user, "/tmp",
+                     "/tmp", user, cmdline,
+                     exdata, mintasks, maxtasks);
+    if (job == NULL) {
+        return -errno;
+    }
+
+    return job->job.jobid;
+}
+
+int load_jobs(void* c)
+{
+    job_t* job;
+    int    i;
+    char   cmdline[1024];
+
+    for (i = 1; i < NJOBS; i++) {
+        sprintf(cmdline, "echo normal; sleep %d", i);
+        job = create_job(c, jobid++, "normal",
+                         "rshen", "/home/rshen",
+                         "/tmp", "rshen", (char*)cmdline,
+                         NULL, 1, 1);
+        if (job == NULL) {
+            return -errno;
+        }
+    }
+
+    for (i = 1; i < NJOBS; i++) {
+        job = create_job(c, jobid++, "xce",
+                         "rshen", "/home/rshen",
+                         "/tmp", "rshen", "echo xce; sleep 30",
+                         NULL, 1, 1);
+        if (job == NULL) {
+            return -errno;
+        }
+    }
+
+    return 0;
+}
+
+static int get_nslots_in_alloc(xce_host_alloc_t* alloc)
+{
+    int i;
+
+    for (i = 0; i < alloc->nallocs; i++) {
+        if (strcmp(alloc->allocs[i]->resname, "slots") == 0) {
+            return (int)alloc->allocs[i]->value;
+        }
+    }
+
+    return 1;
+}
+
+static void free_launchinfo(xce_launch_t* launch)
+{
+    int i;
+
+    for (i = 0; i < launch->nhosts; i++) {
+        FREEUP(launch->hosts[i]);
+    }
+
+    FREEUP(launch->hosts);
+}
+
+static int mk_launchinfo(xce_launch_t*    launch,
+                         xce_job_alloc_t* alloc)
+{
+    int i;
+    int rc = 0;
+
+    memset(launch, 0x0, sizeof(*launch));
+
+    launch->hosts = (xce_host_launch_t**)calloc(alloc->nhosts,
+                                                sizeof(xce_host_launch_t*));
+    if (launch->hosts == NULL) {
+        return -ENOMEM;
+    }
+
+    launch->nhosts = alloc->nhosts;
+    for (i = 0; i < alloc->nhosts; i++) {
+        launch->hosts[i] = (xce_host_launch_t*)calloc(1, sizeof(xce_host_launch_t));
+        if (launch->hosts[i] == NULL) {
+            rc = -ENOMEM;
+            break;
+        }
+
+        launch->hosts[i]->host   = alloc->allocs[i]->host;
+        launch->hosts[i]->nranks = get_nslots_in_alloc(alloc->allocs[i]);
+    }
+
+    if (rc < 0) {
+        free_launchinfo(launch);
+    }
+
+    return rc;
+}
+
+static int dispatch_a_job_(cluster_t*       c,
+                           endpoint_t*      ep,
+                           xce_job_alloc_t* alloc)
+{
+    int                rc = 0;
+    job_t*             j;
+    worker_t*          w;
+    ioctx_t*           ioctx;
+    xce_host_alloc_t** allocs;
+    int                nc;
+    xce_job_exec_t     einfo;
+    xce_launch_t       launchinfo;
+
+    TRACE("%s jobid %ld", __func__, alloc->jobid);
+
+    j = jobmgr_lockjob(&(c->jobmgr), alloc->jobid, OBJREF_UPDATE);
+
+    if (j == NULL) {
+        PERROR("jobmgr_lockjob");
+
+        return -errno;
+    }
+
+    nc = alloc->nhosts;
+    if (nc <= 0) {
+        obj_deref(j);
+        return -EINVAL;
+    }
+
+    /* FIXME: change case of nc > 1 just hard coded mpi as
+     * the real job type later
+     */
+    mk_launchinfo(&launchinfo, alloc);
+
+    allocs = (xce_host_alloc_t**)alloc->allocs;
+    /* using head node of job */
+    w = workermgr_lock(&(c->workermgr),
+                       allocs[0]->host,
+                       OBJREF_READ);
+    if (w == NULL) {
+        PERROR("workermgr_lock");
+        free_launchinfo(&launchinfo);
+        obj_deref(j);
+        return -errno;
+    }
+
+    if ((w->ep == NULL) || (w->host.status != 0)) {
+        ERROR("worker %s is not in correct state, status %x ep %p",
+              w->host.name,
+              w->host.status,
+              w->ep);
+        free_launchinfo(&launchinfo);
+
+        obj_deref(w);
+        obj_deref(j);
+
+        SET_ERROR(E_XCE_BAD_WORKER);
+        return -errno;
+    }
+
+    /* FIXME: we may consider to optimize when a job has enter a queue? */
+    if (1) {
+        char* q_pre_exec;
+        char* q_post_exec;
+
+        rc = get_queue_exec(c, j->job.queue,
+                            &q_pre_exec, &q_post_exec);
+        if (rc < 0) {
+            obj_deref(j);
+            obj_deref(w);
+            return rc;
+        }
+
+        if (j->job.start_info->q_pre_exec != NULL) {
+            obj_deref(j->job.start_info->q_pre_exec);
+        }
+
+        if (j->job.start_info->q_post_exec != NULL) {
+            obj_deref(j->job.start_info->q_post_exec);
+        }
+
+        j->job.start_info->q_pre_exec  = q_pre_exec;
+        j->job.start_info->q_post_exec = q_post_exec;
+    }
+
+    ioctx = ioctx_alloc(w->ep, c->myhost);
+    if (ioctx == NULL) {
+        ERROR("memory allocate failed");
+        free_launchinfo(&launchinfo);
+        obj_deref(j);
+        DEREF_OBJ(w);
+
+        return -ENOMEM;
+    }
+
+    einfo.jobid      = alloc->jobid;
+    einfo.start_info = j->job.start_info;
+    einfo.launchinfo = &launchinfo;
+
+    rc = xce_startjob_req(ioctx, &einfo);
+
+    free_launchinfo(&launchinfo);
+
+    if (j->job.allocs != NULL) {
+        free_JobAlloc_ptr(&(j->job.allocs));
+    }
+    j->job.allocs         = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t));
+    j->job.allocs->jobid  = alloc->jobid;
+    j->job.allocs->nhosts = nc;
+    MOVE_PTR(j->job.allocs->allocs, allocs);
+    alloc->allocs = NULL;
+
+    DEREF_OBJ(w);
+    DEREF_OBJ(j);
+
+    if (rc < 0) {
+        SET_ERROR(-rc);
+        PERROR("xce_startjob_req");
+        return rc;
+    }
+
+    return rc;
+}
+
+int job_ctrl(cluster_t*     c,
+             xce_jobctrl_t* ctrinfo)
+{
+    uint64_t jobid = ctrinfo->jobid;
+    job_t*   j;
+    int      rc = 0;
+
+    TRACE("%s job_ctrl job %ld", __func__, jobid);
+    j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (j == NULL) {
+        PERROR("jobmgr_lockjob");
+        return -ENOENT;
+    }
+
+    if (j->job.allocs != NULL) {
+        xce_host_alloc_t** allocs = j->job.allocs->allocs;
+        worker_t*          w;
+        ioctx_t*           ioctx;
+
+        /* job control command will be sent to first node
+         *  for parallel jobs
+         */
+        w = workermgr_lock(&(c->workermgr),
+                           allocs[0]->host,
+                           OBJREF_READ);
+        if (w != NULL) {
+            /* FIXME: better method to check host status */
+            if ((w->ep != NULL) && (w->host.status == 0)) {
+                ioctx = ioctx_alloc(w->ep, c->myhost);
+                if (ioctx != NULL) {
+                    rc = xce_jobctrl_req(ioctx, ctrinfo);
+                    if (rc < 0) {
+                        ERROR("job ctrl %ld on host %s returns %d",
+                              jobid, w->host.name, rc);
+                    }
+                } else {
+                    ERROR("memory allocate failed");
+                    rc = -ENOMEM;
+                }
+            } else {
+                ERROR("worker %s is not in correct state, status %x ep %p",
+                      w->host.name,
+                      w->host.status,
+                      w->ep);
+                rc = -errno;
+            }
+            obj_deref(w);
+        } else {
+            PERROR("workermgr_lock");
+            rc = -errno;
+        }
+    } else {
+        /* job has not been started yet */
+        job_failinfo_t failinfo = {
+            .jobid = jobid,
+        };
+
+        xce_use(&failinfo);
+
+        switch (ctrinfo->action) {
+        case JobControl_u_siginfo:
+            switch (ctrinfo->u.siginfo->signo) {
+            case SIGCONT:
+                if (j->job.state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP)) {
+                    int state = j->job.state;
+
+                    /* resume suspend job? */
+                    state &= ~(JOB_STAT_PSUSP | JOB_STAT_USUSP);
+                    mark_job_update(c, j, state);
+                }
+                break;
+            case SIGSTOP:
+            case SIGTSTP:
+                if (!(j->job.state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP))) {
+                    int state = j->job.state;
+
+                    state |= JOB_STAT_PSUSP;
+                    mark_job_update(c, j, state);
+                }
+                break;
+            case SIGKILL:
+            case SIGTERM:
+            default:
+                if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
+                    break;
+                }
+                /* FIXME: */
+                failinfo.code   = 128 + ctrinfo->u.siginfo->signo;
+                failinfo.detail = "Job killed by signal";
+
+                mark_job_fail(c, j, &failinfo);
+                break;
+            }
+            break;
+        case JobControl_u_ckptoption:
+            if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
+                ERROR("job <%ld> has been finished, no checkpoint can be done.",
+                      jobid);
+            } else {
+                ERROR("job %ld has not been started, no checkpoint can be done",
+                      jobid);
+            }
+            rc = -EINVAL;
+            break;
+        default:
+            rc = -EINVAL;
+            break;
+        }
+    }
+
+    obj_deref(j);
+    return rc;
+}
+
+static void notify_dispatch(cluster_t*  c,
+                            endpoint_t* ep,
+                            uint64_t    jobid,
+                            int         code)
+{
+    /* clang-format off */
+    job_failinfo_t info = {
+        .jobid = jobid,
+        .code  = code
+    };
+    xce_notify_data_t data = {
+        .event = NotifyData_data_jobDispatchFail,
+        .data  = {
+             .dispatchfail = &info
+	}
+    };
+    /* clang-format on */
+
+    notify(c, ep, &data);
+}
+
+int handle_jobdispatch(cluster_t*     c,
+                       endpoint_t*    ep,
+                       xce_request_t* request)
+{
+    int                 rc = 0;
+    int                 i;
+    xce_job_dispatch_t* dispatchmsg = request->body.dispatchmsg;
+    xce_reply_t         reply;
+
+    /* reply job dispatch that we'll handle request */
+    init_reply_by_req(&reply, request);
+    reply.status     = 0;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+
+    for (i = 0; i < dispatchmsg->njobs; i++) {
+        int              rc1;
+        xce_job_alloc_t* alloc = NULL;
+
+        MOVE_PTR(alloc, dispatchmsg->allocs[i]);
+
+        rc1 = dispatch_a_job_(c, ep, alloc);
+        if (rc1 < 0) {
+            /* notify dispatch fail */
+            notify_dispatch(c, ep, alloc->jobid, rc1);
+        } else {
+            /* notify dispatch ok */
+            notify_dispatch(c, ep, alloc->jobid, 0);
+            free_JobAlloc_ptr(&(alloc));
+        }
+    }
+    return rc;
+}
+
+int handle_jobsubmit(cluster_t*     c,
+                     endpoint_t*    ep,
+                     xce_request_t* request)
+{
+    uint64_t          jobid;
+    xce_job_submit_t* submit;
+    xce_reply_t       reply;
+    xlist_t           jlist;
+    job_t*            j = NULL;
+
+    submit = request->body.jobsubmsg;
+
+    init_reply_by_req(&reply, request);
+
+    if (NULLSTR(submit->user) || NULLSTR(submit->queue) || NULLSTR(submit->cmdline)) {
+        jobid = 0;
+        errno = EINVAL;
+    } else {
+        xce_run_hooks(&(c->hooksys),
+                      XCE_CONTROLLER_HOOK_JOB,
+                      HOOK_BEFORE_EVENT,
+                      HOOK_DETAIL_JOB_NEW,
+                      c, submit);
+
+        jobid = job_submit(c, (char*)submit->user,
+                           (char*)submit->queue,
+                           (char*)submit->resreq,
+                           (char*)submit->cmdline,
+                           (char*)submit->exdata,
+                           submit->mintasks,
+                           submit->maxtasks);
+    }
+    if (jobid == 0) {
+        reply.status     = -errno;
+        reply.data_which = RequestReply_data_nodata;
+
+        xce_reply(ep, c->myhost, &reply);
+        return -errno;
+    }
+
+    reply.status     = 0;
+    reply.data_which = RequestReply_data_jobid;
+    reply.data.jobid = jobid;
+
+    xce_reply(ep, c->myhost, &reply);
+
+    /* Run hooks to tell scheduler */
+    j = jobmgr_lockjob(&(c->jobmgr),
+                       jobid, OBJREF_READ);
+    if (j == NULL) {
+        ERROR("%s fail to lock job %lu",
+              __func__, jobid);
+    } else {
+        xlist_init(&jlist, 2);
+        if (xlist_store(&jlist, j) >= 0) {
+            xce_run_hooks(&(c->hooksys),
+                          XCE_CONTROLLER_HOOK_JOB,
+                          HOOK_AFTER_EVENT,
+                          HOOK_DETAIL_JOB_NEW,
+                          c, &jlist);
+        } else {
+            PERROR("xlist_store");
+        }
+
+        obj_deref(j);
+        xlist_fini(&jlist);
+    }
+
+    return 0;
+}
+
+static int mark_job_start(cluster_t*     c,
+                          job_t*         job,
+                          job_pidinfo_t* info)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_START,
+                  c, info, job);
+    /* update job information */
+    job->job.state      = JOB_STAT_RUN;
+    job->job.start_time = time(NULL);
+    if (job->job.pidinfo == NULL) {
+        job->job.pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t));
+    }
+
+    job->job.pidinfo->npids  = copy_int_list((int**)&(job->job.pidinfo->pids),
+                                             (int*)info->pids, info->npids);
+    job->job.pidinfo->npgids = copy_int_list((int**)&(job->job.pidinfo->pgids),
+                                             (int*)info->pgids, info->npgids);
+    if (info->exdata != NULL) {
+        job->job.pidinfo->exdata = create_obj_by_typename(&(c->objmgr), "xstring",
+                                                          info->exdata);
+    } else {
+        job->job.pidinfo->exdata = NULL;
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_START,
+                  c, info, job);
+
+    return 0;
+}
+
+int on_job_start(cluster_t*     c,
+                 job_pidinfo_t* info)
+{
+    job_t* job = NULL;
+
+    TRACE("%s jobid %ld", __func__, info->jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), info->jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", info->jobid);
+        return -errno;
+    }
+
+    mark_job_start(c, job, info);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+static int mark_job_finish(cluster_t*       c,
+                           job_t*           job,
+                           xce_jobfinish_t* finish)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_EXIT,
+                  c, finish, job);
+    /* update job information */
+    if (finish->exit_code == 0) {
+        job->job.state = JOB_STAT_DONE;
+    } else {
+        job->job.state = JOB_STAT_EXIT;
+    }
+    job->job.finish_time = time(NULL);
+
+    /* fill job run time/rusage information */
+
+    if (job->job.exit_info == NULL) {
+        job->job.exit_info = (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
+    }
+
+    if (job->job.exit_info != NULL) {
+        job->job.exit_info->exitcode = finish->exit_code;
+    }
+
+    job->job.run_time = job->job.finish_time - job->job.start_time;
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_EXIT,
+                  c, finish, job);
+
+    return 0;
+}
+
+int on_job_finish(cluster_t*       c,
+                  xce_jobfinish_t* finish)
+{
+    job_t* job = NULL;
+
+    TRACE("%s job finish jobid %ld exit code %d",
+          __func__, finish->jobid, finish->exit_code);
+
+    job = jobmgr_lockjob(&(c->jobmgr), finish->jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", finish->jobid);
+        return -errno;
+    }
+
+    mark_job_finish(c, job, finish);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+static int mark_job_fail(cluster_t*      c,
+                         job_t*          job,
+                         job_failinfo_t* failinfo)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_FAIL,
+                  c, failinfo, job);
+    /* update job information */
+
+    switch (failinfo->stage) {
+    case JOB_EXEC_STAGE_Q_POST_EXEC:
+    case JOB_EXEC_STAGE_POST_EXEC:
+        if ((job->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
+            WARN("job %ld got post exec fail notification but job's exit status has not been set",
+                 job->job.jobid);
+            job->job.state |= JOB_STAT_EXIT;
+        }
+        job->job.state |= JOB_STAT_PERR;
+        break;
+    default:
+        job->job.state = JOB_STAT_EXIT;
+        break;
+    }
+
+    job->job.finish_time = time(NULL);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_FAIL,
+                  c, failinfo, job);
+
+    return 0;
+}
+
+static int mark_job_update(cluster_t* c,
+                           job_t*     job,
+                           int        state)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, job);
+    /* update job state */
+    job->job.state = state;
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, job);
+
+    return 0;
+}
+
+int on_job_fail(cluster_t*      c,
+                job_failinfo_t* failinfo)
+{
+    job_t* job = NULL;
+
+    TRACE("%s job fail jobid %ld code %d",
+          __func__, failinfo->jobid, failinfo->code);
+
+    job = jobmgr_lockjob(&(c->jobmgr), failinfo->jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", failinfo->jobid);
+        return -errno;
+    }
+
+    mark_job_fail(c, job, failinfo);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_stop(cluster_t* c,
+                uint64_t   jobid)
+{
+    job_t* job = NULL;
+    int    state;
+
+    TRACE("%s job stop jobid %ld",
+          __func__, jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", jobid);
+        return -errno;
+    }
+
+    state = job->job.state;
+
+    state |= JOB_STAT_USUSP;
+    mark_job_update(c, job, state);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_resume(cluster_t* c,
+                  uint64_t   jobid)
+{
+    job_t* job = NULL;
+    int    state;
+
+    TRACE("%s job resume jobid %ld",
+          __func__, jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", jobid);
+        return -errno;
+    }
+
+    state = job->job.state;
+    state &= ~JOB_STAT_USUSP;
+    mark_job_update(c, job, state);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_postdone(cluster_t*  c,
+                    endpoint_t* ep,
+                    uint64_t    jobid)
+{
+    job_t* job = NULL;
+    int    state;
+
+    TRACE("%s job post done jobid %ld",
+          __func__, jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", jobid);
+        return -errno;
+    }
+
+    state = job->job.state;
+    if ((state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
+        WARN("job %ld got post exec fail notification but job's exit status has not been set",
+             job->job.jobid);
+        state |= JOB_STAT_EXIT;
+        /* ensure that job finish time has been set */
+        job->job.finish_time = time(NULL);
+    }
+
+    state |= JOB_STAT_PDONE;
+    mark_job_update(c, job, state);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_ckpt(cluster_t*      c,
+                endpoint_t*     ep,
+                xce_ckptinfo_t* ckptinfo)
+{
+    uint64_t jobid;
+    char*    info = NULL;
+    job_t*   j;
+
+    if (ckptinfo == NULL) {
+        ERROR("%s ckptinfo should not be NULL",
+              __func__);
+        return -EINVAL;
+    }
+
+    jobid = ckptinfo->jobid;
+    info  = ckptinfo->ckptinfo;
+
+    TRACE("%s job %ld ckptinfo %s",
+          __func__, jobid, info);
+
+    j = jobmgr_lockjob(&(c->jobmgr), jobid,
+                       OBJREF_UPDATE);
+    if (j == NULL) {
+        ERROR("%s can not find job %ld",
+              __func__, jobid);
+        return -errno;
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_CKPT,
+                  c, j);
+    if (j->job.ckptinfo != NULL) {
+        obj_deref(j->job.ckptinfo);
+    }
+
+    j->job.ckptinfo = create_obj_by_typename(&(c->objmgr), "xstring",
+                                             info);
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_CKPT,
+                  c, j);
+
+    obj_deref(j);
+
+    return 0;
+}
+
+int do_jobcontrol_op(cluster_t*     c,
+                     endpoint_t*    ep,
+                     xce_request_t* request)
+{
+    int         rc = 0;
+    xce_reply_t reply;
+
+    init_reply_by_req(&reply, request);
+
+    rc               = job_ctrl(c, request->body.jobctrlmsg);
+    reply.status     = rc;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+    return rc;
+}
+
+int update_job_rusages(cluster_t*     c,
+                       endpoint_t*    ep,
+                       xce_request_t* request)
+{
+    xce_job_rusages_t* rusages = request->body.updatemsg->data.jobrusages;
+    job_t*             j;
+    worker_t*          w;
+    char*              host = NULL;
+
+    TRACE("%s trying to update job(%ld) rusage",
+          __func__, rusages->jobid);
+
+    j = jobmgr_lockjob(&(c->jobmgr), rusages->jobid, OBJREF_UPDATE);
+    if (j == NULL) {
+        PERROR("jobmgr_lockjob");
+        return -errno;
+    }
+
+    w = get_ep_worker(c, ep, OBJREF_READ);
+    if (w == NULL) {
+        obj_deref(j);
+        return errno;
+    }
+
+    host = create_obj_by_typename(&(c->objmgr), "xstring",
+                                  w->host.name);
+    obj_deref(w);
+
+    if (host == NULL) {
+        return -ENOMEM;
+    }
+
+    TRACE("%s update job (%ld) rusage for host %s",
+          __func__, rusages->jobid, host);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, j);
+    update_host_runusage(&(j->job.run_usage),
+                         host,
+                         rusages->count,
+                         rusages->usages);
+
+    update_pids(&(j->job.pidinfo),
+                rusages->npids,
+                rusages->pids);
+
+    update_pgids(&(j->job.pidinfo),
+                 rusages->npgids,
+                 rusages->pgids);
+
+    /* FIXME
+     * update other fields of job rusage later
+     */
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, j);
+
+    obj_deref(j);
+    return 0;
+}