update controller code

src/controller/cluster.c

@@ -0,0 +1,686 @@
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "htab.h"
+#include "mylock.h"
+#include "request.h"
+#include "vtaskmgr.h"
+#include "xce/cluster_def.h"
+#include "zarray.h"
+
+#include "xce/types.h"
+
+#include "utils.h"
+#include "worker.h"
+#include "xce/errnox.h"
+#include "xstring.h"
+
+static time_t        last_print_time = 0;
+static unsigned long npkts_old       = 0;
+static unsigned long npkts           = 0;
+
+static xce_peerinfo_t from_controller = {
+    .name = "controller",
+    .key  = "keepSecret"
+};
+
+static ep_private_t*
+ep_get_private()
+{
+    ep_private_t* priv;
+
+    priv = (ep_private_t*)calloc(1, sizeof(*priv));
+    return priv;
+}
+
+static int handle_client_leave(cluster_t* c)
+{
+    return 0;
+}
+
+static void handle_endpoint_error(endpoint_t* ep)
+{
+    cluster_t*    c    = (cluster_t*)ep->owner;
+    ep_private_t* priv = NULL;
+    int           role = -1;
+
+    if (c == NULL) {
+        ERROR("endpoint %p has no owner cluster", ep);
+
+        return;
+    }
+
+    iomgr_unmonitor(&(c->iomgr), ep);
+
+    priv = (ep_private_t*)ep->data;
+
+    TRACE("endpoint %p priv data %p",
+          ep, priv);
+
+    if (priv != NULL) {
+        role = priv->role;
+        switch (role) {
+        case HostRole_worker:
+            if (priv->u.idx >= 0) {
+                handle_worker_leave(c, priv->u.idx);
+            } else {
+                ERROR("incorrect worker idx %d",
+                      priv->u.idx);
+            }
+            break;
+        case HostRole_scheduler:
+            handle_scheduler_leave(c);
+            break;
+        case HostRole_controller:
+            break;
+        case HostRole_client:
+            handle_client_leave(c);
+            break;
+        default:
+            ERROR("unknown role %d for endpoint %p",
+                  priv->role, ep);
+            break;
+        }
+    }
+
+    if (ep->idx >= 0) {
+        zarray_del(&(c->endpoints), ep->idx);
+        return;
+    }
+
+    endpoint_deref(ep);
+
+    return;
+}
+
+static int do_notify(cluster_t*     c,
+                     endpoint_t*    ep,
+                     xce_request_t* req)
+{
+    xce_notify_data_t* data = req->body.notifymsg;
+
+    switch (data->event) {
+    case NotifyData_data_jobPidInfo:
+        on_job_start(c, data->data.pidinfo);
+        break;
+    case NotifyData_data_jobFailInfo:
+        on_job_fail(c, data->data.failinfo);
+        break;
+    case NotifyData_data_jobFinishInfo:
+        on_job_finish(c, data->data.finishinfo);
+        break;
+    case NotifyData_data_stoppedJob:
+        on_job_stop(c, data->data.stopped_job);
+        break;
+    case NotifyData_data_resumedJob:
+        on_job_resume(c, data->data.resumed_job);
+        break;
+    case NotifyData_data_ckptinfo:
+        on_job_ckpt(c, ep, data->data.ckptinfo);
+        break;
+    case NotifyData_data_postDoneJob:
+        on_job_postdone(c, ep, data->data.pdone_job);
+        break;
+    case NotifyData_data_taskPidInfo:
+        on_task_start(c, data->data.taskinfo);
+        break;
+    case NotifyData_data_taskFailInfo:
+        on_task_fail(c, data->data.taskfailinfo);
+        break;
+    case NotifyData_data_taskFinish:
+        on_task_finish(c, data->data.taskfinish);
+        break;
+    default:
+        break;
+    }
+    return 0;
+}
+static int myreq_handler(vtask_t* task)
+{
+    cluster_t*    c    = (cluster_t*)task->ctx.ep->owner;
+    int           stop = 0;
+    int           rc   = 0;
+    xce_request_t request;
+    xce_reply_t   reply;
+
+    rc = ioctx_decode(&(task->ctx), &request);
+
+    TRACE("%s cpu#%d opcode %d",
+          __func__, task->vcpu,
+          request.opcode);
+
+    init_reply_by_req(&reply, &request);
+
+    switch (request.opcode) {
+    case OPCode_shutdown:
+        reply.status     = 0;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(task->ctx.ep, c->myhost, &reply);
+        stop = 1;
+        cbuf_put(&(c->wbuf), &stop);
+        break;
+    case OPCode_hello:
+    case OPCode_nop:
+        reply.status     = 0;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(task->ctx.ep, c->myhost, &reply);
+        break;
+    case OPCode_join:
+        rc = handle_join(c, task->ctx.ep, &request);
+        break;
+    case OPCode_jobdispatch:
+        rc = handle_jobdispatch(c, task->ctx.ep, &request);
+        break;
+    case OPCode_jobsub:
+        rc = handle_jobsubmit(c, task->ctx.ep, &request);
+        break;
+    case OPCode_get:
+        rc = do_get_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_notify:
+        rc = do_notify(c, task->ctx.ep, &request);
+        break;
+    case OPCode_clusterconf:
+        rc = handle_clusterconf(c, task->ctx.ep, &request);
+        break;
+    case OPCode_update:
+        rc = do_update_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_jobcontrol:
+        rc = do_jobcontrol_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_queuecontrol:
+        rc = do_queuecontrol_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_runtasks:
+        rc = do_runtasks(c, task->ctx.ep, &request);
+        break;
+    default:
+        rc = -E_XCE_BAD_OPCODE;
+        ERROR("Unknown opcode: %d",
+              request.opcode);
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(task->ctx.ep, c->myhost, &reply);
+
+        break;
+    }
+
+    free_Request(&request);
+
+    return rc;
+}
+
+static int vtask_entry(vtask_t* task)
+{
+    int rc;
+
+    rc = myreq_handler(task);
+
+    return rc;
+}
+
+static int handle_input(endpoint_t* ep)
+{
+    cluster_t* c = (cluster_t*)ep->owner;
+    time_t     now;
+    vtask_t*   task;
+    int        rc = 0;
+
+    task = vtask_alloc();
+    if (task == NULL) {
+        PERROR("vtask_alloc");
+
+        return -ENOMEM;
+    }
+
+    rc = vtask_init(task, (vtask_func_t)&vtask_entry, ep);
+    if (rc < 0) {
+        SET_ERROR(-rc);
+
+        INFO("vtask_init failed, maybe peer side closed the connection");
+
+        if (rc == -EPIPE) {
+            /* connection broken */
+            TRACE("broken pipe, ep %p", ep);
+
+            handle_endpoint_error(ep);
+        }
+
+        FREEUP(task);
+        return rc;
+    }
+
+    rc = vtask_enq(&(c->vtaskmgr), task);
+    if (rc < 0) {
+        PERROR("vtask_enq");
+
+        return rc;
+    }
+
+    now = time(NULL);
+    npkts++;
+
+    if (now - last_print_time > 10) {
+        TRACE("%ld npkts:%ld iops: %f\n", now, npkts,
+              (npkts - npkts_old) * 1.0 / (now - last_print_time));
+        npkts_old       = npkts;
+        last_print_time = now;
+    }
+
+    return rc;
+}
+
+static int handle_output(endpoint_t* ep)
+{
+    data_item_t* it;
+    ioctx_t*     ctx;
+    int          rc;
+
+    it = endpoint_deq(ep);
+
+    if (it == NULL) {
+        return 0;
+    }
+
+    ctx = ITPTR_TO_IOCTX(it);
+
+    rc = ioctx_flush(ctx, ep->fd);
+    ioctx_deref(ctx, 1);
+
+    return rc;
+}
+
+static int handle_io(struct epoll_event* p)
+{
+    endpoint_t* ep = (endpoint_t*)p->data.ptr;
+    int         rc = 0;
+
+    if ((p->events & EPOLLIN) != 0) {
+        rc = handle_input(ep);
+    }
+
+    if (rc >= 0) {
+        if ((p->events & EPOLLOUT) != 0) {
+            rc = handle_output(ep);
+        }
+    }
+
+    return rc;
+}
+
+static int handle_connection(struct epoll_event* p)
+{
+    endpoint_t*   ep = (endpoint_t*)p->data.ptr;
+    endpoint_t*   ep2;
+    cluster_t*    c     = (cluster_t*)ep->owner;
+    iomgr_t*      iomgr = &(c->iomgr);
+    int           idx;
+    ep_private_t* priv = NULL;
+
+    ep2 = endpoint_get();
+    if (ep2 == NULL) {
+        PERROR("endpoint_get");
+
+        return -ENOMEM;
+    }
+
+    endpoint_accept(ep, ep2);
+    if (ep2->fd < 0) {
+        PERROR("endpoint_accept");
+        return -1;
+    }
+
+    TRACE("server sock %d accepted %d\n", ep->fd, ep2->fd);
+
+    priv = ep_get_private();
+    if (priv == NULL) {
+        PERROR("ep_get_private");
+
+        endpoint_deref(ep2);
+        return -ENOMEM;
+    }
+
+    /* Assume client as default */
+    priv->role  = HostRole_client;
+    priv->u.idx = -1;
+
+    idx = zarray_store(&(c->endpoints), ep2);
+    if (idx < 0) {
+        SET_ERROR(-idx);
+
+        PERROR("zarray_store");
+
+        FREEUP(priv);
+        endpoint_deref(ep2);
+
+        return idx;
+    }
+
+    ep2->idx  = idx;
+    ep2->data = priv;
+
+    endpoint_set_handler(ep2, handle_io, c);
+
+    iomgr_monitor(iomgr, ep2);
+    return 0;
+}
+
+static int controller_init(controller_t* c, cluster_t* cluster)
+{
+    c->ep = NULL;
+
+    return 0;
+}
+
+static void endpoint_killer(void* p)
+{
+    endpoint_t* ep = (endpoint_t*)p;
+    cluster_t*  c  = (cluster_t*)ep->owner;
+
+    if (ep->idx >= 0) {
+        zarray_clear(&(c->endpoints), ep->idx);
+        ep->idx = -1;
+    }
+    endpoint_deref(ep);
+
+    return;
+}
+
+int cluster_build_config(cluster_t* c)
+{
+    char valbuf[128];
+    int  i;
+
+    snprintf(valbuf, sizeof(valbuf) - 1,
+             "%d", c->myconf->cluster_config->heartbeat_interval);
+    add_config(c, "heartbeat_interval", valbuf);
+    snprintf(valbuf, sizeof(valbuf) - 1,
+             "%d", c->myconf->cluster_config->inactive_heartbeat_count);
+    add_config(c, "inactive_heartbeat_count", valbuf);
+    snprintf(valbuf, sizeof(valbuf) - 1,
+             "%d", c->myconf->cluster_config->ncandidates);
+    add_config(c, "ncandidates", valbuf);
+
+    for (i = 0; i < c->myconf->cluster_config->ncandidates; i++) {
+        char keybuf[128];
+
+        sprintf(keybuf, "candidate_%d_name", i);
+        sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->name);
+
+        add_config(c, keybuf, valbuf);
+
+        sprintf(keybuf, "candidate_%d_host", i);
+        sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->host);
+
+        add_config(c, keybuf, valbuf);
+        sprintf(keybuf, "candidate_%d_port", i);
+        sprintf(valbuf, "%d", c->myconf->cluster_config->candidates[i]->port);
+
+        add_config(c, keybuf, valbuf);
+    }
+
+    return 0;
+}
+
+int cluster_build_resources(cluster_t* c)
+{
+    int i;
+
+    if (c->myconf == NULL) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < c->myconf->cluster_config->nres; i++) {
+        xce_resource_t* res;
+
+        res = create_obj_by_typename(&(c->objmgr), "xresource",
+                                     c->myconf->cluster_config->resources[i]);
+        if (res == NULL) {
+            return -errno;
+        }
+    }
+
+    return 0;
+}
+
+int cluster_load_jobs(cluster_t* c)
+{
+    load_jobs(c);
+
+    return 0;
+}
+
+int cluster_build_queues(cluster_t* c)
+{
+    build_queues(c);
+    return 0;
+}
+
+int cluster_build_groups(cluster_t* c)
+{
+    return 0;
+}
+
+static int xce_config_init(int     type,
+                           int     situation,
+                           void*   data,
+                           va_list ap)
+{
+    int        rc = 0;
+    cluster_t* c  = va_arg(ap, cluster_t*);
+
+    if (situation == HOOK_BEFORE_EVENT) {
+        c->myconf->cluster_config = get_default_conf(c);
+        if (c->myconf->cluster_config == NULL) {
+            return -EINVAL;
+        }
+        return 0;
+    }
+
+    xce_use(c);
+
+    rc = cluster_build_config(c);
+    if (rc < 0) {
+        return rc;
+    }
+    rc = cluster_build_resources(c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cluster_build_queues(c);
+    if (rc < 0) {
+        return rc;
+    }
+    rc = cluster_build_groups(c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cluster_load_jobs(c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    tlog_info("xce controller is running");
+
+    return rc;
+}
+
+static int cluster_hook_fun(int     type,
+                            int     situation,
+                            int     detail,
+                            void*   data,
+                            va_list ap)
+{
+    if (detail == HOOK_DETAIL_CLUSTER_INIT) {
+        return xce_config_init(type, situation, data, ap);
+    }
+
+    return 0;
+}
+
+static int xce_install_hooks(cluster_t* c)
+{
+    xce_add_hook(&(c->hooksys),
+                 XCE_CONTROLLER_HOOK_CLUSTER,
+                 (hook_fn_t)cluster_hook_fun,
+                 NULL);
+    return 0;
+}
+
+int cluster_init(cluster_t* c)
+{
+    int i, rc;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    c->myhost = &from_controller;
+    /* FIXME: */
+    c->ncontrollers = 1;
+    c->myidx        = 0;
+
+    xce_hook_init(&(c->hooksys), XCE_CONTROLLER_HOOK_MAX);
+
+    xce_install_hooks(c);
+
+    rc = xce_run_hooks(&(c->hooksys),
+                       XCE_CONTROLLER_HOOK_CLUSTER,
+                       HOOK_BEFORE_EVENT,
+                       HOOK_DETAIL_CLUSTER_INIT, c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    mylock_init(&(c->sched.lock));
+    c->sched.ep = NULL;
+
+    kvmgr_init(&(c->configmgr),
+               string_hash_key,
+               128, 128,
+               (zarray_item_killer_t)config_obj_killer);
+
+    objmgr_init(&(c->objmgr));
+    jobmgr_init(&(c->jobmgr),
+                c->myconf->cluster_config->jobmgr->nhash_slots,
+                c->myconf->cluster_config->jobmgr->delta);
+    resmgr_init(&(c->resmgr),
+                c->myconf->cluster_config->resmgr->nhash_slots,
+                c->myconf->cluster_config->resmgr->delta,
+                c->myconf->cluster_config->resmgr->inst_delta);
+    workermgr_init(&(c->workermgr),
+                   c->myconf->cluster_config->workermgr->nhash_slots,
+                   c->myconf->cluster_config->workermgr->delta);
+    qmgr_init(&(c->qmgr),
+              c->myconf->cluster_config->qmgr->nhash_slots,
+              c->myconf->cluster_config->qmgr->delta);
+    iomgr_init(&(c->iomgr));
+    register_xstring(&(c->objmgr),
+                     c->myconf->cluster_config->xstrings->nhash_slots,
+                     c->myconf->cluster_config->xstrings->delta);
+    register_types(c);
+
+    zarray_init(&(c->endpoints), 16, endpoint_killer);
+
+    rc = endpoint_init(&(c->ep),
+                       AF_INET,
+                       SOCK_STREAM,
+                       c->myconf->cluster_config->port,
+                       5);
+    if (rc < 0) {
+        return rc;
+    }
+
+    for (i = 0; i < c->ncontrollers; i++) {
+        controller_init(&(c->controllers[i]), c);
+    }
+
+    cbuf_init(&(c->wbuf), 1, sizeof(int));
+
+    zarray_init(&(c->configs), 64, NULL /* FIXME */);
+
+    vtaskmgr_init(&(c->vtaskmgr), c->myconf->cluster_config->vcpus);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_CLUSTER,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_CLUSTER_INIT, c);
+
+    return 0;
+}
+
+int cluster_start(cluster_t* c)
+{
+    int rc;
+
+    endpoint_set_handler(&(c->ep), handle_connection, c);
+    rc = iomgr_monitor(&(c->iomgr), &(c->ep));
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    iomgr_start(&(c->iomgr));
+    vtaskmgr_start(&(c->vtaskmgr));
+
+    c->running = 1;
+    return 0;
+}
+
+int cluster_run(cluster_t* c)
+{
+    for (;;) {
+        int stop;
+
+        cbuf_get(&(c->wbuf), &stop);
+        if (stop) {
+            break;
+        }
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_CLUSTER,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
+    c->running = 0;
+    iomgr_unmonitor(&(c->iomgr), &(c->ep));
+    iomgr_stop(&(c->iomgr));
+
+    return 0;
+}
+
+void cluster_fini(cluster_t* c)
+{
+    vtaskmgr_fini(&(c->vtaskmgr));
+    endpoint_deref(&(c->ep));
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_CLUSTER,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
+    cbuf_fini(&(c->wbuf));
+    iomgr_fini(&(c->iomgr));
+
+    jobmgr_fini(&(c->jobmgr));
+    workermgr_fini(&(c->workermgr));
+    qmgr_fini(&(c->qmgr));
+    resmgr_fini(&(c->resmgr));
+    kvmgr_fini(&(c->configmgr));
+    mylock_fini(&(c->sched.lock));
+
+    zarray_fini(&(c->endpoints));
+    zarray_fini(&(c->configs));
+
+    objmgr_fini(&(c->objmgr));
+
+    xce_hook_fini(&(c->hooksys));
+
+    return;
+}

src/controller/dataex.c

@@ -0,0 +1,537 @@
+#include <errno.h>
+
+#include "cluster.h"
+#include "request.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+
+typedef int (*check_fn_t)(void*);
+
+int check_controller(void* arg)
+{
+    cluster_t* c = (cluster_t*)arg;
+
+    if (!c->running) {
+        return -EINVAL;
+    }
+
+    return 1;
+}
+
+struct intern_array_ {
+    int        count;
+    int        idx;
+    void*      data;
+    check_fn_t checkfn;
+};
+
+int not_null_check(void* arg)
+{
+    if (arg == NULL) {
+        return 0;
+    }
+
+    return 1;
+}
+
+static int collect_resval_with_idx_(zarray_t*             array,
+                                    int*                  nres,
+                                    xce_val_with_idx_t*** resvals)
+{
+    int                  nc;
+    int                  i, j;
+    xce_val_with_idx_t** vals;
+
+    nc = 0;
+
+    for (i = 0; i < array->caps; i++) {
+        if (array->data[i] == NULL) {
+            continue;
+        }
+
+        nc++;
+    }
+
+    if (nc == 0) {
+        *nres    = 0;
+        *resvals = NULL;
+        return 0;
+    }
+
+    vals = (xce_val_with_idx_t**)calloc(nc, sizeof(xce_val_with_idx_t*));
+    if (vals == NULL) {
+        return -ENOMEM;
+    }
+
+    j = 0;
+
+    for (i = 0; i < array->caps; i++) {
+        xce_val_with_idx_t* val;
+        xce_val_t*          src = array->data[i];
+
+        if (array->data[i] == NULL) {
+            continue;
+        }
+
+        val = (xce_val_with_idx_t*)calloc(1, sizeof(*val));
+        if (val == NULL) {
+            free_resval_with_idx_list(&vals, j);
+            return -ENOMEM;
+        }
+
+        vals[j] = val;
+        j++;
+
+        val->idx = i;
+        val->val = (xce_val_t*)calloc(1, sizeof(xce_val_t));
+        if (val->val == NULL) {
+            free_resval_with_idx_list(&vals, j);
+            return -ENOMEM;
+        }
+
+        if (src->which == Value_u_svalue) {
+            val->val->which = Value_u_svalue;
+            if (src->u.svalue != NULL) {
+                val->val->u.svalue = strdup(src->u.svalue);
+            } else {
+                val->val->u.svalue = strdup("-");
+            }
+        } else {
+            memcpy(val->val, array->data[i],
+                   sizeof(xce_val_t));
+        }
+    }
+
+    *nres    = j;
+    *resvals = vals;
+    return j;
+}
+
+int collect_resval_with_idx(zarray_t*             array,
+                            int*                  nres,
+                            xce_val_with_idx_t*** resvals)
+{
+    int rc = 0;
+
+    mylock_lock(&(array->lock));
+
+    rc = collect_resval_with_idx_(array, nres, resvals);
+
+    mylock_unlock(&(array->lock));
+
+    return rc;
+}
+static int count_(void* arg1,
+                  void* arg2)
+{
+    struct intern_array_* arg = (struct intern_array_*)arg1;
+
+    if ((arg == NULL) || (arg->checkfn == NULL)) {
+        return -EINVAL;
+    }
+
+    if (!(arg->checkfn(arg2))) {
+        return 0;
+    }
+
+    arg->count++;
+
+    return 0;
+}
+
+static int add_confvar_(void* arg1,
+                        void* arg2)
+{
+    struct intern_array_* arg = (struct intern_array_*)arg1;
+    xce_config_t**        p   = NULL;
+
+    if ((arg == NULL) || (arg->count < 0)) {
+        return -EINVAL;
+    }
+
+    if (arg->count == 0) {
+        return 0;
+    }
+
+    if (!(arg->checkfn(arg2))) {
+        return 0;
+    }
+
+    p = (xce_config_t**)arg->data;
+    if (p == NULL) {
+        p = (xce_config_t**)calloc(arg->count,
+                                   sizeof(xce_config_t*));
+        if (p == NULL) {
+            return -ENOMEM;
+        }
+
+        arg->data = p;
+    }
+
+    p[arg->idx] = (xce_config_t*)obj_ref(arg2, OBJREF_READ);
+    arg->idx++;
+
+    return 1;
+}
+
+void collect_confvars(cluster_t*         c,
+                      collect_check_fn_t fn,
+                      int*               nconfig,
+                      xce_config_t***    configs)
+{
+    xlist_t              lst;
+    struct intern_array_ args = {
+        .count   = 0,
+        .idx     = 0,
+        .data    = NULL,
+        .checkfn = (fn == NULL) ? not_null_check : fn
+    };
+    int rc;
+
+    *nconfig = 0;
+    *configs = NULL;
+
+    xlist_init(&lst, 3);
+    xlist_store(&lst, count_);
+    xlist_store(&lst, add_confvar_);
+
+    rc = kvmgr_iter(&(c->configmgr), &lst, &args);
+
+    xlist_fini(&lst);
+
+    if (rc >= 0) {
+        *nconfig = args.count;
+        *configs = args.data;
+    }
+}
+
+static int add_queue_(void* arg1,
+                      void* arg2)
+{
+    struct intern_array_* arg = (struct intern_array_*)arg1;
+    xce_queue_t**         p   = NULL;
+
+    if (arg == NULL) {
+        return -EINVAL;
+    }
+
+    if (arg->count < 0) {
+        return -EINVAL;
+    }
+
+    if (arg->count == 0) {
+        return 0;
+    }
+
+    if (!(arg->checkfn(arg2))) {
+        return 0;
+    }
+
+    p = (xce_queue_t**)arg->data;
+    if (p == NULL) {
+        p = (xce_queue_t**)calloc(arg->count,
+                                  sizeof(xce_queue_t*));
+        if (p == NULL) {
+            return -ENOMEM;
+        }
+
+        arg->data = p;
+    }
+
+    p[arg->idx] = (xce_queue_t*)obj_ref(arg2, OBJREF_READ);
+    arg->idx++;
+
+    return 1;
+}
+
+void collect_queues(cluster_t*         c,
+                    collect_check_fn_t fn,
+                    int*               nqueues,
+                    xce_queue_t***     queues)
+{
+    xlist_t              lst;
+    struct intern_array_ args = {
+        .count   = 0,
+        .idx     = 0,
+        .data    = NULL,
+        .checkfn = (fn == NULL) ? not_null_check : fn
+    };
+    int rc;
+
+    *nqueues = 0;
+    *queues  = NULL;
+
+    xlist_init(&lst, 3);
+    xlist_store(&lst, count_);
+    xlist_store(&lst, add_queue_);
+
+    rc = qmgr_iter(&(c->qmgr), &lst, &args);
+
+    xlist_fini(&lst);
+
+    if (rc >= 0) {
+        *nqueues = args.count;
+        *queues  = args.data;
+    }
+}
+
+static int add_resource_(void* arg1,
+                         void* arg2)
+{
+    struct intern_array_* arg = (struct intern_array_*)arg1;
+    xce_resource_t**      p   = NULL;
+
+    if (arg == NULL) {
+        return -EINVAL;
+    }
+
+    if (arg->count < 0) {
+        return -EINVAL;
+    }
+
+    if (arg->count == 0) {
+        return 0;
+    }
+
+    if (!(arg->checkfn(arg2))) {
+        return 0;
+    }
+
+    p = (xce_resource_t**)arg->data;
+    if (p == NULL) {
+        p = (xce_resource_t**)calloc(arg->count,
+                                     sizeof(xce_resource_t*));
+        if (p == NULL) {
+            return -ENOMEM;
+        }
+
+        arg->data = p;
+    }
+
+    p[arg->idx] = (xce_resource_t*)obj_ref(arg2, OBJREF_READ);
+
+    arg->idx++;
+
+    return 1;
+}
+
+void collect_resources(cluster_t*         c,
+                       collect_check_fn_t fn,
+                       int*               nresources,
+                       xce_resource_t***  resources)
+{
+    xlist_t              lst;
+    struct intern_array_ args = {
+        .count   = 0,
+        .idx     = 0,
+        .data    = NULL,
+        .checkfn = (fn == NULL) ? not_null_check : fn
+    };
+    int rc;
+
+    *nresources = 0;
+    *resources  = NULL;
+
+    xlist_init(&lst, 3);
+    xlist_store(&lst, count_);
+    xlist_store(&lst, add_resource_);
+
+    rc = resmgr_iter(&(c->resmgr), &lst, &args);
+
+    xlist_fini(&lst);
+
+    if (rc >= 0) {
+        *nresources = args.count;
+        *resources  = args.data;
+    }
+}
+
+static int add_job_(void* arg1,
+                    void* arg2)
+{
+    struct intern_array_* arg = (struct intern_array_*)arg1;
+    xce_job_t**           p   = NULL;
+
+    if (arg == NULL) {
+        return -EINVAL;
+    }
+
+    if (arg->count < 0) {
+        return -EINVAL;
+    }
+
+    if (arg->count == 0) {
+        return 0;
+    }
+
+    if (!(arg->checkfn(arg2))) {
+        return 0;
+    }
+
+    p = (xce_job_t**)arg->data;
+    if (p == NULL) {
+        p = (xce_job_t**)calloc(arg->count, sizeof(xce_job_t*));
+        if (p == NULL) {
+            return -ENOMEM;
+        }
+
+        arg->data = p;
+    }
+
+    p[arg->idx] = (xce_job_t*)obj_ref(arg2, OBJREF_READ);
+    arg->idx++;
+
+    return 1;
+}
+
+void collect_jobs(cluster_t*         c,
+                  collect_check_fn_t fn,
+                  int*               njobs,
+                  xce_job_t***       jobs)
+{
+    xlist_t              lst;
+    struct intern_array_ args = {
+        .count   = 0,
+        .idx     = 0,
+        .data    = NULL,
+        .checkfn = (check_fn_t)fn
+    };
+    int rc = 0;
+
+    *njobs = 0;
+    *jobs  = NULL;
+
+    xlist_init(&lst, 3);
+    xlist_store(&lst, count_);
+    xlist_store(&lst, add_job_);
+
+    rc = jobmgr_iter(&(c->jobmgr), &lst, &args);
+    xlist_fini(&lst);
+
+    if (rc >= 0) {
+        *njobs = args.count;
+        *jobs  = args.data;
+    }
+
+    return;
+}
+
+static int add_worker_(void* arg1,
+                       void* arg2)
+{
+    struct intern_array_* arg = (struct intern_array_*)arg1;
+    xce_worker_t**        p   = NULL;
+    xce_worker_t*         w;
+
+    if (arg == NULL) {
+        return -EINVAL;
+    }
+
+    if (arg->count < 0) {
+        return -EINVAL;
+    }
+
+    if (arg->count == 0) {
+        return 0;
+    }
+
+    if (!(arg->checkfn(arg2))) {
+        return 0;
+    }
+
+    p = (xce_worker_t**)arg->data;
+    if (p == NULL) {
+        p = (xce_worker_t**)calloc(arg->count, sizeof(xce_worker_t*));
+        if (p == NULL) {
+            return -ENOMEM;
+        }
+
+        arg->data = p;
+    }
+
+    w = (xce_worker_t*)obj_ref(arg2, OBJREF_READ);
+
+    collect_resval_with_idx(&(w->resvals),
+                            &(w->nresvals_to_sched),
+                            &(w->resvals_to_sched));
+
+    p[arg->idx] = w;
+    arg->idx++;
+
+    return 1;
+}
+
+void collect_workers(cluster_t*         c,
+                     collect_check_fn_t fn,
+                     int*               nworkers,
+                     xce_worker_t***    workers)
+{
+    xlist_t              lst;
+    struct intern_array_ args = {
+        .count   = 0,
+        .idx     = 0,
+        .data    = NULL,
+        .checkfn = (check_fn_t)fn
+    };
+    int rc = 0;
+
+    *nworkers = 0;
+    *workers  = NULL;
+
+    xlist_init(&lst, 3);
+    xlist_store(&lst, count_);
+    xlist_store(&lst, add_worker_);
+
+    rc = workermgr_iter(&(c->workermgr), &lst, &args);
+
+    xlist_fini(&lst);
+
+    if (rc >= 0) {
+        *nworkers = args.count;
+        *workers  = args.data;
+    }
+
+    return;
+}
+
+void collect_shared_resvals(cluster_t*            c,
+                            collect_check_fn_t    fn,
+                            int*                  nres,
+                            xce_val_with_idx_t*** resvals)
+{
+
+    collect_resval_with_idx(&(c->resmgr.sharedinst),
+                            nres, resvals);
+
+    return;
+}
+
+void deref_objs(int    count,
+                void** objs)
+{
+    int i;
+
+    for (i = 0; i < count; i++) {
+        void* obj = objs[i];
+
+        if (obj == NULL) {
+            continue;
+        }
+
+        DEREF_OBJ(obj);
+    }
+}
+
+int notify(cluster_t*         c,
+           endpoint_t*        ep,
+           xce_notify_data_t* data)
+{
+    ioctx_t* ctx = NULL;
+
+    ctx = ioctx_alloc(ep, c->myhost);
+    if (ctx == NULL) {
+        return -ENOMEM;
+    }
+
+    return xce_notify_req(ctx, data);
+}

src/controller/include/cluster.h

@@ -0,0 +1,172 @@
+#if !defined(_CLUSTER_H_)
+
+#define _CLUSTER_H_ 1
+#include "config.h"
+
+#include <pthread.h>
+#include <stdint.h>
+
+#include "hook.h"
+#include "htab.h"
+#include "kvmgr.h"
+#include "myconf.h"
+#include "mylock.h"
+#include "netio.h"
+#include "objmgr.h"
+#include "vtaskmgr.h"
+#include "worker.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+#include "xlist.h"
+#include "zarray.h"
+
+typedef int (*collect_check_fn_t)(void*);
+
+extern int       check_controller(void*);
+extern int       cluster_init(cluster_t*);
+extern int       cluster_start(cluster_t*);
+extern int       cluster_run(cluster_t*);
+extern void      cluster_fini(cluster_t*);
+extern int       register_types(cluster_t*);
+extern int       jobmgr_init(jobmgr_t*, int, int);
+extern void      jobmgr_fini(jobmgr_t*);
+extern int       workermgr_init(workermgr_t*, int, int);
+extern void      workermgr_fini(workermgr_t*);
+extern int       workermgr_add(workermgr_t*,
+                               worker_t*);
+extern worker_t* workermgr_lock(workermgr_t*,
+                                char*, int);
+extern worker_t* workermgr_lock_byid(workermgr_t*,
+                                     int, int);
+extern void      workermgr_del(workermgr_t*, worker_t*);
+extern int       worker_ctrl(cluster_t*, xce_hostctrl_t*);
+extern void      qmgr_fini(qmgr_t*);
+extern int       qmgr_init(qmgr_t*, int, int);
+extern int       qmgr_add(qmgr_t*, queue_t*);
+extern queue_t*  qmgr_lockq(qmgr_t*, char*, int);
+extern void      qmgr_delq(qmgr_t*, queue_t*);
+extern int       qmgr_iter(qmgr_t*, xlist_t*, void*);
+
+extern xce_resource_t* get_resource(cluster_t*,
+                                    char*);
+extern int             add_to_xlist(xlist_t*, void*);
+extern int             add_worker_to_xlist(xlist_t*, void*);
+extern int             add_queue_to_xlist(xlist_t*, void*);
+extern int             add_job_to_xlist(xlist_t*, void*);
+
+extern void            resource_killer(void*);
+extern int             add_resource(void*, char*, char*,
+                                    int, xce_order_t,
+                                    int, int);
+extern int             resmgr_init(resmgr_t*, int, int, int);
+extern void            resmgr_fini(resmgr_t*);
+extern int32_t         resmgr_add_res(resmgr_t*,
+                                      xce_resource_t*);
+extern xce_resource_t* resmgr_lockres(resmgr_t* mgr,
+                                      char*, int);
+extern void            res_obj_killer(xce_resource_t*);
+extern void            resmgr_del_res(resmgr_t*,
+                                      xce_resource_t*);
+extern int             resmgr_iter(resmgr_t*, xlist_t*, void*);
+
+extern void config_obj_killer(xce_config_t*);
+
+extern int update_resvals(cluster_t*, worker_t*,
+                          int, xce_resval_t*);
+
+extern int load_jobs(void*);
+extern int update_job(void*, uint64_t, xce_kv_list_t*);
+extern int do_job_started(void*, uint64_t, int, int*, int, int*);
+
+extern queue_t* create_queue(cluster_t*, xce_queue_t*);
+extern int      build_queues(void*);
+extern void     queue_fini(queue_t*);
+extern int      queue_ctrl(cluster_t*, xce_queuectrl_t*);
+extern queue_t* qmgr_lockq(qmgr_t*, char*, int);
+
+extern int xce_update_data(endpoint_t*, cluster_t*,
+                           int, int, int*, int, xce_kv_t*);
+extern int on_scheduler_join(cluster_t*);
+
+extern int     xce_push_reslist(endpoint_t*, cluster_t*,
+                                int, xce_resource_t**);
+extern int     xce_push_confvars(endpoint_t*, cluster_t*,
+                                 int, xce_config_t**);
+extern int     xce_push_workerlist(endpoint_t*, cluster_t*,
+                                   int, xce_worker_t**);
+extern int     xce_push_joblist(endpoint_t*, cluster_t*,
+                                int, xce_job_t**);
+extern int     xce_push_queuelist(endpoint_t*, cluster_t*,
+                                  int, xce_queue_t**);
+extern int     xcesched_push_job(cluster_t*, uint64_t);
+extern int     on_worker_join(void*, endpoint_t*);
+extern int     get_resval(cluster_t*, char*,
+                          char*, xce_val_t*);
+extern job_t*  jobmgr_lockjob(jobmgr_t*, uint64_t, int);
+extern int32_t jobmgr_add_job(jobmgr_t*, job_t*);
+extern void    jobmgr_del_job(jobmgr_t*, job_t*);
+extern void    job_obj_killer(job_t*);
+extern int     handle_push(struct capn*, cluster_t*,
+                           endpoint_t*, struct Request*);
+extern int     do_get_op(cluster_t*, endpoint_t*, xce_request_t*);
+extern int     do_update_op(cluster_t*, endpoint_t*, xce_request_t*);
+extern int     handle_join(cluster_t*,
+                           endpoint_t*,
+                           xce_request_t*);
+extern int     handle_reply(endpoint_t*, struct Request*,
+                            cluster_t*);
+extern int     handle_jobdispatch(cluster_t*, endpoint_t*,
+                                  xce_request_t*);
+extern int     handle_jobsubmit(cluster_t*, endpoint_t*, xce_request_t*);
+extern int     handle_worker_leave(cluster_t*, int);
+extern int     handle_scheduler_leave(cluster_t*);
+extern int     handle_clusterconf(cluster_t*, endpoint_t*, xce_request_t*);
+
+extern void collect_confvars(cluster_t*, collect_check_fn_t, int*, xce_config_t***);
+extern void collect_queues(cluster_t*, collect_check_fn_t, int*, xce_queue_t***);
+extern void collect_resources(cluster_t*, collect_check_fn_t, int*, xce_resource_t***);
+extern void collect_jobs(cluster_t*, collect_check_fn_t, int*, xce_job_t***);
+extern void collect_workers(cluster_t*, collect_check_fn_t, int*, xce_worker_t***);
+extern void deref_objs(int, void**);
+extern int  get_queue_exec(cluster_t*, char*, char**, char**);
+extern int  jobmgr_iter(jobmgr_t*, xlist_t*, void*);
+extern int  workermgr_iter(workermgr_t*, xlist_t*, void*);
+
+extern int       on_job_start(cluster_t*, job_pidinfo_t*);
+extern int       on_job_finish(cluster_t*, xce_jobfinish_t*);
+extern int       on_job_fail(cluster_t*, job_failinfo_t*);
+extern int       on_job_stop(cluster_t*, uint64_t);
+extern int       on_job_resume(cluster_t*, uint64_t);
+extern int       on_job_postdone(cluster_t*, endpoint_t*, uint64_t);
+extern int       on_job_ckpt(cluster_t*, endpoint_t*, xce_ckptinfo_t*);
+extern int       notify(cluster_t*, endpoint_t*, xce_notify_data_t*);
+extern int       not_null_check(void*);
+extern int       check_worker_(xce_worker_t*);
+extern int       update_resval_list(cluster_t*, worker_t*, xce_resval_list_t*);
+extern int       do_jobcontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
+extern int       job_ctrl(cluster_t*, xce_jobctrl_t*);
+extern int       update_job_rusages(cluster_t*, endpoint_t*, xce_request_t*);
+extern int       update_host_runusage(xce_job_runusage_t**, char*,
+                                      int, xce_rusage_t**);
+extern int       update_pids(job_pidinfo_t**, int, int*);
+extern int       update_pgids(job_pidinfo_t**, int, int*);
+extern int       get_ep_role(endpoint_t*);
+extern worker_t* get_ep_worker(cluster_t*, endpoint_t*, int);
+
+extern int not_null_check(void*);
+extern int check_worker_(xce_worker_t*);
+extern int update_resval_list(cluster_t*, worker_t*, xce_resval_list_t*);
+extern int do_jobcontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
+extern int job_ctrl(cluster_t*, xce_jobctrl_t*);
+extern int do_queuecontrol_op(cluster_t*, endpoint_t*, xce_request_t*);
+extern int do_runtasks(cluster_t*, endpoint_t*, xce_request_t*);
+extern int on_task_start(cluster_t*, xce_task_pidinfo_t*);
+extern int on_task_fail(cluster_t*, xce_task_failinfo_t*);
+extern int on_task_finish(cluster_t*, xce_task_finish_t*);
+
+extern int xce_controller_res_init(cluster_t*);
+extern int map_resource_to_host(char*, cluster_t*);
+extern int release_resource_from_worker(char*, cluster_t*);
+extern int collect_resval_with_idx(zarray_t*, int*, xce_val_with_idx_t***);
+
+#endif

src/controller/include/worker.h

@@ -0,0 +1,27 @@
+#if !defined(_WORKER_H_)
+#define _WORKER_H_ 1
+
+#include "config.h"
+#include "mylock.h"
+#include "netio.h"
+#include "xce/types.h"
+#include "zarray.h"
+
+typedef struct {
+    uint64_t    jobid;
+    xce_alloc_t alloc;
+    double      used;
+} res_usage_t;
+
+typedef struct {
+    xce_worker_t host;
+    char         hostkey[XCE_MAXKEY_LEN];
+    endpoint_t*  ep;
+    int          idx;
+    zarray_t     usages;
+} worker_t;
+
+extern int add_worker(void*, endpoint_t*, const char*,
+                      hostattrs_t*);
+extern int restore_worker(void*, xce_worker_t);
+#endif

src/controller/job.c

@@ -0,0 +1,922 @@
+#include <errno.h>
+#include <signal.h>
+#include <time.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+#include "xce/types.h"
+
+static int mark_job_start(cluster_t*, job_t*, job_pidinfo_t*);
+static int mark_job_finish(cluster_t*, job_t*, xce_jobfinish_t*);
+static int mark_job_fail(cluster_t*, job_t*, job_failinfo_t*);
+static int mark_job_update(cluster_t*, job_t*, int);
+
+#define NJOBS (100)
+
+uint64_t jobid = 100;
+
+static job_t* create_job(cluster_t* c,
+                         uint64_t   jobid,
+                         char*      queue,
+                         char*      user,
+                         char*      exechome,
+                         char*      execcwd,
+                         char*      execuser,
+                         char*      cmdline,
+                         char*      exdata,
+                         uint32_t   mintasks,
+                         uint32_t   maxtasks)
+{
+    job_t* job = NULL;
+    return job;
+}
+
+uint64_t job_submit(void*    c,
+                    char*    user,
+                    char*    queue,
+                    char*    resreq,
+                    char*    cmdline,
+                    char*    exdata,
+                    uint32_t mintasks,
+                    uint32_t maxtasks)
+{
+    job_t* job = NULL;
+
+    job = create_job(c, jobid++, queue,
+                     user, "/tmp",
+                     "/tmp", user, cmdline,
+                     exdata, mintasks, maxtasks);
+    if (job == NULL) {
+        return -errno;
+    }
+
+    return job->job.jobid;
+}
+
+int load_jobs(void* c)
+{
+    job_t* job;
+    int    i;
+    char   cmdline[1024];
+
+    for (i = 1; i < NJOBS; i++) {
+        sprintf(cmdline, "echo normal; sleep %d", i);
+        job = create_job(c, jobid++, "normal",
+                         "rshen", "/home/rshen",
+                         "/tmp", "rshen", (char*)cmdline,
+                         NULL, 1, 1);
+        if (job == NULL) {
+            return -errno;
+        }
+    }
+
+    for (i = 1; i < NJOBS; i++) {
+        job = create_job(c, jobid++, "xce",
+                         "rshen", "/home/rshen",
+                         "/tmp", "rshen", "echo xce; sleep 30",
+                         NULL, 1, 1);
+        if (job == NULL) {
+            return -errno;
+        }
+    }
+
+    return 0;
+}
+
+static int get_nslots_in_alloc(xce_host_alloc_t* alloc)
+{
+    int i;
+
+    for (i = 0; i < alloc->nallocs; i++) {
+        if (strcmp(alloc->allocs[i]->resname, "slots") == 0) {
+            return (int)alloc->allocs[i]->value;
+        }
+    }
+
+    return 1;
+}
+
+static void free_launchinfo(xce_launch_t* launch)
+{
+    int i;
+
+    for (i = 0; i < launch->nhosts; i++) {
+        FREEUP(launch->hosts[i]);
+    }
+
+    FREEUP(launch->hosts);
+}
+
+static int mk_launchinfo(xce_launch_t*    launch,
+                         xce_job_alloc_t* alloc)
+{
+    int i;
+    int rc = 0;
+
+    memset(launch, 0x0, sizeof(*launch));
+
+    launch->hosts = (xce_host_launch_t**)calloc(alloc->nhosts,
+                                                sizeof(xce_host_launch_t*));
+    if (launch->hosts == NULL) {
+        return -ENOMEM;
+    }
+
+    launch->nhosts = alloc->nhosts;
+    for (i = 0; i < alloc->nhosts; i++) {
+        launch->hosts[i] = (xce_host_launch_t*)calloc(1, sizeof(xce_host_launch_t));
+        if (launch->hosts[i] == NULL) {
+            rc = -ENOMEM;
+            break;
+        }
+
+        launch->hosts[i]->host   = alloc->allocs[i]->host;
+        launch->hosts[i]->nranks = get_nslots_in_alloc(alloc->allocs[i]);
+    }
+
+    if (rc < 0) {
+        free_launchinfo(launch);
+    }
+
+    return rc;
+}
+
+static int dispatch_a_job_(cluster_t*       c,
+                           endpoint_t*      ep,
+                           xce_job_alloc_t* alloc)
+{
+    int                rc = 0;
+    job_t*             j;
+    worker_t*          w;
+    ioctx_t*           ioctx;
+    xce_host_alloc_t** allocs;
+    int                nc;
+    xce_job_exec_t     einfo;
+    xce_launch_t       launchinfo;
+
+    TRACE("%s jobid %ld", __func__, alloc->jobid);
+
+    j = jobmgr_lockjob(&(c->jobmgr), alloc->jobid, OBJREF_UPDATE);
+
+    if (j == NULL) {
+        PERROR("jobmgr_lockjob");
+
+        return -errno;
+    }
+
+    nc = alloc->nhosts;
+    if (nc <= 0) {
+        obj_deref(j);
+        return -EINVAL;
+    }
+
+    /* FIXME: change case of nc > 1 just hard coded mpi as
+     * the real job type later
+     */
+    mk_launchinfo(&launchinfo, alloc);
+
+    allocs = (xce_host_alloc_t**)alloc->allocs;
+    /* using head node of job */
+    w = workermgr_lock(&(c->workermgr),
+                       allocs[0]->host,
+                       OBJREF_READ);
+    if (w == NULL) {
+        PERROR("workermgr_lock");
+        free_launchinfo(&launchinfo);
+        obj_deref(j);
+        return -errno;
+    }
+
+    if ((w->ep == NULL) || (w->host.status != 0)) {
+        ERROR("worker %s is not in correct state, status %x ep %p",
+              w->host.name,
+              w->host.status,
+              w->ep);
+        free_launchinfo(&launchinfo);
+
+        obj_deref(w);
+        obj_deref(j);
+
+        SET_ERROR(E_XCE_BAD_WORKER);
+        return -errno;
+    }
+
+    /* FIXME: we may consider to optimize when a job has enter a queue? */
+    if (1) {
+        char* q_pre_exec;
+        char* q_post_exec;
+
+        rc = get_queue_exec(c, j->job.queue,
+                            &q_pre_exec, &q_post_exec);
+        if (rc < 0) {
+            obj_deref(j);
+            obj_deref(w);
+            return rc;
+        }
+
+        if (j->job.start_info->q_pre_exec != NULL) {
+            obj_deref(j->job.start_info->q_pre_exec);
+        }
+
+        if (j->job.start_info->q_post_exec != NULL) {
+            obj_deref(j->job.start_info->q_post_exec);
+        }
+
+        j->job.start_info->q_pre_exec  = q_pre_exec;
+        j->job.start_info->q_post_exec = q_post_exec;
+    }
+
+    ioctx = ioctx_alloc(w->ep, c->myhost);
+    if (ioctx == NULL) {
+        ERROR("memory allocate failed");
+        free_launchinfo(&launchinfo);
+        obj_deref(j);
+        DEREF_OBJ(w);
+
+        return -ENOMEM;
+    }
+
+    einfo.jobid      = alloc->jobid;
+    einfo.start_info = j->job.start_info;
+    einfo.launchinfo = &launchinfo;
+
+    rc = xce_startjob_req(ioctx, &einfo);
+
+    free_launchinfo(&launchinfo);
+
+    if (j->job.allocs != NULL) {
+        free_JobAlloc_ptr(&(j->job.allocs));
+    }
+    j->job.allocs         = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t));
+    j->job.allocs->jobid  = alloc->jobid;
+    j->job.allocs->nhosts = nc;
+    MOVE_PTR(j->job.allocs->allocs, allocs);
+    alloc->allocs = NULL;
+
+    DEREF_OBJ(w);
+    DEREF_OBJ(j);
+
+    if (rc < 0) {
+        SET_ERROR(-rc);
+        PERROR("xce_startjob_req");
+        return rc;
+    }
+
+    return rc;
+}
+
+int job_ctrl(cluster_t*     c,
+             xce_jobctrl_t* ctrinfo)
+{
+    uint64_t jobid = ctrinfo->jobid;
+    job_t*   j;
+    int      rc = 0;
+
+    TRACE("%s job_ctrl job %ld", __func__, jobid);
+    j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (j == NULL) {
+        PERROR("jobmgr_lockjob");
+        return -ENOENT;
+    }
+
+    if (j->job.allocs != NULL) {
+        xce_host_alloc_t** allocs = j->job.allocs->allocs;
+        worker_t*          w;
+        ioctx_t*           ioctx;
+
+        /* job control command will be sent to first node
+         *  for parallel jobs
+         */
+        w = workermgr_lock(&(c->workermgr),
+                           allocs[0]->host,
+                           OBJREF_READ);
+        if (w != NULL) {
+            /* FIXME: better method to check host status */
+            if ((w->ep != NULL) && (w->host.status == 0)) {
+                ioctx = ioctx_alloc(w->ep, c->myhost);
+                if (ioctx != NULL) {
+                    rc = xce_jobctrl_req(ioctx, ctrinfo);
+                    if (rc < 0) {
+                        ERROR("job ctrl %ld on host %s returns %d",
+                              jobid, w->host.name, rc);
+                    }
+                } else {
+                    ERROR("memory allocate failed");
+                    rc = -ENOMEM;
+                }
+            } else {
+                ERROR("worker %s is not in correct state, status %x ep %p",
+                      w->host.name,
+                      w->host.status,
+                      w->ep);
+                rc = -errno;
+            }
+            obj_deref(w);
+        } else {
+            PERROR("workermgr_lock");
+            rc = -errno;
+        }
+    } else {
+        /* job has not been started yet */
+        job_failinfo_t failinfo = {
+            .jobid = jobid,
+        };
+
+        xce_use(&failinfo);
+
+        switch (ctrinfo->action) {
+        case JobControl_u_siginfo:
+            switch (ctrinfo->u.siginfo->signo) {
+            case SIGCONT:
+                if (j->job.state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP)) {
+                    int state = j->job.state;
+
+                    /* resume suspend job? */
+                    state &= ~(JOB_STAT_PSUSP | JOB_STAT_USUSP);
+                    mark_job_update(c, j, state);
+                }
+                break;
+            case SIGSTOP:
+            case SIGTSTP:
+                if (!(j->job.state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP))) {
+                    int state = j->job.state;
+
+                    state |= JOB_STAT_PSUSP;
+                    mark_job_update(c, j, state);
+                }
+                break;
+            case SIGKILL:
+            case SIGTERM:
+            default:
+                if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
+                    break;
+                }
+                /* FIXME: */
+                failinfo.code   = 128 + ctrinfo->u.siginfo->signo;
+                failinfo.detail = "Job killed by signal";
+
+                mark_job_fail(c, j, &failinfo);
+                break;
+            }
+            break;
+        case JobControl_u_ckptoption:
+            if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) {
+                ERROR("job <%ld> has been finished, no checkpoint can be done.",
+                      jobid);
+            } else {
+                ERROR("job %ld has not been started, no checkpoint can be done",
+                      jobid);
+            }
+            rc = -EINVAL;
+            break;
+        default:
+            rc = -EINVAL;
+            break;
+        }
+    }
+
+    obj_deref(j);
+    return rc;
+}
+
+static void notify_dispatch(cluster_t*  c,
+                            endpoint_t* ep,
+                            uint64_t    jobid,
+                            int         code)
+{
+    /* clang-format off */
+    job_failinfo_t info = {
+        .jobid = jobid,
+        .code  = code
+    };
+    xce_notify_data_t data = {
+        .event = NotifyData_data_jobDispatchFail,
+        .data  = {
+             .dispatchfail = &info
+	}
+    };
+    /* clang-format on */
+
+    notify(c, ep, &data);
+}
+
+int handle_jobdispatch(cluster_t*     c,
+                       endpoint_t*    ep,
+                       xce_request_t* request)
+{
+    int                 rc = 0;
+    int                 i;
+    xce_job_dispatch_t* dispatchmsg = request->body.dispatchmsg;
+    xce_reply_t         reply;
+
+    /* reply job dispatch that we'll handle request */
+    init_reply_by_req(&reply, request);
+    reply.status     = 0;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+
+    for (i = 0; i < dispatchmsg->njobs; i++) {
+        int              rc1;
+        xce_job_alloc_t* alloc = NULL;
+
+        MOVE_PTR(alloc, dispatchmsg->allocs[i]);
+
+        rc1 = dispatch_a_job_(c, ep, alloc);
+        if (rc1 < 0) {
+            /* notify dispatch fail */
+            notify_dispatch(c, ep, alloc->jobid, rc1);
+        } else {
+            /* notify dispatch ok */
+            notify_dispatch(c, ep, alloc->jobid, 0);
+            free_JobAlloc_ptr(&(alloc));
+        }
+    }
+    return rc;
+}
+
+int handle_jobsubmit(cluster_t*     c,
+                     endpoint_t*    ep,
+                     xce_request_t* request)
+{
+    uint64_t          jobid;
+    xce_job_submit_t* submit;
+    xce_reply_t       reply;
+    xlist_t           jlist;
+    job_t*            j = NULL;
+
+    submit = request->body.jobsubmsg;
+
+    init_reply_by_req(&reply, request);
+
+    if (NULLSTR(submit->user) || NULLSTR(submit->queue) || NULLSTR(submit->cmdline)) {
+        jobid = 0;
+        errno = EINVAL;
+    } else {
+        xce_run_hooks(&(c->hooksys),
+                      XCE_CONTROLLER_HOOK_JOB,
+                      HOOK_BEFORE_EVENT,
+                      HOOK_DETAIL_JOB_NEW,
+                      c, submit);
+
+        jobid = job_submit(c, (char*)submit->user,
+                           (char*)submit->queue,
+                           (char*)submit->resreq,
+                           (char*)submit->cmdline,
+                           (char*)submit->exdata,
+                           submit->mintasks,
+                           submit->maxtasks);
+    }
+    if (jobid == 0) {
+        reply.status     = -errno;
+        reply.data_which = RequestReply_data_nodata;
+
+        xce_reply(ep, c->myhost, &reply);
+        return -errno;
+    }
+
+    reply.status     = 0;
+    reply.data_which = RequestReply_data_jobid;
+    reply.data.jobid = jobid;
+
+    xce_reply(ep, c->myhost, &reply);
+
+    /* Run hooks to tell scheduler */
+    j = jobmgr_lockjob(&(c->jobmgr),
+                       jobid, OBJREF_READ);
+    if (j == NULL) {
+        ERROR("%s fail to lock job %lu",
+              __func__, jobid);
+    } else {
+        xlist_init(&jlist, 2);
+        if (xlist_store(&jlist, j) >= 0) {
+            xce_run_hooks(&(c->hooksys),
+                          XCE_CONTROLLER_HOOK_JOB,
+                          HOOK_AFTER_EVENT,
+                          HOOK_DETAIL_JOB_NEW,
+                          c, &jlist);
+        } else {
+            PERROR("xlist_store");
+        }
+
+        obj_deref(j);
+        xlist_fini(&jlist);
+    }
+
+    return 0;
+}
+
+static int mark_job_start(cluster_t*     c,
+                          job_t*         job,
+                          job_pidinfo_t* info)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_START,
+                  c, info, job);
+    /* update job information */
+    job->job.state      = JOB_STAT_RUN;
+    job->job.start_time = time(NULL);
+    if (job->job.pidinfo == NULL) {
+        job->job.pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t));
+    }
+
+    job->job.pidinfo->npids  = copy_int_list((int**)&(job->job.pidinfo->pids),
+                                             (int*)info->pids, info->npids);
+    job->job.pidinfo->npgids = copy_int_list((int**)&(job->job.pidinfo->pgids),
+                                             (int*)info->pgids, info->npgids);
+    if (info->exdata != NULL) {
+        job->job.pidinfo->exdata = create_obj_by_typename(&(c->objmgr), "xstring",
+                                                          info->exdata);
+    } else {
+        job->job.pidinfo->exdata = NULL;
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_START,
+                  c, info, job);
+
+    return 0;
+}
+
+int on_job_start(cluster_t*     c,
+                 job_pidinfo_t* info)
+{
+    job_t* job = NULL;
+
+    TRACE("%s jobid %ld", __func__, info->jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), info->jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", info->jobid);
+        return -errno;
+    }
+
+    mark_job_start(c, job, info);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+static int mark_job_finish(cluster_t*       c,
+                           job_t*           job,
+                           xce_jobfinish_t* finish)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_EXIT,
+                  c, finish, job);
+    /* update job information */
+    if (finish->exit_code == 0) {
+        job->job.state = JOB_STAT_DONE;
+    } else {
+        job->job.state = JOB_STAT_EXIT;
+    }
+    job->job.finish_time = time(NULL);
+
+    /* fill job run time/rusage information */
+
+    if (job->job.exit_info == NULL) {
+        job->job.exit_info = (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
+    }
+
+    if (job->job.exit_info != NULL) {
+        job->job.exit_info->exitcode = finish->exit_code;
+    }
+
+    job->job.run_time = job->job.finish_time - job->job.start_time;
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_EXIT,
+                  c, finish, job);
+
+    return 0;
+}
+
+int on_job_finish(cluster_t*       c,
+                  xce_jobfinish_t* finish)
+{
+    job_t* job = NULL;
+
+    TRACE("%s job finish jobid %ld exit code %d",
+          __func__, finish->jobid, finish->exit_code);
+
+    job = jobmgr_lockjob(&(c->jobmgr), finish->jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", finish->jobid);
+        return -errno;
+    }
+
+    mark_job_finish(c, job, finish);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+static int mark_job_fail(cluster_t*      c,
+                         job_t*          job,
+                         job_failinfo_t* failinfo)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_FAIL,
+                  c, failinfo, job);
+    /* update job information */
+
+    switch (failinfo->stage) {
+    case JOB_EXEC_STAGE_Q_POST_EXEC:
+    case JOB_EXEC_STAGE_POST_EXEC:
+        if ((job->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
+            WARN("job %ld got post exec fail notification but job's exit status has not been set",
+                 job->job.jobid);
+            job->job.state |= JOB_STAT_EXIT;
+        }
+        job->job.state |= JOB_STAT_PERR;
+        break;
+    default:
+        job->job.state = JOB_STAT_EXIT;
+        break;
+    }
+
+    job->job.finish_time = time(NULL);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_FAIL,
+                  c, failinfo, job);
+
+    return 0;
+}
+
+static int mark_job_update(cluster_t* c,
+                           job_t*     job,
+                           int        state)
+{
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, job);
+    /* update job state */
+    job->job.state = state;
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, job);
+
+    return 0;
+}
+
+int on_job_fail(cluster_t*      c,
+                job_failinfo_t* failinfo)
+{
+    job_t* job = NULL;
+
+    TRACE("%s job fail jobid %ld code %d",
+          __func__, failinfo->jobid, failinfo->code);
+
+    job = jobmgr_lockjob(&(c->jobmgr), failinfo->jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", failinfo->jobid);
+        return -errno;
+    }
+
+    mark_job_fail(c, job, failinfo);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_stop(cluster_t* c,
+                uint64_t   jobid)
+{
+    job_t* job = NULL;
+    int    state;
+
+    TRACE("%s job stop jobid %ld",
+          __func__, jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", jobid);
+        return -errno;
+    }
+
+    state = job->job.state;
+
+    state |= JOB_STAT_USUSP;
+    mark_job_update(c, job, state);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_resume(cluster_t* c,
+                  uint64_t   jobid)
+{
+    job_t* job = NULL;
+    int    state;
+
+    TRACE("%s job resume jobid %ld",
+          __func__, jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", jobid);
+        return -errno;
+    }
+
+    state = job->job.state;
+    state &= ~JOB_STAT_USUSP;
+    mark_job_update(c, job, state);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_postdone(cluster_t*  c,
+                    endpoint_t* ep,
+                    uint64_t    jobid)
+{
+    job_t* job = NULL;
+    int    state;
+
+    TRACE("%s job post done jobid %ld",
+          __func__, jobid);
+
+    job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE);
+    if (job == NULL) {
+        ERROR("can not find job %ld", jobid);
+        return -errno;
+    }
+
+    state = job->job.state;
+    if ((state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) {
+        WARN("job %ld got post exec fail notification but job's exit status has not been set",
+             job->job.jobid);
+        state |= JOB_STAT_EXIT;
+        /* ensure that job finish time has been set */
+        job->job.finish_time = time(NULL);
+    }
+
+    state |= JOB_STAT_PDONE;
+    mark_job_update(c, job, state);
+
+    /* unlock the job */
+    DEREF_OBJ(job);
+
+    return 0;
+}
+
+int on_job_ckpt(cluster_t*      c,
+                endpoint_t*     ep,
+                xce_ckptinfo_t* ckptinfo)
+{
+    uint64_t jobid;
+    char*    info = NULL;
+    job_t*   j;
+
+    if (ckptinfo == NULL) {
+        ERROR("%s ckptinfo should not be NULL",
+              __func__);
+        return -EINVAL;
+    }
+
+    jobid = ckptinfo->jobid;
+    info  = ckptinfo->ckptinfo;
+
+    TRACE("%s job %ld ckptinfo %s",
+          __func__, jobid, info);
+
+    j = jobmgr_lockjob(&(c->jobmgr), jobid,
+                       OBJREF_UPDATE);
+    if (j == NULL) {
+        ERROR("%s can not find job %ld",
+              __func__, jobid);
+        return -errno;
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_CKPT,
+                  c, j);
+    if (j->job.ckptinfo != NULL) {
+        obj_deref(j->job.ckptinfo);
+    }
+
+    j->job.ckptinfo = create_obj_by_typename(&(c->objmgr), "xstring",
+                                             info);
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_CKPT,
+                  c, j);
+
+    obj_deref(j);
+
+    return 0;
+}
+
+int do_jobcontrol_op(cluster_t*     c,
+                     endpoint_t*    ep,
+                     xce_request_t* request)
+{
+    int         rc = 0;
+    xce_reply_t reply;
+
+    init_reply_by_req(&reply, request);
+
+    rc               = job_ctrl(c, request->body.jobctrlmsg);
+    reply.status     = rc;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+    return rc;
+}
+
+int update_job_rusages(cluster_t*     c,
+                       endpoint_t*    ep,
+                       xce_request_t* request)
+{
+    xce_job_rusages_t* rusages = request->body.updatemsg->data.jobrusages;
+    job_t*             j;
+    worker_t*          w;
+    char*              host = NULL;
+
+    TRACE("%s trying to update job(%ld) rusage",
+          __func__, rusages->jobid);
+
+    j = jobmgr_lockjob(&(c->jobmgr), rusages->jobid, OBJREF_UPDATE);
+    if (j == NULL) {
+        PERROR("jobmgr_lockjob");
+        return -errno;
+    }
+
+    w = get_ep_worker(c, ep, OBJREF_READ);
+    if (w == NULL) {
+        obj_deref(j);
+        return errno;
+    }
+
+    host = create_obj_by_typename(&(c->objmgr), "xstring",
+                                  w->host.name);
+    obj_deref(w);
+
+    if (host == NULL) {
+        return -ENOMEM;
+    }
+
+    TRACE("%s update job (%ld) rusage for host %s",
+          __func__, rusages->jobid, host);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, j);
+    update_host_runusage(&(j->job.run_usage),
+                         host,
+                         rusages->count,
+                         rusages->usages);
+
+    update_pids(&(j->job.pidinfo),
+                rusages->npids,
+                rusages->pids);
+
+    update_pgids(&(j->job.pidinfo),
+                 rusages->npgids,
+                 rusages->pgids);
+
+    /* FIXME
+     * update other fields of job rusage later
+     */
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_JOB,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_JOB_UPDATE,
+                  c, j);
+
+    obj_deref(j);
+    return 0;
+}

src/controller/jobmgr.c

@@ -0,0 +1,172 @@
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cluster.h"
+#include "htab.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+#include "xstring.h"
+
+int jobmgr_init(jobmgr_t* mgr,
+                int       nhslots,
+                int       delta)
+{
+    ht_init(&(mgr->htab), nhslots,
+            int_hash_key);
+    zarray_init(&(mgr->jobs), delta,
+                (zarray_item_killer_t)job_obj_killer);
+    mylock_init(&(mgr->lock));
+    return 0;
+}
+
+void jobmgr_fini(jobmgr_t* mgr)
+{
+    ht_fini(&(mgr->htab));
+    zarray_fini(&(mgr->jobs));
+    mylock_fini(&(mgr->lock));
+}
+
+job_t* jobmgr_lockjob_(jobmgr_t* mgr,
+                       uint64_t  jobid,
+                       int       op)
+{
+    hent_t* e = NULL;
+    int     idx;
+    job_t*  job = NULL;
+
+    e = ht_get(&(mgr->htab), (void*)jobid);
+    if (e == NULL) {
+        SET_ERROR(E_XCE_BAD_JOB);
+        return NULL;
+    }
+
+    idx = e->data;
+
+    if (idx < 0) {
+        SET_ERROR(E_XCE_BAD_JOB);
+        return NULL;
+    }
+
+    job = (job_t*)zarray_get(&(mgr->jobs), idx);
+
+    return obj_ref(job, op);
+}
+
+job_t* jobmgr_lockjob(jobmgr_t* mgr,
+                      uint64_t  jobid,
+                      int       op)
+{
+    job_t* job = NULL;
+
+    if (mgr == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    /* clang-format off */
+    if ((op != OBJREF_NOP) &&
+	(op != OBJREF_READ) &&
+	(op != OBJREF_UPDATE)) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(mgr->lock));
+    job = jobmgr_lockjob_(mgr, jobid, op);
+    mylock_unlock(&(mgr->lock));
+
+    return job;
+}
+
+static int32_t jobmgr_add_job_(jobmgr_t* mgr,
+                               job_t*    job)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab),
+               (void*)job->job.jobid);
+    if (e != NULL) {
+        return -EEXIST;
+    }
+
+    idx = zarray_store(&(mgr->jobs), job);
+    if (idx < 0) {
+        return idx;
+    }
+
+    e = ht_add(&(mgr->htab),
+               (void*)job->job.jobid);
+    if (e == NULL) {
+        zarray_del(&(mgr->jobs), idx);
+        return -errno;
+    }
+
+    e->data = idx;
+    return idx;
+}
+
+int32_t jobmgr_add_job(jobmgr_t* mgr,
+                       job_t*    job)
+{
+    int32_t idx;
+
+    if ((mgr == NULL) || (job == NULL)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(mgr->lock));
+    idx = jobmgr_add_job_(mgr, job);
+    mylock_unlock(&(mgr->lock));
+
+    return idx;
+}
+
+static void jobmgr_del_job_(jobmgr_t* mgr,
+                            job_t*    job)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab),
+               (void*)job->job.jobid);
+    if (e == NULL) {
+        return;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        return;
+    }
+
+    zarray_del(&(mgr->jobs), idx);
+
+    return;
+}
+
+void jobmgr_del_job(jobmgr_t* mgr,
+                    job_t*    job)
+{
+    if ((mgr == NULL) || (job == NULL)) {
+        return;
+    }
+    mylock_lock(&(mgr->lock));
+    jobmgr_del_job_(mgr, job);
+    mylock_unlock(&(mgr->lock));
+}
+
+int jobmgr_iter(jobmgr_t* mgr,
+                xlist_t*  l,
+                void*     arg)
+{
+    int rc = 0;
+
+    mylock_lock(&(mgr->lock));
+    rc = zarray_iter_ex(&(mgr->jobs), l, arg);
+    mylock_unlock(&(mgr->lock));
+
+    return rc;
+}

src/controller/join.c

@@ -0,0 +1,202 @@
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+
+static int validate_host(cluster_t*  c,
+                         const char* host,
+                         const char* key)
+{
+    return 1;
+}
+
+static int handle_client_join(cluster_t*     c,
+                              endpoint_t*    ep,
+                              xce_request_t* request)
+{
+    xce_reply_t reply;
+
+    init_reply_by_req(&reply, request);
+
+    reply.status     = 0;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+
+    return 0;
+}
+
+static int handle_worker_join(cluster_t*     c,
+                              endpoint_t*    ep,
+                              xce_request_t* request)
+{
+    int             idx;
+    ep_private_t*   priv = (ep_private_t*)ep->data;
+    xce_reply_t     reply;
+    xce_peerinfo_t* from    = request->from;
+    int             rc      = 0;
+    xce_joindata_t* joinmsg = NULL;
+
+    TRACE("%s worker %s joining...",
+          __func__, from->name);
+
+    init_reply_by_req(&reply, request);
+
+    if (!validate_host(c, from->name, from->key)) {
+        ERROR("%s reject join of host %s",
+              __func__, from->name);
+        rc = -EPERM;
+
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+
+        xce_reply(ep, c->myhost, &reply);
+        return rc;
+    }
+
+    joinmsg = request->body.joinmsg;
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_WORKER,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_WORKER_JOIN,
+                  c, from->name, 0);
+    idx = add_worker(c, ep, from->name, joinmsg->data.attrs);
+
+    reply.status     = idx;
+    reply.data_which = RequestReply_data_nodata;
+    xce_reply(ep, c->myhost, &reply);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_WORKER,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_WORKER_JOIN,
+                  c, from->name, idx);
+    if (idx < 0) {
+        ERROR("worker %s join failed, rc %d",
+              from->name, idx);
+
+        return idx;
+    }
+
+    priv->u.idx = idx;
+    on_worker_join(c, ep);
+
+    return 0;
+}
+
+static int handle_scheduler_join(cluster_t*     c,
+                                 endpoint_t*    ep,
+                                 xce_request_t* request)
+{
+    xce_reply_t   reply;
+    ep_private_t* priv = (ep_private_t*)ep->data;
+
+    TRACE("%s scheduler join, ep %p",
+          __func__, ep);
+
+    init_reply_by_req(&reply, request);
+
+    c->sched.ep = ep;
+    priv->u.idx = 0;
+
+    reply.status     = 0;
+    reply.data_which = RequestReply_data_nodata;
+    xce_reply(ep, c->myhost, &reply);
+
+    on_scheduler_join(c);
+
+    return 0;
+}
+
+static int handle_controller_join(cluster_t*     c,
+                                  endpoint_t*    ep,
+                                  xce_request_t* request)
+{
+    xce_reply_t reply;
+    int         rc = -ENOSYS;
+
+    ERROR("we still have not support join of controller");
+
+    init_reply_by_req(&reply, request);
+    reply.status     = rc;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+
+    return rc;
+}
+
+static int handle_join_(cluster_t*     c,
+                        endpoint_t*    ep,
+                        xce_request_t* request)
+{
+    ep_private_t*   priv;
+    int             rc = -EINVAL;
+    int             role;
+    xce_reply_t     reply;
+    xce_peerinfo_t* from = request->from;
+
+    init_reply_by_req(&reply, request);
+
+    priv = (ep_private_t*)ep->data;
+    if (priv == NULL) {
+        rc = -E_XCE_INTERNAL;
+
+        ERROR("priv data of endpoint should not be NULL");
+
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(ep, c->myhost, &reply);
+
+        return rc;
+    }
+
+    role       = request->body.joinmsg->role;
+    priv->role = role;
+
+    TRACE("%s role %d join from %s",
+          __func__, role, from->name);
+
+    switch (role) {
+    case HostRole_client:
+        rc = handle_client_join(c, ep, request);
+        break;
+    case HostRole_worker:
+        rc = handle_worker_join(c, ep, request);
+        break;
+    case HostRole_scheduler:
+        rc = handle_scheduler_join(c, ep, request);
+        break;
+    case HostRole_controller:
+        rc = handle_controller_join(c, ep, request);
+        break;
+    default:
+        rc = -E_XCE_BAD_ROLE;
+        ERROR("unknown role %d from host %s",
+              role, from->name);
+
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(ep, c->myhost, &reply);
+        break;
+    }
+
+    return rc;
+}
+
+int handle_join(cluster_t*     c,
+                endpoint_t*    ep,
+                xce_request_t* request)
+{
+    int rc = 0;
+
+    rc = handle_join_(c, ep, request);
+
+    return rc;
+}

src/controller/main.c

@@ -0,0 +1,162 @@
+#include <argp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <tlog.h>
+
+#include "myconf.h"
+#include "utils.h"
+
+extern int myapp_run(myconf_t*);
+
+static char  doc[]                    = "xce_controller";
+static char  args_doc[]               = "";
+const char*  argp_program_version     = MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID));
+const char*  argp_program_bug_address = BUG_EMAIL;
+static char* default_config           = "/tmp/xce/xce.yaml";
+
+struct arguments {
+    char* config;
+    char* logdir;
+    char* clusterconf;
+    int   verbose;
+};
+
+static struct argp_option options[] = {
+    { "config", 'c', "FILE", 0, "Load initial configuration from FILE" },
+    { "logdir", 'd', "DIR", 0, "Directory for log file" },
+    { "verbose", 'v', 0, 0, "Produce verbose logs" },
+    { "clusterconf", 'f', "FILE", 0, "Load Cluster Configuration from FILE" },
+    { 0 }
+};
+
+static error_t parse_opt(int                key,
+                         char*              arg,
+                         struct argp_state* state)
+{
+    struct arguments* arguments = state->input;
+
+    switch (key) {
+    case 'c':
+        arguments->config = arg;
+        break;
+    case 'd':
+        arguments->logdir = arg;
+        break;
+    case 'v':
+        arguments->verbose++;
+        break;
+    case 'f':
+        arguments->clusterconf = arg;
+        break;
+    default:
+        return ARGP_ERR_UNKNOWN;
+    }
+
+    return 0;
+}
+
+static struct argp argp = {
+    options,
+    parse_opt,
+    args_doc,
+    doc
+};
+
+static char* verbose_to_loglevel(int verbose)
+{
+    char* loglevel = "info";
+
+    switch (verbose) {
+    case 1:
+        loglevel = "debug";
+        break;
+    case 2:
+        loglevel = "debug2";
+        break;
+    case 3:
+        loglevel = "debug3";
+        break;
+    default:
+        if (verbose > 3) {
+            loglevel = "debug3";
+        }
+        break;
+    }
+
+    return loglevel;
+}
+
+int main(int argc, char* argv[])
+{
+    struct arguments  arguments;
+    initial_config_t* init_config;
+    myconf_t          myconf;
+    char*             logdir   = "/tmp";
+    char*             loglevel = "info";
+    int               logsize  = 64 * 1024 * 1024;
+    int               logcount = 10;
+    char*             myname   = "xce_controller";
+
+    memset(&arguments, 0x0, sizeof(arguments));
+
+    argp_parse(&argp, argc, argv, 0, 0, &arguments);
+
+    if (arguments.clusterconf == NULL) {
+        tlog_error("no cluster configuration specified. please use -f to specify cluster configuration file");
+        return -1;
+    }
+
+    if (arguments.config == NULL) {
+        arguments.config = default_config;
+    }
+
+    if (arguments.logdir != NULL) {
+        logdir = arguments.logdir;
+    }
+
+    if (arguments.verbose != 0) {
+        loglevel = verbose_to_loglevel(arguments.verbose);
+    }
+
+    mylog_init(logdir, myname, loglevel, logsize, logcount);
+
+    load_initial_config(arguments.config,
+                        &init_config,
+                        arguments.verbose);
+
+    if (init_config == NULL) {
+        tlog_error("Fail to read initial log");
+        return -1;
+    }
+    myconf.init_config    = init_config;
+    myconf.cluster_config = NULL;
+    myconf.conffile       = arguments.clusterconf;
+
+    if (init_config != NULL) {
+        if (init_config->logging != NULL) {
+            if ((init_config->logging->logdir != NULL) && (arguments.logdir == NULL)) {
+                char logfile[XCE_MAXPATH_LEN + 1];
+
+                logdir = init_config->logging->logdir;
+                snprintf(logfile, sizeof(logfile) - 1,
+                         "%s/%s.log", logdir, myname);
+
+                tlog_set_logfile(logfile);
+            }
+
+            if ((init_config->logging->loglevel != NULL) && (arguments.verbose == 0)) {
+                loglevel = init_config->logging->loglevel;
+
+                mylog_setlevel(loglevel);
+            }
+        }
+    }
+
+    myapp_run(&myconf);
+
+    free_initial_config(init_config);
+    mylog_fini();
+
+    return 0;
+}

src/controller/meson.build

@@ -0,0 +1,52 @@
+controller_src = [
+  'cluster.c',
+  'dataex.c',
+  'job.c',
+  'jobmgr.c',
+  'join.c',
+  'main.c',
+  'misc.c',
+  'myapp.c',
+  'opget.c',
+  'opupd.c',
+  'qmgr.c',
+  'queue.c',
+  'reply.c',
+  'resmgr.c',
+  'resources.c',
+  'sched.c',
+  'task.c',
+  'worker.c',
+  'workermgr.c',
+  'wrap.c',
+  'xceobjs.c',
+  'myconf.c'
+]
+
+incdir = include_directories('.', '../include', '../lib',
+                             'include') 
+
+cc = meson.get_compiler('c')
+math_dep = cc.find_library('m', required : false)
+
+controller_deps = [libxce_dep, math_dep]
+
+controller = executable('xce_controller',
+                        controller_src,
+                        include_directories: incdir,
+                        dependencies: controller_deps,
+                        install: true)
+
+xce_test_src = [ 'testsuite/test.c']
+
+readline_dep = dependency('readline')
+ncurses_dep = dependency('ncurses')
+history_dep = dependency('history')
+
+xce_test_deps = [libxce_dep, readline_dep, history_dep, ncurses_dep]
+xce_test = executable('xce_test',
+                      xce_test_src,
+                      include_directories: [incdir],
+                      dependencies: xce_test_deps,
+                      install: true)
+

src/controller/misc.c

@@ -0,0 +1,215 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "cluster.h"
+#include "objmgr.h"
+#include "worker.h"
+#include "xce/types.h"
+#include "xlist.h"
+
+int add_to_xlist(xlist_t* l, void* data)
+{
+    return xlist_store(l, data);
+}
+
+int add_worker_to_xlist(xlist_t* l, void* data)
+{
+    worker_t* w = (worker_t*)data;
+
+    return xlist_store(l, &(w->host));
+}
+
+int add_queue_to_xlist(xlist_t* l, void* data)
+{
+    queue_t* q = (queue_t*)data;
+
+    return xlist_store(l, &(q->queue));
+}
+
+int add_job_to_xlist(xlist_t* l, void* data)
+{
+    job_t* j = (job_t*)data;
+
+    return xlist_store(l, &(j->job));
+}
+
+int get_ep_role(endpoint_t* ep)
+{
+    ep_private_t* priv;
+
+    if (ep == NULL) {
+        return -EINVAL;
+    }
+
+    priv = (ep_private_t*)ep->data;
+    if (priv == NULL) {
+        ERROR("priv data of endpoint %p is NULL",
+              ep);
+        return -EINVAL;
+    }
+
+    return priv->role;
+}
+
+worker_t* get_ep_worker(cluster_t*  c,
+                        endpoint_t* ep,
+                        int         op)
+{
+    int       role, idx;
+    worker_t* w = NULL;
+
+    role = get_ep_role(ep);
+    if (role != HostRole_worker) {
+        ERROR("endpoint %p is not a worker", ep);
+        return NULL;
+    }
+
+    idx = ((ep_private_t*)ep->data)->u.idx;
+    if (idx < 0) {
+        ERROR("invalid idx (%d) for endpoint %p", idx, ep);
+        return NULL;
+    }
+
+    w = workermgr_lock_byid(&(c->workermgr), idx, op);
+    if (w == NULL) {
+        ERROR("can not find worker with id (%d)", idx);
+    }
+
+    return w;
+}
+
+int update_host_runusage(xce_job_runusage_t** runusage,
+                         char*                host,
+                         int                  count,
+                         xce_rusage_t**       usages)
+{
+    xce_job_runusage_t* rusages = NULL;
+    int                 i, j;
+
+    if ((*runusage) == NULL) {
+        *runusage = (xce_job_runusage_t*)calloc(1, sizeof(xce_job_runusage_t));
+        if ((*runusage) == NULL) {
+            return -ENOMEM;
+        }
+    }
+
+    rusages = *runusage;
+    j       = -1;
+
+    for (i = 0; i < rusages->num_hosts; i++) {
+        if (strcmp(host, rusages->host_runusage[i]->host_name) == 0) {
+            j = i;
+            break;
+        }
+    }
+
+    if (j < 0) {
+        int                   cnt = rusages->num_hosts + 1;
+        xce_host_runusage_t** hru = (xce_host_runusage_t**)realloc(rusages->host_runusage,
+                                                                   cnt * sizeof(xce_host_runusage_t*));
+        if (hru == NULL) {
+            return -ENOMEM;
+        }
+
+        rusages->host_runusage = hru;
+        j                      = cnt - 1;
+
+        hru[j] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
+        if (hru[j] == NULL) {
+            return -ENOMEM;
+        }
+
+        rusages->num_hosts++;
+        hru[j]->host_name = host;
+    }
+
+    for (i = 0; i < count; i++) {
+        if (strcmp(usages[i]->resname, "cputime") == 0) {
+            rusages->host_runusage[j]->cpu_time = usages[i]->value;
+            continue;
+        }
+
+        if (strcmp(usages[i]->resname, "mem") == 0) {
+            rusages->host_runusage[j]->mem_usage = usages[i]->value;
+            continue;
+        }
+    }
+
+    return 0;
+}
+
+int update_pids(job_pidinfo_t** pidinfo,
+                int             npids,
+                int*            pids)
+{
+    job_pidinfo_t* info;
+    int            i;
+
+    if ((*pidinfo) == NULL) {
+        *pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t));
+        if ((*pidinfo) == NULL) {
+            return -ENOMEM;
+        }
+    }
+
+    info = *pidinfo;
+
+    if (info->pids == NULL) {
+        info->pids = (unsigned int*)calloc(npids, sizeof(unsigned int));
+    } else if (info->npids < npids) {
+        unsigned int* ptr;
+        ptr = (unsigned int*)realloc(info->pids,
+                                     npids * sizeof(unsigned int));
+        if (ptr == NULL) {
+            return -ENOMEM;
+        }
+
+        info->pids = ptr;
+    }
+
+    for (i = 0; i < npids; i++) {
+        info->pids[i] = pids[i];
+    }
+
+    info->npids = npids;
+
+    return 0;
+}
+
+int update_pgids(job_pidinfo_t** pidinfo,
+                 int             npgids,
+                 int*            pgids)
+{
+    job_pidinfo_t* info;
+    int            i;
+
+    if ((*pidinfo) == NULL) {
+        *pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t));
+        if ((*pidinfo) == NULL) {
+            return -ENOMEM;
+        }
+    }
+
+    info = *pidinfo;
+
+    if (info->pgids == NULL) {
+        info->pgids = (unsigned int*)calloc(npgids, sizeof(unsigned int));
+    } else if (info->npgids < npgids) {
+        unsigned int* ptr;
+        ptr = (unsigned int*)realloc(info->pgids,
+                                     npgids * sizeof(unsigned int));
+        if (ptr == NULL) {
+            return -ENOMEM;
+        }
+
+        info->pgids = ptr;
+    }
+
+    for (i = 0; i < npgids; i++) {
+        info->pgids[i] = pgids[i];
+    }
+
+    info->npgids = npgids;
+
+    return 0;
+}

src/controller/myapp.c

@@ -0,0 +1,52 @@
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "myconf.h"
+#include "netio.h"
+#include "xce/cluster_def.h"
+
+#include "request.h"
+#include "utils.h"
+
+time_t        last_print_time = 0;
+unsigned long npkts           = 0;
+unsigned long npkts_old       = 0;
+
+int myapp_run(myconf_t* myconf)
+{
+    cluster_t c;
+    int       rc;
+
+    c.myconf = myconf;
+
+    rc = cluster_init(&c);
+    if (rc < 0) {
+        SET_ERROR(-rc);
+
+        ERROR("cluster_init: %s", strerror(-rc));
+
+        cluster_fini(&c);
+
+        return rc;
+    }
+
+    rc = cluster_start(&c);
+    if (rc < 0) {
+        SET_ERROR(-rc);
+        perror("cluster_start");
+        cluster_fini(&c);
+
+        return rc;
+    }
+
+    rc = cluster_run(&c);
+
+    cluster_fini(&c);
+
+    return rc;
+}

src/controller/myconf.c

@@ -0,0 +1,173 @@
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <tlog.h>
+
+#include "controller.capnp.h"
+#include "myconf.h"
+#include "objmgr.h"
+#include "utils.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+
+int add_config(void* c,
+               char* name,
+               char* value)
+{
+    cluster_t*   cls = (cluster_t*)c;
+    xce_config_t var = {
+        .name  = name,
+        .value = value
+    };
+    xce_config_t* cv = NULL;
+
+    cv = create_obj_by_typename(&(cls->objmgr), "xconfig",
+                                &var);
+    if (cv == NULL) {
+        return -errno;
+    }
+
+    return 0;
+}
+
+struct cluster_configuration {
+    cluster_config_t* cluster;
+};
+
+static int config_fini(int     type,
+                       int     situation,
+                       void*   data,
+                       va_list ap)
+{
+    struct cluster_configuration* cf = (struct cluster_configuration*)data;
+
+    switch (situation) {
+    case HOOK_AFTER_EVENT:
+        if (cf->cluster != NULL) {
+            free_ClusterConfig(cf->cluster);
+            FREEUP(cf->cluster);
+        }
+        break;
+    default:
+        break;
+    }
+    return 0;
+}
+
+static int cluster_hook_fun(int     type,
+                            int     situation,
+                            int     detail,
+                            void*   data,
+                            va_list ap)
+{
+    if (detail == HOOK_DETAIL_CLUSTER_SHUTDOWN) {
+        return config_fini(type, situation, data, ap);
+    }
+
+    return 0;
+}
+
+#define SET_DEFAULT_IF_UNSET(var, unset, val) \
+    do {                                      \
+        if ((var) == (unset)) {               \
+            (var) = (val);                    \
+        }                                     \
+    } while (0)
+
+static void fill_default_cluster_config(cluster_config_t* c)
+{
+    SET_DEFAULT_IF_UNSET(c->port, 0, 9990);
+    SET_DEFAULT_IF_UNSET(c->heartbeat_interval, 0, 5);
+    SET_DEFAULT_IF_UNSET(c->vcpus, 0, 8);
+    SET_DEFAULT_IF_UNSET(c->fetch_job_interval, 0, 10);
+}
+
+static int load_cluster_config_from_file(char*              file,
+                                         cluster_config_t** config)
+{
+    struct capn       c;
+    ClusterConfig_ptr p;
+    FILE*             fp;
+    int               rc = 0;
+
+    fp = fopen(file, "rb");
+    if (fp == NULL) {
+        rc = -errno;
+        perror("fopen");
+        return rc;
+    }
+
+    capn_init_fp(&c, fp, 0);
+    p.p = capn_getp(capn_root(&c), 0, 1);
+
+    decode_ClusterConfig_ptr(config, p);
+
+    capn_free(&c);
+
+    fclose(fp);
+
+    return 0;
+}
+
+int load_cluster_config(cluster_t*                     c,
+                        struct cluster_configuration** cluster_config)
+{
+    cluster_config_t* cfg;
+    int               rc;
+
+    *cluster_config = (struct cluster_configuration*)calloc(1, sizeof(*cluster_config));
+    if ((*cluster_config) == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = load_cluster_config_from_file(c->myconf->conffile, &((*cluster_config)->cluster));
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    if ((*cluster_config)->cluster == NULL) {
+        return -errno;
+    }
+
+    cfg = (*cluster_config)->cluster;
+
+    fill_default_cluster_config(cfg);
+
+    xce_add_hook(&(c->hooksys),
+                 XCE_CONTROLLER_HOOK_CLUSTER,
+                 (hook_fn_t)cluster_hook_fun,
+                 (*cluster_config));
+
+    return 0;
+}
+
+static struct cluster_configuration* cf = NULL;
+
+cluster_config_t* get_default_conf(void* cl)
+{
+    cluster_config_t* c;
+    cluster_t*        cls = (cluster_t*)cl;
+
+    load_cluster_config(cls, &cf);
+    if (cf == NULL) {
+        PERROR("load_cluster_config");
+        return NULL;
+    }
+
+    c = cf->cluster;
+
+    return c;
+}
+
+int load_initial_config(char*              file,
+                        initial_config_t** conf,
+                        int                verbose)
+{
+    return 0;
+}
+
+void free_initial_config(initial_config_t* conf)
+{
+}

src/controller/opget.c

@@ -0,0 +1,225 @@
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "controller.capnp.h"
+#include "utils.h"
+#include "xce/cluster_def.h"
+
+static int
+do_reply_reslist(cluster_t*     c,
+                 endpoint_t*    ep,
+                 xce_request_t* request)
+{
+    xce_reply_t         reply;
+    xce_resource_data_t resdata;
+    xce_data_t          getdata = {
+                 .table = request->body.getmsg->table,
+                 .data  = {
+                      .resources = &resdata }
+    };
+    int rc = 0;
+
+    init_reply_by_req(&reply, request);
+    collect_resources(c, NULL, &(resdata.n_total),
+                      &(resdata.resources));
+
+    reply.status       = 0;
+    reply.data_which   = RequestReply_data_getdata;
+    reply.data.getdata = &getdata;
+
+    rc = xce_reply_get(ep, c->myhost, &reply, NULL, NULL, -1);
+
+    deref_objs(resdata.n_total, (void**)resdata.resources);
+    FREEUP(resdata.resources);
+
+    return rc;
+}
+
+static int
+do_reply_resvals(cluster_t*     c,
+                 endpoint_t*    ep,
+                 xce_request_t* request)
+{
+    xce_reply_t       reply;
+    xce_resval_data_t resvaldata;
+    xce_get_t*        getmsg  = request->body.getmsg;
+    xce_data_t        getdata = {
+               .data = {
+                   .resvals = &resvaldata }
+    };
+    int           rc = 0;
+    xce_val_t     val;
+    xce_resval_t  resval;
+    xce_resval_t* vals[] = {
+        &resval
+    };
+
+    init_reply_by_req(&reply, request);
+
+    resvaldata.n_total = 0;
+    resvaldata.vals    = NULL;
+
+    if (getmsg != NULL) {
+        xce_match_t* condition = getmsg->condition;
+
+        if (condition != NULL) {
+            getdata.table = getmsg->table;
+            rc            = get_resval(c, condition->condition.reskey->resname,
+                                       condition->condition.reskey->hostname,
+                                       &val);
+            if (rc >= 0) {
+                resval.val         = &val;
+                resval.name        = condition->condition.reskey->resname;
+                resval.host        = condition->condition.reskey->hostname;
+                resvaldata.n_total = 1;
+                resvaldata.vals    = vals;
+            }
+        } else {
+            rc = -EINVAL;
+        }
+    } else {
+        rc = -EINVAL;
+    }
+
+    reply.status       = rc;
+    reply.data_which   = RequestReply_data_getdata;
+    reply.data.getdata = &getdata;
+
+    rc = xce_reply_get(ep, c->myhost, &reply, NULL, NULL, -1);
+
+    return rc;
+}
+
+int check_worker_(xce_worker_t* w)
+{
+    if (w == NULL) {
+        return 0;
+    }
+
+    return 1;
+}
+
+static int
+do_reply_workerlist(cluster_t*     c,
+                    endpoint_t*    ep,
+                    xce_request_t* request)
+{
+    xce_reply_t        reply;
+    xce_workers_data_t workerdata;
+    /* clang-format off */
+    xce_data_t         getdata = {
+                .table = request->body.getmsg->table,
+                .data  = {
+                     .workers = &workerdata }
+    };
+    /* clang-format on */
+    int rc = 0;
+    int i;
+
+    init_reply_by_req(&reply, request);
+    collect_workers(c, (collect_check_fn_t)check_worker_,
+                    &(workerdata.n_total), &(workerdata.workers));
+
+    reply.status       = 0;
+    reply.data_which   = RequestReply_data_getdata;
+    reply.data.getdata = &getdata;
+
+    rc = xce_reply_get(ep, c->myhost, &reply,
+                       check_controller, c, 100);
+
+    for (i = 0; i < workerdata.n_total; i++) {
+        xce_worker_t* w = workerdata.workers[i];
+
+        free_resval_with_idx_list(&(w->resvals_to_sched),
+                                  w->nresvals_to_sched);
+    }
+
+    deref_objs(workerdata.n_total, (void**)workerdata.workers);
+    FREEUP(workerdata.workers);
+
+    return rc;
+}
+
+static int check_job_(xce_job_t* j)
+{
+    if (j == NULL) {
+        return 0;
+    }
+
+    return 1;
+}
+
+static int
+do_reply_joblist(cluster_t*     c,
+                 endpoint_t*    ep,
+                 xce_request_t* request)
+{
+    xce_reply_t     reply;
+    xce_jobs_data_t jobdata;
+    xce_data_t      getdata = {
+             .table = request->body.getmsg->table,
+             .data  = {
+                  .jobs = &jobdata }
+    };
+    int rc = 0;
+
+    init_reply_by_req(&reply, request);
+    collect_jobs(c, (collect_check_fn_t)check_job_,
+                 &(jobdata.n_total), &(jobdata.jobs));
+
+    reply.status       = 0;
+    reply.data_which   = RequestReply_data_getdata;
+    reply.data.getdata = &getdata;
+
+    rc = xce_reply_get(ep, c->myhost, &reply,
+                       check_controller, c, 100);
+
+    deref_objs(jobdata.n_total, (void**)jobdata.jobs);
+    FREEUP(jobdata.jobs);
+
+    return rc;
+}
+
+int do_get_op(cluster_t*     c,
+              endpoint_t*    ep,
+              xce_request_t* request)
+{
+    int         rc = 0;
+    xce_reply_t reply;
+
+    TRACE("%s table %d",
+          __func__, request->body.getmsg->table);
+
+    init_reply_by_req(&reply, request);
+
+    switch (request->body.getmsg->table) {
+    case Data_data_resources:
+        rc = do_reply_reslist(c, ep, request);
+        break;
+    case Data_data_workers:
+    case Data_data_schedworkers:
+        rc = do_reply_workerlist(c, ep, request);
+        break;
+    case Data_data_schedjobs:
+    case Data_data_jobs:
+        rc = do_reply_joblist(c, ep, request);
+        break;
+    case Data_data_resvals:
+        rc = do_reply_resvals(c, ep, request);
+        break;
+    default:
+        rc = -ENOENT;
+
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+
+        xce_reply(ep, c->myhost, &reply);
+        break;
+    }
+
+    return rc;
+}

src/controller/opupd.c

@@ -0,0 +1,103 @@
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+
+static int
+do_update_resvals(cluster_t*     c,
+                  endpoint_t*    ep,
+                  xce_request_t* request)
+{
+    ep_private_t* priv = NULL;
+    int           rc   = -EPERM;
+    xce_reply_t   reply;
+
+    priv = (ep_private_t*)ep->data;
+    TRACE("endpoint %p priv data %p", ep, priv);
+
+    init_reply_by_req(&reply, request);
+
+    if (priv != NULL) {
+        int idx = priv->u.idx;
+
+        if ((priv->role == HostRole_worker) && (idx >= 0)) {
+            worker_t* w;
+
+            w = workermgr_lock_byid(&(c->workermgr),
+                                    idx, OBJREF_UPDATE);
+            if (w == NULL) {
+                rc = -E_XCE_BAD_WORKER;
+            } else {
+                xce_update_data_t* updatemsg = request->body.updatemsg;
+                xce_resval_list_t* resvals   = updatemsg->data.resvals;
+
+                xce_run_hooks(&(c->hooksys),
+                              XCE_CONTROLLER_HOOK_RESOURCE,
+                              HOOK_BEFORE_EVENT,
+                              HOOK_DETAIL_RESOURCE_UPDATE,
+                              c, resvals);
+                rc = update_resval_list(c, w, resvals);
+                xce_run_hooks(&(c->hooksys),
+                              XCE_CONTROLLER_HOOK_RESOURCE,
+                              HOOK_AFTER_EVENT,
+                              HOOK_DETAIL_RESOURCE_UPDATE,
+                              c, resvals);
+                DEREF_OBJ(w);
+            }
+        }
+    }
+
+    reply.status     = rc;
+    reply.data_which = RequestReply_data_nodata;
+    xce_reply(ep, c->myhost, &reply);
+
+    return 0;
+}
+
+static int do_update_jobrusage(cluster_t*     c,
+                               endpoint_t*    ep,
+                               xce_request_t* request)
+{
+    int         rc = 0;
+    xce_reply_t reply;
+
+    init_reply_by_req(&reply, request);
+    rc = update_job_rusages(c, ep, request);
+
+    reply.status     = rc;
+    reply.data_which = RequestReply_data_nodata;
+    xce_reply(ep, c->myhost, &reply);
+
+    return rc;
+}
+
+int do_update_op(cluster_t*     c,
+                 endpoint_t*    ep,
+                 xce_request_t* request)
+{
+    int         rc = 0;
+    xce_reply_t reply;
+
+    init_reply_by_req(&reply, request);
+
+    switch (request->body.updatemsg->table) {
+    case Update_data_resvals:
+        rc = do_update_resvals(c, ep, request);
+        break;
+    case Update_data_jobrusages:
+        rc = do_update_jobrusage(c, ep, request);
+        break;
+    default:
+        rc = -ENOENT;
+
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(ep, c->myhost, &reply);
+        break;
+    }
+    return rc;
+}

src/controller/qmgr.c

@@ -0,0 +1,171 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cluster.h"
+#include "htab.h"
+#include "xce/cluster_def.h"
+#include "xstring.h"
+
+extern void queue_obj_killer(queue_t*);
+
+int qmgr_init(qmgr_t* mgr,
+              int     nslots,
+              int     delta)
+{
+    mylock_init(&(mgr->lock));
+    ht_init(&(mgr->htab), nslots,
+            string_hash_key);
+    zarray_init(&(mgr->queues), delta,
+                (zarray_item_killer_t)queue_obj_killer);
+    return 0;
+}
+
+void qmgr_fini(qmgr_t* mgr)
+{
+    zarray_fini(&(mgr->queues));
+    ht_fini(&(mgr->htab));
+    mylock_fini(&(mgr->lock));
+}
+
+static int qmgr_add_(qmgr_t*  mgr,
+                     queue_t* q)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab), (void*)q->queue.queue);
+    if (e != NULL) {
+        return -EEXIST;
+    }
+
+    idx = zarray_store(&(mgr->queues), q);
+    if (idx < 0) {
+        return idx;
+    }
+
+    e = ht_add(&(mgr->htab), (void*)q->queue.queue);
+    if (e == NULL) {
+        zarray_del(&(mgr->queues), idx);
+        return -errno;
+    }
+
+    e->data = idx;
+    return idx;
+}
+
+int qmgr_add(qmgr_t*  mgr,
+             queue_t* q)
+{
+    int idx;
+
+    if ((mgr == NULL) || (q == NULL)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(mgr->lock));
+    idx = qmgr_add_(mgr, q);
+    mylock_unlock(&(mgr->lock));
+
+    return idx;
+}
+static queue_t* qmgr_lockq_(qmgr_t* mgr,
+                            char*   name,
+                            int     op)
+{
+    hent_t*  e = NULL;
+    int      idx;
+    queue_t* q = NULL;
+
+    e = ht_get(&(mgr->htab), (void*)name);
+    if (e == NULL) {
+        SET_ERROR(ENOENT);
+        return NULL;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        SET_ERROR(ENOENT);
+        return NULL;
+    }
+
+    q = (queue_t*)zarray_get(&(mgr->queues), idx);
+
+    return obj_ref(q, op);
+}
+queue_t* qmgr_lockq(qmgr_t* mgr,
+                    char*   name,
+                    int     op)
+{
+    queue_t* q = NULL;
+
+    if (mgr == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    if (name == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    /* clang-format off */
+    if ((op != OBJREF_NOP) &&
+	(op != OBJREF_READ) &&
+	(op != OBJREF_UPDATE)) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(mgr->lock));
+    q = qmgr_lockq_(mgr, name, op);
+    mylock_unlock(&(mgr->lock));
+
+    return q;
+}
+
+static void qmgr_delq_(qmgr_t*  mgr,
+                       queue_t* q)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab),
+               (void*)q->queue.queue);
+    if (e == NULL) {
+        return;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        return;
+    }
+
+    zarray_del(&(mgr->queues), idx);
+}
+
+void qmgr_delq(qmgr_t*  mgr,
+               queue_t* q)
+{
+    if ((mgr == NULL) || (q == NULL)) {
+        return;
+    }
+
+    mylock_lock(&(mgr->lock));
+    qmgr_delq_(mgr, q);
+    mylock_unlock(&(mgr->lock));
+}
+
+int qmgr_iter(qmgr_t*  mgr,
+              xlist_t* l,
+              void*    arg)
+{
+    int rc = 0;
+
+    mylock_lock(&(mgr->lock));
+    rc = zarray_iter_ex(&(mgr->queues), l, arg);
+    mylock_unlock(&(mgr->lock));
+
+    return rc;
+}

src/controller/queue.c

@@ -0,0 +1,163 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+
+void queue_fini(queue_t* q)
+{
+    if (q != NULL) {
+        free(q);
+    }
+}
+
+queue_t* create_queue(cluster_t*   c,
+                      xce_queue_t* p)
+{
+    queue_t  q1;
+    queue_t* q;
+
+    memcpy(&(q1.queue), p, sizeof(q1.queue));
+
+    q = create_obj_by_typename(&(c->objmgr), "xqueue",
+                               &q1);
+    return q;
+}
+
+int build_queues(void* c)
+{
+    cluster_t* cls = (cluster_t*)c;
+    int        i;
+
+    if (cls == NULL) {
+        return -EINVAL;
+    }
+
+    if (cls->myconf == NULL) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < cls->myconf->cluster_config->nqueues; i++) {
+        xce_queue_t* q = cls->myconf->cluster_config->queues[i];
+
+        create_queue(cls, q);
+    }
+
+    return 0;
+}
+
+int queue_ctrl(cluster_t* c, xce_queuectrl_t* ctrl)
+{
+    queue_t* q  = NULL;
+    int      rc = 0;
+
+    if (!c || !ctrl || !ctrl->queue || *ctrl->queue == '\0') {
+        TRACE("%s invalid arguments.", __func__);
+        return -EINVAL;
+    }
+    TRACE("%s queue_ctrl queue %s", __func__, ctrl->queue);
+    q = qmgr_lockq(&(c->qmgr), ctrl->queue, OBJREF_UPDATE);
+    if (q == NULL) {
+        PERROR("qmgr_lockq");
+        return -ENOENT;
+    }
+
+    switch (ctrl->action) {
+    case QUEUE_OPEN:
+        if (!(q->queue.status & QUEUE_STAT_CLOSED)) {
+            goto clean;
+        }
+        q->queue.status &= ~QUEUE_STAT_CLOSED;
+        break;
+    case QUEUE_CLOSE:
+        if (q->queue.status & QUEUE_STAT_CLOSED) {
+            goto clean;
+        }
+        q->queue.status |= QUEUE_STAT_CLOSED;
+        break;
+    case QUEUE_INACTIVE:
+        if (q->queue.status & QUEUE_STAT_INACT_ADMIN) {
+            goto clean;
+        }
+        q->queue.status |= QUEUE_STAT_INACT_ADMIN;
+        break;
+    case QUEUE_ACTIVE:
+        if (!(q->queue.status & QUEUE_STAT_INACT_ADMIN)) {
+            goto clean;
+        }
+        q->queue.status &= ~QUEUE_STAT_INACT_ADMIN;
+        break;
+    default:
+        rc = -1;
+        break;
+    }
+    if (rc == 0) {
+        xce_run_hooks(&(c->hooksys),
+                      XCE_CONTROLLER_HOOK_QUEUE,
+                      HOOK_AFTER_EVENT,
+                      HOOK_DETAIL_QUEUE_UPDATE,
+                      c, q);
+    }
+clean:
+    obj_deref(q);
+    return rc;
+}
+
+int do_queuecontrol_op(cluster_t*     c,
+                       endpoint_t*    ep,
+                       xce_request_t* request)
+{
+    int         rc = 0;
+    xce_reply_t reply;
+
+    init_reply_by_req(&reply, request);
+
+    rc               = queue_ctrl(c, request->body.queuectrlmsg);
+    reply.status     = rc;
+    reply.data_which = RequestReply_data_nodata;
+
+    xce_reply(ep, c->myhost, &reply);
+    return rc;
+}
+
+int get_queue_exec(cluster_t* c,
+                   char*      queue,
+                   char**     pre_exec,
+                   char**     post_exec)
+{
+    queue_t* q = NULL;
+
+    TRACE("entering %s", __func__);
+
+    if ((c == NULL) || (queue == NULL) || (pre_exec == NULL) || (post_exec == NULL)) {
+        return -EINVAL;
+    }
+
+    TRACE("%s queue %s", __func__, queue);
+
+    q = qmgr_lockq(&(c->qmgr), queue, OBJREF_READ);
+    if (q == NULL) {
+        PERROR("qmgr_lockq");
+        return -ENOENT;
+    }
+
+    if (q->queue.pre_exec != NULL) {
+        *pre_exec = create_obj_by_typename(&(c->objmgr), "xstring",
+                                           q->queue.pre_exec);
+    } else {
+        *pre_exec = NULL;
+    }
+
+    if (q->queue.post_exec != NULL) {
+        *post_exec = create_obj_by_typename(&(c->objmgr), "xstring",
+                                            q->queue.post_exec);
+    } else {
+        *post_exec = NULL;
+    }
+
+    obj_deref(q);
+
+    return 0;
+}

src/controller/reply.c

@@ -0,0 +1,75 @@
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+
+int handle_clusterconf(cluster_t*     c,
+                       endpoint_t*    ep,
+                       xce_request_t* request)
+{
+    /* clang-format off */
+    xce_reply_t       reply;
+    int               rc          = 0;
+    xce_clusterconf_t clusterconf = {
+        .name = c->myconf->cluster_config->clustername,
+        .wlist = {
+            .count   = 0,
+            .workers = NULL
+        },
+        .uglist = {
+            .count  = 0,
+            .groups = NULL
+        },
+        .hglist = {
+            .count  = 0,
+            .groups = NULL
+        }
+    };
+    /* clang-format on */
+    int i;
+
+    init_reply_by_req(&reply, request);
+
+    reply.status           = 0;
+    reply.data_which       = RequestReply_data_clusterconf;
+    reply.data.clusterconf = &clusterconf;
+
+    collect_confvars(c, NULL, &(clusterconf.cfglist.count),
+                     &(clusterconf.cfglist.configs));
+    collect_queues(c, NULL, &(clusterconf.qlist.count),
+                   &(clusterconf.qlist.queues));
+    collect_resources(c, NULL, &(clusterconf.reslist.count),
+                      &(clusterconf.reslist.res));
+    collect_workers(c, (collect_check_fn_t)check_worker_,
+                    &(clusterconf.wlist.count),
+                    &(clusterconf.wlist.workers));
+
+    xce_reply(ep, c->myhost, &reply);
+
+    for (i = 0; i < clusterconf.wlist.count; i++) {
+        xce_worker_t* w = clusterconf.wlist.workers[i];
+
+        free_resval_with_idx_list(&(w->resvals_to_sched),
+                                  w->nresvals_to_sched);
+    }
+
+    deref_objs(clusterconf.cfglist.count,
+               (void**)clusterconf.cfglist.configs);
+    deref_objs(clusterconf.qlist.count,
+               (void**)clusterconf.qlist.queues);
+    deref_objs(clusterconf.reslist.count,
+               (void**)clusterconf.reslist.res);
+    deref_objs(clusterconf.wlist.count,
+               (void**)clusterconf.wlist.workers);
+
+    FREEUP(clusterconf.cfglist.configs);
+    FREEUP(clusterconf.qlist.queues);
+    FREEUP(clusterconf.reslist.res);
+    FREEUP(clusterconf.wlist.workers);
+
+    return rc;
+}

src/controller/resmgr.c

@@ -0,0 +1,209 @@
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cluster.h"
+#include "htab.h"
+#include "utils.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+#include "xstring.h"
+
+extern void res_obj_killer(xce_resource_t*);
+extern void resval_obj_killer(xce_resval_t*);
+
+int resmgr_init(resmgr_t* mgr,
+                int       nslots,
+                int       delta,
+                int       inst_delta)
+{
+    TRACE("%s nslots %d delta %d inst_delta %d",
+          __func__, nslots, delta,
+          inst_delta);
+
+    mylock_init(&(mgr->lock));
+
+    ht_init(&(mgr->htab), nslots,
+            string_hash_key);
+    zarray_init(&(mgr->defs), delta,
+                (zarray_item_killer_t)res_obj_killer);
+    zarray_init(&(mgr->sharedinst), inst_delta,
+                (zarray_item_killer_t)resval_obj_killer);
+    mgr->nshared   = 0;
+    mgr->nunshared = 0;
+
+    return 0;
+}
+
+void resmgr_fini(resmgr_t* mgr)
+{
+
+    zarray_fini(&(mgr->sharedinst));
+    zarray_fini(&(mgr->defs));
+    ht_fini(&(mgr->htab));
+    mylock_fini(&(mgr->lock));
+
+    return;
+}
+
+static int32_t resmgr_add_res_(resmgr_t*       mgr,
+                               xce_resource_t* res)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab), (void*)res->name);
+    if (e != NULL) {
+        return -EEXIST;
+    }
+
+    idx = zarray_store(&(mgr->defs), res);
+    if (idx < 0) {
+        return idx;
+    }
+
+    e = ht_add(&(mgr->htab),
+               (void*)res->name);
+    if (e == NULL) {
+        zarray_del(&(mgr->defs), idx);
+        return -errno;
+    }
+
+    e->data = idx;
+    if (res->flags & XCE_RESF_SHARED) {
+        res->idx = mgr->nshared;
+        mgr->nshared++;
+    } else {
+        res->idx = mgr->nunshared;
+        mgr->nunshared++;
+    }
+    return idx;
+}
+
+int32_t resmgr_add_res(resmgr_t*       mgr,
+                       xce_resource_t* res)
+{
+    int32_t idx;
+
+    if ((mgr == NULL) || (res == NULL)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(mgr->lock));
+    idx = resmgr_add_res_(mgr, res);
+    mylock_unlock(&(mgr->lock));
+
+    return idx;
+}
+
+static xce_resource_t* resmgr_lockres_(resmgr_t* mgr,
+                                       char*     resname,
+                                       int       op)
+{
+    hent_t*         e = NULL;
+    int             idx;
+    xce_resource_t* r = NULL;
+
+    e = ht_get(&(mgr->htab), (void*)resname);
+    if (e == NULL) {
+        SET_ERROR(E_XCE_BAD_RES);
+        return NULL;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        SET_ERROR(E_XCE_INTERNAL);
+        return NULL;
+    }
+
+    r = (xce_resource_t*)zarray_get(&(mgr->defs), idx);
+    if (r == NULL) {
+        SET_ERROR(E_XCE_INTERNAL);
+        return NULL;
+    }
+
+    return obj_ref(r, op);
+}
+xce_resource_t* resmgr_lockres(resmgr_t* mgr,
+                               char*     resname,
+                               int       op)
+{
+    xce_resource_t* r = NULL;
+
+    if (mgr == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    if (resname == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    /* clang-format off */
+    if ((op != OBJREF_NOP) &&
+	(op != OBJREF_READ) &&
+	(op != OBJREF_UPDATE)) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(mgr->lock));
+    r = resmgr_lockres_(mgr, resname, op);
+    mylock_unlock(&(mgr->lock));
+
+    return r;
+}
+
+static void resmgr_del_res_(resmgr_t*       mgr,
+                            xce_resource_t* r)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab),
+               (void*)r->name);
+    if (e == NULL) {
+        SET_ERROR(E_XCE_BAD_RES);
+
+        return;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        SET_ERROR(E_XCE_INTERNAL);
+
+        return;
+    }
+
+    zarray_del(&(mgr->defs), idx);
+}
+
+void resmgr_del_res(resmgr_t*       mgr,
+                    xce_resource_t* r)
+{
+    if ((mgr == NULL) || (r == NULL)) {
+        SET_ERROR(EINVAL);
+
+        return;
+    }
+
+    mylock_lock(&(mgr->lock));
+    resmgr_del_res_(mgr, r);
+    mylock_unlock(&(mgr->lock));
+}
+
+int resmgr_iter(resmgr_t* mgr,
+                xlist_t*  l,
+                void*     arg)
+{
+    int rc = 0;
+
+    mylock_lock(&(mgr->lock));
+    rc = zarray_iter_ex(&(mgr->defs), l, arg);
+    mylock_unlock(&(mgr->lock));
+
+    return rc;
+}

src/controller/resources.c

@@ -0,0 +1,10 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+
+#include "htab.h"
+#include "worker.h"
+#include "xce/types.h"

src/controller/sched.c

@@ -0,0 +1,620 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "cluster.h"
+#include "controller.capnp.h"
+#include "objmgr.h"
+#include "request.h"
+#include "worker.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+
+extern void collect_shared_resvals(cluster_t*, collect_check_fn_t,
+                                   int*, xce_val_with_idx_t***);
+
+static int xcesched_push_joblist(cluster_t* c,
+                                 xlist_t*   l)
+{
+    xce_data_t      data;
+    xce_jobs_data_t jobs = {
+        .n_total = l->count,
+        .pos     = 0,
+        .count   = l->count,
+        .jobs    = (xce_job_t**)l->data
+    };
+    int rc = 0;
+
+    memset(&data, 0x0, sizeof(data));
+
+    data.table     = Data_data_schedjobs;
+    data.data.jobs = &jobs;
+
+    rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
+                           check_controller, c, 100);
+    return rc;
+}
+
+static int xcesched_push_workerlist(cluster_t* c,
+                                    xlist_t*   l)
+{
+    xce_data_t         data;
+    xce_workers_data_t workers = {
+        .n_total = l->count,
+        .pos     = 0,
+        .count   = l->count,
+        .workers = (xce_worker_t**)l->data
+    };
+    int rc = 0;
+
+    memset(&data, 0x0, sizeof(data));
+
+    data.table        = Data_data_schedworkers;
+    data.data.workers = &workers;
+
+    rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
+                           check_controller, c, 100);
+    return rc;
+}
+
+static int xcesched_push_configs(cluster_t* c)
+{
+    xce_configdata_t configdata;
+    int              rc = 0;
+
+    memset(&configdata, 0x0, sizeof(configdata));
+
+    configdata.data.conf4sched = (xce_conf4sched_t*)calloc(1, sizeof(xce_conf4sched_t));
+    if (configdata.data.conf4sched == NULL) {
+        return -ENOMEM;
+    }
+
+    collect_confvars(c, NULL, &(configdata.data.conf4sched->n_confvars),
+                     &(configdata.data.conf4sched->confvars));
+    collect_queues(c, NULL, &(configdata.data.conf4sched->n_queues),
+                   &(configdata.data.conf4sched->queues));
+    collect_resources(c, NULL, &(configdata.data.conf4sched->n_resources),
+                      &(configdata.data.conf4sched->resources));
+    collect_shared_resvals(c, NULL, &(configdata.data.conf4sched->nresvals),
+                           &(configdata.data.conf4sched->resvals));
+
+    configdata.data.conf4sched->n_ugroups = c->myconf->cluster_config->nugroups;
+    configdata.data.conf4sched->ugroups   = c->myconf->cluster_config->ugroups;
+    configdata.data.conf4sched->n_hgroups = c->myconf->cluster_config->nhgroups;
+    configdata.data.conf4sched->hgroups   = c->myconf->cluster_config->hgroups;
+
+    /* add resources */
+    configdata.data_which                   = ConfigData_data_schedconfig;
+    configdata.data.conf4sched->clustername = c->myconf->cluster_config->clustername;
+
+    rc = xce_pushconfig_req(c->sched.ep, c->myhost,
+                            &configdata);
+
+    free_resval_with_idx_list(&(configdata.data.conf4sched->resvals),
+                              configdata.data.conf4sched->nresvals);
+
+    deref_objs(configdata.data.conf4sched->n_confvars,
+               (void**)configdata.data.conf4sched->confvars);
+    deref_objs(configdata.data.conf4sched->n_queues,
+               (void**)configdata.data.conf4sched->queues);
+    deref_objs(configdata.data.conf4sched->n_resources,
+               (void**)configdata.data.conf4sched->resources);
+
+    FREEUP(configdata.data.conf4sched->confvars);
+    FREEUP(configdata.data.conf4sched->queues);
+    FREEUP(configdata.data.conf4sched->resources);
+
+    FREEUP(configdata.data.conf4sched);
+
+    return rc;
+}
+
+static int check_job_(xce_job_t* j)
+{
+    if (j == NULL) {
+        return 0;
+    }
+
+    return 1;
+}
+
+static int xcesched_push_jobs(cluster_t* c)
+{
+    xce_data_t      data;
+    xce_jobs_data_t jobs = {
+        .n_total = 0,
+        .pos     = 0,
+        .count   = 0,
+        .jobs    = NULL
+    };
+    int rc = 0;
+
+    memset(&data, 0x0, sizeof(data));
+
+    data.table     = Data_data_schedjobs;
+    data.data.jobs = &jobs;
+
+    collect_jobs(c, (collect_check_fn_t)check_job_,
+                 &(jobs.n_total), &(jobs.jobs));
+    jobs.count = jobs.n_total;
+
+    rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
+                           check_controller, c, 100);
+    deref_objs(jobs.n_total, (void**)jobs.jobs);
+    FREEUP(jobs.jobs);
+
+    return rc;
+}
+
+static int xcesched_push_workers(cluster_t* c)
+{
+    xce_data_t         data;
+    xce_workers_data_t workers = {
+        .n_total = 0,
+        .pos     = 0,
+        .count   = 0,
+        .workers = NULL
+    };
+    int rc = 0;
+    int i;
+
+    memset(&data, 0x0, sizeof(data));
+
+    data.table        = Data_data_schedworkers;
+    data.data.workers = &workers;
+
+    collect_workers(c, (collect_check_fn_t)check_worker_,
+                    &(workers.n_total), &(workers.workers));
+    workers.count = workers.n_total;
+
+    rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
+                           check_controller, c, 100);
+
+    /* free up collected resource value which used
+       passed to scheduler
+    */
+    for (i = 0; i < workers.n_total; i++) {
+        xce_worker_t* w = workers.workers[i];
+
+        free_resval_with_idx_list(&(w->resvals_to_sched),
+                                  w->nresvals_to_sched);
+    }
+
+    deref_objs(workers.n_total, (void**)workers.workers);
+    FREEUP(workers.workers);
+
+    return rc;
+}
+
+static int sched_job_new_fun(int     type,
+                             int     situation,
+                             void*   data,
+                             va_list ap)
+{
+    cluster_t* c  = va_arg(ap, cluster_t*);
+    xlist_t*   l  = va_arg(ap, xlist_t*);
+    int        rc = 0;
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = -EINVAL;
+            if (l != NULL) {
+                rc = xcesched_push_joblist(c, l);
+            }
+            break;
+        default:
+            break;
+        }
+    }
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+
+static int sched_job_start_fun(int     type,
+                               int     situation,
+                               void*   data,
+                               va_list ap)
+{
+    cluster_t*     c    = va_arg(ap, cluster_t*);
+    job_pidinfo_t* info = va_arg(ap, job_pidinfo_t*);
+    int            rc   = 0;
+    /* clang-format off */
+    xce_notify_data_t xdata = {
+        .event = NotifyData_data_schedStartJob,
+        .data  = {
+             .sched_start_job = info->jobid
+	}
+    };
+    /* clang-format on */
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = notify(c, c->sched.ep, &xdata);
+            break;
+        default:
+            break;
+        }
+    }
+
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+
+static int sched_job_exit_fun(int     type,
+                              int     situation,
+                              void*   data,
+                              va_list ap)
+{
+    cluster_t*       c    = va_arg(ap, cluster_t*);
+    xce_jobfinish_t* info = va_arg(ap, xce_jobfinish_t*);
+    int              rc   = 0;
+    /* clang-format off */
+    xce_notify_data_t xdata = {
+        .event = NotifyData_data_schedExitJob,
+        .data  = {
+             .sched_exit_job = info->jobid
+	}
+    };
+    /* clang-format on */
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = notify(c, c->sched.ep, &xdata);
+            break;
+        default:
+            break;
+        }
+    }
+
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+
+static int sched_job_fail_fun(int     type,
+                              int     situation,
+                              void*   data,
+                              va_list ap)
+{
+    cluster_t*      c    = va_arg(ap, cluster_t*);
+    job_failinfo_t* info = va_arg(ap, job_failinfo_t*);
+    int             rc   = 0;
+    /* clang-format off */
+    xce_notify_data_t xdata = {
+        .event = NotifyData_data_schedFailJob,
+        .data  = {
+             .sched_fail_job = info->jobid
+	}
+    };
+    /* clang-format on */
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = notify(c, c->sched.ep, &xdata);
+            break;
+        default:
+            break;
+        }
+    }
+
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+
+static int sched_job_update_fun(int     type,
+                                int     situation,
+                                void*   data,
+                                va_list ap)
+{
+    cluster_t* c   = va_arg(ap, cluster_t*);
+    job_t*     job = va_arg(ap, job_t*);
+    int        rc  = 0;
+    /* clang-format off */
+    xce_notify_data_t xdata = {
+        .event = NotifyData_data_schedUpdateJob,
+        .data  = {
+	    .sched_update_job = &(job->job)
+	}
+    };
+    /* clang-format on */
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = notify(c, c->sched.ep, &xdata);
+            break;
+        default:
+            break;
+        }
+    }
+
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+
+static int sched_job_hook_fun(int     type,
+                              int     situation,
+                              int     detail,
+                              void*   data,
+                              va_list ap)
+{
+    switch (detail) {
+    case HOOK_DETAIL_JOB_NEW:
+        return sched_job_new_fun(type, situation,
+                                 data, ap);
+    case HOOK_DETAIL_JOB_START:
+        return sched_job_start_fun(type, situation,
+                                   data, ap);
+    case HOOK_DETAIL_JOB_EXIT:
+        return sched_job_exit_fun(type, situation,
+                                  data, ap);
+    case HOOK_DETAIL_JOB_UPDATE:
+        return sched_job_update_fun(type, situation,
+                                    data, ap);
+    case HOOK_DETAIL_JOB_FAIL:
+        return sched_job_fail_fun(type, situation,
+                                  data, ap);
+    }
+
+    return 0;
+}
+static int sched_worker_change(int        type,
+                               int        situation,
+                               void*      data,
+                               cluster_t* c,
+                               worker_t*  w)
+{
+    int     rc = 0;
+    xlist_t wlist;
+
+    TRACE("%s worker %s idx %d situation %d",
+          __func__, w->host.name,
+          w->idx, situation);
+
+    collect_resval_with_idx(&(w->host.resvals),
+                            &(w->host.nresvals_to_sched),
+                            &(w->host.resvals_to_sched));
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            xlist_init(&wlist, 2);
+            xlist_store(&wlist, w);
+            rc = xcesched_push_workerlist(c, &wlist);
+            xlist_fini(&wlist);
+            break;
+        default:
+            break;
+        }
+    }
+    mylock_unlock(&(c->sched.lock));
+
+    free_resval_with_idx_list(&(w->host.resvals_to_sched),
+                              w->host.nresvals_to_sched);
+
+    return rc;
+}
+
+static int sched_worker_join(int     type,
+                             int     situation,
+                             void*   data,
+                             va_list ap)
+{
+    cluster_t* c    = va_arg(ap, cluster_t*);
+    char*      name = va_arg(ap, char*);
+    worker_t*  w    = NULL;
+    int        rc;
+
+    if (situation == HOOK_BEFORE_EVENT) {
+        /* nothing to do  */
+        return 0;
+    }
+
+    w = workermgr_lock(&(c->workermgr), name,
+                       OBJREF_READ);
+
+    rc = sched_worker_change(type, situation, data,
+                             c, w);
+
+    /* FIXME: need to mark the jobs in this worker */
+    obj_deref(w);
+
+    return rc;
+}
+
+static int sched_worker_dead(int     type,
+                             int     situation,
+                             void*   data,
+                             va_list ap)
+{
+    cluster_t* c = va_arg(ap, cluster_t*);
+    worker_t*  w = va_arg(ap, worker_t*);
+
+    /* FIXME: need to handle job status in this worker */
+    if (situation == HOOK_BEFORE_EVENT) {
+        return 0;
+    }
+
+    return sched_worker_change(type, situation, data,
+                               c, w);
+}
+
+static int sched_worker_update(int     type,
+                               int     situation,
+                               void*   data,
+                               va_list ap)
+{
+    cluster_t* c = va_arg(ap, cluster_t*);
+    worker_t*  w = va_arg(ap, worker_t*);
+
+    if (situation == HOOK_BEFORE_EVENT) {
+        return 0;
+    }
+
+    return sched_worker_change(type, situation, data,
+                               c, w);
+}
+static int sched_worker_hook_fun(int     type,
+                                 int     situation,
+                                 int     detail,
+                                 void*   data,
+                                 va_list ap)
+{
+    switch (detail) {
+    case HOOK_DETAIL_WORKER_JOIN:
+        return sched_worker_join(type, situation,
+                                 data, ap);
+    case HOOK_DETAIL_WORKER_DEAD:
+        return sched_worker_dead(type, situation,
+                                 data, ap);
+    case HOOK_DETAIL_WORKER_UPDATE:
+        return sched_worker_update(type, situation,
+                                   data, ap);
+    }
+
+    return 0;
+}
+
+static int sched_resource_hook_fun(int     type,
+                                   int     situation,
+                                   int     detail,
+                                   void*   data,
+                                   va_list ap)
+{
+    cluster_t*         c       = va_arg(ap, cluster_t*);
+    xce_resval_list_t* resvals = va_arg(ap, xce_resval_list_t*);
+    /* clang-format on */
+    xce_notify_data_t xdata = {
+        .event = NotifyData_data_schedUpdateResvals,
+        .data  = {
+             .sched_update_resvals = resvals }
+    };
+    /* clang-format off */
+    int rc = 0;
+
+    if (detail != HOOK_DETAIL_RESOURCE_UPDATE) {
+        return 0;
+    }
+
+    xce_use(resvals);
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = notify(c, c->sched.ep, &xdata);
+            break;
+        default:
+            break;
+        }
+    }
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+
+static int sched_queue_update(int     type,
+                              int     situation,
+                              void*   data,
+                              va_list ap)
+{
+    cluster_t*   c = va_arg(ap, cluster_t*);
+    queue_t* q = va_arg(ap, queue_t*);
+    int          rc = 0;
+    /* clang-format off */
+    xce_notify_data_t xdata = {
+	.event = NotifyData_data_schedUpdateQueue,
+	.data = {
+	    .sched_update_queue = &(q->queue)
+	}
+    };
+    /* clang-format on */
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        switch (situation) {
+        case HOOK_AFTER_EVENT:
+            rc = notify(c, c->sched.ep, &xdata);
+            break;
+        default:
+            break;
+        }
+    }
+    mylock_unlock(&(c->sched.lock));
+
+    return rc;
+}
+static int sched_queue_hook_fun(int     type,
+                                int     situation,
+                                int     detail,
+                                void*   data,
+                                va_list ap)
+{
+    switch (detail) {
+    case HOOK_DETAIL_QUEUE_UPDATE:
+        return sched_queue_update(type, situation, data, ap);
+    default:
+        break;
+    }
+
+    return 0;
+}
+
+int on_scheduler_join(cluster_t* c)
+{
+    static int hook_installed = 0;
+
+    mylock_lock(&(c->sched.lock));
+    if (c->sched.ep != NULL) {
+        xcesched_push_configs(c);
+        xcesched_push_workers(c);
+        xcesched_push_jobs(c);
+    }
+    mylock_unlock(&(c->sched.lock));
+
+    if (!hook_installed) {
+        hook_installed = 1;
+
+        xce_add_hook(&(c->hooksys),
+                     XCE_CONTROLLER_HOOK_JOB,
+                     (hook_fn_t)sched_job_hook_fun,
+                     NULL);
+        xce_add_hook(&(c->hooksys),
+                     XCE_CONTROLLER_HOOK_WORKER,
+                     (hook_fn_t)sched_worker_hook_fun,
+                     NULL);
+        xce_add_hook(&(c->hooksys),
+                     XCE_CONTROLLER_HOOK_RESOURCE,
+                     (hook_fn_t)sched_resource_hook_fun,
+                     NULL);
+        xce_add_hook(&(c->hooksys),
+                     XCE_CONTROLLER_HOOK_QUEUE,
+                     (hook_fn_t)sched_queue_hook_fun,
+                     NULL);
+    }
+    return 0;
+}
+
+int handle_scheduler_leave(cluster_t* c)
+{
+    /* we need to mark the cluster status */
+    TRACE("%s scheduler leave",
+          __func__);
+    mylock_lock(&(c->sched.lock));
+    c->sched.ep = NULL;
+    mylock_unlock(&(c->sched.lock));
+    return 0;
+}

src/controller/task.c

@@ -0,0 +1,203 @@
+#include <errno.h>
+#include <signal.h>
+#include <time.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+#include "xce/types.h"
+
+static int run_tasks_on_host(cluster_t*      c,
+                             endpoint_t*     ep,
+                             uint64_t        jobid,
+                             xce_taskhost_t* info)
+{
+    worker_t*     w;
+    ioctx_t*      ioctx;
+    int           rc;
+    xce_taskrun_t runmsg = {
+        .nhosts    = 1,
+        .jobid     = jobid,
+        .clientidx = ep->idx,
+        .taskhosts = (xce_taskhost_t*[]) { info }
+    };
+
+    w = workermgr_lock(&(c->workermgr), info->host, OBJREF_READ);
+    if (w == NULL) {
+        return -ENOENT;
+    }
+
+    if ((w->ep == NULL) || (IS_WORKER_UNAVAIL(w->host.status))) {
+        obj_deref(w);
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(w->ep, c->myhost);
+    if (ioctx == NULL) {
+        obj_deref(w);
+        return -ENOMEM;
+    }
+
+    rc = xce_runtask_req(ioctx, &runmsg);
+
+    obj_deref(w);
+
+    return rc;
+}
+
+static int check_task_hosts(cluster_t*     c,
+                            xce_taskrun_t* runmsg)
+{
+    int i;
+
+    for (i = 0; i < runmsg->nhosts; i++) {
+        worker_t* w;
+
+        w = workermgr_lock(&(c->workermgr),
+                           runmsg->taskhosts[i]->host,
+                           OBJREF_READ);
+        if (w == NULL) {
+            return -ENOENT;
+        }
+
+        if ((w->ep == NULL) || IS_WORKER_UNAVAIL(w->host.status)) {
+            obj_deref(w);
+            return -ENOENT;
+        }
+    }
+
+    return 0;
+}
+
+int do_runtasks(cluster_t*     c,
+                endpoint_t*    ep,
+                xce_request_t* req)
+{
+    int            rc = 0;
+    xce_reply_t    reply;
+    int            i;
+    xce_taskrun_t* runmsg = req->body.runtaskmsg;
+
+    init_reply_by_req(&reply, req);
+    reply.data_which = RequestReply_data_nodata;
+    rc               = check_task_hosts(c, runmsg);
+
+    reply.status = rc;
+    xce_reply(ep, c->myhost, &reply);
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = 0;
+
+    for (i = 0; i < runmsg->nhosts; i++) {
+        int rc1;
+
+        rc1 = run_tasks_on_host(c, ep,
+                                runmsg->jobid,
+                                runmsg->taskhosts[i]);
+        if (rc1 < 0) {
+            xce_task_failinfo_t failinfo = {
+                .jobid   = runmsg->jobid,
+                .host    = runmsg->taskhosts[i]->host,
+                .taskidx = -1, /* all tasks failed */
+                .code    = rc1,
+                .detail  = NULL
+            };
+            xce_notify_data_t data = {
+                .event = NotifyData_data_taskFailInfo,
+                .data  = {
+                     .taskfailinfo = &failinfo }
+            };
+
+            /* notify client that run tasks on host fail */
+            notify(c, ep, &data);
+
+            break;
+        }
+    }
+
+    return rc;
+}
+
+int on_task_start(cluster_t*          c,
+                  xce_task_pidinfo_t* info)
+{
+    endpoint_t*       ep;
+    xce_notify_data_t data = {
+        .event = NotifyData_data_taskPidInfo,
+        .data  = {
+             .taskinfo = info }
+    };
+    int clientidx = info->clientidx;
+
+    if (clientidx < 0) {
+        ERROR("incorrect client idx %d", clientidx);
+        return -EINVAL;
+    }
+
+    ep = (endpoint_t*)zarray_get(&(c->endpoints), clientidx);
+    if (ep == NULL) {
+        ERROR("incorrect client idx %d", clientidx);
+        return -EINVAL;
+    }
+
+    notify(c, ep, &data);
+
+    return 0;
+}
+
+int on_task_fail(cluster_t*           c,
+                 xce_task_failinfo_t* info)
+{
+    endpoint_t*       ep;
+    xce_notify_data_t data = {
+        .event = NotifyData_data_taskFailInfo,
+        .data  = {
+             .taskfailinfo = info }
+    };
+    int clientidx = info->clientidx;
+
+    if (clientidx < 0) {
+        ERROR("incorrect client idx %d", clientidx);
+        return -EINVAL;
+    }
+
+    ep = (endpoint_t*)zarray_get(&(c->endpoints), clientidx);
+    if (ep == NULL) {
+        ERROR("incorrect client idx %d", clientidx);
+        return -EINVAL;
+    }
+
+    notify(c, ep, &data);
+
+    return 0;
+}
+
+int on_task_finish(cluster_t*         c,
+                   xce_task_finish_t* info)
+{
+    endpoint_t*       ep;
+    xce_notify_data_t data = {
+        .event = NotifyData_data_taskFinish,
+        .data  = {
+             .taskfinish = info }
+    };
+    int clientidx = info->clientidx;
+
+    if (clientidx < 0) {
+        ERROR("incorrect client idx %d", clientidx);
+        return -EINVAL;
+    }
+
+    ep = (endpoint_t*)zarray_get(&(c->endpoints), clientidx);
+    if (ep == NULL) {
+        ERROR("incorrect client idx %d", clientidx);
+        return -EINVAL;
+    }
+
+    notify(c, ep, &data);
+
+    return 0;
+}

src/controller/worker.c

@@ -0,0 +1,599 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "cluster.h"
+#include "worker.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+
+static int hostmatched(const char* pattern,
+                       const char* name)
+{
+    if (strcmp(pattern, name) == 0) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static xce_hostconfig_t* find_matched_hostconfig(cluster_t*  c,
+                                                 const char* name)
+{
+    xce_hostconfig_t*  defaultconfig = NULL;
+    xce_hostconfig_t** hconfigs      = NULL;
+    int                i, nhconfigs;
+
+    if (c->myconf == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (c->myconf->cluster_config == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    if (c->myconf->cluster_config->hconfigs == NULL) {
+        return NULL;
+    }
+
+    nhconfigs = c->myconf->cluster_config->nhconfigs;
+    hconfigs  = c->myconf->cluster_config->hconfigs;
+
+    for (i = 0; i < nhconfigs; i++) {
+        xce_hostconfig_t* config = hconfigs[i];
+
+        if (strcmp(config->name, "default") == 0) {
+            defaultconfig = config;
+            continue;
+        }
+
+        if (hostmatched(config->name, name)) {
+            return config;
+        }
+    }
+
+    return defaultconfig;
+}
+
+static int update_val_res(cluster_t*      c,
+                          worker_t*       w,
+                          xce_resource_t* r,
+                          xce_val_t*      val)
+{
+    int        idx;
+    zarray_t*  array = NULL;
+    xce_val_t* v;
+
+    idx = r->idx;
+
+    if (r->flags & XCE_RESF_SHARED) {
+        /* update shared resource */
+        array = &(c->resmgr.sharedinst);
+    } else {
+        /* update host based resource */
+        array = &(w->host.resvals);
+    }
+
+    if (idx < array->caps) {
+        v = (xce_val_t*)array->data[idx];
+    } else {
+        v = NULL;
+    }
+
+    if (v == NULL) {
+        int rc;
+
+        v  = (xce_val_t*)calloc(1, sizeof(*v));
+        rc = zarray_put(array, idx, v);
+        if (rc < 0) {
+            FREEUP(v);
+            return rc;
+        }
+    }
+
+    memcpy(v, val, sizeof(*v));
+
+    return 0;
+}
+
+static int update_resval(cluster_t*    c,
+                         worker_t*     w,
+                         xce_resval_t* val)
+{
+    xce_resource_t* r;
+    xce_val_t*      v = NULL;
+    int             rc;
+
+    r = resmgr_lockres(&(c->resmgr),
+                       val->name,
+                       OBJREF_READ);
+    if (r == NULL) {
+        ERROR("resource `%s' can not be found",
+              val->name);
+        return -errno;
+    }
+
+    if (r->idx < 0) {
+        ERROR("invalid resource definition for %s",
+              val->name);
+        return -EINVAL;
+    }
+
+    v = val->val;
+
+    rc = update_val_res(c, w, r, v);
+
+    obj_deref(r);
+
+    return rc;
+}
+
+int update_resvals(cluster_t*    c,
+                   worker_t*     w,
+                   int           nres,
+                   xce_resval_t* vals)
+{
+    int rc = 0;
+    int i;
+
+    for (i = 0; i < nres; i++) {
+        rc = update_resval(c, w, &(vals[i]));
+        if (rc < 0) {
+            PERROR("update_resval");
+        }
+    }
+
+    return 0;
+}
+
+int update_resval_list(cluster_t*         c,
+                       worker_t*          w,
+                       xce_resval_list_t* l)
+{
+    int rc = 0;
+    int i;
+
+    for (i = 0; i < l->count; i++) {
+        rc = update_resval(c, w, l->vals[i]);
+        if (rc < 0) {
+            PERROR("update_resval");
+        }
+    }
+
+    return 0;
+}
+
+static int mark_host_res(cluster_t*    c,
+                         worker_t*     w,
+                         xce_config_t* config)
+{
+    xce_resource_t* r;
+    char*           n = config->name;
+    int             rc;
+    /* clang-format-off */
+    xce_val_t val;
+    /* clang-format-on */
+
+    r = resmgr_lockres(&(c->resmgr), n, OBJREF_READ);
+    if (r == NULL) {
+        ERROR("resource `%s' can not be found",
+              n);
+        return -ENOENT;
+    }
+
+    if (r->idx < 0) {
+        ERROR("invalid resource definition for `%s'",
+              n);
+        return -EINVAL;
+    }
+
+    if (r->flags & XCE_RESF_SHARED) {
+        ERROR("resource `%s' should not be a shared resource", n);
+        return -EINVAL;
+    }
+
+    switch (r->valtype) {
+    case XCE_VAL_STRING:
+        val.which = Value_u_svalue;
+        if (config->value == NULL) {
+            val.u.svalue = "";
+        } else {
+            val.u.svalue = config->value;
+        }
+
+        break;
+    case XCE_VAL_NUMERIC:
+        val.which = Value_u_fvalue;
+        if (config->value == NULL) {
+            val.u.fvalue = 0.0;
+        } else {
+            val.u.fvalue = atof(config->value);
+        }
+        break;
+    case XCE_VAL_BOOLEAN:
+        val.which    = Value_u_bvalue;
+        val.u.bvalue = 1;
+        break;
+    }
+    rc = update_val_res(c, w, r, &val);
+
+    obj_deref(r);
+    return rc;
+}
+
+static int add_hostattrs_resvals(cluster_t* c,
+                                 worker_t*  w)
+{
+#define N_HOST_ATTRS (8)
+    xce_val_t values[N_HOST_ATTRS] = {
+        { .which = Value_u_svalue, .u = { .svalue = w->host.attrs->hosttype }  },
+        { .which = Value_u_svalue, .u = { .svalue = w->host.attrs->hostmodel } },
+        { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->cpufactor } },
+        { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxcpus }   },
+        { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxmem }    },
+        { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxswap }   },
+        { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxtmp }    },
+        { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->ndisks }    }
+    };
+    xce_resval_t resvals[N_HOST_ATTRS] = {
+        { .name = "type",    .val = &values[0] },
+        { .name = "model",   .val = &values[1] },
+        { .name = "cpuf",    .val = &values[2] },
+        { .name = "maxcpus", .val = &values[3] },
+        { .name = "maxmem",  .val = &values[4] },
+        { .name = "maxswap", .val = &values[5] },
+        { .name = "maxtmp",  .val = &values[6] },
+        { .name = "ndisks",  .val = &values[7] }
+    };
+
+    return update_resvals(c, w, N_HOST_ATTRS,
+                          &(resvals[0]));
+}
+
+static int set_host_slots(cluster_t* c,
+                          worker_t*  w,
+                          int        nslots)
+{
+    /* clang-format off */
+    xce_val_t values[1] = {
+        {
+            .which = Value_u_fvalue,
+            .u = {
+                .fvalue = (float)nslots
+            }
+        }
+    };
+    xce_resval_t resvals[1] = {
+        {
+            .name = "slots",
+            .val = &values[0]
+        }
+    };
+    /* clang-format on */
+
+    return update_resvals(c, w, 1, &(resvals[0]));
+}
+
+int add_worker(void*        cls,
+               endpoint_t*  ep,
+               const char*  host,
+               hostattrs_t* attrs)
+{
+    cluster_t* c = (cluster_t*)cls;
+    /* clang-format off */
+    worker_t   worker = {
+        .host = {
+            .name = (char*)host,
+        }
+    };
+    /* clang-format on */
+    worker_t*         w = NULL;
+    int               idx, i;
+    xce_hostconfig_t* hconfig = NULL;
+    int               nslots;
+
+    if ((cls == NULL) || (host == NULL)) {
+        return -EINVAL;
+    }
+
+    hconfig = find_matched_hostconfig(c, host);
+    if (hconfig == NULL) {
+        return -errno;
+    }
+
+    worker.host.attrs = attrs;
+
+    w = create_obj_by_typename(&(c->objmgr), "xworker", &worker);
+    if (w == NULL) {
+        return -errno;
+    }
+
+    /* we should hold the update lock */
+    /* If admin makes the host in locked or closed, we need keep this */
+    w->host.status &= ~XCE_WORKER_UNAVAIL; /* status OK */
+    w->ep = ep;
+
+    /* override host type? */
+    if (strcmp(hconfig->hosttype, "!") != 0) {
+        obj_deref(w->host.attrs->hosttype);
+        w->host.attrs->hosttype = create_obj_by_typename(&(c->objmgr),
+                                                         "xstring",
+                                                         hconfig->hosttype);
+    }
+
+    /* override host model */
+    if (strcmp(hconfig->hostmodel, "!") != 0) {
+        obj_deref(w->host.attrs->hostmodel);
+        w->host.attrs->hostmodel = create_obj_by_typename(&(c->objmgr),
+                                                          "xstring",
+                                                          hconfig->hostmodel);
+    }
+    add_hostattrs_resvals(c, w);
+    idx = w->idx;
+
+    /* using auto detected or pre configured MXJ? */
+    nslots = hconfig->mxj;
+    if (nslots < 0) {
+        nslots = w->host.attrs->maxcpus;
+    }
+
+    set_host_slots(c, w, nslots);
+
+    if (w->host.dispatch_window_config != NULL) {
+        free_timewindow_config(w->host.dispatch_window_config);
+        w->host.dispatch_window_config = NULL;
+    }
+
+    if (hconfig->dispatchwindow != NULL) {
+        w->host.dispatch_window_config = dup_timewindow_config(hconfig->dispatchwindow);
+    }
+
+    if (w->host.thresholds != NULL) {
+        free_resource_thresholds(w->host.thresholds,
+                                 w->host.nthresholds);
+        w->host.nthresholds = 0;
+        w->host.thresholds  = NULL;
+    }
+
+    if (hconfig->thresholds != NULL) {
+        w->host.thresholds  = dup_resource_thresholds(hconfig->thresholds,
+                                                      hconfig->nthresholds);
+        w->host.nthresholds = hconfig->nthresholds;
+    }
+
+    /* add bool resources to host */
+    for (i = 0; i < hconfig->nresources; i++) {
+        mark_host_res(c, w, hconfig->resources[i]);
+    }
+
+    DEREF_OBJ(w); /* release update lock */
+
+    return idx;
+}
+
+int restore_worker(void*        cls,
+                   xce_worker_t host)
+{
+    cluster_t* c   = (cluster_t*)cls;
+    int        idx = 0;
+    worker_t*  w   = NULL;
+    /* do work in add_worker() doing for normal worker join */
+    /* clang-format off */
+    worker_t worker = {
+        .host = {
+            .name   = (char*)host.name,
+            .status = host.status,
+        },
+    };
+    /* clang-format on */
+
+    w = create_obj_by_typename(&(c->objmgr), "xworker", &worker);
+    if (w == NULL) {
+        return -errno;
+    }
+    add_hostattrs_resvals(c, w);
+    idx = w->idx;
+    w->host.status |= XCE_WORKER_UNAVAIL;
+
+    DEREF_OBJ(w);
+
+    return idx;
+}
+
+struct worker_config_publish_args {
+    cluster_t*  c;
+    endpoint_t* ep;
+};
+
+static int check_conf4worker_(xce_config_t* c)
+{
+    if (c != NULL) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static int worker_push_config(cluster_t*  c,
+                              endpoint_t* ep)
+{
+    xce_configdata_t configdata;
+    int              rc = 0;
+
+    memset(&configdata, 0x0, sizeof(configdata));
+
+    configdata.data.conf4worker = (xce_conf4worker_t*)calloc(1, sizeof(xce_conf4worker_t));
+    if (configdata.data.conf4worker == NULL) {
+        return -ENOMEM;
+    }
+
+    collect_confvars(c, (collect_check_fn_t)check_conf4worker_,
+                     &(configdata.data.conf4worker->n_confvars),
+                     &(configdata.data.conf4worker->confvars));
+    configdata.data_which                    = ConfigData_data_workerconfig;
+    configdata.data.conf4worker->clustername = c->myconf->cluster_config->clustername;
+
+    rc = xce_pushconfig_req(ep, c->myhost, &configdata);
+
+    deref_objs(configdata.data.conf4worker->n_confvars,
+               (void**)configdata.data.conf4worker->confvars);
+
+    FREEUP(configdata.data.conf4worker->confvars);
+    FREEUP(configdata.data.conf4worker);
+
+    return rc;
+}
+
+/* consider to push jobs on hosts to */
+int on_worker_join(void*       cls,
+                   endpoint_t* ep)
+{
+    cluster_t* c  = (cluster_t*)cls;
+    int        rc = 0;
+
+    rc = worker_push_config(c, ep);
+
+    return rc;
+}
+
+int get_resval(cluster_t* c,
+               char*      resname,
+               char*      host,
+               xce_val_t* val)
+{
+    xce_resource_t* r;
+    zarray_t*       array = NULL;
+    xce_val_t*      v     = NULL;
+    worker_t*       w     = NULL;
+
+    r = resmgr_lockres(&(c->resmgr),
+                       resname,
+                       OBJREF_READ);
+    if (r == NULL) {
+        return -ENOENT;
+    }
+
+    if ((r->flags & XCE_RESF_SHARED) == 0) {
+        if ((host == NULL) || (strlen(host) == 0)) {
+            obj_deref(r);
+
+            return -EINVAL;
+        }
+
+        w = workermgr_lock(&(c->workermgr),
+                           host, OBJREF_READ);
+        if (w == NULL) {
+            DEREF_OBJ(r);
+            return -ENOENT;
+        }
+
+        array = &(w->host.resvals);
+    } else {
+        array = &(c->resmgr.sharedinst);
+    }
+
+    if (r->idx < array->caps) {
+        v = (xce_val_t*)array->data[r->idx];
+    }
+
+    obj_deref(r);
+
+    if (v == NULL) {
+        DEREF_OBJ(w);
+        return -ENOENT;
+    }
+
+    memcpy(val, v, sizeof(*val));
+    DEREF_OBJ(w);
+    return 1;
+}
+
+int handle_worker_leave(cluster_t* c,
+                        int        idx)
+{
+    worker_t* w = NULL;
+
+    TRACE("%s worker %d broken connection",
+          __func__, idx);
+
+    if (c == NULL) {
+        SET_ERROR(EINVAL);
+        return -errno;
+    }
+
+    if (idx < 0) {
+        SET_ERROR(EINVAL);
+        return -errno;
+    }
+
+    w = workermgr_lock_byid(&(c->workermgr),
+                            idx, OBJREF_UPDATE);
+    if (w == NULL) {
+        ERROR("fail to find worker by id %d", idx);
+        SET_ERROR(E_XCE_BAD_WORKER);
+        return -errno;
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_WORKER,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_WORKER_DEAD,
+                  c, w);
+    w->host.status |= XCE_WORKER_UNAVAIL;
+    w->ep = NULL;
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_WORKER,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_WORKER_DEAD,
+                  c, w);
+    obj_deref(w);
+    return 0;
+}
+
+int worker_ctrl(cluster_t* c, xce_hostctrl_t* ctrl)
+{
+    worker_t* w  = NULL;
+    int       rc = 0;
+
+    if (!c || !ctrl || !ctrl->hostname || *ctrl->hostname == '\0') {
+        TRACE("%s invalid arguments.", __func__);
+        return -EINVAL;
+    }
+    TRACE("%s worker_ctrl worker %s", __func__, ctrl->hostname);
+    w = workermgr_lock(&(c->workermgr), ctrl->hostname, OBJREF_UPDATE);
+    if (w == NULL) {
+        PERROR("workermgr_lock");
+        return -ENOENT;
+    }
+
+    switch (ctrl->action) {
+    case HOST_OPEN:
+        if (!(w->host.status & XCE_WORKER_CLOSED)) {
+            goto clean;
+        }
+        w->host.status &= ~XCE_WORKER_CLOSED;
+        break;
+    case HOST_CLOSE:
+        if (w->host.status & XCE_WORKER_CLOSED) {
+            goto clean;
+        }
+        w->host.status |= XCE_WORKER_CLOSED;
+        break;
+        break;
+    default:
+        rc = -1;
+        break;
+    }
+    if (rc == 0) {
+        xce_run_hooks(&(c->hooksys),
+                      XCE_CONTROLLER_HOOK_WORKER,
+                      HOOK_AFTER_EVENT,
+                      HOOK_DETAIL_WORKER_UPDATE,
+                      c, w);
+    }
+clean:
+    obj_deref(w);
+    return rc;
+}

src/controller/workermgr.c

@@ -0,0 +1,216 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cluster.h"
+#include "htab.h"
+#include "mylock.h"
+#include "xce/cluster_def.h"
+#include "xce/errnox.h"
+#include "xstring.h"
+
+extern void worker_obj_killer(worker_t*);
+
+int workermgr_init(workermgr_t* mgr,
+                   int          nslots,
+                   int          delta)
+{
+    TRACE("%s nslots %d delta %d",
+          __func__, nslots, delta);
+
+    mylock_init(&(mgr->lock));
+    ht_init(&(mgr->htab), nslots,
+            string_hash_key);
+    zarray_init(&(mgr->workers), delta,
+                (zarray_item_killer_t)worker_obj_killer);
+    return 0;
+}
+
+void workermgr_fini(workermgr_t* mgr)
+{
+    zarray_fini(&(mgr->workers));
+    ht_fini(&(mgr->htab));
+    mylock_fini(&(mgr->lock));
+}
+
+static int32_t workermgr_add_(workermgr_t* mgr,
+                              worker_t*    w)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab), (void*)w->host.name);
+    if (e != NULL) {
+        return (int)e->data;
+    }
+
+    idx = zarray_store(&(mgr->workers), w);
+    if (idx < 0) {
+        return idx;
+    }
+
+    e = ht_add(&(mgr->htab),
+               (void*)w->host.name);
+    if (e == NULL) {
+        zarray_del(&(mgr->workers), idx);
+        return -errno;
+    }
+
+    e->data = idx;
+
+    return idx;
+}
+
+int32_t workermgr_add(workermgr_t* mgr,
+                      worker_t*    w)
+{
+    int idx;
+
+    if ((mgr == NULL) || (w == NULL)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(mgr->lock));
+    idx = workermgr_add_(mgr, w);
+    mylock_unlock(&(mgr->lock));
+
+    return idx;
+}
+
+static worker_t* workermgr_lock_(workermgr_t* mgr,
+                                 char*        name,
+                                 int          op)
+{
+    hent_t*   e = NULL;
+    int       idx;
+    worker_t* w = NULL;
+
+    e = ht_get(&(mgr->htab), (void*)name);
+    if (e == NULL) {
+        SET_ERROR(E_XCE_BAD_WORKER);
+        return NULL;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        SET_ERROR(E_XCE_BAD_WORKER);
+        return NULL;
+    }
+
+    w = zarray_get(&(mgr->workers), idx);
+
+    return obj_ref(w, op);
+}
+
+worker_t* workermgr_lock(workermgr_t* mgr,
+                         char*        name,
+                         int          op)
+{
+    worker_t* w = NULL;
+
+    if (mgr == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    if (name == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    /* clang-format off */
+    if ((op != OBJREF_NOP) &&
+	(op != OBJREF_READ) &&
+	(op != OBJREF_UPDATE)) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(mgr->lock));
+    w = workermgr_lock_(mgr, name, op);
+    mylock_unlock(&(mgr->lock));
+
+    return w;
+}
+
+worker_t* workermgr_lock_byid(workermgr_t* mgr,
+                              int          idx,
+                              int          op)
+{
+    worker_t* w = NULL;
+
+    if (mgr == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    if ((idx < 0) || (idx >= mgr->workers.caps)) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    /* clang-format off */
+    if ((op != OBJREF_NOP) &&
+	(op != OBJREF_READ) &&
+	(op != OBJREF_UPDATE)) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(mgr->lock));
+    w = (worker_t*)zarray_get(&(mgr->workers), idx);
+    if (w != NULL) {
+        w = obj_ref(w, op);
+    }
+    mylock_unlock(&(mgr->lock));
+
+    return w;
+}
+
+static void workermgr_del_(workermgr_t* mgr,
+                           worker_t*    w)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab),
+               (void*)w->host.name);
+    if (e == NULL) {
+        return;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        return;
+    }
+
+    zarray_del(&(mgr->workers), idx);
+}
+
+void workermgr_del(workermgr_t* mgr,
+                   worker_t*    w)
+{
+    if ((mgr == NULL) || (w == NULL)) {
+        SET_ERROR(EINVAL);
+        return;
+    }
+
+    mylock_lock(&(mgr->lock));
+    workermgr_del_(mgr, w);
+    mylock_unlock(&(mgr->lock));
+}
+
+int workermgr_iter(workermgr_t* mgr,
+                   xlist_t*     l,
+                   void*        arg)
+{
+    int rc = 0;
+
+    mylock_lock(&(mgr->lock));
+    rc = zarray_iter_ex(&(mgr->workers), l, arg);
+    mylock_unlock(&(mgr->lock));
+
+    return rc;
+}

src/controller/wrap.c

@@ -0,0 +1,59 @@
+#include <string.h>
+
+#include "utils.h"
+#include "xce/errnox.h"
+
+int __wrap___xpg_strerror_r(int    errnum,
+                            char*  buf,
+                            size_t bufsize)
+{
+    return xce_strerror_r(errnum, buf, bufsize);
+}
+
+static void xce_perror_(const char* s, int errnum)
+{
+    char    buf[XCE_MAX_ERRMSG_LEN];
+    ectx_t* ectx = get_ectx();
+
+    xce_strerror_r(errnum, buf, sizeof(buf));
+
+    if (tlog_get_root() == NULL) {
+        if ((ectx->file != NULL) && (ectx->func != NULL) && (ectx->line > 0) && (ectx->code == errnum)) {
+            fprintf(stderr, "%s: %s at %s:%d %s\n",
+                    s, buf, ectx->file,
+                    ectx->line, ectx->func);
+        } else {
+            fprintf(stderr, "%s: %s", s, buf);
+        }
+    } else {
+        if ((ectx->file != NULL) && (ectx->func != NULL) && (ectx->line > 0) && (ectx->code == errnum)) {
+            tlog_error("%s: %s at %s:%d %s\n",
+                       s, buf, ectx->file,
+                       ectx->line, ectx->func);
+        } else {
+            tlog_error("%s: %s", s, buf);
+        }
+    }
+}
+
+void xce_perror(const char* s)
+{
+    xce_perror_(s, errno);
+}
+
+void __wrap_perror(const char* s)
+{
+    xce_perror(s);
+}
+
+int __wrap_printf(const char* fmt, ...)
+{
+    va_list ap;
+    int     rc;
+
+    va_start(ap, fmt);
+    rc = xce_vprintf(fmt, ap);
+    va_end(ap);
+
+    return rc;
+}