use nix build & meson & zlog

src/lib/client.c

@@ -0,0 +1,629 @@
+#include <errno.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <sys/socket.h>
+
+#include "client.h"
+#include "controller.capnp.h"
+#include "mylock.h"
+
+#include "request.h"
+#include "xce/types.h"
+#include "xceapi.h"
+#include "xcectx.h"
+#include "xlist.h"
+
+typedef enum {
+    API_CTX_INIT = 0,
+    API_CTX_RUN,
+    API_CTX_DOWN,
+    API_CTX_ERR
+} api_ctx_state_t;
+
+typedef struct {
+    xce_client_config_t* conf;
+    endpoint_t           ep;
+    xce_ctx_t            ctx;
+} api_ctx_t;
+
+int xcecli_ctx_init(xcecli_ctx_t*        c,
+                    xce_client_config_t* cfg)
+{
+    api_ctx_t* ctx;
+    int        rc;
+
+    if ((c == NULL) || (cfg == NULL)) {
+        return -EINVAL;
+    }
+
+    if (cfg->nmasters < 1) {
+        return -EINVAL;
+    }
+
+    ctx = (api_ctx_t*)calloc(1, sizeof(*ctx));
+    if (ctx == NULL) {
+        return -ENOMEM;
+    }
+
+    ctx->conf = cfg;
+    rc        = xce_ctx_init(&(ctx->ctx),
+                             ctx->conf->from,
+                             ctx->conf->key);
+    if (rc < 0) {
+        free(ctx);
+        return rc;
+    }
+
+    endpoint_init(&(ctx->ep), AF_INET,
+                  SOCK_STREAM, 0, 0);
+
+    xce_ctx_set_endpoint(&(ctx->ctx), &(ctx->ep));
+
+    *c = (void*)ctx;
+
+    return 0;
+}
+
+void xcecli_ctx_fini(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    if (ctx == NULL) {
+        return;
+    }
+
+    xce_ctx_fini(&(ctx->ctx));
+    endpoint_deref(&(ctx->ep));
+
+    free(ctx);
+}
+
+int xcecli_set_handler(xcecli_ctx_t          c,
+                       xce_request_handler_t handler)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if ((c == NULL) || (handler == NULL)) {
+        return -EINVAL;
+    }
+
+    return xce_ctx_set_handler(&(ctx->ctx), handler);
+}
+
+int xcecli_wakeup(xcecli_ctx_t c,
+                  int          status)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_wakeup(&(ctx->ctx), status);
+}
+
+int xcecli_try_wakeup(xcecli_ctx_t c,
+                      int          status)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_try_wakeup(&(ctx->ctx), status);
+}
+
+int xcecli_start(xcecli_ctx_t c)
+{
+    int        rc;
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    rc = endpoint_connect(&(ctx->ep),
+                          ctx->conf->masters[0],
+                          ctx->conf->master_ports[0]);
+    if (rc < 0) {
+        endpoint_deref(&(ctx->ep));
+        return rc;
+    }
+
+    return xce_ctx_start(&(ctx->ctx));
+}
+
+int xcecli_stop(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    xce_ctx_stop(&(ctx->ctx));
+    endpoint_deref(&(ctx->ep));
+
+    return 0;
+}
+
+endpoint_t* xcecli_get_my_endpoint(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return NULL;
+    }
+
+    return &(ctx->ep);
+}
+
+xce_peerinfo_t* xcecli_get_myinfo(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return NULL;
+    }
+
+    return &(ctx->ctx.myinfo);
+}
+
+int xcecli_hello(xcecli_ctx_t c,
+                 char*        msg)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_hello(&(ctx->ctx), msg);
+}
+
+int xcecli_shutdown(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_shutdown(&(ctx->ctx));
+}
+
+int xcecli_reslist(xcecli_ctx_t      c,
+                   xce_resource_t*** res)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_reslist(&(ctx->ctx), res);
+}
+
+int xcecli_join(xcecli_ctx_t c,
+                int          role,
+                hostattrs_t* attrs)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_join(&(ctx->ctx), role, attrs);
+}
+
+int xcecli_workerlist(xcecli_ctx_t    c,
+                      xce_worker_t*** workers)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_workerlist(&(ctx->ctx), workers);
+}
+
+int xcecli_jobdispatch(xcecli_ctx_t        c,
+                       xce_job_dispatch_t* info)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_jobdispatch(&(ctx->ctx), info);
+}
+
+int xcecli_jobsubmit(xcecli_ctx_t      c,
+                     xce_job_submit_t* submit,
+                     uint64_t*         jobid)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_jobsubmit(&(ctx->ctx), submit, jobid);
+}
+
+int xcecli_joblist(xcecli_ctx_t c,
+                   xce_job_t*** jobs)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_joblist(&(ctx->ctx), jobs);
+}
+
+int xcecli_clusterconf(xcecli_ctx_t        c,
+                       xce_clusterconf_t** conf)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_clusterconf(&(ctx->ctx), conf);
+}
+
+int xcecli_notify_jobfinish(xcecli_ctx_t     c,
+                            xce_jobfinish_t* jfinish)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_jobFinishInfo,
+        .data  = {
+             .finishinfo = jfinish
+	}
+    };
+    /* clang-format on */
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    if (jfinish == NULL) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_update_resvals(xcecli_ctx_t   c,
+                          int            nres,
+                          xce_resval_t** vals)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    int        rc;
+
+    if ((ctx == NULL) || (vals == NULL)) {
+        return -EINVAL;
+    }
+
+    rc = xce_update_resvals(&(ctx->ctx), nres, vals);
+
+    return rc;
+}
+
+int xcecli_update_jusage(xcecli_ctx_t   c,
+                         uint64_t       jobid,
+                         int            npids,
+                         int*           pids,
+                         int            npgids,
+                         int*           pgids,
+                         int            nusage,
+                         xce_rusage_t** usages)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    int        rc;
+
+    if ((ctx == NULL) || (jobid == 0)) {
+        return -EINVAL;
+    }
+
+    rc = xce_update_jusage(&(ctx->ctx), jobid,
+                           npids, pids, npgids, pgids,
+                           nusage, usages);
+
+    return rc;
+}
+
+int xcecli_resval(xcecli_ctx_t    c,
+                  char*           host,
+                  char*           res,
+                  xce_resval_t*** val)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    if (res == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_resval(&(ctx->ctx), host, res, val);
+}
+
+int xcecli_notify(xcecli_ctx_t       c,
+                  xce_notify_data_t* data)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    if (data == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_notify(&(ctx->ctx), data);
+}
+
+int xcecli_notify_jobstart(xcecli_ctx_t   c,
+                           job_pidinfo_t* pidinfo)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_jobPidInfo,
+        .data  = {
+             .pidinfo = pidinfo
+	}
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (pidinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_notify_jobfail(xcecli_ctx_t    c,
+                          job_failinfo_t* failinfo)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_jobFailInfo,
+        .data  = {
+            .failinfo = failinfo
+        }
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (failinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_jobctrl(xcecli_ctx_t   c,
+                   xce_jobctrl_t* ctrlinfo)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    int        rc;
+
+    if ((ctx == NULL) || (ctrlinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    rc = xce_jobctrl(&(ctx->ctx), ctrlinfo);
+
+    return rc;
+}
+
+int xcecli_notify_jobstop(xcecli_ctx_t c,
+                          uint64_t     jobid)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_stoppedJob,
+        .data  = {
+             .stopped_job = jobid
+	}
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (jobid == 0)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_notify_jobresume(xcecli_ctx_t c,
+                            uint64_t     jobid)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_resumedJob,
+        .data  = {
+             .resumed_job = jobid
+	}
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (jobid == 0)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_notify_jobchkpt(xcecli_ctx_t c,
+                           uint64_t     jobid,
+                           char*        info)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_ckptinfo_t ckptinfo = {
+	.jobid = jobid,
+	.ckptinfo = info
+    };
+    xce_notify_data_t data = {
+        .event = NotifyData_data_ckptinfo,
+        .data  = {
+             .ckptinfo = &ckptinfo
+	}
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (jobid == 0) || (info == NULL)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_notify_jobpostdone(xcecli_ctx_t c,
+                              uint64_t     jobid)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_postDoneJob,
+        .data  = {
+             .pdone_job = jobid
+	}
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (jobid == 0)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_queuectrl(xcecli_ctx_t     c,
+                     xce_queuectrl_t* ctrlinfo)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    int        rc;
+
+    if ((ctx == NULL) || (ctrlinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    rc = xce_queuectrl(&(ctx->ctx), ctrlinfo);
+
+    return rc;
+}
+
+int xcecli_runtask(xcecli_ctx_t   c,
+                   xce_taskrun_t* info)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    int        rc;
+
+    if ((ctx == NULL) || (info == NULL)) {
+        return -EINVAL;
+    }
+
+    info->clientidx = -1; /* this field will be used by
+                           * controller to trace back
+                           * client
+                           */
+    rc = xce_runtask(&(ctx->ctx), info);
+
+    return rc;
+}
+
+int xcecli_notify_taskstart(xcecli_ctx_t        c,
+                            xce_task_pidinfo_t* pidinfo)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_taskPidInfo,
+        .data  = {
+             .taskinfo = pidinfo
+	}
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (pidinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_notify_taskfail(xcecli_ctx_t         c,
+                           xce_task_failinfo_t* failinfo)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_taskFailInfo,
+        .data  = {
+             .taskfailinfo = failinfo
+	  }
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (failinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+int xcecli_notify_taskfinish(xcecli_ctx_t       c,
+                             xce_task_finish_t* info)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    /* clang-format off */
+    xce_notify_data_t data = {
+        .event = NotifyData_data_taskFinish,
+        .data  = {
+             .taskfinish = info
+	  }
+    };
+    /* clang-format on */
+
+    if ((c == NULL) || (info == NULL)) {
+        return -EINVAL;
+    }
+
+    return xcecli_notify(ctx, &data);
+}
+
+void xcecli_block_tasks(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+    if (c == NULL) {
+        return;
+    }
+
+    xce_block_tasks(&(ctx->ctx));
+}
+
+void xcecli_unblock_tasks(xcecli_ctx_t c)
+{
+    api_ctx_t* ctx = (api_ctx_t*)c;
+
+    if (c == NULL) {
+        return;
+    }
+
+    xce_unblock_tasks(&(ctx->ctx));
+}

src/lib/dataq.c

@@ -0,0 +1,119 @@
+#include <stdlib.h>
+
+#include "dataq.h"
+
+int dataq_init(dataq_t*           q,
+               data_item_killer_t killer)
+{
+    q->head = q->tail = NULL;
+    q->killer         = killer;
+    mylock_init(&(q->lock));
+    return 0;
+}
+
+static data_item_t* dataq_deq_head(dataq_t* q)
+{
+    if (q->head != NULL) {
+        data_item_t* b = NULL;
+        b              = q->head;
+        q->head        = b->next;
+        b->next        = NULL;
+        if (q->head == NULL) {
+            q->tail = NULL;
+        }
+
+        return b;
+    }
+
+    return NULL;
+}
+
+static data_item_t* dataq_deq_first_match(dataq_t*             q,
+                                          data_item_match_fn_t f)
+{
+    data_item_t* prev = NULL;
+    data_item_t* curr = q->head;
+
+    while (curr != NULL) {
+        if ((*f)(curr)) { // Found the item to remove
+            if (prev == NULL) {
+                // Removing the head item
+                q->head = curr->next;
+                if (q->head == NULL) {
+                    // Queue became empty after removing head
+                    q->tail = NULL;
+                }
+                // else: tail remains unchanged (it wasn't the head)
+            } else {
+                // Removing an item after prev (middle or tail)
+                prev->next = curr->next;
+                if (curr == q->tail) { // Check if we are removing the tail
+                    // We removed the tail item, prev is the new tail
+                    q->tail = prev;
+                }
+                // else: tail remains unchanged (it was further down the list)
+            }
+            curr->next = NULL; // Isolate the removed item
+            return curr;       // Return the removed item
+        }
+        // Move to the next item
+        prev = curr;
+        curr = curr->next;
+    }
+
+    return NULL;
+}
+
+data_item_t* dataq_deq(dataq_t*             q,
+                       data_item_match_fn_t f)
+{
+    data_item_t* b = NULL;
+
+    mylock_lock(&(q->lock));
+    if (f == NULL) {
+        b = dataq_deq_head(q);
+    } else {
+        b = dataq_deq_first_match(q, f);
+    }
+
+    mylock_unlock(&(q->lock));
+
+    return b;
+}
+
+int dataq_enq(dataq_t* q, data_item_t* b)
+{
+    int rc = 0;
+
+    b->next = NULL;
+
+    mylock_lock(&(q->lock));
+    if (q->tail != NULL) {
+        q->tail->next = b;
+    }
+
+    if (q->head == NULL) {
+        q->head = b;
+    }
+
+    q->tail = b;
+    mylock_unlock(&(q->lock));
+
+    return rc;
+}
+
+void dataq_fini(dataq_t* q)
+{
+    data_item_t* it;
+    while ((it = dataq_deq_head(q)) != NULL) {
+         if (q->killer != NULL) {
+            (q->killer)(it);
+        }
+        // Note: dataq_deq_head already isolates 'it' (it->next = NULL)
+    }
+
+    q->head = q->tail = NULL; // Explicitly set just in case.
+    q->killer = NULL;
+   
+    return;
+}

src/lib/hook.c

@@ -0,0 +1,136 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "hook.h"
+#include "utils.h"
+
+int xce_hook_init(hooksys_t* s,
+                  int        maxtypes)
+{
+    int i;
+    if ((s == NULL) || (maxtypes <= 0)) {
+        return -EINVAL;
+    }
+
+    s->maxtypes = 0;
+    s->hooks    = (xlist_t*)calloc(maxtypes, sizeof(xlist_t));
+    if (s->hooks == NULL) {
+        return -ENOMEM;
+    }
+
+    s->maxtypes = maxtypes;
+    for (i = 0; i < s->maxtypes; i++) {
+        xlist_init(&(s->hooks[i]), 8);
+    }
+    return 0;
+}
+
+int xce_add_hook(hooksys_t* s,
+                 int        typ,
+                 hook_fn_t  fn,
+                 void*      data)
+{
+    hook_item_t* it;
+
+    if (s == NULL) {
+        return -EINVAL;
+    }
+
+    if (fn == NULL) {
+        return -EINVAL;
+    }
+
+    if ((typ < 0) || (typ >= s->maxtypes)) {
+        return -EINVAL;
+    }
+
+    it = (hook_item_t*)calloc(1, sizeof(*it));
+    if (it == NULL) {
+        return -ENOMEM;
+    }
+
+    it->fn   = fn;
+    it->data = data;
+
+    xlist_store(&(s->hooks[typ]), it);
+
+    return 0;
+}
+
+/* HOOK_BEFORE_EVENT returns
+ * negative value will cause
+ * fail
+ */
+int xce_run_hooks(hooksys_t* s,
+                  int        type,
+                  int        situation,
+		  int        detail,
+                  ...)
+{
+    xlist_t* l;
+    int      i;
+    int      rc = 0;
+
+    if (s == NULL) {
+        return -EINVAL;
+    }
+
+    if ((type < 0) || (type >= s->maxtypes)) {
+        return -EINVAL;
+    }
+
+    l = &(s->hooks[type]);
+
+    for (i = 0; i < l->count; i++) {
+        hook_item_t* it = l->data[i];
+	va_list ap;
+
+        if (it == NULL) {
+            continue;
+        }
+
+	va_start(ap, detail);
+        rc = (*(it->fn))(type, situation, detail,
+                         it->data, ap);
+	va_end(ap);
+	
+        if ((situation == HOOK_BEFORE_EVENT) && (rc < 0)) {
+            break;
+        }
+    }
+
+    return rc;
+}
+
+static int free_hook_item(void *data1,
+			  void *data2) {
+    if (data1 == NULL) {
+	return -EINVAL;
+    }
+
+    free(data1);
+    return 0;
+}
+
+void xce_hook_fini(hooksys_t* s)
+{
+    int i;
+
+    if (s == NULL) {
+        return;
+    }
+
+    if (s->hooks == NULL) {
+        return;
+    }
+
+    for (i = 0; i < s->maxtypes; i++) {
+	xlist_iter(&(s->hooks[i]),
+		   (xlist_iter_fn_t)free_hook_item,
+		   NULL);
+        xlist_fini(&(s->hooks[i]));
+    }
+
+    FREEUP(s->hooks);
+}

src/lib/htab.c

@@ -0,0 +1,215 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "htab.h"
+#include "utils.h"
+
+#define HASH_KEY_HASH(t, key) \
+    (((t)->fn.hash != NULL) ? ((t)->fn.hash)(key) : (unsigned long)key)
+
+#define HASH_KEY_CMP(t, k1, k2) \
+    (((t)->fn.cmp != NULL) ? ((t)->fn.cmp)((k1), (k2)) : ((k1) != (k2)))
+
+#define HASH_KEY_COPY(t, key) \
+    (((t)->fn.copy != NULL) ? ((t)->fn.copy)(key) : (key))
+
+#define HASH_KEY_FREE(t, key)                            \
+    do {                                                 \
+        if (((t)->fn.free != NULL) && ((key) != NULL)) { \
+            (t)->fn.free(key);                           \
+        }                                                \
+    } while (0)
+
+static unsigned int hash_code(const char* key)
+{
+    unsigned int hc = 5381;
+    int          c;
+
+    while ((c = *key++)) {
+        hc = ((hc << 5) + hc) + c;
+    }
+
+    return hc;
+}
+
+static unsigned int string_hash(void* data)
+{
+    return hash_code((const char*)data);
+}
+
+static int string_cmp(void* d1,
+                      void* d2)
+{
+    return (int)strcmp((char*)d1, (char*)d2);
+}
+
+static void* string_dup(void* data)
+{
+    return (void*)strdup((char*)data);
+}
+
+static void string_free(void* data)
+{
+    if (data != NULL) {
+        free(data);
+    }
+}
+
+hash_key_fn_t string_hash_key = {
+    .hash = (hash_key_hash_fn_t)string_hash,
+    .cmp  = (hash_key_cmp_fn_t)string_cmp,
+    .copy = (hash_key_copy_fn_t)string_dup,
+    .free = (hash_key_free_fn_t)string_free
+};
+
+hash_key_fn_t int_hash_key = {
+    .hash = NULL,
+    .cmp  = NULL,
+    .copy = NULL,
+    .free = NULL
+};
+
+int ht_init(htab_t*       t,
+            int           nslots,
+            hash_key_fn_t fn)
+{
+    if ((t == NULL) || (nslots <= 1)) {
+        return -EINVAL;
+    }
+
+    mylock_init(&(t->lock));
+    t->slots = (hent_t**)calloc(nslots, sizeof(hent_t*));
+    if (t->slots == NULL) {
+        return -ENOMEM;
+    }
+
+    t->nslots = nslots;
+    t->fn     = fn;
+    return 0;
+}
+
+void ht_fini(htab_t* t)
+{
+    int     i;
+    hent_t* e;
+
+    if (t == NULL) {
+        return;
+    }
+
+    for (i = 0; i < t->nslots; i++) {
+        while ((e = t->slots[i]) != NULL) {
+            t->slots[i] = e->next;
+            e->next     = NULL;
+            HASH_KEY_FREE(t, e->key);
+            free(e);
+        }
+    }
+
+    free(t->slots);
+
+    return;
+}
+
+hent_t* ht_get(htab_t* t,
+               void*   key)
+{
+    unsigned int hc;
+    hent_t*      e;
+
+    if (t == NULL) {
+        SET_ERROR(EINVAL);
+
+        return NULL;
+    }
+
+    hc = HASH_KEY_HASH(t, key) % (t->nslots);
+
+    mylock_lock(&(t->lock));
+    e = t->slots[hc];
+
+    while (e != NULL) {
+        if (HASH_KEY_CMP(t, e->key, key) == 0) {
+            break;
+        }
+
+        e = e->next;
+    }
+    mylock_unlock(&(t->lock));
+    return e;
+}
+
+hent_t* ht_add(htab_t* t,
+               void*   key)
+{
+    unsigned int hc;
+    hent_t*      e;
+
+    if ((t == NULL) || (key == NULL)) {
+        return NULL;
+    }
+
+    e = ht_get(t, key);
+    if (e != NULL) {
+        return e;
+    }
+
+    e = (hent_t*)calloc(1, sizeof(*e));
+    if (e == NULL) {
+        return NULL;
+    }
+
+    e->key = HASH_KEY_COPY(t, key);
+    if (e->key == NULL) {
+        free(e);
+        return NULL;
+    }
+
+    e->data = -1;
+
+    hc = HASH_KEY_HASH(t, key) % (t->nslots);
+    mylock_init(&(t->lock));
+    e->next      = t->slots[hc];
+    t->slots[hc] = e;
+    mylock_unlock(&(t->lock));
+
+    return e;
+}
+
+int ht_del(htab_t* t,
+           void*   key)
+{
+    unsigned int hc;
+    hent_t**     e;
+    int          deleted = 0;
+
+    if ((t == NULL) || (key == NULL)) {
+        return -EINVAL;
+    }
+
+    hc = HASH_KEY_HASH(t, key) % (t->nslots);
+    mylock_init(&(t->lock));
+    e = &(t->slots[hc]);
+    while ((*e) != NULL) {
+        if (HASH_KEY_CMP(t, (*e)->key, key) == 0) {
+            hent_t* e1 = *e;
+
+            *e = e1->next;
+            HASH_KEY_FREE(t, e1->key);
+            free(e1);
+            deleted = 1;
+            break;
+        }
+
+        e = &((*e)->next);
+    }
+
+    mylock_unlock(&(t->lock));
+
+    if (deleted) {
+        return 1;
+    }
+
+    return -ENOENT;
+}

src/lib/kvmgr.c

@@ -0,0 +1,211 @@
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "htab.h"
+#include "kvmgr.h"
+#include "objmgr.h"
+#include "utils.h"
+#include "xce/types.h"
+#include "zarray.h"
+
+int kvmgr_init(kvmgr_t*             mgr,
+               hash_key_fn_t        keyfn,
+               int                  nslots,
+               int                  delta,
+               zarray_item_killer_t killer)
+{
+    ht_init(&(mgr->htab), nslots, keyfn);
+    zarray_init(&(mgr->data), delta, killer);
+    mylock_init(&(mgr->lock));
+    return 0;
+}
+
+void kvmgr_fini(kvmgr_t* mgr)
+{
+    ht_fini(&(mgr->htab));
+    zarray_fini(&(mgr->data));
+    mylock_fini(&(mgr->lock));
+}
+
+static void* kvmgr_lock_(kvmgr_t* mgr,
+                         void*    key,
+                         int      op)
+{
+    hent_t* e = NULL;
+    int32_t idx;
+    void*   ptr;
+
+    e = ht_get(&(mgr->htab), key);
+    if (e == NULL) {
+        SET_ERROR(ENOENT);
+        return NULL;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        SET_ERROR(ENOENT);
+        return NULL;
+    }
+
+    ptr = (void*)zarray_get(&(mgr->data), idx);
+
+    return obj_ref(ptr, op);
+}
+
+void* kvmgr_lock(kvmgr_t* mgr,
+                 void*    key,
+                 int      op)
+{
+    void* ptr = NULL;
+
+    if (mgr == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    /* clang-format off */
+    if ((op != OBJREF_NOP) &&
+	(op != OBJREF_READ) &&
+	(op != OBJREF_UPDATE)) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(mgr->lock));
+    ptr = kvmgr_lock_(mgr, key, op);
+    mylock_unlock(&(mgr->lock));
+
+    return ptr;
+}
+
+static int32_t kvmgr_add_(kvmgr_t* mgr,
+                          void*    key,
+                          void*    p)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab), key);
+    if (e != NULL) {
+        return -EEXIST;
+    }
+
+    idx = zarray_store(&(mgr->data), p);
+    if (idx < 0) {
+        return idx;
+    }
+
+    e = ht_add(&(mgr->htab),
+               (void*)key);
+    if (e == NULL) {
+        zarray_del(&(mgr->data), idx);
+        return -errno;
+    }
+
+    e->data = idx;
+    return idx;
+}
+
+int32_t kvmgr_add(kvmgr_t* mgr,
+                  void*    key,
+                  void*    p)
+{
+    int32_t idx;
+
+    if ((mgr == NULL) || (p == NULL)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(mgr->lock));
+    idx = kvmgr_add_(mgr, key, p);
+    mylock_unlock(&(mgr->lock));
+
+    return idx;
+}
+
+static void kvmgr_del_(kvmgr_t* mgr,
+                       void*    key)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->htab), key);
+    if (e == NULL) {
+        return;
+    }
+
+    idx = e->data;
+    if (idx < 0) {
+        return;
+    }
+
+    zarray_del(&(mgr->data), idx);
+
+    return;
+}
+
+void kvmgr_del(kvmgr_t* mgr,
+               void*    key)
+{
+    if (mgr == NULL) {
+        return;
+    }
+    mylock_lock(&(mgr->lock));
+    kvmgr_del_(mgr, key);
+    mylock_unlock(&(mgr->lock));
+}
+
+static int add_config_(xlist_t* l, void* data)
+{
+    xce_config_t* config;
+
+    if (data == NULL) {
+        return -EINVAL;
+    }
+
+    config = (xce_config_t*)obj_ref(data, OBJREF_READ);
+    return xlist_store(l, config);
+}
+
+int kvmgr_publish(kvmgr_t*          mgr,
+                  xlist_action_fn_t fn,
+                  void*             arg)
+{
+    xlist_t l;
+    int     rc = 0;
+    int     i;
+
+    xlist_init(&l, 128);
+
+    mylock_lock(&(mgr->lock));
+    zarray_iter(&(mgr->data),
+                (zarray_iter_fn_t)add_config_,
+                &l);
+    mylock_unlock(&(mgr->lock));
+
+    rc = (*fn)(&l, arg);
+
+    for (i = 0; i < l.count; i++) {
+        xce_config_t* config = l.data[i];
+
+        DEREF_OBJ(config);
+    }
+
+    xlist_fini(&l);
+    return rc;
+}
+
+int kvmgr_iter(kvmgr_t* mgr,
+               xlist_t* l,
+               void*    arg)
+{
+    int rc = 0;
+
+    mylock_lock(&(mgr->lock));
+    rc = zarray_iter_ex(&(mgr->data), l, arg);
+    mylock_unlock(&(mgr->lock));
+
+    return rc;
+}

src/lib/meson.build

@@ -0,0 +1,69 @@
+capnp = find_program('capnp')
+
+gen_srcs = custom_target(
+  'controllercapnp.[ch]',
+  output:['controller.capnp.c', 'controller.capnp.h'],
+  input: '../schema/controller.capnp',
+  command: [capnp, 'compile','-o' ,'c:lib', '-I', '../schema','--src-prefix', '../schema', '@INPUT@']
+)
+
+libxce_src = [ gen_srcs[0],
+               'client.c',
+               'dataq.c',
+               'hook.c',
+               'htab.c',
+               'kvmgr.c',
+               'mylock.c',
+               'netio.c',
+               'objmgr.c',
+               'request.c',
+               'threshold.c',
+               'utils.c',
+               'vtaskmgr.c',
+               'window.c',
+               'xceapi.c',
+               'xcectx.c',
+               'xlist.c',
+               'xstring.c',
+               'zarray.c'
+             ]
+
+incdir = include_directories('../include')
+build_inc = include_directories('.')
+
+capnpc_c_prog = find_program('capnpc-c')
+
+# debug('capnpc_c_prog %s\n', capnpc_c_prog)
+
+thread_dep = dependency('threads')
+c_capnproto_dep = dependency('c-capnproto')
+zlog_dep = dependency('zlog')
+unity_dep = dependency('unity')
+
+libxce_deps = [c_capnproto_dep, zlog_dep, thread_dep]
+
+libxce = static_library('xce',
+                        libxce_src,
+                        include_directories: [incdir, build_inc],
+                        dependencies: libxce_deps,
+                        install: true)
+
+libxce_dep = declare_dependency(link_with: libxce)
+
+libxce_test_src = ['testsuite/test.c',
+                   'testsuite/test_error.c',
+                   'testsuite/test_htab.c',
+                   'testsuite/test_htab_int.c',
+                   'testsuite/test_utils.c',
+                   'testsuite/test_xstring.c',
+                   'testsuite/test_zarray.c'
+                  ]
+
+libxce_test_deps = [libxce_deps, unity_dep, libxce_dep]
+libxce_test = executable('xcelib_test',
+                         libxce_test_src,
+                         include_directories: [incdir, build_inc],
+                         dependencies: libxce_test_deps,
+                         install: true)
+
+test('libxce test', libxce_test)

src/lib/mylock.c

@@ -0,0 +1,212 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "mylock.h"
+
+int mylock_init(mylock_t* l)
+{
+    if (l == NULL) {
+        return -EINVAL;
+    }
+
+    return pthread_mutex_init(l, NULL);
+}
+
+void mylock_fini(mylock_t* l)
+{
+    if (l == NULL) {
+        return;
+    }
+
+    pthread_mutex_destroy(l);
+}
+int mylock_try(mylock_t* l)
+{
+    if (l == NULL) {
+        return -EINVAL;
+    }
+
+    return pthread_mutex_trylock(l);
+}
+
+int mylock_lock(mylock_t* l)
+{
+    if (l == NULL) {
+        return -EINVAL;
+    }
+
+    return pthread_mutex_lock(l);
+}
+
+int mylock_unlock(mylock_t* l)
+{
+    if (l == NULL) {
+        return -EINVAL;
+    }
+
+    return pthread_mutex_unlock(l);
+}
+
+int cbuf_init(cbuf_t* b,
+              int     cap,
+              int     isize)
+{
+    if ((b == NULL) || (cap <= 0) || (isize <= 0)) {
+        return -EINVAL;
+    }
+
+    b->ncap  = cap;
+    b->isize = isize;
+    b->data  = (void*)calloc(cap, isize);
+
+    if (b->data == NULL) {
+        return -EINVAL;
+    }
+
+    b->head = 0;
+    b->len  = 0;
+
+    pthread_mutex_init(&(b->mutex), NULL);
+    pthread_cond_init(&(b->condget), NULL);
+    pthread_cond_init(&(b->condput), NULL);
+
+    return 0;
+}
+
+void cbuf_fini(cbuf_t* b)
+{
+    if (b == NULL) {
+        return;
+    }
+
+    if (b->data != NULL) {
+        free(b->data);
+        b->data = NULL;
+    }
+
+    pthread_mutex_destroy(&(b->mutex));
+    pthread_cond_destroy(&(b->condput));
+    pthread_cond_destroy(&(b->condget));
+}
+
+int cbuf_put(cbuf_t* b,
+             void*   it)
+{
+    int i;
+
+    if ((b == NULL) || (it == NULL)) {
+        return -EINVAL;
+    }
+
+    pthread_mutex_lock(&(b->mutex));
+    while (b->len == b->ncap) {
+        pthread_cond_wait(&(b->condput),
+                          &(b->mutex));
+    }
+
+    i = (b->head + b->len) % b->ncap;
+
+    memcpy((void*)((char*)b->data) + b->isize * i,
+           it, b->isize);
+
+    b->len++;
+
+    pthread_cond_signal(&(b->condget));
+
+    pthread_mutex_unlock(&(b->mutex));
+    return 0;
+}
+
+int cbuf_tryput(cbuf_t* b,
+                void*   it)
+{
+    int i;
+    int rc = 0;
+
+    if ((b == NULL) || (it == NULL)) {
+        return -EINVAL;
+    }
+
+    pthread_mutex_lock(&(b->mutex));
+    if (b->len < b->ncap) {
+        i = (b->head + b->len) % b->ncap;
+
+        memcpy((void*)((char*)b->data) + b->isize * i,
+               it, b->isize);
+
+        b->len++;
+
+        pthread_cond_signal(&(b->condget));
+    } else {
+        rc = -EAGAIN;
+    }
+
+    pthread_mutex_unlock(&(b->mutex));
+    return rc;
+}
+
+int cbuf_get(cbuf_t* b,
+             void*   it)
+{
+    if ((b == NULL) || (it == NULL)) {
+        return -EINVAL;
+    }
+
+    pthread_mutex_lock(&(b->mutex));
+    while (b->len == 0) {
+        pthread_cond_wait(&(b->condget),
+                          &(b->mutex));
+    }
+
+    memcpy(it, (void*)((char*)b->data) + b->isize * b->head,
+           b->isize);
+
+    b->head = (b->head + 1) % b->ncap;
+    b->len--;
+
+    pthread_cond_signal(&(b->condput));
+
+    pthread_mutex_unlock(&(b->mutex));
+    return 0;
+}
+
+int my_rwlock_init(my_rwlock_t *l) {
+    if ( l == NULL) {
+	return -EINVAL;
+    }
+
+    return pthread_rwlock_init(l, NULL);
+}
+
+void my_rwlock_fini(my_rwlock_t *l) {
+    if (l == NULL) {
+	return;
+    }
+
+    pthread_rwlock_destroy(l);
+}
+
+int my_rwlock_unlock(my_rwlock_t *l) {
+    if (l == NULL)  {
+	return -EINVAL;
+    }
+
+    return pthread_rwlock_unlock(l);
+}
+
+int my_rwlock_rlock(my_rwlock_t *l) {
+    if (l == NULL) {
+	return -EINVAL;
+    }
+
+    return pthread_rwlock_rdlock(l);
+}
+
+int my_rwlock_wlock(my_rwlock_t *l) {
+    if (l==NULL) {
+	return -EINVAL;
+    }
+
+    return pthread_rwlock_wrlock(l);
+}

src/lib/netio.c

@@ -0,0 +1,687 @@
+#include <arpa/inet.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/epoll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "dataq.h"
+#include "netio.h"
+#include "request.h"
+#include "utils.h"
+
+static int  iomgr_update_poll(iomgr_t*, endpoint_t*, int);
+static void endpoint_fini(endpoint_t*);
+
+static int socket_create(int domain,
+                         int type,
+                         int port,
+                         int backlog)
+{
+    int sock;
+    int opt = 1;
+
+    if ((domain < 0) || (type < 0)) {
+        return -EINVAL;
+    }
+
+    sock = socket(domain, type, 0);
+    if (sock < 0) {
+        SET_ERROR(errno);
+
+        return -errno;
+    }
+
+    fcntl(sock, F_SETFD, (fcntl(sock, F_GETFD) | FD_CLOEXEC));
+
+    if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR | SO_REUSEPORT, &opt, sizeof(opt)) < 0) {
+        close(sock);
+        return -1;
+    }
+
+    if (port > 0) {
+        struct sockaddr_in sin;
+
+        memset((char*)&sin, 0, sizeof(sin));
+        sin.sin_family      = domain;
+        sin.sin_port        = htons(port);
+        sin.sin_addr.s_addr = htonl(INADDR_ANY);
+
+        if (bind(sock, (struct sockaddr*)&sin, sizeof(sin)) < 0) {
+            int rc;
+
+            SET_ERROR(errno);
+
+            rc = -errno;
+            (void)close(sock);
+
+            return rc;
+        }
+
+        if (backlog > 0) {
+            if (listen(sock, backlog) < 0) {
+                int rc;
+
+                SET_ERROR(errno);
+
+                rc = -errno;
+                (void)close(sock);
+
+                return rc;
+            }
+        }
+    }
+
+    return sock;
+}
+
+static int socket_connect(int   sock,
+                          char* host,
+                          int   port)
+{
+    struct sockaddr_in server;
+    struct hostent*    hp;
+    int                rc;
+
+    server.sin_family = AF_INET;
+    hp                = gethostbyname(host);
+    if (hp == NULL) {
+        SET_ERROR(errno);
+
+        rc = -errno;
+
+        return rc;
+    }
+
+    memcpy((char*)&server.sin_addr, (char*)hp->h_addr, (int)hp->h_length);
+
+    server.sin_port = htons(port);
+
+    rc = connect(sock, (struct sockaddr*)&server, sizeof(server));
+    if (rc < 0) {
+        SET_ERROR(errno);
+
+        return -errno;
+    }
+
+    return rc;
+}
+
+static int socket_accept(int sock)
+{
+    socklen_t          len;
+    int                s;
+    struct sockaddr_in from;
+
+    len = sizeof(from);
+    s   = accept(sock, (struct sockaddr*)&from, &len);
+    if (s < 0) {
+        SET_ERROR(errno);
+
+        return -errno;
+    }
+
+    fcntl(s, F_SETFD, (fcntl(s, F_GETFD) | FD_CLOEXEC));
+
+    return s;
+}
+
+int netio_write(int   fd,
+                char* data,
+                int   sz)
+{
+    int nc = 0;
+
+    while (sz > 0) {
+        int rc;
+        rc = write(fd, data + nc, sz);
+        if (rc < 0) {
+            SET_ERROR(errno);
+
+            return -errno;
+        }
+
+        sz -= rc;
+        nc += rc;
+    }
+
+    return nc;
+}
+
+int netio_read(int   fd,
+               char* data,
+               int   sz)
+{
+    int nc = 0;
+
+    while (sz > 0) {
+        int rc;
+
+        rc = read(fd, data + nc, sz);
+        if (rc < 0) {
+            SET_ERROR(errno);
+
+            return -errno;
+        }
+
+        if (rc == 0) {
+            /* network read return 0, it seems remote
+             * size has tear down the connection
+             */
+            return -EPIPE;
+        }
+
+        sz -= rc;
+        nc += rc;
+    }
+
+    return nc;
+}
+
+int write_packet(int          fd,
+                 io_packet_t* pkt)
+{
+    int rc;
+
+    rc = netio_write(fd, (char*)&(pkt->sz),
+                     sizeof(pkt->sz));
+    if (rc < 0) {
+        return -errno;
+    }
+
+    rc = netio_write(fd, pkt->data, pkt->sz);
+    if (rc < 0) {
+        return -errno;
+    }
+    return rc;
+}
+
+int alloc_packet(io_packet_t* pkt)
+{
+    if (pkt->sz <= 0) {
+        SET_ERROR(EINVAL);
+
+        return -EINVAL;
+    }
+
+    pkt->data = (char*)malloc(pkt->sz);
+    if (pkt->data == NULL) {
+        SET_ERROR(ENOMEM);
+
+        return -ENOMEM;
+    }
+
+    return 1;
+}
+int read_packet(int          fd,
+                io_packet_t* pkt)
+{
+    int rc;
+
+    rc = netio_read(fd, (char*)&(pkt->sz),
+                    sizeof(pkt->sz));
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = alloc_packet(pkt);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = netio_read(fd, pkt->data, pkt->sz);
+    if (rc < 0) {
+        free_packet(pkt);
+    }
+
+    return rc;
+}
+
+void free_packet(io_packet_t* pkt)
+{
+    if (pkt->data != NULL) {
+        free(pkt->data);
+        pkt->data = NULL;
+    }
+}
+
+#define MAX_EPOLL_EVENTS (5)
+
+static void* io_worker(void* args)
+{
+    iomgr_t* iomgr = (iomgr_t*)args;
+
+    while (1) {
+        int                nfds = -1;
+        int                i;
+        struct epoll_event events[MAX_EPOLL_EVENTS];
+
+        nfds = epoll_wait(iomgr->epoll_fd, events,
+                          MAX_EPOLL_EVENTS, -1);
+        if (nfds == -1) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            perror("epoll");
+
+            return NULL;
+        }
+
+        for (i = 0; i < nfds; i++) {
+            endpoint_t* p = (endpoint_t*)events[i].data.ptr;
+
+            if (p->handler != NULL) {
+                (p->handler)(&events[i]);
+            }
+        }
+
+        pthread_testcancel();
+    }
+
+    return NULL;
+}
+
+static int endpoint_calc_flags(endpoint_t* ep)
+{
+    int flags = EPOLLIN;
+
+    if (ep->outq.head != NULL) {
+        flags = flags | EPOLLOUT;
+    }
+
+    return flags;
+}
+
+endpoint_t* endpoint_get()
+{
+    endpoint_t* ep;
+
+    ep = (endpoint_t*)calloc(1, sizeof(*ep));
+    if (ep == NULL) {
+        SET_ERROR(errno);
+
+        return ep;
+    }
+
+    ep->fd       = -1;
+    ep->idx      = -1;
+    ep->iomgr    = NULL;
+    ep->busy     = 0;
+    ep->needfree = 1;
+
+    mylock_init(&(ep->lock));
+
+    return endpoint_ref(ep);
+}
+
+static void outq_item_killer(data_item_t* it)
+{
+    if (it != NULL) {
+        ioctx_t* ctx = ITPTR_TO_IOCTX(it);
+        ioctx_deref(ctx, 1);
+    }
+}
+int endpoint_init(endpoint_t* ep,
+                  int         domain,
+                  int         type,
+                  int         port,
+                  int         backlog)
+{
+    int fd = -1;
+
+    if (ep == NULL) {
+        SET_ERROR(EINVAL);
+
+        return -EINVAL;
+    }
+
+    fd = socket_create(domain, type,
+                       port, backlog);
+    if (fd < 0) {
+        return -errno;
+    }
+
+    ep->flags    = 0;
+    ep->fd       = fd;
+    ep->owner    = NULL;
+    ep->data     = NULL;
+    ep->iomgr    = NULL;
+    ep->busy     = 0;
+    ep->needfree = 0;
+
+    dataq_init(&(ep->outq),
+               outq_item_killer);
+    mylock_init(&(ep->lock));
+
+    endpoint_ref(ep);
+
+    return fd;
+}
+
+int endpoint_connect(endpoint_t* ep,
+                     char*       host,
+                     int         port)
+{
+    int rc = -1;
+
+    rc = socket_connect(ep->fd, host, port);
+    if (rc < 0) {
+        return rc;
+    }
+
+    ep->flags = 0;
+    return rc;
+}
+
+int endpoint_accept(endpoint_t* ep,
+                    endpoint_t* ep2)
+{
+    int sock = -1;
+
+    sock = socket_accept(ep->fd);
+    if (sock < 0) {
+        return sock;
+    }
+
+    ep2->fd    = sock;
+    ep2->flags = 0;
+
+    return sock;
+}
+
+int endpoint_set_handler(endpoint_t*  ep,
+                         msghandler_t h,
+                         void*        owner)
+{
+    if (ep->fd < 0) {
+        SET_ERROR(EINVAL);
+
+        return -EINVAL;
+    }
+
+    mylock_lock(&(ep->lock));
+    ep->handler = h;
+    ep->owner   = owner;
+    mylock_unlock(&(ep->lock));
+
+    return 1;
+}
+
+int endpoint_enq(endpoint_t*  ep,
+                 data_item_t* it)
+{
+    int rc = 0;
+    int flags;
+
+    if ((ep == NULL) || (it == NULL)) {
+        SET_ERROR(EINVAL);
+
+        return -EINVAL;
+    }
+
+    mylock_lock(&(ep->lock));
+    rc = dataq_enq(&(ep->outq), it);
+
+    flags = endpoint_calc_flags(ep);
+    if (flags != ep->flags) {
+        if (ep->iomgr != NULL) {
+            int rc1;
+
+            mylock_lock(&(ep->iomgr->lock));
+            rc1 = iomgr_update_poll(ep->iomgr,
+                                    ep, flags);
+            mylock_unlock(&(ep->iomgr->lock));
+
+            if (rc1 >= 0) {
+                ep->flags = flags;
+            } else {
+                WARN("rc1=%d < 0", rc1);
+            }
+        }
+    }
+    mylock_unlock(&(ep->lock));
+
+    return rc;
+}
+
+data_item_t* endpoint_deq(endpoint_t* ep)
+{
+    data_item_t* it = NULL;
+    int          flags;
+
+    if (ep == NULL) {
+        SET_ERROR(EINVAL);
+
+        return NULL;
+    }
+
+    mylock_lock(&(ep->lock));
+    it    = dataq_deq(&(ep->outq), NULL);
+    flags = endpoint_calc_flags(ep);
+    if (flags != ep->flags) {
+        if (ep->iomgr != NULL) {
+            int rc;
+
+            mylock_lock(&(ep->iomgr->lock));
+            rc = iomgr_update_poll(ep->iomgr,
+                                   ep, flags);
+            mylock_unlock(&(ep->iomgr->lock));
+
+            if (rc >= 0) {
+                ep->flags = flags;
+            }
+        }
+    }
+    mylock_unlock(&(ep->lock));
+
+    return it;
+}
+
+endpoint_t* endpoint_ref(endpoint_t* ep)
+{
+    mylock_lock(&(ep->lock));
+    ep->refcount++;
+    mylock_unlock(&(ep->lock));
+
+    return ep;
+}
+
+void endpoint_deref(endpoint_t* ep)
+{
+    uint32_t refcount;
+
+    mylock_lock(&(ep->lock));
+    ep->refcount--;
+
+    refcount = ep->refcount;
+    mylock_unlock(&(ep->lock));
+
+    if (refcount == 0) {
+        endpoint_fini(ep);
+    }
+}
+
+uint64_t endpoint_getseq(endpoint_t* ep)
+{
+    uint64_t seq;
+
+    mylock_lock(&(ep->lock));
+    seq = ep->seq++;
+    mylock_unlock(&(ep->lock));
+
+    return seq;
+}
+
+void endpoint_block_tasks(endpoint_t* ep)
+{
+    TRACE("%s[%ld] block tasks for endpoint %p",
+          __func__, pthread_self(), ep);
+    mylock_lock(&(ep->lock));
+    ep->busy = 1;
+    mylock_unlock(&(ep->lock));
+}
+
+void endpoint_unblock_tasks(endpoint_t* ep)
+{
+    TRACE("%s[%ld] unblock tasks for endpoint %p",
+          __func__, pthread_self(), ep);
+    mylock_lock(&(ep->lock));
+    ep->busy = 0;
+    mylock_unlock(&(ep->lock));
+}
+
+int endpoint_is_task_blocked(endpoint_t* ep)
+{
+    int busy;
+
+    mylock_lock(&(ep->lock));
+    busy = ep->busy;
+    mylock_unlock(&(ep->lock));
+
+    TRACE("%s[%ld] endpoint %p task %s",
+          __func__, pthread_self(),
+          ep, (busy != 0) ? "blocked" : "not blocked");
+
+    return (busy != 0);
+}
+
+static void endpoint_fini(endpoint_t* ep)
+{
+    int needfree = ep->needfree;
+
+    dataq_fini(&(ep->outq));
+    FREEUP(ep->data);
+
+    if (ep->fd >= 0) {
+        close(ep->fd);
+    }
+
+    ep->flags   = 0;
+    ep->handler = NULL;
+    ep->owner   = NULL;
+
+    if (needfree) {
+        FREEUP(ep);
+    }
+}
+
+int iomgr_init(iomgr_t* iomgr)
+{
+    mylock_init(&(iomgr->lock));
+    iomgr->epoll_fd = epoll_create1(0);
+    if (iomgr->epoll_fd < 0) {
+        SET_ERROR(errno);
+
+        return -errno;
+    }
+
+    return 0;
+}
+
+static int iomgr_update_poll(iomgr_t*    iomgr,
+                             endpoint_t* ep,
+                             int         flags)
+{
+    struct epoll_event ev;
+
+    ev.events   = flags;
+    ev.data.ptr = ep;
+
+    return epoll_ctl(iomgr->epoll_fd,
+                     EPOLL_CTL_MOD,
+                     ep->fd, &ev);
+}
+
+int iomgr_monitor(iomgr_t*    iomgr,
+                  endpoint_t* ep)
+{
+    struct epoll_event ev;
+    int                rc;
+    int                flags;
+
+    flags       = EPOLLIN;
+    ev.events   = flags;
+    ev.data.ptr = ep;
+
+    mylock_lock(&(ep->lock));
+    ep->iomgr = iomgr;
+    mylock_lock(&(iomgr->lock));
+    rc = epoll_ctl(iomgr->epoll_fd, EPOLL_CTL_ADD,
+                   ep->fd, &ev);
+    mylock_unlock(&(iomgr->lock));
+
+    if (rc >= 0) {
+        ep->flags = flags;
+    }
+
+    mylock_unlock(&(ep->lock));
+
+    return rc;
+}
+
+int iomgr_unmonitor(iomgr_t*    iomgr,
+                    endpoint_t* ep)
+{
+    int rc;
+
+    mylock_lock(&(iomgr->lock));
+    rc = epoll_ctl(iomgr->epoll_fd, EPOLL_CTL_DEL,
+                   ep->fd, NULL);
+    if (rc < 0) {
+        SET_ERROR(errno);
+
+        rc = -errno;
+    }
+    mylock_unlock(&(iomgr->lock));
+
+    return rc;
+}
+
+int iomgr_start(iomgr_t* iomgr)
+{
+    int rc;
+
+    rc = pthread_create(&(iomgr->worker), NULL, &io_worker, iomgr);
+    return rc;
+}
+
+int iomgr_fini(iomgr_t* iomgr)
+{
+    return 0;
+}
+
+int iomgr_stop(iomgr_t* iomgr)
+{
+    int   rc;
+    void* res;
+
+    if (iomgr->worker == 0) {
+        return 0;
+    }
+    
+    rc = pthread_cancel(iomgr->worker);
+    if (rc != 0) {
+        SET_ERROR(errno);
+
+        return -errno;
+    }
+
+    rc = pthread_join(iomgr->worker, &res);
+    if (rc != 0) {
+        SET_ERROR(errno);
+
+        return -errno;
+    }
+
+    if (res == PTHREAD_CANCELED) {
+        iomgr->worker = 0;
+        return 0;
+    }
+
+    return -1;
+}

src/lib/objmgr.c

@@ -0,0 +1,322 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "htab.h"
+#include "objmgr.h"
+#include "utils.h"
+#include "zarray.h"
+
+static objmgr_t* objmgr_default = NULL;
+
+static void objtype_killer(objtype_t* typeval)
+{
+    if (typeval == NULL) {
+        return;
+    }
+
+    if (typeval->fn_type_fini == NULL) {
+        return;
+    }
+
+    typeval->fn_type_fini(typeval);
+}
+
+int objmgr_init(objmgr_t* mgr)
+{
+    if (mgr == NULL) {
+        return -EINVAL;
+    }
+
+    mylock_init(&(mgr->lock));
+
+    ht_init(&(mgr->names), 32,
+            string_hash_key);
+    zarray_init(&(mgr->types), 32,
+                (zarray_item_killer_t)objtype_killer);
+    return 0;
+}
+
+void objmgr_fini(objmgr_t* mgr)
+{
+    if (mgr == NULL) {
+        return;
+    }
+
+    zarray_fini(&(mgr->types));
+    ht_fini(&(mgr->names));
+
+    mylock_fini(&(mgr->lock));
+}
+
+static int objmgr_register_internal(objmgr_t*  mgr,
+                                    objtype_t* type)
+{
+    hent_t* e = NULL;
+    int     idx;
+
+    e = ht_get(&(mgr->names), (void*)type->type);
+    if (e != NULL) {
+        return e->data;
+    }
+
+    e = ht_add(&(mgr->names), (void*)type->type);
+    if (e == NULL) {
+        return -ENOMEM;
+    }
+
+    idx = zarray_store(&(mgr->types), type);
+    if (idx < 0) {
+        ht_del(&(mgr->names), (void*)type->type);
+        return idx;
+    }
+
+    e->data = idx;
+    return idx;
+}
+
+int obj_init(objmgr_t*  mgr,
+             objtype_t* type,
+             int32_t    exdata,
+             myobj_t*   obj)
+{
+    obj->type   = type;
+    obj->nref   = 1;
+    obj->nread  = 0;
+    obj->nwrite = 0;
+    obj->exdata = exdata;
+    obj->magic  = OBJ_MAGIC;
+
+    my_rwlock_init(&(obj->rwlock));
+    mylock_init(&(obj->lock));
+
+    return 1;
+}
+
+int objmgr_register(objmgr_t*  mgr,
+                    objtype_t* type)
+{
+    int rc = 0;
+
+    /* clang-format off */
+    if ((mgr == NULL) ||
+	(type == NULL) ||
+	(type->type == NULL) ||
+	(type->fn_create == NULL)) {
+	return -EINVAL;
+    }
+    /* clang-format on */
+
+    mylock_init(&(type->lock));
+
+    mylock_lock(&(mgr->lock));
+    rc = objmgr_register_internal(mgr, type);
+    mylock_unlock(&(mgr->lock));
+
+    return rc;
+}
+
+static void* create_obj_ex(objmgr_t* mgr,
+                           int       type,
+                           va_list   ap)
+{
+    objtype_t* typeval = NULL;
+    void*      obj;
+
+    if ((type < 0) || (type >= mgr->types.caps)) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    typeval = (objtype_t*)zarray_get(&(mgr->types), type);
+    if (typeval == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    obj = typeval->fn_create(mgr, typeval, ap);
+
+    return obj;
+}
+
+void* create_obj_by_typeid(objmgr_t* mgr,
+                           int       type,
+                           ...)
+{
+    va_list ap;
+    void*   p = NULL;
+
+    va_start(ap, type);
+
+    p = create_obj_ex(mgr, type, ap);
+
+    va_end(ap);
+
+    return p;
+}
+
+void* create_obj_by_typename(objmgr_t* mgr,
+                             char*     type,
+                             ...)
+{
+    hent_t* e;
+    va_list ap;
+    void*   p = NULL;
+
+    mylock_lock(&(mgr->lock));
+    e = ht_get(&(mgr->names), (void*)type);
+    mylock_unlock(&(mgr->lock));
+
+    if (e == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    if (e->data < 0) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    va_start(ap, type);
+
+    p = create_obj_ex(mgr, e->data, ap);
+
+    va_end(ap);
+
+    return p;
+}
+
+int typeid_of_name(objmgr_t* mgr,
+                   char*     type)
+{
+    hent_t* e;
+    int typeid;
+
+    mylock_lock(&(mgr->lock));
+    e = ht_get(&(mgr->names), (void*)type);
+    if (e == NULL) {
+        typeid = -ENOENT;
+    } else {
+        typeid = e->data;
+    }
+    mylock_unlock(&(mgr->lock));
+
+    return typeid;
+}
+void* obj_ref(void* obj, int op)
+{
+    myobj_t* myobj;
+
+    if (obj == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    myobj = TO_MYOBJ(obj);
+    if (myobj->magic != OBJ_MAGIC) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    switch (op) {
+    case OBJREF_READ:
+        my_rwlock_rlock(&(myobj->rwlock));
+        break;
+    case OBJREF_UPDATE:
+        my_rwlock_wlock(&(myobj->rwlock));
+        break;
+    default:
+        break;
+    }
+
+    mylock_lock(&(myobj->lock));
+    switch (op) {
+    case OBJREF_READ:
+        myobj->nread++;
+        break;
+    case OBJREF_UPDATE:
+        myobj->nwrite++;
+        break;
+    default:
+        break;
+    }
+
+    myobj->nref++;
+    mylock_unlock(&(myobj->lock));
+
+    return obj;
+}
+
+void obj_deref(void* obj)
+{
+    myobj_t*   myobj;
+    int        nc;
+    objtype_t* typeval;
+    int        release_rwlock = 0;
+
+    if (obj == NULL) {
+        SET_ERROR(EINVAL);
+        return;
+    }
+
+    myobj = TO_MYOBJ(obj);
+    if (myobj->magic != OBJ_MAGIC) {
+        SET_ERROR(EINVAL);
+        return;
+    }
+
+    /* the object lock should be held */
+    mylock_lock(&(myobj->lock));
+    myobj->nref--;
+    nc = myobj->nref;
+    if ((myobj->nread > 0) || (myobj->nwrite > 0)) {
+        release_rwlock = 1;
+        if (myobj->nread > 0) {
+            myobj->nread--;
+        } else {
+            myobj->nwrite--;
+        }
+    }
+    mylock_unlock(&(myobj->lock));
+
+    if (release_rwlock) {
+        my_rwlock_unlock(&(myobj->rwlock));
+    }
+
+    if (nc > 0) {
+        return;
+    }
+
+    typeval = myobj->type;
+    if (typeval == NULL) {
+        return;
+    }
+
+    if (typeval->fn_fini != NULL) {
+        typeval->fn_fini(obj);
+    }
+
+    return;
+}
+
+objmgr_t* objmgr_get_default()
+{
+    return objmgr_default;
+}
+
+int objmgr_set_default(objmgr_t* mgr)
+{
+    objmgr_default = mgr;
+    return 0;
+}
+
+myobj_t *obj_alloc(size_t size) {
+    myobj_t *obj;
+    
+    if (size <= 0) {
+	SET_ERROR(EINVAL);
+	return NULL;
+    }
+
+    obj = (myobj_t *)calloc(1, sizeof(*obj) + size);
+    return obj;
+}

src/lib/request.c

@@ -0,0 +1,931 @@
+#include <errno.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "mylock.h"
+#include "netio.h"
+#include "request.h"
+#include "typemap.h"
+#include "utils.h"
+#include "xce/types.h"
+
+ioctx_t* ioctx_alloc(endpoint_t*     ep,
+                     xce_peerinfo_t* peer)
+{
+    ioctx_t* ctx;
+
+    ctx = (ioctx_t*)calloc(1, sizeof(*ctx));
+    if (ctx == NULL) {
+        errno = -ENOMEM;
+        return NULL;
+    }
+
+    ctx->ep   = endpoint_ref(ep);
+    ctx->from = peer;
+    ctx->nrefs++;
+    return ctx;
+}
+
+int ioctx_ref(ioctx_t* ctx)
+{
+    int old = ctx->nrefs;
+
+    ctx->nrefs++;
+
+    return old;
+}
+
+static void ioctx_kill(ioctx_t* ctx,
+                       int      free_ctx)
+{
+    if (ctx == NULL) {
+        return;
+    }
+
+    endpoint_deref(ctx->ep);
+
+    free_packet(&(ctx->pkt));
+    capn_free(&(ctx->c));
+    if (free_ctx) {
+        free(ctx);
+    }
+}
+
+void ioctx_deref(ioctx_t* ctx,
+                 int      free_ctx)
+{
+    if (ctx == NULL) {
+        return;
+    }
+
+    if (ctx->nrefs == 0) {
+        WARN("Something wrong. Have you forgotten to call  ioctx_ref()?");
+        return;
+    }
+    ctx->nrefs--;
+    if (ctx->nrefs == 0) {
+        ioctx_kill(ctx, free_ctx);
+    }
+}
+
+int ioctx_encode(ioctx_t*       ctx,
+                 xce_request_t* req)
+{
+    struct capn_segment* cs;
+    Request_ptr          rp;
+    int                  rc;
+
+    capn_init_malloc(&(ctx->c));
+    cs = capn_root(&(ctx->c)).seg;
+
+    encode_Request(cs, &(ctx->rq), req);
+
+    cs = capn_root(&(ctx->c)).seg;
+    rp = new_Request(cs);
+    write_Request(&(ctx->rq), rp);
+
+    capn_setp(capn_root(&(ctx->c)), 0, rp.p);
+
+    ctx->pkt.sz = capn_size(&(ctx->c));
+    rc          = alloc_packet(&(ctx->pkt));
+    if (rc < 0) {
+        return rc;
+    }
+
+    capn_write_mem(&(ctx->c),
+                   (uint8_t*)ctx->pkt.data,
+                   ctx->pkt.sz, 0);
+    return 0;
+}
+
+int ioctx_decode(ioctx_t*       ctx,
+                 xce_request_t* req)
+{
+    Request_ptr rp;
+
+    capn_init_mem(&(ctx->c), (const uint8_t*)ctx->pkt.data,
+                  ctx->pkt.sz, 0);
+    rp.p = capn_getp(capn_root(&(ctx->c)), 0, 1);
+
+    read_Request(&(ctx->rq), rp);
+
+    decode_Request(req, &(ctx->rq));
+
+    return 0;
+}
+
+int ioctx_flush(ioctx_t* ctx, int fd)
+{
+    int rc;
+
+    rc = write_packet(fd, &(ctx->pkt));
+    return rc;
+}
+
+void init_reply_by_req(xce_reply_t*   reply,
+                       xce_request_t* req)
+{
+    reply->seq    = req->seq;
+    reply->opcode = req->opcode;
+}
+
+int xce_request(ioctx_t*       ctx,
+                xce_request_t* req)
+{
+    int rc;
+
+    req->seq  = endpoint_getseq(ctx->ep);
+    req->from = ctx->from;
+
+    rc = ioctx_encode(ctx, req);
+    if (rc < 0) {
+        return rc;
+    }
+
+    endpoint_enq(ctx->ep, &(ctx->link));
+    return 0;
+}
+
+int xce_reply_with_ctx(ioctx_t*        ctx,
+                       xce_peerinfo_t* from,
+                       xce_reply_t*    reply)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_reply,
+        .body_which = Request_body_replymsg,
+    };
+    /* clang-format on */
+    int rc;
+
+    TRACE("%s seq %lu opcode %d status %d data which %d",
+          __func__, reply->seq, reply->opcode,
+          reply->status, reply->data_which);
+    
+    request.from          = from;
+    request.body.replymsg = reply;
+
+    request.seq = endpoint_getseq(ctx->ep);
+
+    rc = ioctx_encode(ctx, &request);
+    if (rc < 0) {
+        return rc;
+    }
+
+    endpoint_enq(ctx->ep, &(ctx->link));
+    return 0;
+}
+
+int xce_reply(endpoint_t*     ep,
+              xce_peerinfo_t* from,
+              xce_reply_t*    reply)
+{
+    ioctx_t* ctx;
+
+    ctx = ioctx_alloc(ep, from);
+    if (ctx == NULL) {
+        return -ENOMEM;
+    }
+
+    return xce_reply_with_ctx(ctx, from, reply);
+}
+
+int xce_nop_req(ioctx_t* ctx)
+{
+    xce_request_t request = {
+        .opcode     = OPCode_nop,
+        .body_which = Request_body_nomsg
+    };
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    return xce_request(ctx, &request);
+}
+
+int xce_hello_req(ioctx_t* ctx,
+                  char*    msg)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode     = OPCode_hello,
+        .body_which = Request_body_hellomsg,
+        .body       = {
+                  .hellomsg = msg
+	}
+    };
+    /* clang-format on */
+
+    if ((ctx == NULL) || (msg == NULL)) {
+        return -EINVAL;
+    }
+    return xce_request(ctx, &request);
+}
+
+int xce_join_req(ioctx_t*     ctx,
+                 int          role,
+                 hostattrs_t* attrs)
+{
+    /* clang-format off */
+    xce_joindata_t joinmsg = {
+        .role = role
+    };
+    xce_request_t request = {
+        .opcode = OPCode_join,
+        .body_which = Request_body_joinmsg,
+        .body = {
+            .joinmsg = &joinmsg 
+        }
+    };
+
+    if ((role != HostRole_client) &&
+	(role != HostRole_worker) &&
+	(role != HostRole_scheduler)) {
+        return -EINVAL;
+    }
+    /* clang-format on */
+
+    if (role == HostRole_worker) {
+        if (attrs == NULL) {
+            return -EINVAL;
+        }
+
+        joinmsg.data_which = JoinData_data_attrs;
+        joinmsg.data.attrs = attrs;
+    }
+    return xce_request(ctx, &request);
+}
+
+int xce_shutdown_req(ioctx_t* ctx)
+{
+    int           rc;
+    dataq_t*      outq;
+    xce_request_t request = {
+        .opcode     = OPCode_shutdown,
+        .body_which = Request_body_nomsg
+    };
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    rc = xce_request(ctx, &request);
+    if (rc < 0) {
+        return rc;
+    }
+
+    outq = &(ctx->ep->outq);
+
+    while (!DATAQ_EMPTY(outq)) {
+        sleep(1);
+    }
+
+    return 0;
+}
+
+int xce_pushconfig_req(endpoint_t*       ep,
+                       xce_peerinfo_t*   from,
+                       xce_configdata_t* config)
+{
+    int           rc      = 0;
+    xce_request_t request = {
+        .opcode     = OPCode_push,
+        .body_which = Request_body_configpushmsg
+    };
+    ioctx_t* ctx = NULL;
+
+    if ((ep == NULL) || (config == NULL)) {
+        return -EINVAL;
+    }
+
+    ctx = ioctx_alloc(ep, from);
+    if (ctx == NULL) {
+        return -ENOMEM;
+    }
+
+    request.body.configpushmsg = config;
+    rc                         = xce_request(ctx, &request);
+
+    return rc;
+}
+
+static int xce_push_req(endpoint_t*     ep,
+                        xce_peerinfo_t* from,
+                        xce_data_t*     data)
+{
+    int           rc      = 0;
+    ioctx_t*      ctx     = NULL;
+    xce_request_t request = {
+        .opcode     = OPCode_push,
+        .body_which = Request_body_pushmsg
+    };
+
+    if ((ep == NULL) || (from == NULL) || (data == NULL)) {
+        return -EINVAL;
+    }
+
+    ctx = ioctx_alloc(ep, from);
+    if (ctx == NULL) {
+        return -ENOMEM;
+    }
+
+    request.body.pushmsg = data;
+
+    rc = xce_request(ctx, &request);
+    return rc;
+}
+
+int xce_push_data_req(endpoint_t*     ep,
+                      xce_peerinfo_t* from,
+                      xce_data_t*     data,
+                      xce_check_fn_t  checkf,
+                      void*           checkarg,
+                      int             pktlimit)
+{
+    int        nrows = 0;
+    int        pos   = 0;
+    int        rc    = 0;
+    xce_data_t data1;
+    int        checkstatus = 0;
+
+    if ((ep == NULL) || (from == NULL) || (data == NULL)) {
+        return -EINVAL;
+    }
+
+    data1.table = data->table;
+    switch (data->table) {
+    case Data_data_schedjobs:
+        nrows           = data->data.jobs->n_total;
+        data1.data.jobs = (xce_jobs_data_t*)calloc(1, sizeof(xce_jobs_data_t));
+        if (data1.data.jobs == NULL) {
+            rc = -ENOMEM;
+        }
+        break;
+    case Data_data_schedworkers:
+        nrows              = data->data.workers->n_total;
+        data1.data.workers = (xce_workers_data_t*)calloc(1, sizeof(xce_workers_data_t));
+        if (data1.data.workers == NULL) {
+            rc = -ENOMEM;
+        }
+        break;
+    default:
+        rc = -EINVAL;
+        break;
+    }
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (pktlimit <= 0) {
+        pktlimit = nrows;
+    }
+
+    do {
+        int nc = nrows - pos;
+
+        if (nc > pktlimit) {
+            nc = pktlimit;
+        }
+
+        if (checkf != NULL) {
+            checkstatus = (*checkf)(checkarg);
+            if (checkstatus < 0) {
+                nc = 0;
+            }
+        }
+
+        switch (data->table) {
+        case Data_data_schedjobs:
+            data1.data.jobs->n_total = data->data.jobs->n_total;
+            data1.data.jobs->pos     = pos;
+            data1.data.jobs->count   = nc;
+            if (nc == 0) {
+                data1.data.jobs->jobs = NULL;
+            } else {
+                data1.data.jobs->jobs = &(data->data.jobs->jobs[pos]);
+            }
+            break;
+        case Data_data_schedworkers:
+            data1.data.workers->n_total = data->data.workers->n_total;
+            data1.data.workers->pos     = pos;
+            data1.data.workers->count   = nc;
+            if (nc == 0) {
+                data1.data.workers->workers = NULL;
+            } else {
+                data1.data.workers->workers = &(data->data.workers->workers[pos]);
+            }
+            break;
+        default:
+            break;
+        }
+
+        rc = xce_push_req(ep, from, &data1);
+
+        if (rc < 0) {
+            break;
+        }
+
+        pos += nc;
+
+    } while ((pos < nrows) && (checkstatus >= 0));
+
+    switch (data->table) {
+    case Data_data_schedjobs:
+        FREEUP(data1.data.jobs);
+        break;
+    case Data_data_schedworkers:
+        FREEUP(data1.data.workers);
+        break;
+    default:
+        break;
+    }
+
+    return rc;
+}
+
+int xce_get_req(ioctx_t*   ctx,
+                xce_get_t* get)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_get,
+        .body_which = Request_body_getmsg,
+        .body = {
+            .getmsg = get
+        }
+    };
+    /* clang-format on */
+
+    return xce_request(ctx, &request);
+}
+
+int xce_reply_get(endpoint_t*     ep,
+                  xce_peerinfo_t* from,
+                  xce_reply_t*    reply,
+                  xce_check_fn_t  checkf,
+                  void*           checkarg,
+                  int             pktlimit)
+{
+    int rc    = 0;
+    int nrows = 0;
+    int pos   = 0;
+    /* clang-format off */
+    xce_data_t getdata = {
+        .table = reply->data.getdata->table
+    };
+    xce_reply_t reply1 = {
+        .seq        = reply->seq,
+        .opcode     = reply->opcode,
+        .status     = reply->status,
+        .data_which = reply->data_which,
+        .data       = {
+            .getdata = &getdata
+        }
+    };
+    /* clang-format on */
+    int         checkstatus = 0;
+    xce_data_t* data        = reply->data.getdata;
+
+    if ((ep == NULL) || (from == NULL) || (reply == NULL)) {
+        return -EINVAL;
+    }
+
+    switch (getdata.table) {
+    case Data_data_schedjobs:
+    case Data_data_jobs:
+        nrows             = data->data.jobs->n_total;
+        getdata.data.jobs = (xce_jobs_data_t*)calloc(1, sizeof(xce_jobs_data_t));
+        break;
+    case Data_data_schedworkers:
+    case Data_data_workers:
+        nrows                = data->data.workers->n_total;
+        getdata.data.workers = (xce_workers_data_t*)calloc(1, sizeof(xce_workers_data_t));
+        break;
+    case Data_data_resources:
+        nrows                  = data->data.resources->n_total;
+        getdata.data.resources = (xce_resource_data_t*)calloc(1, sizeof(xce_resource_data_t));
+        break;
+    case Data_data_resvals:
+        nrows                = data->data.resvals->n_total;
+        getdata.data.resvals = (xce_resval_data_t*)calloc(1, sizeof(xce_resval_data_t));
+        break;
+    default:
+        rc = -EINVAL;
+        break;
+    }
+
+    if (rc < 0) {
+        /* FIXME we need to reply no data */
+        return rc;
+    }
+
+    if (pktlimit <= 0) {
+        pktlimit = nrows;
+    }
+
+    do {
+        int nc = nrows - pos;
+
+        if (nc > pktlimit) {
+            nc = pktlimit;
+        }
+
+        if (checkf != NULL) {
+            checkstatus = (*checkf)(checkarg);
+
+            if (checkstatus < 0) {
+                nc = 0;
+            }
+        }
+
+        switch (getdata.table) {
+        case Data_data_schedjobs:
+        case Data_data_jobs:
+            getdata.data.jobs->n_total = data->data.jobs->n_total;
+            getdata.data.jobs->pos     = pos;
+            getdata.data.jobs->count   = nc;
+            if (nc == 0) {
+                getdata.data.jobs->jobs = NULL;
+            } else {
+                getdata.data.jobs->jobs = &(data->data.jobs->jobs[pos]);
+            }
+            break;
+        case Data_data_schedworkers:
+        case Data_data_workers:
+            getdata.data.workers->n_total = data->data.workers->n_total;
+            getdata.data.workers->pos     = pos;
+            getdata.data.workers->count   = nc;
+            if (nc == 0) {
+                getdata.data.workers->workers = NULL;
+            } else {
+                getdata.data.workers->workers = &(data->data.workers->workers[pos]);
+            }
+            break;
+        case Data_data_resources:
+            getdata.data.resources->n_total = data->data.resources->n_total;
+            getdata.data.resources->pos     = pos;
+            getdata.data.workers->count     = nc;
+            if (nc == 0) {
+                getdata.data.resources->resources = NULL;
+            } else {
+                getdata.data.resources->resources = &(data->data.resources->resources[pos]);
+            }
+            break;
+        case Data_data_resvals:
+            getdata.data.resvals->n_total = data->data.resvals->n_total;
+            getdata.data.resvals->pos     = pos;
+            getdata.data.resvals->count   = nc;
+            if (nc == 0) {
+                getdata.data.resvals->vals = NULL;
+            } else {
+                getdata.data.resvals->vals = &(data->data.resvals->vals[pos]);
+            }
+            break;
+        default:
+            /* should never goes here */
+            break;
+        }
+
+        rc = xce_reply(ep, from, &reply1);
+        if (rc < 0) {
+            break;
+        }
+
+        pos += nc;
+    } while ((pos < nrows) && (checkstatus >= 0));
+
+    switch (getdata.table) {
+    case Data_data_schedjobs:
+    case Data_data_jobs:
+        FREEUP(getdata.data.jobs);
+        break;
+    case Data_data_schedworkers:
+    case Data_data_workers:
+        FREEUP(getdata.data.workers);
+        break;
+    case Data_data_resources:
+        FREEUP(getdata.data.resources);
+        break;
+    case Data_data_resvals:
+        FREEUP(getdata.data.resvals);
+        break;
+    default:
+        break;
+    }
+
+    return rc;
+}
+
+int extract_data(dataset_t*  dataset,
+                 xce_data_t* getdata)
+{
+    int ntotal;
+    int rc    = 0;
+    int pos   = 0;
+    int count = 0;
+
+    TRACE("%s ...", __func__);
+    
+    switch (getdata->table) {
+    case Data_data_schedjobs:
+    case Data_data_jobs:
+        if (getdata->data.jobs->count <= 0) {
+            return 1;
+        }
+
+        ntotal = getdata->data.jobs->n_total;
+        pos    = getdata->data.jobs->pos;
+        count  = getdata->data.jobs->count;
+
+        if (dataset->rows == NULL) {
+            dataset->rows  = (xce_job_t**)calloc(ntotal, sizeof(xce_job_t*));
+            dataset->nrows = 0;
+        }
+
+        memcpy((void*)(((char*)dataset->rows) + sizeof(xce_job_t*) * pos),
+               getdata->data.jobs->jobs, count * sizeof(xce_job_t*));
+        memset(getdata->data.jobs->jobs, 0x0, count * sizeof(xce_job_t*));
+
+        break;
+    case Data_data_schedworkers:
+    case Data_data_workers:
+        if (getdata->data.workers->count <= 0) {
+            return 1;
+        }
+
+        ntotal = getdata->data.workers->n_total;
+        if (dataset->rows == NULL) {
+            dataset->rows  = (xce_worker_t**)calloc(ntotal, sizeof(xce_worker_t*));
+            dataset->nrows = 0;
+        }
+        pos   = getdata->data.workers->pos;
+        count = getdata->data.workers->count;
+
+        memcpy((void*)(((char*)dataset->rows) + sizeof(xce_worker_t*) * pos),
+               getdata->data.workers->workers, count * sizeof(xce_worker_t*));
+        memset(getdata->data.workers->workers, 0x0, count * sizeof(xce_worker_t*));
+
+        break;
+    case Data_data_resources:
+        if (getdata->data.resources->count <= 0) {
+            return 1;
+        }
+
+        ntotal = getdata->data.resources->n_total;
+        if (dataset->rows == NULL) {
+            dataset->rows  = (xce_resource_t**)calloc(ntotal, sizeof(xce_resource_t*));
+            dataset->nrows = 0;
+        }
+        pos   = getdata->data.resources->pos;
+        count = getdata->data.resources->count;
+
+        memcpy((void*)(((char*)dataset->rows) + sizeof(xce_resource_t*) * pos),
+               getdata->data.resources->resources, count * sizeof(xce_resource_t*));
+
+        memset(getdata->data.resources->resources, 0x0, count * sizeof(xce_resource_t*));
+        break;
+    case Data_data_resvals:
+        if (getdata->data.resvals->count <= 0) {
+            return 1;
+        }
+
+        ntotal = getdata->data.resvals->n_total;
+        if (dataset->rows == NULL) {
+            dataset->rows  = (xce_resval_t**)calloc(ntotal, sizeof(xce_resval_t*));
+            dataset->nrows = 0;
+        }
+
+        pos   = getdata->data.resvals->pos;
+        count = getdata->data.resvals->count;
+
+        memcpy((void*)(((char*)dataset->rows) + sizeof(xce_resval_t*) * pos),
+               getdata->data.resvals->vals, count * sizeof(xce_resval_t*));
+
+        memset(getdata->data.resvals->vals, 0x0, count * sizeof(xce_resval_t*));
+        break;
+    default:
+        rc = -EINVAL;
+        break;
+    }
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    dataset->nrows += count;
+    if (dataset->nrows < ntotal) {
+        return 0;
+    }
+
+    return 1;
+}
+
+int xce_jobdispatch_req(ioctx_t*            ctx,
+                        xce_job_dispatch_t* info)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_jobdispatch,
+        .body_which = Request_body_dispatchmsg,
+        .body = {
+            .dispatchmsg = info 
+        }
+    };
+    /* clang-format on */
+
+    /* clang-format off */
+    if ((ctx == NULL) ||
+	(info == NULL)) {
+        return -EINVAL;
+    }
+    /* clang-format on */
+
+    return xce_request(ctx, &request);
+}
+
+int xce_notify_req(ioctx_t*           ctx,
+                   xce_notify_data_t* data)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_notify,
+        .body_which = Request_body_notifymsg,
+        .body = {
+            .notifymsg = data
+        }
+    };
+    /* clang-forn on */
+
+    if ((ctx == NULL) || (data == NULL)) {
+	return -EINVAL;
+    }
+
+    return xce_request(ctx, &request);
+}
+
+int xce_jobsubmit_req(ioctx_t*          ctx,
+                      xce_job_submit_t* submit)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_jobsub,
+        .body_which = Request_body_jobsubmsg,
+        .body = {
+            .jobsubmsg = submit
+        }
+    };
+    /*clang-format on*/
+    
+    /* clang-format off */
+    if ((ctx == NULL) ||
+	(submit == NULL)) {
+	return -EINVAL;
+    }
+    /* clang-format on */
+
+    return xce_request(ctx, &request);
+}
+
+int xce_startjob_req(ioctx_t*        ctx,
+                     xce_job_exec_t* execinfo)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_startjob,
+        .body_which = Request_body_jobstartmsg,
+        .body = {
+            .jobstartmsg = execinfo
+        }
+    };
+    /* clang-format on */
+
+    return xce_request(ctx, &request);
+}
+
+int xce_clusterconf_req(ioctx_t* ctx)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_clusterconf,
+        .body_which = Request_body_nomsg
+    };
+    /* clang-format on */
+
+    return xce_request(ctx, &request);
+}
+
+int xce_update_req(ioctx_t*           ctx,
+                   xce_update_data_t* data)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_update,
+        .body_which = Request_body_updatemsg,
+        .body = {
+            .updatemsg = data
+        }
+    };
+    /* clang-format on */
+
+    if ((ctx == NULL) || (data == NULL)) {
+        return -EINVAL;
+    }
+
+    return xce_request(ctx, &request);
+}
+
+int xce_jobctrl_req(ioctx_t*       ctx,
+                    xce_jobctrl_t* ctrinfo)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_jobcontrol,
+        .body_which = Request_body_jobctrlmsg,
+        .body = {
+            .jobctrlmsg = ctrinfo
+        }
+    };
+    /* clang-format on */
+
+    if ((ctx == NULL) || (ctrinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    if (ctrinfo->jobid == 0) {
+        return -EINVAL;
+    }
+
+    switch (ctrinfo->action) {
+    case JobControl_u_siginfo:
+        break;
+    case JobControl_u_ckptoption:
+        if (ctrinfo->u.ckptoption->method == NULL) {
+            ctrinfo->u.ckptoption->method = "default";
+        }
+        break;
+    default:
+        return -EINVAL;
+        break;
+    }
+    return xce_request(ctx, &request);
+}
+
+int xce_queuectrl_req(ioctx_t*         ctx,
+                      xce_queuectrl_t* ctrlinfo)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_queuecontrol,
+        .body_which = Request_body_queuectrlmsg,
+        .body = {
+            .queuectrlmsg = ctrlinfo
+        }
+    };
+    /* clang-format on */
+
+    if ((ctx == NULL) || (ctrlinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    if ((ctrlinfo->queue == NULL) || (strlen(ctrlinfo->queue) == 0)) {
+        return -EINVAL;
+    }
+
+    if (ctrlinfo->action == QUEUE_DO_NONE) {
+        return 0;
+    }
+
+    if ((ctrlinfo->action < 0) || (ctrlinfo->action > QUEUE_ACTIVE)) {
+        return -EINVAL;
+    }
+
+    return xce_request(ctx, &request);
+}
+
+int xce_runtask_req(ioctx_t*       ctx,
+                    xce_taskrun_t* runinfo)
+{
+    /* clang-format off */
+    xce_request_t request = {
+        .opcode = OPCode_runtasks,
+        .body_which = Request_body_runtaskmsg,
+        .body = {
+            .runtaskmsg = runinfo
+        }
+    };
+    /* clang-format on */
+
+    if ((ctx == NULL) || (runinfo == NULL)) {
+        return -EINVAL;
+    }
+
+    if (runinfo->nhosts <= 0) {
+        return -EINVAL;
+    }
+
+    return xce_request(ctx, &request);
+}

src/lib/testsuite/test.c

@@ -0,0 +1,35 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "unity.h"
+
+extern void test_xstrings(void);
+extern void test_zarray(void);
+extern void test_ht_string(void);
+extern void test_ht_int(void);
+extern void test_error(void);
+extern void test_utils(void);
+
+void setUp(void)
+{
+    return;
+}
+
+void tearDown(void)
+{
+    return;
+}
+
+int main(int argc, char* argv[])
+{
+    UNITY_BEGIN();
+    test_ht_string();
+    test_ht_int();
+    test_zarray();
+    test_xstrings();
+    test_error();
+    test_utils();
+
+    return UNITY_END();
+}

src/lib/testsuite/test_error.c

@@ -0,0 +1,44 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "xce/errnox.h"
+#include "unity.h"
+#include "utils.h"
+
+static void test_error_setget(void) {
+    int myerrno =1;
+    ectx_t *e;
+
+    SET_ERROR(myerrno);
+    e = get_ectx();
+
+    TEST_ASSERT_TRUE(e != NULL);
+    TEST_ASSERT_TRUE(errno == myerrno);
+    TEST_ASSERT_TRUE(e->file != NULL);
+    TEST_ASSERT_TRUE(e->func != NULL);
+    TEST_ASSERT_TRUE(strcmp(e->file, __FILE__) == 0);
+    TEST_ASSERT_TRUE(strcmp(e->func, __func__) == 0);
+}
+
+#if 0
+static void test_strerror_r(void) {
+    char errmsg1[256];
+    char errmsg2[256];
+    int  errors[] ={ENOENT, EINVAL, E_XCE_BAD_JOB, E_XCE_BAD_WORKER, 0};
+    int  i;
+
+    for(i = 0; errors[i] != 0; i ++) {
+	strerror_r(errors[i], errmsg1, sizeof(errmsg1));
+	xce_strerror_r(errors[i], errmsg2, sizeof(errmsg2));
+	TEST_ASSERT_TRUE(strcmp(errmsg1, errmsg2) == 0);
+    }
+}
+#endif
+void test_error(void) {
+    printf("\n>> error test\n\n");
+
+    RUN_TEST(test_error_setget);
+#if 0
+    RUN_TEST(test_strerror_r);
+#endif
+}

src/lib/testsuite/test_htab.c

@@ -0,0 +1,50 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "unity.h"
+#include "htab.h"
+
+static htab_t htab;
+static char  *s = "Hello,world";
+
+static void test_ht_add_string(void) {
+    hent_t *e = NULL;
+
+    e = ht_add(&(htab), (void *)s);
+    TEST_ASSERT_NOT_NULL(e);
+    
+    return;
+}
+
+static void test_ht_get_string(void) {
+    hent_t *e = NULL;
+
+    e = ht_get(&htab, (void *)s);
+    TEST_ASSERT_NOT_NULL(e);
+
+    TEST_ASSERT_TRUE(0 == strcmp(e->key, s));
+    
+    return;
+}
+
+static void test_ht_del_string(void) {
+    hent_t *e = NULL;
+    
+    ht_del(&htab, (void *)s);
+
+    e = ht_get(&htab, (void *)s);
+    TEST_ASSERT_TRUE( e == NULL);
+    
+    return;
+}
+
+void test_ht_string(void) {
+    printf("\n>> htab test string\n\n");
+
+    ht_init(&htab, 13, string_hash_key);
+    
+    RUN_TEST(test_ht_add_string);
+    RUN_TEST(test_ht_get_string);
+    RUN_TEST(test_ht_del_string);
+}

src/lib/testsuite/test_htab_int.c

@@ -0,0 +1,52 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "unity.h"
+#include "htab.h"
+
+static htab_t htab;
+
+uint64_t key = (uint64_t)0xfabcde6666666666;
+
+static void test_ht_add_string(void) {
+    hent_t *e = NULL;
+
+    e = ht_add(&(htab), (void *)key);
+    TEST_ASSERT_NOT_NULL(e);
+    
+    return;
+}
+
+static void test_ht_get_string(void) {
+    hent_t *e = NULL;
+
+    e = ht_get(&htab, (void *)key);
+    TEST_ASSERT_NOT_NULL(e);
+
+    TEST_ASSERT_TRUE((uint64_t)e->key == (uint64_t)key);
+    
+    return;
+}
+
+static void test_ht_del_string(void) {
+    hent_t *e = NULL;
+    
+    ht_del(&htab, (void *)key);
+
+    e = ht_get(&htab, (void *)key);
+    TEST_ASSERT_TRUE( e == NULL);
+    
+    return;
+}
+
+void test_ht_int(void) {
+    printf("\n>> htab test int\n\n");
+
+    ht_init(&htab, 13, int_hash_key);
+    
+    RUN_TEST(test_ht_add_string);
+    RUN_TEST(test_ht_get_string);
+    RUN_TEST(test_ht_del_string);
+}

src/lib/testsuite/test_utils.c

@@ -0,0 +1,105 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "unity.h"
+#include "utils.h"
+#include "xce/types.h"
+
+static void test_timewindow_config_dup()
+{
+    /* clang-format off */
+    xce_timewindow_config_t win1 = {
+        .nsegs = 2,
+        .segs  = (xce_timewindow_seg_config_t*[]) {
+            &(xce_timewindow_seg_config_t) {
+                .tz   = "GMT-8",
+                .open = &(xce_timewindow_time_t) {
+                    .day  = 0,
+                    .hour = 9,
+                    .min  = 0 },
+                .close = &(xce_timewindow_time_t) {
+                    .day = 0,
+                    .hour = 17,
+                    .min = 59 } },
+            &(xce_timewindow_seg_config_t) {
+                .tz = "GMT-8",
+                .open = &(xce_timewindow_time_t) {
+                    .day = 5,
+                    .hour = 21,
+                    .min = 0 },
+                .close = &(xce_timewindow_time_t) {
+                    .day = 7,
+                    .hour = 20,
+                    .min = 59 } } }
+    };
+    /* clang-format on */
+    xce_timewindow_config_t* win2 = NULL;
+
+    win2 = dup_timewindow_config(&win1);
+
+    TEST_ASSERT_TRUE(win2 != NULL);
+    TEST_ASSERT_TRUE(win2->nsegs == 2);
+    TEST_ASSERT_TRUE((strcmp(win2->segs[0]->tz, "GMT-8") == 0));
+    TEST_ASSERT_TRUE(win2->segs[0]->open->day == 0);
+    TEST_ASSERT_TRUE(win2->segs[0]->open->hour == 9);
+    TEST_ASSERT_TRUE(win2->segs[0]->open->min == 0);
+    TEST_ASSERT_TRUE(win2->segs[0]->close->day == 0);
+    TEST_ASSERT_TRUE(win2->segs[0]->close->hour == 17);
+    TEST_ASSERT_TRUE(win2->segs[0]->close->min == 59);
+
+    TEST_ASSERT_TRUE((strcmp(win2->segs[1]->tz, "GMT-8") == 0));
+    TEST_ASSERT_TRUE(win2->segs[1]->open->day == 5);
+    TEST_ASSERT_TRUE(win2->segs[1]->open->hour == 21);
+    TEST_ASSERT_TRUE(win2->segs[1]->open->min == 0);
+    TEST_ASSERT_TRUE(win2->segs[1]->close->day == 7);
+    TEST_ASSERT_TRUE(win2->segs[1]->close->hour == 20);
+    TEST_ASSERT_TRUE(win2->segs[1]->close->min == 59);
+
+    free_timewindow_config(win2);
+
+    return;
+}
+
+void test_dup_threshold()
+{
+    /* clang-format off */
+    xce_res_threshold_t* threshold[2] = {
+        &(xce_res_threshold_t) {
+            .name  = "r15s",
+            .sched = 1.0,
+            .stop  = 5.0 },
+        &(xce_res_threshold_t) {
+            .name  = "r1m",
+            .sched = 0.9,
+            .stop  = 4.5 }
+    };
+    /* clang-format on */
+    xce_res_threshold_t** threshold2 = NULL;
+
+    threshold2 = dup_resource_thresholds(threshold, 2);
+
+    TEST_ASSERT_TRUE(threshold2 != NULL);
+
+    TEST_ASSERT_TRUE(strcmp(threshold[0]->name, threshold2[0]->name) == 0);
+    TEST_ASSERT_TRUE(strcmp(threshold[1]->name, threshold2[1]->name) == 0);
+    TEST_ASSERT_TRUE(float_equal(threshold[0]->sched,
+                                 threshold2[0]->sched));
+    TEST_ASSERT_TRUE(float_equal(threshold[0]->stop,
+                                 threshold2[0]->stop));
+    TEST_ASSERT_TRUE(float_equal(threshold[1]->sched,
+                                 threshold2[1]->sched));
+    TEST_ASSERT_TRUE(float_equal(threshold[1]->stop,
+                                 threshold2[1]->stop));
+
+    free_resource_thresholds(threshold2, 2);
+    return;
+}
+
+void test_utils(void)
+{
+    printf("\n>> utils test\n\n");
+
+    RUN_TEST(test_timewindow_config_dup);
+    RUN_TEST(test_dup_threshold);
+}

src/lib/testsuite/test_xstring.c

@@ -0,0 +1,128 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "objmgr.h"
+#include "unity.h"
+#include "xstring.h"
+
+void test_xstring_create(void)
+{
+    objmgr_t mgr;
+    char*    s1 = "Hello,world";
+    char*    s2 = NULL;
+    int      rc = 0;
+    myobj_t* obj;
+
+    objmgr_init(&mgr);
+    register_xstring(&mgr,4,4);
+    TEST_ASSERT_EQUAL(1, mgr.types.count);
+
+    s2 = create_obj_by_typename(&mgr, "xstring", s1);
+    TEST_ASSERT_NOT_NULL(s2);
+    obj = TO_MYOBJ(s2);
+    TEST_ASSERT_EQUAL(1, XSTRING_OBJ_TYPE_STRINGS(obj).count);
+
+    rc = strcmp(s1, s2);
+    TEST_ASSERT_EQUAL(0, rc);
+
+    objmgr_fini(&mgr);
+}
+
+void test_xstring_ref(void)
+{
+    objmgr_t mgr;
+    char*    s1 = NULL;
+    char*    s2 = NULL;
+    myobj_t* obj;
+
+    objmgr_init(&mgr);
+    register_xstring(&mgr,4,4);
+    TEST_ASSERT_EQUAL(1, mgr.types.count);
+
+    s1 = create_obj_by_typename(&mgr, "xstring",
+                                "Hello,world");
+    TEST_ASSERT_NOT_NULL(s1);
+
+    obj = TO_MYOBJ(s1);
+    TEST_ASSERT_EQUAL(1, XSTRING_OBJ_TYPE_STRINGS(obj).count);
+
+    s2 = (char*)obj_ref(s1, OBJREF_READ);
+    TEST_ASSERT_EQUAL(s1, s2);
+
+    TEST_ASSERT_EQUAL(2, obj->nref);
+    objmgr_fini(&mgr);
+}
+
+void test_xstring_deref(void)
+{
+    objmgr_t mgr;
+    char*    s1 = "Hello,world";
+    char*    s2 = NULL;
+    int      rc = 0;
+    myobj_t* obj;
+
+    objmgr_init(&mgr);
+    register_xstring(&mgr,4,4);
+    TEST_ASSERT_EQUAL(1, mgr.types.count);
+
+    s2 = create_obj_by_typename(&mgr, "xstring", s1);
+    TEST_ASSERT_NOT_NULL(s2);
+
+    rc = strcmp(s1, s2);
+
+    TEST_ASSERT_EQUAL(0, rc);
+    if (rc == 0) {
+	objtype_t *typeval;
+
+        obj = TO_MYOBJ(s2);
+	typeval = obj->type;
+
+	obj_deref(s2);
+
+        TEST_ASSERT_EQUAL(0, XSTRING_TYPE_STRINGS(typeval).count);
+    }
+
+    objmgr_fini(&mgr);
+}
+
+void test_xstring_share(void)
+{
+    objmgr_t mgr;
+    char*    s1 = "Hello,world";
+    char*    s2 = NULL;
+    char*    s3 = NULL;
+    int      rc = 0;
+
+    objmgr_init(&mgr);
+    register_xstring(&mgr,4,4);
+    TEST_ASSERT_EQUAL(1, mgr.types.count);
+
+    s2 = create_obj_by_typename(&mgr, "xstring", s1);
+    TEST_ASSERT_NOT_NULL(s2);
+
+    rc = strcmp(s1, s2);
+
+    TEST_ASSERT_EQUAL(0, rc);
+    s3 = create_obj_by_typename(&mgr, "xstring", s1);
+    TEST_ASSERT_EQUAL(s2, s3);
+    if (1) {
+        myobj_t* obj;
+
+        obj = TO_MYOBJ(s3);
+        TEST_ASSERT_EQUAL(2, obj->nref);
+    }
+
+    objmgr_fini(&mgr);
+}
+
+void test_xstrings(void)
+{
+    printf("\n>> xstring test\n\n");
+    RUN_TEST(test_xstring_create);
+    RUN_TEST(test_xstring_ref);
+    RUN_TEST(test_xstring_deref);
+    RUN_TEST(test_xstring_share);
+
+    return;
+}

src/lib/testsuite/test_zarray.c

@@ -0,0 +1,102 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "unity.h"
+#include "zarray.h"
+
+static void test_zarray_create(void) {
+    zarray_t array;
+
+    zarray_init(&array, 8, NULL);
+
+    TEST_ASSERT_EQUAL(0,array.count);
+    TEST_ASSERT_NULL(array.data);
+}
+
+static void test_zarray_store(void) {
+    zarray_t array;
+    int idx;
+    char *s1 = "hello";
+    char *s2 = NULL;
+
+    zarray_init(&array, 8, NULL);
+    idx = zarray_store(&array, (void *)s1);
+    TEST_ASSERT_TRUE(idx >= 0);
+    s2 = (char *)zarray_get(&array, idx);
+    TEST_ASSERT_NOT_NULL(s2);
+    TEST_ASSERT_EQUAL(0, strcmp(s1, s2));
+
+    zarray_fini(&(array));
+}
+
+static void test_zarray_del(void) {
+    zarray_t array;
+    int idx;
+    char *s1 = "hello";
+
+    zarray_init(&array, 8, NULL);
+    idx = zarray_store(&array, (void *)s1);
+    TEST_ASSERT_TRUE(idx >= 0);
+    TEST_ASSERT_TRUE(array.count == 1);
+    zarray_del(&array, idx);
+    TEST_ASSERT_TRUE(zarray_get(&array, idx) == NULL);
+    TEST_ASSERT_TRUE(array.count == 0);
+
+    zarray_fini(&array);
+}
+
+static void test_zarray_put(void) {
+    zarray_t array;
+    int idx;
+    char *s1 = "hello";
+    char *s2 = "test";
+
+    zarray_init(&array, 8, NULL);
+    idx = zarray_store(&array, (void *)s1);
+    TEST_ASSERT_TRUE(idx >= 0);
+    TEST_ASSERT_TRUE(array.count == 1);
+    TEST_ASSERT_TRUE(zarray_get(&array, idx) != NULL);
+    
+    TEST_ASSERT_TRUE(strcmp((char *)zarray_get(&array, idx), s1) == 0);
+
+    zarray_put(&array, idx, (void *)s2);
+    TEST_ASSERT_TRUE(zarray_get(&array, idx) != NULL);
+    TEST_ASSERT_TRUE(strcmp((char *)zarray_get(&array, idx), s2) == 0);
+
+    zarray_fini(&array);
+}
+
+static int sum_(void *arg1, void *arg2) {
+    int *pv = (int *)arg1;
+    int x = (int)((long)arg2);
+
+    (*pv) += x;
+    return 0;
+}
+
+static void test_zarray_iter(void) {
+    zarray_t array;
+    int i;
+    int val = 0;
+
+    zarray_init(&array, 10, NULL);
+    for(i = 0; i < 10; i ++) {
+	zarray_store(&array, (void *)(long)i);
+    }
+
+    zarray_iter(&array, (zarray_iter_fn_t)sum_, &val);
+
+    TEST_ASSERT_TRUE(val == 45);
+    
+    zarray_fini(&(array));
+}
+void test_zarray(void) {
+    printf("\n>> zarray test\n\n");
+
+    RUN_TEST(test_zarray_create);
+    RUN_TEST(test_zarray_store);
+    RUN_TEST(test_zarray_del);
+    RUN_TEST(test_zarray_put);
+    RUN_TEST(test_zarray_iter);
+}

src/lib/threshold.c

@@ -0,0 +1,81 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "utils.h"
+#include "xce/errnox.h"
+#include "xce/types.h"
+
+static void kill_thresholds(xce_res_threshold_t** thresholds,
+                            int                   count)
+{
+    int i;
+
+    if (thresholds == NULL) {
+        return;
+    }
+
+    if (count > 0) {
+        for (i = 0; i < count; i++) {
+            xce_res_threshold_t* it = thresholds[i];
+
+            if (it == NULL) {
+                continue;
+            }
+
+            FREEUP(it->name);
+            FREEUP(it);
+        }
+    }
+
+    FREEUP(thresholds);
+}
+
+void free_resource_thresholds(xce_res_threshold_t** thresholds,
+                              int                   count)
+{
+    kill_thresholds(thresholds, count);
+}
+
+xce_res_threshold_t** dup_resource_thresholds(xce_res_threshold_t** src,
+                                              int                   count)
+{
+    xce_res_threshold_t** thresholds = NULL;
+    int                   i;
+
+    if ((src == NULL) || (count <= 0)) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    thresholds = (xce_res_threshold_t**)calloc(count, sizeof(xce_res_threshold_t*));
+    if (thresholds == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    for (i = 0; i < count; i++) {
+        xce_res_threshold_t* it = (xce_res_threshold_t*)calloc(1, sizeof(*it));
+
+        if (it == NULL) {
+            kill_thresholds(thresholds, i);
+            errno = ENOMEM;
+
+            return NULL;
+        }
+
+        thresholds[i] = it;
+
+        it->name = strdup(src[i]->name);
+        if (it->name == NULL) {
+            kill_thresholds(thresholds, i + 1);
+            errno = ENOMEM;
+
+            return NULL;
+        }
+
+        it->sched = src[i]->sched;
+        it->stop  = src[i]->stop;
+    }
+
+    return thresholds;
+}

src/lib/utils.c

@@ -0,0 +1,371 @@
+#include <ctype.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#if defined(__GLIBC__) && (__GLIBC__ == 2 && __GLIBC_MINOR__ < 28)
+#include <pthread.h>
+#else
+#include <threads.h>
+#endif
+#include <sys/param.h>
+#include <unistd.h>
+
+#if defined(__linux__)
+#include <execinfo.h>
+#endif
+
+#include "config.h"
+#include "controller.capnp.h"
+#include <capnp_c.h>
+
+#include "utils.h"
+#include "xce/errnox.h"
+#include "xce/types.h"
+
+static capn_text chars_to_text_internal(const char* chars)
+{
+    capn_text data = {
+        .len = strlen(chars),
+        .str = chars,
+        .seg = NULL,
+    };
+
+    return data;
+}
+
+capn_text chars_to_text(const char* chars)
+{
+    if (chars == NULL) {
+        chars = "";
+    }
+
+    return chars_to_text_internal(chars);
+}
+
+#define ARGV_INCR_DELTA (16)
+
+#define FILL_ARGV(rc, argv, argc, maxargc, delta, s)          \
+    do {                                                      \
+        (rc) = 0;                                             \
+        char* s1;                                             \
+        if (((argc) >= (maxargc))) {                          \
+            char** p;                                         \
+            int    nc, i;                                     \
+            nc = (maxargc) + (delta);                         \
+            p  = (char**)realloc((argv), nc * sizeof(char*)); \
+            if (p == NULL) {                                  \
+                (rc) = -ENOMEM;                               \
+                break;                                        \
+            }                                                 \
+            for (i = (maxargc); i < nc; i++) {                \
+                p[i] = NULL;                                  \
+            }                                                 \
+            (argv)    = p;                                    \
+            (maxargc) = nc;                                   \
+        }                                                     \
+        s1 = strdup(s);                                       \
+        if (s1 == NULL) {                                     \
+            (rc) = -ENOMEM;                                   \
+            break;                                            \
+        }                                                     \
+        (argv)[argc] = s1;                                    \
+        (argc)++;                                             \
+    } while (0)
+
+#define FREE_ARGV(argv, argc)          \
+    do {                               \
+        int i;                         \
+        if ((argv) == NULL) {          \
+            break;                     \
+        }                              \
+        for (i = 0; i < (argc); i++) { \
+            if ((argv)[i] != NULL) {   \
+                free((argv)[i]);       \
+                (argv)[i] = NULL;      \
+            }                          \
+        }                              \
+        free(argv);                    \
+    } while (0)
+
+int parse_argv(char*   input,
+               char*** argv)
+{
+    int   max_argc   = 0;
+    char* arg        = NULL;
+    int   i          = 0;
+    int   in_quotes  = 0;
+    char  quote_char = 0;
+    int   argc       = 0;
+    int   rc         = 0;
+
+    rc  = 0;
+    arg = (char*)malloc(strlen(input) + 1);
+    if (arg == NULL) {
+        return -ENOMEM;
+    }
+
+    *argv = NULL;
+    while (*input) {
+        if (in_quotes) {
+            if (*input == quote_char) {
+                in_quotes = 0;
+            } else if (*input == '\\') {
+                input++;
+                if (*input) {
+                    arg[i++] = *input;
+                }
+            } else {
+                arg[i++] = *input;
+            }
+        } else {
+            if (isspace(*input)) {
+                if (i > 0) {
+                    arg[i] = '\0';
+                    FILL_ARGV(rc, *argv, argc, max_argc,
+                              ARGV_INCR_DELTA, arg);
+                    if (rc < 0) {
+                        FREE_ARGV(*argv, argc);
+                        free(arg);
+                        return rc;
+                    }
+                    i = 0;
+                }
+            } else if ((*input == '"') || (*input == '\'')) {
+                in_quotes  = 1;
+                quote_char = *input;
+            } else if (*input == '\\') {
+                input++;
+                if (*input) {
+                    arg[i++] = *input;
+                }
+            } else {
+                arg[i++] = *input;
+            }
+        }
+        input++;
+    }
+
+    if (i > 0) {
+        arg[i] = '\0';
+        FILL_ARGV(rc, *argv, argc, max_argc,
+                  ARGV_INCR_DELTA, arg);
+        if (rc < 0) {
+            FREE_ARGV(*argv, argc);
+            free(arg);
+            return rc;
+        }
+    }
+
+    free(arg);
+    return argc;
+}
+
+void free_argv(char** argv, int argc)
+{
+    FREE_ARGV(argv, argc);
+}
+
+#if 0
+void freeup_kvs(int       count,
+                xce_kv_t* vals)
+{
+    int i;
+
+    if (vals == NULL) {
+        return;
+    }
+
+    for (i = 0; i < count; i++) {
+        if (vals[i].value.which != XceFieldValue_data_svalue) {
+            continue;
+        }
+
+        FREEUP(vals[i].value.u.svalue);
+    }
+
+    FREEUP(vals);
+}
+#endif
+
+char* frame_symbol(int k)
+{
+#if defined(__linux__)
+    void*  array[k + 1];
+    size_t size    = backtrace(array, k + 1);
+    char** symbols = backtrace_symbols(array, size);
+    char*  s       = NULL;
+
+    if (symbols == NULL) {
+        return NULL;
+    }
+
+    s = strdup(symbols[MIN(size - 1, k)]);
+    free(symbols);
+    return s;
+#else
+    return NULL;
+#endif
+}
+
+int xce_vprintf(const char* fmt, va_list ap)
+{
+    int rc = 0;
+
+    /* FIXME: */
+    return rc;
+}
+
+static _Thread_local ectx_t ectx;
+
+void set_error(char* file,
+               char* func,
+               int   line,
+               int   code)
+{
+    ectx.file = file;
+    ectx.func = func;
+    ectx.line = line;
+    ectx.code = code;
+    errno     = code;
+}
+
+ectx_t* get_ectx()
+{
+    return &ectx;
+}
+
+static char* xce_err_str[] = {
+    "invalid job",
+    "invalid worker",
+    "duplicated jobs",
+    "duplicated workers",
+    "bad opcode",
+    "unknown role",
+    "unknown resource",
+    "unknown queue",
+    "unknown group",
+    "internal error",
+    NULL
+};
+
+int xce_strerror_r(int    errnum,
+                   char*  buf,
+                   size_t bufsize)
+{
+    if (errnum < XCE_ERRNO_BASE) {
+        return strerror_r(errnum, buf, bufsize);
+    }
+
+    if (errnum >= E_XCE_MAX) {
+        strncpy(buf, "unknown error", bufsize);
+        return -1;
+    }
+
+    strncpy(buf, xce_err_str[errnum - XCE_ERRNO_BASE],
+            bufsize);
+    return 0;
+}
+
+int copy_int_list(int** dst,
+                  int*  src,
+                  int   count)
+{
+    int i;
+
+    FREEUP(*dst);
+    if (count == 0) {
+        return 0;
+    }
+
+    *dst = (int*)calloc(count, sizeof(int));
+    if ((*dst) == NULL) {
+        return -ENOMEM;
+    }
+
+    for (i = 0; i < count; i++) {
+        (*dst)[i] = (int)src[i];
+    }
+
+    return count;
+}
+
+int copy_strlist(xce_strlist_t* dst,
+                 xce_strlist_t  src)
+{
+    if (!dst) {
+        return -1;
+    }
+    for (int i = 0; i < dst->count; i++) {
+        FREEUP(dst->data[i]);
+    }
+    FREEUP(dst->data);
+    dst->count = src.count;
+    if (dst->count == 0) {
+        return 0;
+    }
+
+    dst->data = (char**)calloc(dst->count, sizeof(char*));
+    if (dst->data == NULL) {
+        return -ENOMEM;
+    }
+    for (int i = 0; i < src.count; i++) {
+        if (src.data[i]) {
+            dst->data[i] = strndup(src.data[i], strlen(src.data[i]));
+            if (dst->data[i] == NULL) {
+                return -ENOMEM;
+            }
+        }
+    }
+    return 0;
+}
+void free_resval_with_idx_list(xce_val_with_idx_t*** vals,
+                               int                   count)
+{
+    xce_val_with_idx_t** resvals = *vals;
+    int                  i;
+
+    if (resvals == NULL) {
+        return;
+    }
+
+    if (count <= 0) {
+        return;
+    }
+
+    for (i = 0; i < count; i++) {
+        xce_val_with_idx_t* val = resvals[i];
+
+        if (val == NULL) {
+            continue;
+        }
+
+        if (val->val->which == Value_u_svalue) {
+            FREEUP(val->val->u.svalue);
+        }
+        
+        FREEUP(val->val);
+        FREEUP(val);
+    }
+
+    FREEUP(resvals);
+    *vals = NULL;
+}
+
+void free_strlist(xce_strlist_t *l) {
+    int i;
+    
+    if ( l == NULL) {
+        return;
+    }
+
+    if ((l->count <= 0) || (l->data == NULL)) {
+        return;
+    }
+
+    for(i = 0; i < l->count; i ++) {
+        FREEUP(l->data[i]);
+    }
+
+    FREEUP(l->data);
+}

src/lib/vtaskmgr.c

@@ -0,0 +1,362 @@
+#include <errno.h>
+#include <pthread.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "mylock.h"
+#include "netio.h"
+#include "utils.h"
+#include "vtaskmgr.h"
+
+static int bind_vtask(vtask_t*, vcpu_t*);
+
+vtask_t* vtask_alloc()
+{
+    vtask_t* task;
+
+    task = calloc(1, sizeof(*task));
+    return task;
+}
+
+int vtask_init(vtask_t*     task,
+               vtask_func_t entry,
+               endpoint_t*  ep)
+{
+    int rc;
+
+    task->entry  = entry;
+    task->vcpu   = -1;
+    task->ctx.ep = endpoint_ref(ep);
+
+    rc = read_packet(ep->fd, &(task->ctx.pkt));
+    if (rc < 0) {
+        free_packet(&(task->ctx.pkt));
+        endpoint_deref(task->ctx.ep);
+        task->ctx.ep = NULL;
+    }
+
+    return rc;
+}
+
+void vtask_fini(vtask_t* task)
+{
+    if (task == NULL) {
+        return;
+    }
+
+    ioctx_deref(&(task->ctx), 0);
+
+    return;
+}
+
+void vcpu_idle(vcpu_t* vcpu)
+{
+    vtaskmgr_t* vtaskmgr = (vtaskmgr_t*)vcpu->owner;
+
+    vcpu->task = NULL;
+    cbuf_put(&(vtaskmgr->wbuf),
+             (void*)&(vcpu->cpuid));
+    cbuf_get(&(vcpu->wbuf),
+             (void*)&(vcpu->task));
+    return;
+}
+
+void run_task(vtask_t* task)
+{
+    vtaskmgr_t *vtaskmgr;
+    
+    if (task->entry != NULL) {
+        (task->entry)(task);
+    }
+
+    endpoint_unblock_tasks(task->ctx.ep);
+    vtaskmgr = (vtaskmgr_t *)task->owner;
+    if (vtaskmgr != NULL) {
+        /* wake up dispatcher */
+        vtaskmgr_wakeup(vtaskmgr);
+    }
+
+    return;
+}
+
+static void* vcpu_entry(void* arg)
+{
+    vcpu_t* my = (vcpu_t*)arg;
+
+    while (1) {
+        if (my->task != NULL) {
+            vtask_t* t = my->task;
+
+            pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, NULL);
+
+            my->task = NULL;
+            run_task(t);
+            vtask_fini(t);
+            free(t);
+            my->state = VCPU_IDLE;
+
+            pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
+        } else {
+            vcpu_idle(my);
+        }
+        pthread_testcancel();
+    }
+    return NULL;
+}
+
+int vcpu_init(vcpu_t* vcpu,
+              int     cpuid,
+              void*   owner)
+{
+    vcpu->state = VCPU_IDLE;
+    vcpu->vcpu  = 0;
+    vcpu->cpuid = cpuid;
+    vcpu->owner = owner;
+    vcpu->task  = NULL;
+    cbuf_init(&(vcpu->wbuf), 1, sizeof(void*));
+
+    return 0;
+}
+
+int vcpu_start(vcpu_t* vcpu)
+{
+    int rc;
+
+    rc = pthread_create(&(vcpu->vcpu), NULL,
+                        &vcpu_entry, vcpu);
+    if (rc != 0) {
+        vcpu->state = VCPU_ERR;
+        return -errno;
+    }
+    return rc;
+}
+
+void vcpu_fini(vcpu_t* vcpu)
+{
+    int   rc;
+    void* res;
+
+    if (vcpu->vcpu) {
+        /* wakeup vcpu first */
+        bind_vtask(NULL, vcpu);
+        rc = pthread_cancel(vcpu->vcpu);
+        if (rc != 0) {
+            cbuf_fini(&(vcpu->wbuf));
+            return;
+        }
+
+        rc = pthread_join(vcpu->vcpu, &res);
+        if (rc != 0) {
+            cbuf_fini(&(vcpu->wbuf));
+            return;
+        }
+    }
+
+    cbuf_fini(&(vcpu->wbuf));
+
+    return;
+}
+
+static void taskq_item_killer(data_item_t* it)
+{
+    if (it != NULL) {
+        vtask_t* task = ITPTR_TO_VTASK(it);
+        vtask_fini(task);
+        free(task);
+    }
+}
+
+int vtaskmgr_init(vtaskmgr_t* vtaskmgr,
+                  int         nvcpus)
+{
+    int i;
+
+    if ((vtaskmgr == NULL) || (nvcpus <= 0)) {
+        return -EINVAL;
+    }
+
+    vtaskmgr->dispatcher = 0;
+    vtaskmgr->n_vcpus    = nvcpus;
+    vtaskmgr->vcpus      = (vcpu_t*)calloc(nvcpus,
+                                           sizeof(vcpu_t));
+    if (vtaskmgr->vcpus == NULL) {
+        return -ENOMEM;
+    }
+    for (i = 0; i < nvcpus; i++) {
+        vcpu_init(&(vtaskmgr->vcpus[i]),
+                  i, vtaskmgr);
+    }
+
+    cbuf_init(&(vtaskmgr->wbuf),
+              nvcpus, sizeof(int));
+    cbuf_init(&(vtaskmgr->wtbuf),
+              1, sizeof(int));
+
+    dataq_init(&(vtaskmgr->taskq),
+               taskq_item_killer);
+
+    return 0;
+}
+
+static vcpu_t* pick_idle_vcpu(vtaskmgr_t* vtaskmgr)
+{
+    int i;
+
+    cbuf_get(&(vtaskmgr->wbuf), &i);
+    if ((i < 0) || (i >= vtaskmgr->n_vcpus)) {
+        return NULL;
+    }
+
+    TRACE("%s pick idle cpu #%d",
+          __func__, i);
+    
+    return &(vtaskmgr->vcpus[i]);
+}
+
+static int vtask_eligible(data_item_t* data)
+{
+    vtask_t* task = ITPTR_TO_VTASK(data);
+    int eligible = 0;
+
+    if (task->ctx.ep == NULL) {
+        WARN("no endpoint for task 0x%p", task);
+        return 0;
+    }
+
+    eligible = !(endpoint_is_task_blocked(task->ctx.ep));
+    TRACE("%s vtask %p is %s",
+          __func__, task,
+          eligible?"eligible":"not eligible");
+
+    return eligible;
+}
+
+static vtask_t* pick_eligible_task(vtaskmgr_t* vtaskmgr)
+{
+    data_item_t* data;
+    vtask_t*     task = NULL;
+
+    data = dataq_deq(&(vtaskmgr->taskq), vtask_eligible);
+    if (data == NULL) {
+        return NULL;
+    }
+
+    task = ITPTR_TO_VTASK(data);
+
+    return task;
+}
+
+static int bind_vtask(vtask_t* task,
+                      vcpu_t*  vcpu)
+{
+    TRACE("binding task %p to cpu#%d\n",
+          task, vcpu->cpuid);
+
+    if (task != NULL) {
+        endpoint_block_tasks(task->ctx.ep);
+
+        task->vcpu = vcpu->cpuid;
+    }
+
+    cbuf_put(&(vcpu->wbuf), (void*)&task);
+
+    return 0;
+}
+
+static void* vdispatcher_entry(void* arg)
+{
+    vtaskmgr_t* vtaskmgr = (vtaskmgr_t*)arg;
+    vcpu_t*     vcpu     = NULL;
+    vtask_t*    task     = NULL;
+
+    while (1) {
+        pthread_testcancel();
+        if (vcpu == NULL) {
+            vcpu = pick_idle_vcpu(vtaskmgr);
+            if (vcpu == NULL) {
+                continue;
+            }
+        }
+
+        task = pick_eligible_task(vtaskmgr);
+        if (task != NULL) {
+            bind_vtask(task, vcpu);
+            vcpu = NULL;
+            task = NULL;
+        }
+        else {
+            vtaskmgr_wait(vtaskmgr);
+        }
+    }
+    return NULL;
+}
+
+int vtaskmgr_start(vtaskmgr_t* vtaskmgr)
+{
+    int i, rc;
+
+    rc = pthread_create(&(vtaskmgr->dispatcher), NULL,
+                        &vdispatcher_entry, vtaskmgr);
+    if (rc != 0) {
+        return -errno;
+    }
+
+    for (i = 0; i < vtaskmgr->n_vcpus; i++) {
+        vcpu_start(&(vtaskmgr->vcpus[i]));
+    }
+
+    return 0;
+}
+
+void vtaskmgr_wait(vtaskmgr_t *vtaskmgr) {
+    int dummy = 0;
+    
+    cbuf_get(&(vtaskmgr->wtbuf), &dummy);
+}
+
+void vtaskmgr_wakeup(vtaskmgr_t *vtaskmgr) {
+    int dummy = 0;
+
+    cbuf_tryput(&(vtaskmgr->wtbuf), &dummy);
+}
+
+void vtaskmgr_fini(vtaskmgr_t* vtaskmgr)
+{
+    int   i;
+    void* res;
+
+    if (vtaskmgr->dispatcher != 0) {
+        pthread_cancel(vtaskmgr->dispatcher);
+        pthread_join(vtaskmgr->dispatcher, &res);
+    }
+
+    if (vtaskmgr->vcpus != NULL) {
+        for (i = 0; i < vtaskmgr->n_vcpus; i++) {
+            vcpu_fini(&(vtaskmgr->vcpus[i]));
+        }
+        free(vtaskmgr->vcpus);
+    }
+
+    dataq_fini(&(vtaskmgr->taskq));
+    cbuf_fini(&(vtaskmgr->wtbuf));
+    cbuf_fini(&(vtaskmgr->wbuf));
+
+    return;
+}
+
+int vtask_enq(vtaskmgr_t* vtaskmgr,
+              vtask_t*    task)
+{
+    int rc = 0;
+    ioctx_ref(&(task->ctx));
+
+    rc = dataq_enq(&(vtaskmgr->taskq),
+                     &(task->ctx.link));
+    if (rc >= 0) {
+        /* wake up task dispatcher */
+        task->owner = vtaskmgr;
+        vtaskmgr_wakeup(vtaskmgr);
+    }
+
+    return rc;
+}

src/lib/window.c

@@ -0,0 +1,126 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "utils.h"
+#include "xce/errnox.h"
+#include "xce/types.h"
+
+static void kill_timewindow_time(xce_timewindow_time_t* t)
+{
+    FREEUP(t);
+}
+
+static void kill_timewindow_seg_config(xce_timewindow_seg_config_t* s)
+{
+    if (s == NULL) {
+        return;
+    }
+
+    FREEUP(s->tz);
+
+    kill_timewindow_time(s->open);
+    kill_timewindow_time(s->close);
+    FREEUP(s);
+}
+
+static void kill_timewindow_config(xce_timewindow_config_t* w)
+{
+    int i;
+
+    if (w == NULL) {
+        return;
+    }
+
+    if (w->segs != NULL) {
+        for (i = 0; i < w->nsegs; i++) {
+            xce_timewindow_seg_config_t* seg = w->segs[i];
+
+            kill_timewindow_seg_config(seg);
+        }
+
+        FREEUP(w->segs);
+    }
+
+    FREEUP(w);
+}
+
+void free_timewindow_config(xce_timewindow_config_t* w)
+{
+    kill_timewindow_config(w);
+}
+
+static xce_timewindow_seg_config_t* dup_timewindow_seg_config(xce_timewindow_seg_config_t* src)
+{
+    xce_timewindow_seg_config_t* seg = NULL;
+
+    if (src == NULL) {
+        errno = EINVAL;
+
+        return NULL;
+    }
+
+    seg = (xce_timewindow_seg_config_t*)calloc(1, sizeof(*seg));
+    if (seg == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    seg->tz = strdup(src->tz);
+    if (seg->tz == NULL) {
+        kill_timewindow_seg_config(seg);
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    seg->open  = (xce_timewindow_time_t*)calloc(1, sizeof(xce_timewindow_time_t));
+    seg->close = (xce_timewindow_time_t*)calloc(1, sizeof(xce_timewindow_time_t));
+    if ((seg->open == NULL) || (seg->close == NULL)) {
+        kill_timewindow_seg_config(seg);
+        errno = ENOMEM;
+
+        return NULL;
+    }
+
+    memcpy(seg->open, src->open, sizeof(xce_timewindow_time_t));
+    memcpy(seg->close, src->close, sizeof(xce_timewindow_time_t));
+
+    return seg;
+}
+
+xce_timewindow_config_t* dup_timewindow_config(xce_timewindow_config_t* src)
+{
+    xce_timewindow_config_t* w;
+    int                      i;
+
+    if (src == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    w = (xce_timewindow_config_t*)calloc(1, sizeof(*w));
+    if (w == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    w->segs = (xce_timewindow_seg_config_t**)calloc(src->nsegs,
+                                                    sizeof(xce_timewindow_seg_config_t*));
+    if (w->segs == NULL) {
+        kill_timewindow_config(w);
+        errno = ENOMEM;
+
+        return NULL;
+    }
+
+    for (i = 0; i < src->nsegs; i++) {
+        w->segs[i] = dup_timewindow_seg_config(src->segs[i]);
+        if (w->segs[i] == NULL) {
+            kill_timewindow_config(w);
+            return NULL;
+        }
+    }
+
+    w->nsegs = src->nsegs;
+
+    return w;
+}

src/lib/xceapi.c

@@ -0,0 +1,667 @@
+#include <errno.h>
+
+#include "utils.h"
+#include "xceapi.h"
+
+int xce_hello(xce_ctx_t* ctx,
+              char*      msg)
+{
+    ioctx_t* ioctx;
+    int      rc, status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_hello_req(ioctx, msg);
+    if (rc < 0) {
+        return rc;
+    }
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_shutdown(xce_ctx_t* ctx)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_shutdown_req(ioctx);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_get(xce_ctx_t* ctx,
+            int        table,
+            void***    data)
+{
+    ioctx_t*    ioctx;
+    int         rc;
+    int         status;
+    xce_match_t condition = {
+        .type = MatchCondition_condition_matchall
+    };
+    xce_get_t get = {
+        .table     = table,
+        .condition = &condition
+    };
+    dataset_t getreply = {
+        .table = table,
+        .nrows = 0,
+        .rows  = NULL
+    };
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    ctx->reply.u.get = &getreply;
+    rc               = xce_get_req(ioctx, &get);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (status >= 0) {
+        *data = getreply.rows;
+        rc    = getreply.nrows;
+    } else {
+        rc = status;
+    }
+
+    ctx->reply.u.get = NULL;
+
+    return rc;
+}
+
+int xce_reslist(xce_ctx_t*        ctx,
+                xce_resource_t*** res)
+{
+    int table = Data_data_resources;
+
+    return xce_get(ctx, table, (void***)res);
+}
+
+int xce_resval(xce_ctx_t*      ctx,
+               char*           host,
+               char*           res,
+               xce_resval_t*** vals)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+    /* clang-format off */
+    xce_reskey_t reskey = {
+	.resname = res,
+	.hostname = host
+    };
+    xce_match_t condition = {
+	.type = MatchCondition_condition_matchreskey,
+	.condition = {
+	    .reskey = &reskey
+	}
+    };
+    xce_get_t get = {
+	.table = Data_data_resvals,
+	.condition = &condition
+    };
+    dataset_t getreply = {
+	.table = Data_data_resvals,
+	.nrows = 0,
+	.rows = NULL
+    };
+    /* clang-format on */
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    ctx->reply.u.get = &getreply;
+    rc               = xce_get_req(ioctx, &get);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (status >= 0) {
+        *vals = (xce_resval_t**)getreply.rows;
+        rc    = getreply.nrows;
+    } else {
+        rc = status;
+    }
+
+    ctx->reply.u.get = NULL;
+
+    return rc;
+}
+
+int xce_join(xce_ctx_t*   ctx,
+             int          role,
+             hostattrs_t* attrs)
+{
+    ioctx_t* ioctx;
+    int      rc, status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_join_req(ioctx, role, attrs);
+    if (rc < 0) {
+        return rc;
+    }
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_workerlist(xce_ctx_t*      ctx,
+                   xce_worker_t*** workers)
+{
+    int table = Data_data_workers;
+
+    return xce_get(ctx, table, (void***)workers);
+}
+
+int xce_free_workers(int           nworkers,
+                     xce_worker_t* workers)
+{
+    return 0;
+}
+
+int xce_jobdispatch(xce_ctx_t*          ctx,
+                    xce_job_dispatch_t* info)
+{
+    ioctx_t* ioctx;
+    int      rc, status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_jobdispatch_req(ioctx, info);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_notify(xce_ctx_t*         ctx,
+               xce_notify_data_t* data)
+{
+    ioctx_t* ioctx;
+    int      rc;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_notify_req(ioctx, data);
+
+    return rc;
+}
+
+int xce_jobsubmit(xce_ctx_t*        ctx,
+                  xce_job_submit_t* submit,
+                  uint64_t*         jobid)
+{
+    ioctx_t* ioctx;
+    int      rc, status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_jobsubmit_req(ioctx, submit);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (status >= 0) {
+        *jobid = ctx->reply.u.jobid;
+    }
+    return status;
+}
+
+int xce_joblist_ex(xce_ctx_t*   ctx,
+                   int          sched,
+                   xce_job_t*** jobs)
+{
+    int table;
+
+    if (sched) {
+        table = Data_data_schedjobs;
+    } else {
+        table = Data_data_jobs;
+    }
+
+    return xce_get(ctx, table, (void***)jobs);
+}
+
+int xce_sched_joblist(xce_ctx_t*   ctx,
+                      xce_job_t*** jobs)
+{
+    return xce_joblist_ex(ctx, 1, jobs);
+}
+
+int xce_joblist(xce_ctx_t*   ctx,
+                xce_job_t*** jobs)
+{
+    return xce_joblist_ex(ctx, 0, jobs);
+}
+
+int xce_varlist(xce_ctx_t*      ctx,
+                xce_config_t*** vars)
+{
+#if 0
+    int            table = XCETables_conf;
+
+    return xce_get(ctx, table, (void***)vars);
+#endif
+    return 0;
+}
+
+int xce_queuelist(xce_ctx_t*     ctx,
+                  xce_queue_t*** queues)
+{
+#if 0
+    int            table = XCETables_queues;
+
+    return xce_get(ctx, table, (void***)queues);
+#else
+    return 0;
+#endif
+}
+
+int xce_free_queues(int          nqueues,
+                    xce_queue_t* queues)
+{
+    int i;
+
+    if (queues == NULL) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < nqueues; i++) {
+        FREEUP(queues[i].resreq);
+    }
+    return 0;
+}
+
+int xce_grouplist(xce_ctx_t*     ctx,
+                  int            group_type,
+                  xce_group_t*** groups)
+{
+#if 0
+    int            table;
+
+    switch (group_type) {
+    case 0:
+        table = XCETables_ugroups;
+        break;
+    default:
+        table = XCETables_hgroups;
+        break;
+    }
+
+    return xce_get(ctx, table, (void***)groups);
+#else
+    return 0;
+#endif
+}
+
+int xce_free_groups(int          ngroups,
+                    xce_group_t* groups)
+{
+    /* TODO */
+    return 0;
+}
+
+int xce_clusterconf(xce_ctx_t*          ctx,
+                    xce_clusterconf_t** conf)
+{
+    ioctx_t* ioctx;
+    int      rc, status;
+
+    if ((ctx == NULL) || (conf == NULL)) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_clusterconf_req(ioctx);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (status < 0) {
+        return status;
+    }
+
+    MOVE_PTR((*conf), ctx->reply.u.clusterconf);
+    return 0;
+}
+
+int xce_update_resvals(xce_ctx_t*     ctx,
+                       int            count,
+                       xce_resval_t** vals)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+
+    /* clang-format off */
+    xce_resval_list_t valist = {
+        .count = count,
+        .vals  = vals
+    };
+    xce_update_data_t data = {
+        .table = Update_data_resvals,
+        .data  = {
+             .resvals = &valist
+	}
+    };
+    /* clang-format on */
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_update_req(ioctx, &data);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_update_jusage(xce_ctx_t*     ctx,
+                      uint64_t       jobid,
+                      int            npids,
+                      int*           pids,
+                      int            ngids,
+                      int*           pgids,
+                      int            nusage,
+                      xce_rusage_t** usages)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+
+    /* clang-format off */
+    xce_job_rusages_t rusages = {
+        .jobid  = jobid,
+        .npids  = npids,
+        .pids   = pids,
+        .pgids  = pgids,
+        .count  = nusage,
+        .usages = usages
+    };
+    xce_update_data_t data = {
+        .table = Update_data_jobrusages,
+        .data  = {
+             .jobrusages = &rusages }
+    };
+    /* clang-format on */
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_update_req(ioctx, &data);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_jobctrl(xce_ctx_t*     ctx,
+                xce_jobctrl_t* ctrl)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_jobctrl_req(ioctx, ctrl);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_queuectrl(xce_ctx_t*       ctx,
+                  xce_queuectrl_t* ctrl)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_queuectrl_req(ioctx, ctrl);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}
+
+int xce_runtask(xce_ctx_t*       ctx,
+		xce_taskrun_t* runinfo)
+{
+    ioctx_t* ioctx;
+    int      rc;
+    int      status;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->state != XCE_CTX_RUN) {
+        return -EINVAL;
+    }
+
+    ioctx = ioctx_alloc(ctx->ep, &(ctx->myinfo));
+    if (ioctx == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = xce_runtask_req(ioctx, runinfo);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cbuf_get(&(ctx->wbuf), &status);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return status;
+}

src/lib/xcectx.c

@@ -0,0 +1,371 @@
+#include <errno.h>
+#include <string.h>
+#include <sys/socket.h>
+
+#include "dataq.h"
+#include "mylock.h"
+#include "netio.h"
+#include "typemap.h"
+#include "utils.h"
+#include "xcectx.h"
+
+static int handle_get_reply(xce_ctx_t*   ctx,
+                            xce_reply_t* reply)
+{
+    xce_data_t* getdata = reply->data.getdata;
+    dataset_t*  dataset = ctx->reply.u.get;
+
+    return extract_data(dataset, getdata);
+}
+
+static int handle_sub_reply(xce_ctx_t*   ctx,
+                            xce_reply_t* reply)
+{
+    if (reply->data_which != RequestReply_data_jobid) {
+        return -1;
+    }
+
+    ctx->reply.u.jobid = reply->data.jobid;
+    return 1;
+}
+
+static int handle_reply(xce_ctx_t*   ctx,
+                        xce_reply_t* reply)
+{
+    int rc = 0;
+
+    if (reply->status < 0) {
+        return reply->status;
+    }
+
+    switch (reply->opcode) {
+    case OPCode_get:
+        rc = handle_get_reply(ctx, reply);
+        break;
+    case OPCode_jobsub:
+        rc = handle_sub_reply(ctx, reply);
+        break;
+    case OPCode_clusterconf:
+        rc = -1;
+        if (reply->data_which == RequestReply_data_clusterconf) {
+            MOVE_PTR(ctx->reply.u.clusterconf,
+                     reply->data.clusterconf);
+            rc = 1;
+        }
+        break;
+    default:
+        rc = -ENOENT;
+        break;
+    }
+
+    return rc;
+}
+
+static int vtask_entry(vtask_t* task)
+{
+    endpoint_t*   ep  = task->ctx.ep;
+    xce_ctx_t*    ctx = NULL;
+    int           rc  = 0;
+    xce_request_t request;
+
+    if (ep == NULL) {
+        return -EINVAL;
+    }
+
+    ctx = (xce_ctx_t*)ep->owner;
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    rc = ioctx_decode(&(task->ctx), &request);
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (request.opcode == OPCode_reply) {
+        int need_wake = 1;
+        int status    = request.body.replymsg->status;
+
+        need_wake = handle_reply(ctx, request.body.replymsg);
+        if (need_wake) {
+            rc = xce_wakeup(ctx, (int)status);
+        }
+    } else {
+        int callnext = 1;
+
+        if ((request.opcode == OPCode_push) && (request.body_which == Request_body_pushmsg)) {
+
+            if (ctx->dataset == NULL) {
+                ctx->dataset        = (dataset_t*)calloc(1, sizeof(dataset_t));
+                ctx->dataset->table = request.body.pushmsg->table;
+            }
+
+            callnext = extract_data(ctx->dataset,
+                                    request.body.pushmsg);
+            if (callnext) {
+                FREEUP(request.body.pushmsg);
+                request.body_which = Request_body_datamsg;
+                MOVE_PTR(request.body.datamsg, ctx->dataset);
+            }
+        }
+
+        if (callnext) {
+            rc = (ctx->handler)(&request);
+        }
+    }
+
+    free_Request(&request);
+
+    return 0;
+}
+static int handle_input(endpoint_t* ep)
+{
+    xce_ctx_t* ctx = (xce_ctx_t*)ep->owner;
+    vtask_t*   task;
+    int        rc = 0;
+
+    task = vtask_alloc();
+    if (task == NULL) {
+        return -ENOMEM;
+    }
+
+    rc = vtask_init(task, (vtask_func_t)&vtask_entry, ep);
+    if (rc < 0) {
+        SET_ERROR(-rc);
+
+        if (rc == -EPIPE) {
+            /* connection broken */
+            TRACE("broken pipe, ep %p", ep);
+        }
+
+        FREEUP(task);
+        return rc;
+    }
+
+    rc = vtask_enq(&(ctx->vtaskmgr), task);
+    if (rc < 0) {
+        return rc;
+    }
+
+    return 0;
+}
+
+static int handle_output(endpoint_t* ep)
+{
+    data_item_t* it;
+    ioctx_t*     ctx;
+    int          rc;
+
+    it = endpoint_deq(ep);
+    if (it == NULL) {
+        return 0;
+    }
+
+    ctx = ITPTR_TO_IOCTX(it);
+
+    rc = ioctx_flush(ctx, ep->fd);
+    ioctx_deref(ctx, 1);
+
+    return rc;
+}
+
+static int handle_io(struct epoll_event* p)
+{
+    endpoint_t* ep = (endpoint_t*)p->data.ptr;
+    int         rc = 0;
+
+    if ((p->events & EPOLLIN) != 0) {
+        rc = handle_input(ep);
+    }
+
+    if (rc >= 0) {
+        if ((p->events & EPOLLOUT) != 0) {
+            rc = handle_output(ep);
+        }
+    }
+
+    if (rc < 0) {
+        xce_ctx_t* ctx   = (xce_ctx_t*)ep->owner;
+        iomgr_t*   iomgr = &(ctx->iomgr);
+        iomgr_unmonitor(iomgr, ep);
+        endpoint_deref(ep);
+
+        xce_try_wakeup(ctx, -EIO);
+
+        ctx->state = XCE_CTX_DOWN;
+
+        return rc;
+    }
+
+    return 0;
+}
+
+int xce_ctx_init(xce_ctx_t* ctx,
+                 char*      name,
+                 char*      key)
+{
+    int rc;
+
+    if ((ctx == NULL) || (name == NULL) || (key == NULL)) {
+        return -EINVAL;
+    }
+
+    memset(ctx, 0x0, sizeof(*ctx));
+    ctx->state       = XCE_CTX_ERR;
+    ctx->myinfo.name = strdup(name);
+    ctx->myinfo.key  = strdup(key);
+
+    if ((ctx->myinfo.name == NULL) || (ctx->myinfo.key == NULL)) {
+        FREEUP(ctx->myinfo.name);
+        FREEUP(ctx->myinfo.key);
+        return -ENOMEM;
+    }
+
+    rc = cbuf_init(&(ctx->wbuf), 1, sizeof(int));
+    if (rc < 0) {
+        FREEUP(ctx->myinfo.name);
+        FREEUP(ctx->myinfo.key);
+        return rc;
+    }
+
+    iomgr_init(&(ctx->iomgr));
+    vtaskmgr_init(&(ctx->vtaskmgr), 4);
+
+    ctx->ep    = NULL;
+    ctx->state = XCE_CTX_INIT;
+    return 0;
+}
+
+void xce_ctx_fini(xce_ctx_t* ctx)
+{
+    if (ctx == NULL) {
+        return;
+    }
+
+    xce_ctx_stop(ctx);
+
+    vtaskmgr_fini(&(ctx->vtaskmgr));
+    iomgr_fini(&(ctx->iomgr));
+    cbuf_fini(&(ctx->wbuf));
+
+    FREEUP(ctx->myinfo.name);
+    FREEUP(ctx->myinfo.key);
+
+    return;
+}
+
+int xce_ctx_set_handler(xce_ctx_t*            ctx,
+                        xce_request_handler_t handler)
+{
+    if ((ctx == NULL) || (handler == NULL)) {
+        return -EINVAL;
+    }
+
+    ctx->handler = handler;
+    return 0;
+}
+
+int xce_ctx_set_endpoint(xce_ctx_t*  ctx,
+                         endpoint_t* ep)
+{
+    if ((ctx == NULL) || (ep == NULL)) {
+        return -EINVAL;
+    }
+
+    ctx->ep = ep;
+    return 0;
+}
+
+int xce_ctx_start(xce_ctx_t* ctx)
+{
+    int rc;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if ((ctx->state != XCE_CTX_INIT) || (ctx->ep == NULL)) {
+        return -EINVAL;
+    }
+
+    endpoint_set_handler(ctx->ep,
+                         handle_io, ctx);
+    rc = iomgr_monitor(&(ctx->iomgr),
+                       ctx->ep);
+    if (rc < 0) {
+        ctx->state = XCE_CTX_ERR;
+        return rc;
+    }
+
+    iomgr_start(&(ctx->iomgr));
+    vtaskmgr_start(&(ctx->vtaskmgr));
+
+    ctx->state = XCE_CTX_RUN;
+
+    return 0;
+}
+
+int xce_ctx_stop(xce_ctx_t* ctx)
+{
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    if (ctx->ep != NULL) {
+        iomgr_unmonitor(&(ctx->iomgr),
+                        ctx->ep);
+        ctx->ep = NULL;
+    }
+    iomgr_stop(&(ctx->iomgr));
+
+    ctx->state = XCE_CTX_DOWN;
+
+    return 0;
+}
+
+int xce_wakeup(xce_ctx_t* ctx,
+               int        status)
+{
+    int rc = 0;
+
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    rc = cbuf_tryput(&(ctx->wbuf), &status);
+    if (rc == -EAGAIN) {
+        /* pending data in wbuf, this is normal */
+        rc = 0;
+    }
+
+    return rc;
+}
+
+int xce_try_wakeup(xce_ctx_t* ctx,
+                   int        status)
+{
+    if (ctx == NULL) {
+        return -EINVAL;
+    }
+
+    return cbuf_tryput(&(ctx->wbuf), &status);
+}
+
+void xce_block_tasks(xce_ctx_t* ctx)
+{
+    if (ctx == NULL) {
+        return;
+    }
+
+    endpoint_block_tasks(ctx->ep);
+}
+
+void xce_unblock_tasks(xce_ctx_t* ctx)
+{
+    if (ctx == NULL) {
+        return;
+    }
+
+    endpoint_unblock_tasks(ctx->ep);
+    vtaskmgr_wakeup(&(ctx->vtaskmgr));
+}

src/lib/xlist.c

@@ -0,0 +1,99 @@
+#include <errno.h>
+#include <stdlib.h>
+
+#include "utils.h"
+#include "xlist.h"
+
+int xlist_init(xlist_t* l,
+               int      delta)
+{
+    if ((l == NULL) || (delta <= 0)) {
+        return -EINVAL;
+    }
+
+    l->delta = delta;
+    l->data  = NULL;
+    l->count = 0;
+    l->caps  = 0;
+
+    return 0;
+}
+
+static int xlist_ensure(xlist_t* l)
+{
+    void* p;
+    int   nc, i;
+
+    if (l->count < l->caps) {
+        return 1;
+    }
+
+    nc = l->caps + l->delta;
+    p  = (void*)realloc(l->data, nc * sizeof(void*));
+    if (p == NULL) {
+        return -ENOMEM;
+    }
+
+    l->data = p;
+    for(i = l->caps; i < nc; i ++) {
+	l->data[i] = NULL;
+    }
+    
+    l->caps = nc;
+
+    return 1;
+}
+
+int xlist_store(xlist_t* l,
+                void*    data)
+{
+    int rc;
+
+    if (l == NULL) {
+        return -EINVAL;
+    }
+
+    rc = xlist_ensure(l);
+    if (rc < 0) {
+        return rc;
+    }
+
+    l->data[l->count] = data;
+    l->count++;
+
+    return 1;
+}
+
+void xlist_clear(xlist_t *l) {
+    if (l == NULL) {
+	return;
+    }
+
+    l->count = 0;
+}
+
+int xlist_iter(xlist_t *l,
+	       xlist_iter_fn_t fn,
+	       void *data) {
+    int i;
+    
+    if ((l == NULL) || (fn == NULL)) {
+	return -EINVAL;
+    }
+
+    for(i = 0; i < l->count; i ++) {
+	(*fn)((void *)l->data[i], data);
+    }
+
+    return 0;
+}
+
+void xlist_fini(xlist_t* l)
+{
+    if (l == NULL) {
+        return;
+    }
+
+    FREEUP(l->data);
+    return;
+}

src/lib/xstring.c

@@ -0,0 +1,147 @@
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "htab.h"
+#include "utils.h"
+#include "xstring.h"
+
+static void* xstring_create(objmgr_t*, objtype_t*, va_list);
+static void  xstring_fini(void*);
+static void  xstring_type_fini(objtype_t*);
+
+static xstring_typedata_t xstring_typedata;
+
+static objtype_t xstring = {
+    .type         = "xstring",
+    .fn_create    = (obj_create_fn_t)xstring_create,
+    .fn_fini      = xstring_fini,
+    .fn_type_fini = (objtype_fini_fn_t)xstring_type_fini,
+    .typedata     = &xstring_typedata
+};
+
+static void xstring_obj_killer(myobj_t* obj)
+{
+    if (obj == NULL) {
+        return;
+    }
+
+    memset(obj, 0x0, sizeof(*obj));
+    free(obj);
+}
+
+int register_xstring(objmgr_t* mgr,
+                     int       nslots,
+                     int       delta)
+{
+    ht_init(&(xstring_typedata.tab), nslots,
+            string_hash_key);
+    zarray_init(&(xstring_typedata.strings), delta,
+                (zarray_item_killer_t)xstring_obj_killer);
+
+    return objmgr_register(mgr, &xstring);
+}
+
+static void* xstring_create(objmgr_t*  mgr,
+                            objtype_t* type,
+                            va_list    ap)
+{
+    char*               s        = va_arg(ap, char*);
+    char*               p        = NULL;
+    myobj_t*            obj      = NULL;
+    hent_t*             e        = NULL;
+    xstring_typedata_t* typedata = NULL;
+    int32_t             idx;
+
+    if (s == NULL) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    typedata = (xstring_typedata_t*)type->typedata;
+
+    e = ht_get((htab_t*)&(typedata->tab), (void*)s);
+    if (e != NULL) {
+        idx = e->data;
+        obj = (myobj_t*)zarray_get(&(typedata->strings), idx);
+
+        if (obj != NULL) {
+            p = MYOBJ_TO_OBJ(char*, obj);
+            return obj_ref(p, OBJREF_NOP);
+        }
+
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+
+    obj = (myobj_t*)calloc(1, sizeof(*obj) + strlen(s) + 1);
+    if (obj == NULL) {
+        return NULL;
+    }
+
+    idx = zarray_store(&(typedata->strings), obj);
+    if (idx < 0) {
+        free(obj);
+        SET_ERROR(-idx);
+        return NULL;
+    }
+
+    obj_init(mgr, type, idx, obj);
+
+    p = MYOBJ_TO_OBJ(char*, obj);
+
+    strcpy(p, s);
+
+    e = ht_add((htab_t*)&(typedata->tab), (void*)s);
+    if (e == NULL) {
+        obj_deref(p);
+        return NULL;
+    }
+
+    e->data = idx;
+
+    return p;
+}
+
+/* this will ONLY be called by obj_deref() */
+static void xstring_fini(void* s)
+{
+    myobj_t*            myobj;
+    objtype_t*          typeval;
+    xstring_typedata_t* typedata;
+    int32_t             idx;
+
+    if (s == NULL) {
+        return;
+    }
+
+    myobj    = TO_MYOBJ(s);
+    typeval  = myobj->type;
+    typedata = (xstring_typedata_t*)typeval->typedata;
+
+    if (typedata == NULL) {
+        return;
+    }
+
+    ht_del(&(typedata->tab), (void*)s);
+
+    idx = (int32_t)myobj->exdata;
+    if (idx >= 0) {
+        zarray_del(&(typedata->strings), idx);
+    }
+
+    return;
+}
+
+static void xstring_type_fini(objtype_t* typeval)
+{
+    xstring_typedata_t* typedata = (xstring_typedata_t*)typeval->typedata;
+
+    if (typedata == NULL) {
+        return;
+    }
+
+    ht_fini(&(typedata->tab));
+    zarray_fini(&(typedata->strings));
+}

src/lib/zarray.c

@@ -0,0 +1,277 @@
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "utils.h"
+#include "zarray.h"
+
+int zarray_init(zarray_t*            array,
+                int                  delta,
+                zarray_item_killer_t killer)
+{
+    if ((array == NULL) || (delta <= 0)) {
+        return -EINVAL;
+    }
+
+    mylock_init(&(array->lock));
+
+    array->delta  = delta;
+    array->data   = NULL;
+    array->killer = killer;
+    array->count  = 0;
+    array->caps   = 0;
+    array->pos    = 0;
+    return 0;
+}
+
+static int zarray_ensure_items(zarray_t* array,
+                               int       mincap)
+{
+    int   nc, i;
+    void* p;
+
+    if (mincap < array->caps) {
+        return 1;
+    }
+
+    nc = mincap;
+    if (nc < array->caps + array->delta) {
+        nc = array->caps + array->delta;
+    }
+
+    p = (void*)realloc(array->data, nc * sizeof(void*));
+    if (p == NULL) {
+        return -ENOMEM;
+    }
+
+    array->data = p;
+    for (i = array->caps; i < nc; i++) {
+        array->data[i] = NULL;
+    }
+
+    /* this will make we can get next free
+     * location quickly
+     */
+    array->pos  = array->caps;
+    array->caps = nc;
+
+    return 1;
+}
+
+int zarray_store(zarray_t* array,
+                 void*     data)
+{
+    int rc, i;
+
+    if (array == NULL) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(array->lock));
+    rc = zarray_ensure_items(array, array->count + 1);
+
+    if (rc >= 0) {
+        int idx;
+
+        rc  = -ENOMEM;
+        idx = array->pos;
+
+        for (i = 0; i < array->caps; i++) {
+            if (array->data[idx] == NULL) {
+                array->data[idx] = data;
+                array->count++;
+                rc = idx;
+                break;
+            }
+
+            idx++;
+            if (idx >= array->caps) {
+                idx = 0;
+            }
+        }
+        if (rc >= 0) {
+            array->pos = (rc + 1) % array->caps;
+        }
+    }
+
+    mylock_unlock(&(array->lock));
+
+    return rc;
+}
+
+int zarray_clear(zarray_t* array,
+                 int       idx)
+{
+    if (array == NULL) {
+        return -EINVAL;
+    }
+
+    if ((idx < 0) || (idx >= array->caps)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(array->lock));
+    array->data[idx] = NULL;
+    mylock_unlock(&(array->lock));
+
+    return 0;
+}
+
+void zarray_fini(zarray_t* array)
+{
+    if (array == NULL) {
+        return;
+    }
+
+    if (array->data != NULL) {
+        int i;
+
+        if (array->killer != NULL) {
+            for (i = 0; i < array->caps; i++) {
+                if (array->data[i] != NULL) {
+                    void* ptr      = array->data[i];
+                    array->data[i] = NULL;
+                    (array->killer)(ptr);
+                }
+            }
+        }
+
+        free(array->data);
+        array->data = NULL;
+    }
+}
+
+int zarray_iter_ex(zarray_t* array,
+                   xlist_t*  l,
+                   void*     arg)
+{
+    int i, j;
+
+    if ((array == NULL) || (l == NULL)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(array->lock));
+
+    for (j = 0; j < l->count; j++) {
+        zarray_iter_fn_t f = (zarray_iter_fn_t)l->data[j];
+
+        if (f == NULL) {
+            continue;
+        }
+
+        for (i = 0; i < array->caps; i++) {
+            (*f)(arg, array->data[i]);
+        }
+    }
+    mylock_unlock(&(array->lock));
+
+    return 0;
+}
+
+int zarray_iter(zarray_t*        array,
+                zarray_iter_fn_t f,
+                void*            arg)
+{
+    int     rc;
+    xlist_t l;
+
+    if ((array == NULL) || (f == NULL)) {
+        return -EINVAL;
+    }
+
+    xlist_init(&l, 2);
+    xlist_store(&l, f);
+
+    rc = zarray_iter_ex(array, &l, arg);
+
+    xlist_fini(&l);
+    return rc;
+}
+
+int zarray_del(zarray_t* array,
+               int       idx)
+{
+    void* ptr = NULL;
+
+    if (array == NULL) {
+        return -EINVAL;
+    }
+
+    if ((idx < 0) && (idx >= array->caps)) {
+        return -EINVAL;
+    }
+
+    mylock_lock(&(array->lock));
+    ptr              = array->data[idx];
+    array->data[idx] = NULL;
+    if (ptr != NULL) {
+        array->count--;
+    }
+
+    /* idx will be a usable position */
+    array->pos = idx;
+    mylock_unlock(&(array->lock));
+
+    if (array->killer != NULL) {
+        if (ptr != NULL) {
+            (array->killer)(ptr);
+        }
+    }
+
+    return 0;
+}
+
+int zarray_put(zarray_t* array,
+               int       idx,
+               void*     data)
+{
+    int rc = 0;
+
+    /* clang-format off */
+    if ((array == NULL) || 
+	(idx < 0) ||
+	(data == NULL)) {
+	return -EINVAL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(array->lock));
+    rc = zarray_ensure_items(array, idx + 1);
+    if (rc >= 0) {
+        void* ptr        = array->data[idx];
+        array->data[idx] = data;
+        rc               = 0;
+        if (ptr != NULL) {
+            if (array->killer != NULL) {
+                array->killer(ptr);
+            }
+        } else {
+            array->count++;
+        }
+    }
+    mylock_unlock(&(array->lock));
+
+    return rc;
+}
+
+void* zarray_get(zarray_t* array,
+                 int       idx)
+{
+    void* ptr = NULL;
+
+    /* clang-format off */
+    if ((array == NULL) || 
+        (idx < 0) || 
+        (idx >= array->caps)) {
+        SET_ERROR(EINVAL);
+        return NULL;
+    }
+    /* clang-format on */
+
+    mylock_lock(&(array->lock));
+    ptr = array->data[idx];
+    mylock_unlock(&(array->lock));
+
+    return ptr;
+}