update controller code

src/controller/cluster.c

@@ -0,0 +1,686 @@
+#include "config.h"
+
+#include <errno.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "cluster.h"
+#include "htab.h"
+#include "mylock.h"
+#include "request.h"
+#include "vtaskmgr.h"
+#include "xce/cluster_def.h"
+#include "zarray.h"
+
+#include "xce/types.h"
+
+#include "utils.h"
+#include "worker.h"
+#include "xce/errnox.h"
+#include "xstring.h"
+
+static time_t        last_print_time = 0;
+static unsigned long npkts_old       = 0;
+static unsigned long npkts           = 0;
+
+static xce_peerinfo_t from_controller = {
+    .name = "controller",
+    .key  = "keepSecret"
+};
+
+static ep_private_t*
+ep_get_private()
+{
+    ep_private_t* priv;
+
+    priv = (ep_private_t*)calloc(1, sizeof(*priv));
+    return priv;
+}
+
+static int handle_client_leave(cluster_t* c)
+{
+    return 0;
+}
+
+static void handle_endpoint_error(endpoint_t* ep)
+{
+    cluster_t*    c    = (cluster_t*)ep->owner;
+    ep_private_t* priv = NULL;
+    int           role = -1;
+
+    if (c == NULL) {
+        ERROR("endpoint %p has no owner cluster", ep);
+
+        return;
+    }
+
+    iomgr_unmonitor(&(c->iomgr), ep);
+
+    priv = (ep_private_t*)ep->data;
+
+    TRACE("endpoint %p priv data %p",
+          ep, priv);
+
+    if (priv != NULL) {
+        role = priv->role;
+        switch (role) {
+        case HostRole_worker:
+            if (priv->u.idx >= 0) {
+                handle_worker_leave(c, priv->u.idx);
+            } else {
+                ERROR("incorrect worker idx %d",
+                      priv->u.idx);
+            }
+            break;
+        case HostRole_scheduler:
+            handle_scheduler_leave(c);
+            break;
+        case HostRole_controller:
+            break;
+        case HostRole_client:
+            handle_client_leave(c);
+            break;
+        default:
+            ERROR("unknown role %d for endpoint %p",
+                  priv->role, ep);
+            break;
+        }
+    }
+
+    if (ep->idx >= 0) {
+        zarray_del(&(c->endpoints), ep->idx);
+        return;
+    }
+
+    endpoint_deref(ep);
+
+    return;
+}
+
+static int do_notify(cluster_t*     c,
+                     endpoint_t*    ep,
+                     xce_request_t* req)
+{
+    xce_notify_data_t* data = req->body.notifymsg;
+
+    switch (data->event) {
+    case NotifyData_data_jobPidInfo:
+        on_job_start(c, data->data.pidinfo);
+        break;
+    case NotifyData_data_jobFailInfo:
+        on_job_fail(c, data->data.failinfo);
+        break;
+    case NotifyData_data_jobFinishInfo:
+        on_job_finish(c, data->data.finishinfo);
+        break;
+    case NotifyData_data_stoppedJob:
+        on_job_stop(c, data->data.stopped_job);
+        break;
+    case NotifyData_data_resumedJob:
+        on_job_resume(c, data->data.resumed_job);
+        break;
+    case NotifyData_data_ckptinfo:
+        on_job_ckpt(c, ep, data->data.ckptinfo);
+        break;
+    case NotifyData_data_postDoneJob:
+        on_job_postdone(c, ep, data->data.pdone_job);
+        break;
+    case NotifyData_data_taskPidInfo:
+        on_task_start(c, data->data.taskinfo);
+        break;
+    case NotifyData_data_taskFailInfo:
+        on_task_fail(c, data->data.taskfailinfo);
+        break;
+    case NotifyData_data_taskFinish:
+        on_task_finish(c, data->data.taskfinish);
+        break;
+    default:
+        break;
+    }
+    return 0;
+}
+static int myreq_handler(vtask_t* task)
+{
+    cluster_t*    c    = (cluster_t*)task->ctx.ep->owner;
+    int           stop = 0;
+    int           rc   = 0;
+    xce_request_t request;
+    xce_reply_t   reply;
+
+    rc = ioctx_decode(&(task->ctx), &request);
+
+    TRACE("%s cpu#%d opcode %d",
+          __func__, task->vcpu,
+          request.opcode);
+
+    init_reply_by_req(&reply, &request);
+
+    switch (request.opcode) {
+    case OPCode_shutdown:
+        reply.status     = 0;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(task->ctx.ep, c->myhost, &reply);
+        stop = 1;
+        cbuf_put(&(c->wbuf), &stop);
+        break;
+    case OPCode_hello:
+    case OPCode_nop:
+        reply.status     = 0;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(task->ctx.ep, c->myhost, &reply);
+        break;
+    case OPCode_join:
+        rc = handle_join(c, task->ctx.ep, &request);
+        break;
+    case OPCode_jobdispatch:
+        rc = handle_jobdispatch(c, task->ctx.ep, &request);
+        break;
+    case OPCode_jobsub:
+        rc = handle_jobsubmit(c, task->ctx.ep, &request);
+        break;
+    case OPCode_get:
+        rc = do_get_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_notify:
+        rc = do_notify(c, task->ctx.ep, &request);
+        break;
+    case OPCode_clusterconf:
+        rc = handle_clusterconf(c, task->ctx.ep, &request);
+        break;
+    case OPCode_update:
+        rc = do_update_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_jobcontrol:
+        rc = do_jobcontrol_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_queuecontrol:
+        rc = do_queuecontrol_op(c, task->ctx.ep, &request);
+        break;
+    case OPCode_runtasks:
+        rc = do_runtasks(c, task->ctx.ep, &request);
+        break;
+    default:
+        rc = -E_XCE_BAD_OPCODE;
+        ERROR("Unknown opcode: %d",
+              request.opcode);
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(task->ctx.ep, c->myhost, &reply);
+
+        break;
+    }
+
+    free_Request(&request);
+
+    return rc;
+}
+
+static int vtask_entry(vtask_t* task)
+{
+    int rc;
+
+    rc = myreq_handler(task);
+
+    return rc;
+}
+
+static int handle_input(endpoint_t* ep)
+{
+    cluster_t* c = (cluster_t*)ep->owner;
+    time_t     now;
+    vtask_t*   task;
+    int        rc = 0;
+
+    task = vtask_alloc();
+    if (task == NULL) {
+        PERROR("vtask_alloc");
+
+        return -ENOMEM;
+    }
+
+    rc = vtask_init(task, (vtask_func_t)&vtask_entry, ep);
+    if (rc < 0) {
+        SET_ERROR(-rc);
+
+        INFO("vtask_init failed, maybe peer side closed the connection");
+
+        if (rc == -EPIPE) {
+            /* connection broken */
+            TRACE("broken pipe, ep %p", ep);
+
+            handle_endpoint_error(ep);
+        }
+
+        FREEUP(task);
+        return rc;
+    }
+
+    rc = vtask_enq(&(c->vtaskmgr), task);
+    if (rc < 0) {
+        PERROR("vtask_enq");
+
+        return rc;
+    }
+
+    now = time(NULL);
+    npkts++;
+
+    if (now - last_print_time > 10) {
+        TRACE("%ld npkts:%ld iops: %f\n", now, npkts,
+              (npkts - npkts_old) * 1.0 / (now - last_print_time));
+        npkts_old       = npkts;
+        last_print_time = now;
+    }
+
+    return rc;
+}
+
+static int handle_output(endpoint_t* ep)
+{
+    data_item_t* it;
+    ioctx_t*     ctx;
+    int          rc;
+
+    it = endpoint_deq(ep);
+
+    if (it == NULL) {
+        return 0;
+    }
+
+    ctx = ITPTR_TO_IOCTX(it);
+
+    rc = ioctx_flush(ctx, ep->fd);
+    ioctx_deref(ctx, 1);
+
+    return rc;
+}
+
+static int handle_io(struct epoll_event* p)
+{
+    endpoint_t* ep = (endpoint_t*)p->data.ptr;
+    int         rc = 0;
+
+    if ((p->events & EPOLLIN) != 0) {
+        rc = handle_input(ep);
+    }
+
+    if (rc >= 0) {
+        if ((p->events & EPOLLOUT) != 0) {
+            rc = handle_output(ep);
+        }
+    }
+
+    return rc;
+}
+
+static int handle_connection(struct epoll_event* p)
+{
+    endpoint_t*   ep = (endpoint_t*)p->data.ptr;
+    endpoint_t*   ep2;
+    cluster_t*    c     = (cluster_t*)ep->owner;
+    iomgr_t*      iomgr = &(c->iomgr);
+    int           idx;
+    ep_private_t* priv = NULL;
+
+    ep2 = endpoint_get();
+    if (ep2 == NULL) {
+        PERROR("endpoint_get");
+
+        return -ENOMEM;
+    }
+
+    endpoint_accept(ep, ep2);
+    if (ep2->fd < 0) {
+        PERROR("endpoint_accept");
+        return -1;
+    }
+
+    TRACE("server sock %d accepted %d\n", ep->fd, ep2->fd);
+
+    priv = ep_get_private();
+    if (priv == NULL) {
+        PERROR("ep_get_private");
+
+        endpoint_deref(ep2);
+        return -ENOMEM;
+    }
+
+    /* Assume client as default */
+    priv->role  = HostRole_client;
+    priv->u.idx = -1;
+
+    idx = zarray_store(&(c->endpoints), ep2);
+    if (idx < 0) {
+        SET_ERROR(-idx);
+
+        PERROR("zarray_store");
+
+        FREEUP(priv);
+        endpoint_deref(ep2);
+
+        return idx;
+    }
+
+    ep2->idx  = idx;
+    ep2->data = priv;
+
+    endpoint_set_handler(ep2, handle_io, c);
+
+    iomgr_monitor(iomgr, ep2);
+    return 0;
+}
+
+static int controller_init(controller_t* c, cluster_t* cluster)
+{
+    c->ep = NULL;
+
+    return 0;
+}
+
+static void endpoint_killer(void* p)
+{
+    endpoint_t* ep = (endpoint_t*)p;
+    cluster_t*  c  = (cluster_t*)ep->owner;
+
+    if (ep->idx >= 0) {
+        zarray_clear(&(c->endpoints), ep->idx);
+        ep->idx = -1;
+    }
+    endpoint_deref(ep);
+
+    return;
+}
+
+int cluster_build_config(cluster_t* c)
+{
+    char valbuf[128];
+    int  i;
+
+    snprintf(valbuf, sizeof(valbuf) - 1,
+             "%d", c->myconf->cluster_config->heartbeat_interval);
+    add_config(c, "heartbeat_interval", valbuf);
+    snprintf(valbuf, sizeof(valbuf) - 1,
+             "%d", c->myconf->cluster_config->inactive_heartbeat_count);
+    add_config(c, "inactive_heartbeat_count", valbuf);
+    snprintf(valbuf, sizeof(valbuf) - 1,
+             "%d", c->myconf->cluster_config->ncandidates);
+    add_config(c, "ncandidates", valbuf);
+
+    for (i = 0; i < c->myconf->cluster_config->ncandidates; i++) {
+        char keybuf[128];
+
+        sprintf(keybuf, "candidate_%d_name", i);
+        sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->name);
+
+        add_config(c, keybuf, valbuf);
+
+        sprintf(keybuf, "candidate_%d_host", i);
+        sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->host);
+
+        add_config(c, keybuf, valbuf);
+        sprintf(keybuf, "candidate_%d_port", i);
+        sprintf(valbuf, "%d", c->myconf->cluster_config->candidates[i]->port);
+
+        add_config(c, keybuf, valbuf);
+    }
+
+    return 0;
+}
+
+int cluster_build_resources(cluster_t* c)
+{
+    int i;
+
+    if (c->myconf == NULL) {
+        return -EINVAL;
+    }
+
+    for (i = 0; i < c->myconf->cluster_config->nres; i++) {
+        xce_resource_t* res;
+
+        res = create_obj_by_typename(&(c->objmgr), "xresource",
+                                     c->myconf->cluster_config->resources[i]);
+        if (res == NULL) {
+            return -errno;
+        }
+    }
+
+    return 0;
+}
+
+int cluster_load_jobs(cluster_t* c)
+{
+    load_jobs(c);
+
+    return 0;
+}
+
+int cluster_build_queues(cluster_t* c)
+{
+    build_queues(c);
+    return 0;
+}
+
+int cluster_build_groups(cluster_t* c)
+{
+    return 0;
+}
+
+static int xce_config_init(int     type,
+                           int     situation,
+                           void*   data,
+                           va_list ap)
+{
+    int        rc = 0;
+    cluster_t* c  = va_arg(ap, cluster_t*);
+
+    if (situation == HOOK_BEFORE_EVENT) {
+        c->myconf->cluster_config = get_default_conf(c);
+        if (c->myconf->cluster_config == NULL) {
+            return -EINVAL;
+        }
+        return 0;
+    }
+
+    xce_use(c);
+
+    rc = cluster_build_config(c);
+    if (rc < 0) {
+        return rc;
+    }
+    rc = cluster_build_resources(c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cluster_build_queues(c);
+    if (rc < 0) {
+        return rc;
+    }
+    rc = cluster_build_groups(c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = cluster_load_jobs(c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    tlog_info("xce controller is running");
+
+    return rc;
+}
+
+static int cluster_hook_fun(int     type,
+                            int     situation,
+                            int     detail,
+                            void*   data,
+                            va_list ap)
+{
+    if (detail == HOOK_DETAIL_CLUSTER_INIT) {
+        return xce_config_init(type, situation, data, ap);
+    }
+
+    return 0;
+}
+
+static int xce_install_hooks(cluster_t* c)
+{
+    xce_add_hook(&(c->hooksys),
+                 XCE_CONTROLLER_HOOK_CLUSTER,
+                 (hook_fn_t)cluster_hook_fun,
+                 NULL);
+    return 0;
+}
+
+int cluster_init(cluster_t* c)
+{
+    int i, rc;
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    c->myhost = &from_controller;
+    /* FIXME: */
+    c->ncontrollers = 1;
+    c->myidx        = 0;
+
+    xce_hook_init(&(c->hooksys), XCE_CONTROLLER_HOOK_MAX);
+
+    xce_install_hooks(c);
+
+    rc = xce_run_hooks(&(c->hooksys),
+                       XCE_CONTROLLER_HOOK_CLUSTER,
+                       HOOK_BEFORE_EVENT,
+                       HOOK_DETAIL_CLUSTER_INIT, c);
+    if (rc < 0) {
+        return rc;
+    }
+
+    mylock_init(&(c->sched.lock));
+    c->sched.ep = NULL;
+
+    kvmgr_init(&(c->configmgr),
+               string_hash_key,
+               128, 128,
+               (zarray_item_killer_t)config_obj_killer);
+
+    objmgr_init(&(c->objmgr));
+    jobmgr_init(&(c->jobmgr),
+                c->myconf->cluster_config->jobmgr->nhash_slots,
+                c->myconf->cluster_config->jobmgr->delta);
+    resmgr_init(&(c->resmgr),
+                c->myconf->cluster_config->resmgr->nhash_slots,
+                c->myconf->cluster_config->resmgr->delta,
+                c->myconf->cluster_config->resmgr->inst_delta);
+    workermgr_init(&(c->workermgr),
+                   c->myconf->cluster_config->workermgr->nhash_slots,
+                   c->myconf->cluster_config->workermgr->delta);
+    qmgr_init(&(c->qmgr),
+              c->myconf->cluster_config->qmgr->nhash_slots,
+              c->myconf->cluster_config->qmgr->delta);
+    iomgr_init(&(c->iomgr));
+    register_xstring(&(c->objmgr),
+                     c->myconf->cluster_config->xstrings->nhash_slots,
+                     c->myconf->cluster_config->xstrings->delta);
+    register_types(c);
+
+    zarray_init(&(c->endpoints), 16, endpoint_killer);
+
+    rc = endpoint_init(&(c->ep),
+                       AF_INET,
+                       SOCK_STREAM,
+                       c->myconf->cluster_config->port,
+                       5);
+    if (rc < 0) {
+        return rc;
+    }
+
+    for (i = 0; i < c->ncontrollers; i++) {
+        controller_init(&(c->controllers[i]), c);
+    }
+
+    cbuf_init(&(c->wbuf), 1, sizeof(int));
+
+    zarray_init(&(c->configs), 64, NULL /* FIXME */);
+
+    vtaskmgr_init(&(c->vtaskmgr), c->myconf->cluster_config->vcpus);
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_CLUSTER,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_CLUSTER_INIT, c);
+
+    return 0;
+}
+
+int cluster_start(cluster_t* c)
+{
+    int rc;
+
+    endpoint_set_handler(&(c->ep), handle_connection, c);
+    rc = iomgr_monitor(&(c->iomgr), &(c->ep));
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    iomgr_start(&(c->iomgr));
+    vtaskmgr_start(&(c->vtaskmgr));
+
+    c->running = 1;
+    return 0;
+}
+
+int cluster_run(cluster_t* c)
+{
+    for (;;) {
+        int stop;
+
+        cbuf_get(&(c->wbuf), &stop);
+        if (stop) {
+            break;
+        }
+    }
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_CLUSTER,
+                  HOOK_BEFORE_EVENT,
+                  HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
+    c->running = 0;
+    iomgr_unmonitor(&(c->iomgr), &(c->ep));
+    iomgr_stop(&(c->iomgr));
+
+    return 0;
+}
+
+void cluster_fini(cluster_t* c)
+{
+    vtaskmgr_fini(&(c->vtaskmgr));
+    endpoint_deref(&(c->ep));
+
+    xce_run_hooks(&(c->hooksys),
+                  XCE_CONTROLLER_HOOK_CLUSTER,
+                  HOOK_AFTER_EVENT,
+                  HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
+    cbuf_fini(&(c->wbuf));
+    iomgr_fini(&(c->iomgr));
+
+    jobmgr_fini(&(c->jobmgr));
+    workermgr_fini(&(c->workermgr));
+    qmgr_fini(&(c->qmgr));
+    resmgr_fini(&(c->resmgr));
+    kvmgr_fini(&(c->configmgr));
+    mylock_fini(&(c->sched.lock));
+
+    zarray_fini(&(c->endpoints));
+    zarray_fini(&(c->configs));
+
+    objmgr_fini(&(c->objmgr));
+
+    xce_hook_fini(&(c->hooksys));
+
+    return;
+}