myxce/src/controller/cluster.c

#include "config.h"

#include <errno.h>
#include <string.h>
#include <unistd.h>

#include "cluster.h"
#include "htab.h"
#include "mylock.h"
#include "request.h"
#include "vtaskmgr.h"
#include "xce/cluster_def.h"
#include "zarray.h"

#include "xce/types.h"

#include "utils.h"
#include "worker.h"
#include "xce/errnox.h"
#include "xstring.h"

static time_t        last_print_time = 0;
static unsigned long npkts_old       = 0;
static unsigned long npkts           = 0;

static xce_peerinfo_t from_controller
    = { .name = "controller", .key = "keepSecret" };

static ep_private_t* ep_get_private()
{
    ep_private_t* priv;

    priv = (ep_private_t*)calloc(1, sizeof(*priv));
    return priv;
}

static int handle_client_leave(cluster_t* c) { return 0; }

static void handle_endpoint_error(endpoint_t* ep)
{
    cluster_t*    c    = (cluster_t*)ep->owner;
    ep_private_t* priv = NULL;
    int           role = -1;

    if (c == NULL) {
        ERROR("endpoint %p has no owner cluster", ep);

        return;
    }

    iomgr_unmonitor(&(c->iomgr), ep);

    priv = (ep_private_t*)ep->data;

    TRACE("endpoint %p priv data %p", ep, priv);

    if (priv != NULL) {
        role = priv->role;
        switch (role) {
        case HostRole_worker:
            if (priv->u.idx >= 0) {
                handle_worker_leave(c, priv->u.idx);
            } else {
                ERROR("incorrect worker idx %d", priv->u.idx);
            }
            break;
        case HostRole_scheduler:
            handle_scheduler_leave(c);
            break;
        case HostRole_controller:
            break;
        case HostRole_client:
            handle_client_leave(c);
            break;
        default:
            ERROR("unknown role %d for endpoint %p", priv->role, ep);
            break;
        }
    }

    if (ep->idx >= 0) {
        zarray_del(&(c->endpoints), ep->idx);
        return;
    }

    endpoint_deref(ep);

    return;
}

static int do_notify(cluster_t* c, endpoint_t* ep, xce_request_t* req)
{
    xce_notify_data_t* data = req->body.notifymsg;

    switch (data->event) {
    case NotifyData_data_jobPidInfo:
        on_job_start(c, data->data.pidinfo);
        break;
    case NotifyData_data_jobFailInfo:
        on_job_fail(c, data->data.failinfo);
        break;
    case NotifyData_data_jobFinishInfo:
        on_job_finish(c, data->data.finishinfo);
        break;
    case NotifyData_data_stoppedJob:
        on_job_stop(c, data->data.stopped_job);
        break;
    case NotifyData_data_resumedJob:
        on_job_resume(c, data->data.resumed_job);
        break;
    case NotifyData_data_ckptinfo:
        on_job_ckpt(c, ep, data->data.ckptinfo);
        break;
    case NotifyData_data_postDoneJob:
        on_job_postdone(c, ep, data->data.pdone_job);
        break;
    case NotifyData_data_taskPidInfo:
        on_task_start(c, data->data.taskinfo);
        break;
    case NotifyData_data_taskFailInfo:
        on_task_fail(c, data->data.taskfailinfo);
        break;
    case NotifyData_data_taskFinish:
        on_task_finish(c, data->data.taskfinish);
        break;
    default:
        break;
    }
    return 0;
}
static int myreq_handler(vtask_t* task)
{
    cluster_t*    c    = (cluster_t*)task->ctx.ep->owner;
    int           stop = 0;
    int           rc   = 0;
    xce_request_t request;
    xce_reply_t   reply;

    rc = ioctx_decode(&(task->ctx), &request);

    TRACE("%s cpu#%d opcode %d", __func__, task->vcpu, request.opcode);

    init_reply_by_req(&reply, &request);

    switch (request.opcode) {
    case OPCode_shutdown:
        reply.status     = 0;
        reply.data_which = RequestReply_data_nodata;
        xce_reply(task->ctx.ep, c->myhost, &reply);
        stop = 1;
        cbuf_put(&(c->wbuf), &stop);
        break;
    case OPCode_hello:
    case OPCode_nop:
        reply.status     = 0;
        reply.data_which = RequestReply_data_nodata;
        xce_reply(task->ctx.ep, c->myhost, &reply);
        break;
    case OPCode_join:
        rc = handle_join(c, task->ctx.ep, &request);
        break;
    case OPCode_jobdispatch:
        rc = handle_jobdispatch(c, task->ctx.ep, &request);
        break;
    case OPCode_jobsub:
        rc = handle_jobsubmit(c, task->ctx.ep, &request);
        break;
    case OPCode_get:
        rc = do_get_op(c, task->ctx.ep, &request);
        break;
    case OPCode_notify:
        rc = do_notify(c, task->ctx.ep, &request);
        break;
    case OPCode_clusterconf:
        rc = handle_clusterconf(c, task->ctx.ep, &request);
        break;
    case OPCode_update:
        rc = do_update_op(c, task->ctx.ep, &request);
        break;
    case OPCode_jobcontrol:
        rc = do_jobcontrol_op(c, task->ctx.ep, &request);
        break;
    case OPCode_queuecontrol:
        rc = do_queuecontrol_op(c, task->ctx.ep, &request);
        break;
    case OPCode_runtasks:
        rc = do_runtasks(c, task->ctx.ep, &request);
        break;
    default:
        rc = -E_XCE_BAD_OPCODE;
        ERROR("Unknown opcode: %d", request.opcode);
        reply.status     = rc;
        reply.data_which = RequestReply_data_nodata;
        xce_reply(task->ctx.ep, c->myhost, &reply);

        break;
    }

    free_Request(&request);

    return rc;
}

static int vtask_entry(vtask_t* task)
{
    int rc;

    rc = myreq_handler(task);

    return rc;
}

static int handle_input(endpoint_t* ep)
{
    cluster_t* c = (cluster_t*)ep->owner;
    time_t     now;
    vtask_t*   task;
    int        rc = 0;

    task = vtask_alloc();
    if (task == NULL) {
        PERROR("vtask_alloc");

        return -ENOMEM;
    }

    rc = vtask_init(task, (vtask_func_t)&vtask_entry, ep);
    if (rc < 0) {
        SET_ERROR(-rc);

        INFO("vtask_init failed, maybe peer side closed the connection");

        if (rc == -EPIPE) {
            /* connection broken */
            TRACE("broken pipe, ep %p", ep);

            handle_endpoint_error(ep);
        }

        FREEUP(task);
        return rc;
    }

    rc = vtask_enq(&(c->vtaskmgr), task);
    if (rc < 0) {
        PERROR("vtask_enq");

        return rc;
    }

    now = time(NULL);
    npkts++;

    if (now - last_print_time > 10) {
        TRACE("%ld npkts:%ld iops: %f\n", now, npkts,
              (npkts - npkts_old) * 1.0 / (now - last_print_time));
        npkts_old       = npkts;
        last_print_time = now;
    }

    return rc;
}

static int handle_output(endpoint_t* ep)
{
    data_item_t* it;
    ioctx_t*     ctx;
    int          rc;

    it = endpoint_deq(ep);

    if (it == NULL) {
        return 0;
    }

    ctx = ITPTR_TO_IOCTX(it);

    rc = ioctx_flush(ctx, ep->fd);
    ioctx_deref(ctx, 1);

    return rc;
}

static int handle_io(struct epoll_event* p)
{
    endpoint_t* ep = (endpoint_t*)p->data.ptr;
    int         rc = 0;

    if ((p->events & EPOLLIN) != 0) {
        rc = handle_input(ep);
    }

    if (rc >= 0) {
        if ((p->events & EPOLLOUT) != 0) {
            rc = handle_output(ep);
        }
    }

    return rc;
}

static int handle_connection(struct epoll_event* p)
{
    endpoint_t*   ep = (endpoint_t*)p->data.ptr;
    endpoint_t*   ep2;
    cluster_t*    c     = (cluster_t*)ep->owner;
    iomgr_t*      iomgr = &(c->iomgr);
    int           idx;
    ep_private_t* priv = NULL;

    ep2 = endpoint_get();
    if (ep2 == NULL) {
        PERROR("endpoint_get");

        return -ENOMEM;
    }

    endpoint_accept(ep, ep2);
    if (ep2->fd < 0) {
        PERROR("endpoint_accept");
        return -1;
    }

    TRACE("server sock %d accepted %d\n", ep->fd, ep2->fd);

    priv = ep_get_private();
    if (priv == NULL) {
        PERROR("ep_get_private");

        endpoint_deref(ep2);
        return -ENOMEM;
    }

    /* Assume client as default */
    priv->role  = HostRole_client;
    priv->u.idx = -1;

    idx = zarray_store(&(c->endpoints), ep2);
    if (idx < 0) {
        SET_ERROR(-idx);

        PERROR("zarray_store");

        FREEUP(priv);
        endpoint_deref(ep2);

        return idx;
    }

    ep2->idx  = idx;
    ep2->data = priv;

    endpoint_set_handler(ep2, handle_io, c);

    iomgr_monitor(iomgr, ep2);
    return 0;
}

static int controller_init(controller_t* c, cluster_t* cluster)
{
    c->ep = NULL;

    return 0;
}

static void endpoint_killer(void* p)
{
    endpoint_t* ep = (endpoint_t*)p;
    cluster_t*  c  = (cluster_t*)ep->owner;

    if (ep->idx >= 0) {
        zarray_clear(&(c->endpoints), ep->idx);
        ep->idx = -1;
    }
    endpoint_deref(ep);

    return;
}

int cluster_build_config(cluster_t* c)
{
    char valbuf[128];
    int  i;

    snprintf(valbuf, sizeof(valbuf) - 1, "%d",
             c->myconf->cluster_config->heartbeat_interval);
    add_config(c, "heartbeat_interval", valbuf);
    snprintf(valbuf, sizeof(valbuf) - 1, "%d",
             c->myconf->cluster_config->inactive_heartbeat_count);
    add_config(c, "inactive_heartbeat_count", valbuf);
    snprintf(valbuf, sizeof(valbuf) - 1, "%d",
             c->myconf->cluster_config->ncandidates);
    add_config(c, "ncandidates", valbuf);

    for (i = 0; i < c->myconf->cluster_config->ncandidates; i++) {
        char keybuf[128];

        sprintf(keybuf, "candidate_%d_name", i);
        sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->name);

        add_config(c, keybuf, valbuf);

        sprintf(keybuf, "candidate_%d_host", i);
        sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->host);

        add_config(c, keybuf, valbuf);
        sprintf(keybuf, "candidate_%d_port", i);
        sprintf(valbuf, "%d", c->myconf->cluster_config->candidates[i]->port);

        add_config(c, keybuf, valbuf);
    }

    return 0;
}

int cluster_build_resources(cluster_t* c)
{
    int i;

    if (c->myconf == NULL) {
        return -EINVAL;
    }

    for (i = 0; i < c->myconf->cluster_config->nres; i++) {
        xce_resource_t* res;

        res = create_obj_by_typename(&(c->objmgr), "xresource",
                                     c->myconf->cluster_config->resources[i]);
        if (res == NULL) {
            return -errno;
        }
    }

    return 0;
}

int cluster_load_jobs(cluster_t* c)
{
    load_jobs(c);

    return 0;
}

int cluster_build_queues(cluster_t* c)
{
    build_queues(c);
    return 0;
}

int cluster_build_groups(cluster_t* c) { return 0; }

static int xce_config_init(int type, int situation, void* data, va_list ap)
{
    int        rc = 0;
    cluster_t* c  = va_arg(ap, cluster_t*);

    if (situation == HOOK_BEFORE_EVENT) {
        c->myconf->cluster_config = get_default_conf(c);
        if (c->myconf->cluster_config == NULL) {
            return -EINVAL;
        }
        return 0;
    }

    xce_use(c);

    rc = cluster_build_config(c);
    if (rc < 0) {
        return rc;
    }
    rc = cluster_build_resources(c);
    if (rc < 0) {
        return rc;
    }

    rc = cluster_build_queues(c);
    if (rc < 0) {
        return rc;
    }
    rc = cluster_build_groups(c);
    if (rc < 0) {
        return rc;
    }

    rc = cluster_load_jobs(c);
    if (rc < 0) {
        return rc;
    }

    tlog_info("xce controller is running");

    return rc;
}

static int cluster_hook_fun(int type, int situation, int detail, void* data,
                            va_list ap)
{
    if (detail == HOOK_DETAIL_CLUSTER_INIT) {
        return xce_config_init(type, situation, data, ap);
    }

    return 0;
}

static int xce_install_hooks(cluster_t* c)
{
    xce_add_hook(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER,
                 (hook_fn_t)cluster_hook_fun, NULL);
    return 0;
}

int cluster_init(cluster_t* c)
{
    int i, rc;

    if (c == NULL) {
        return -EINVAL;
    }

    c->myhost = &from_controller;
    /* FIXME: */
    c->ncontrollers = 1;
    c->myidx        = 0;

    xce_hook_init(&(c->hooksys), XCE_CONTROLLER_HOOK_MAX);

    xce_install_hooks(c);

    rc = xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER,
                       HOOK_BEFORE_EVENT, HOOK_DETAIL_CLUSTER_INIT, c);
    if (rc < 0) {
        return rc;
    }

    mylock_init(&(c->sched.lock));
    c->sched.ep = NULL;

    kvmgr_init(&(c->configmgr), string_hash_key, 128, 128,
               (zarray_item_killer_t)config_obj_killer);

    objmgr_init(&(c->objmgr));
    jobmgr_init(&(c->jobmgr), c->myconf->cluster_config->jobmgr->nhash_slots,
                c->myconf->cluster_config->jobmgr->delta);
    resmgr_init(&(c->resmgr), c->myconf->cluster_config->resmgr->nhash_slots,
                c->myconf->cluster_config->resmgr->delta,
                c->myconf->cluster_config->resmgr->inst_delta);
    workermgr_init(&(c->workermgr),
                   c->myconf->cluster_config->workermgr->nhash_slots,
                   c->myconf->cluster_config->workermgr->delta);
    qmgr_init(&(c->qmgr), c->myconf->cluster_config->qmgr->nhash_slots,
              c->myconf->cluster_config->qmgr->delta);
    iomgr_init(&(c->iomgr));
    register_xstring(&(c->objmgr),
                     c->myconf->cluster_config->xstrings->nhash_slots,
                     c->myconf->cluster_config->xstrings->delta);
    register_types(c);

    zarray_init(&(c->endpoints), 16, endpoint_killer);

    rc = endpoint_init(&(c->ep), AF_INET, SOCK_STREAM,
                       c->myconf->cluster_config->port, 5);
    if (rc < 0) {
        return rc;
    }

    for (i = 0; i < c->ncontrollers; i++) {
        controller_init(&(c->controllers[i]), c);
    }

    cbuf_init(&(c->wbuf), 1, sizeof(int));

    zarray_init(&(c->configs), 64, NULL /* FIXME */);

    vtaskmgr_init(&(c->vtaskmgr), c->myconf->cluster_config->vcpus);

    xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_AFTER_EVENT,
                  HOOK_DETAIL_CLUSTER_INIT, c);

    return 0;
}

int cluster_start(cluster_t* c)
{
    int rc;

    endpoint_set_handler(&(c->ep), handle_connection, c);
    rc = iomgr_monitor(&(c->iomgr), &(c->ep));

    if (rc < 0) {
        return rc;
    }

    iomgr_start(&(c->iomgr));
    vtaskmgr_start(&(c->vtaskmgr));

    c->running = 1;
    return 0;
}

int cluster_run(cluster_t* c)
{
    for (;;) {
        int stop;

        cbuf_get(&(c->wbuf), &stop);
        if (stop) {
            break;
        }
    }

    xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_BEFORE_EVENT,
                  HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
    c->running = 0;
    iomgr_unmonitor(&(c->iomgr), &(c->ep));
    iomgr_stop(&(c->iomgr));

    return 0;
}

void cluster_fini(cluster_t* c)
{
    vtaskmgr_fini(&(c->vtaskmgr));
    endpoint_deref(&(c->ep));

    xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_AFTER_EVENT,
                  HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
    cbuf_fini(&(c->wbuf));
    iomgr_fini(&(c->iomgr));

    jobmgr_fini(&(c->jobmgr));
    workermgr_fini(&(c->workermgr));
    qmgr_fini(&(c->qmgr));
    resmgr_fini(&(c->resmgr));
    kvmgr_fini(&(c->configmgr));
    mylock_fini(&(c->sched.lock));

    zarray_fini(&(c->endpoints));
    zarray_fini(&(c->configs));

    objmgr_fini(&(c->objmgr));

    xce_hook_fini(&(c->hooksys));

    return;
}