diff --git a/.clang-format b/.clang-format index a6bfef3..725abf6 100644 --- a/.clang-format +++ b/.clang-format @@ -1,4 +1,35 @@ --- BasedOnStyle: webkit +AlignAfterOpenBracket: Align +AlignArrayOfStructures: Left +AlignConsecutiveAssignments: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false +AlignConsecutiveBitFields: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false +AlignConsecutiveDeclarations: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false +AlignConsecutiveMacros: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false +AlignConsecutiveShortCaseStatements: + Enabled: true + AcrossEmptyLines: true + AcrossComments: true + AlignCaseColons: true +AlignEscapedNewlines: Right +AlignOperands: AlignAfterOperator +AlignTrailingComments: + Kind: Always + OverEmptyLines: 2 +AllowAllArgumentsOnNextLine: false +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortEnumsOnASingleLine: false --- diff --git a/src/controller/cluster.c b/src/controller/cluster.c new file mode 100644 index 0000000..1b6a280 --- /dev/null +++ b/src/controller/cluster.c @@ -0,0 +1,686 @@ +#include "config.h" + +#include +#include +#include + +#include "cluster.h" +#include "htab.h" +#include "mylock.h" +#include "request.h" +#include "vtaskmgr.h" +#include "xce/cluster_def.h" +#include "zarray.h" + +#include "xce/types.h" + +#include "utils.h" +#include "worker.h" +#include "xce/errnox.h" +#include "xstring.h" + +static time_t last_print_time = 0; +static unsigned long npkts_old = 0; +static unsigned long npkts = 0; + +static xce_peerinfo_t from_controller = { + .name = "controller", + .key = "keepSecret" +}; + +static ep_private_t* +ep_get_private() +{ + ep_private_t* priv; + + priv = (ep_private_t*)calloc(1, sizeof(*priv)); + return priv; +} + +static int handle_client_leave(cluster_t* c) +{ + return 0; +} + +static void handle_endpoint_error(endpoint_t* ep) +{ + cluster_t* c = (cluster_t*)ep->owner; + ep_private_t* priv = NULL; + int role = -1; + + if (c == NULL) { + ERROR("endpoint %p has no owner cluster", ep); + + return; + } + + iomgr_unmonitor(&(c->iomgr), ep); + + priv = (ep_private_t*)ep->data; + + TRACE("endpoint %p priv data %p", + ep, priv); + + if (priv != NULL) { + role = priv->role; + switch (role) { + case HostRole_worker: + if (priv->u.idx >= 0) { + handle_worker_leave(c, priv->u.idx); + } else { + ERROR("incorrect worker idx %d", + priv->u.idx); + } + break; + case HostRole_scheduler: + handle_scheduler_leave(c); + break; + case HostRole_controller: + break; + case HostRole_client: + handle_client_leave(c); + break; + default: + ERROR("unknown role %d for endpoint %p", + priv->role, ep); + break; + } + } + + if (ep->idx >= 0) { + zarray_del(&(c->endpoints), ep->idx); + return; + } + + endpoint_deref(ep); + + return; +} + +static int do_notify(cluster_t* c, + endpoint_t* ep, + xce_request_t* req) +{ + xce_notify_data_t* data = req->body.notifymsg; + + switch (data->event) { + case NotifyData_data_jobPidInfo: + on_job_start(c, data->data.pidinfo); + break; + case NotifyData_data_jobFailInfo: + on_job_fail(c, data->data.failinfo); + break; + case NotifyData_data_jobFinishInfo: + on_job_finish(c, data->data.finishinfo); + break; + case NotifyData_data_stoppedJob: + on_job_stop(c, data->data.stopped_job); + break; + case NotifyData_data_resumedJob: + on_job_resume(c, data->data.resumed_job); + break; + case NotifyData_data_ckptinfo: + on_job_ckpt(c, ep, data->data.ckptinfo); + break; + case NotifyData_data_postDoneJob: + on_job_postdone(c, ep, data->data.pdone_job); + break; + case NotifyData_data_taskPidInfo: + on_task_start(c, data->data.taskinfo); + break; + case NotifyData_data_taskFailInfo: + on_task_fail(c, data->data.taskfailinfo); + break; + case NotifyData_data_taskFinish: + on_task_finish(c, data->data.taskfinish); + break; + default: + break; + } + return 0; +} +static int myreq_handler(vtask_t* task) +{ + cluster_t* c = (cluster_t*)task->ctx.ep->owner; + int stop = 0; + int rc = 0; + xce_request_t request; + xce_reply_t reply; + + rc = ioctx_decode(&(task->ctx), &request); + + TRACE("%s cpu#%d opcode %d", + __func__, task->vcpu, + request.opcode); + + init_reply_by_req(&reply, &request); + + switch (request.opcode) { + case OPCode_shutdown: + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + xce_reply(task->ctx.ep, c->myhost, &reply); + stop = 1; + cbuf_put(&(c->wbuf), &stop); + break; + case OPCode_hello: + case OPCode_nop: + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + xce_reply(task->ctx.ep, c->myhost, &reply); + break; + case OPCode_join: + rc = handle_join(c, task->ctx.ep, &request); + break; + case OPCode_jobdispatch: + rc = handle_jobdispatch(c, task->ctx.ep, &request); + break; + case OPCode_jobsub: + rc = handle_jobsubmit(c, task->ctx.ep, &request); + break; + case OPCode_get: + rc = do_get_op(c, task->ctx.ep, &request); + break; + case OPCode_notify: + rc = do_notify(c, task->ctx.ep, &request); + break; + case OPCode_clusterconf: + rc = handle_clusterconf(c, task->ctx.ep, &request); + break; + case OPCode_update: + rc = do_update_op(c, task->ctx.ep, &request); + break; + case OPCode_jobcontrol: + rc = do_jobcontrol_op(c, task->ctx.ep, &request); + break; + case OPCode_queuecontrol: + rc = do_queuecontrol_op(c, task->ctx.ep, &request); + break; + case OPCode_runtasks: + rc = do_runtasks(c, task->ctx.ep, &request); + break; + default: + rc = -E_XCE_BAD_OPCODE; + ERROR("Unknown opcode: %d", + request.opcode); + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(task->ctx.ep, c->myhost, &reply); + + break; + } + + free_Request(&request); + + return rc; +} + +static int vtask_entry(vtask_t* task) +{ + int rc; + + rc = myreq_handler(task); + + return rc; +} + +static int handle_input(endpoint_t* ep) +{ + cluster_t* c = (cluster_t*)ep->owner; + time_t now; + vtask_t* task; + int rc = 0; + + task = vtask_alloc(); + if (task == NULL) { + PERROR("vtask_alloc"); + + return -ENOMEM; + } + + rc = vtask_init(task, (vtask_func_t)&vtask_entry, ep); + if (rc < 0) { + SET_ERROR(-rc); + + INFO("vtask_init failed, maybe peer side closed the connection"); + + if (rc == -EPIPE) { + /* connection broken */ + TRACE("broken pipe, ep %p", ep); + + handle_endpoint_error(ep); + } + + FREEUP(task); + return rc; + } + + rc = vtask_enq(&(c->vtaskmgr), task); + if (rc < 0) { + PERROR("vtask_enq"); + + return rc; + } + + now = time(NULL); + npkts++; + + if (now - last_print_time > 10) { + TRACE("%ld npkts:%ld iops: %f\n", now, npkts, + (npkts - npkts_old) * 1.0 / (now - last_print_time)); + npkts_old = npkts; + last_print_time = now; + } + + return rc; +} + +static int handle_output(endpoint_t* ep) +{ + data_item_t* it; + ioctx_t* ctx; + int rc; + + it = endpoint_deq(ep); + + if (it == NULL) { + return 0; + } + + ctx = ITPTR_TO_IOCTX(it); + + rc = ioctx_flush(ctx, ep->fd); + ioctx_deref(ctx, 1); + + return rc; +} + +static int handle_io(struct epoll_event* p) +{ + endpoint_t* ep = (endpoint_t*)p->data.ptr; + int rc = 0; + + if ((p->events & EPOLLIN) != 0) { + rc = handle_input(ep); + } + + if (rc >= 0) { + if ((p->events & EPOLLOUT) != 0) { + rc = handle_output(ep); + } + } + + return rc; +} + +static int handle_connection(struct epoll_event* p) +{ + endpoint_t* ep = (endpoint_t*)p->data.ptr; + endpoint_t* ep2; + cluster_t* c = (cluster_t*)ep->owner; + iomgr_t* iomgr = &(c->iomgr); + int idx; + ep_private_t* priv = NULL; + + ep2 = endpoint_get(); + if (ep2 == NULL) { + PERROR("endpoint_get"); + + return -ENOMEM; + } + + endpoint_accept(ep, ep2); + if (ep2->fd < 0) { + PERROR("endpoint_accept"); + return -1; + } + + TRACE("server sock %d accepted %d\n", ep->fd, ep2->fd); + + priv = ep_get_private(); + if (priv == NULL) { + PERROR("ep_get_private"); + + endpoint_deref(ep2); + return -ENOMEM; + } + + /* Assume client as default */ + priv->role = HostRole_client; + priv->u.idx = -1; + + idx = zarray_store(&(c->endpoints), ep2); + if (idx < 0) { + SET_ERROR(-idx); + + PERROR("zarray_store"); + + FREEUP(priv); + endpoint_deref(ep2); + + return idx; + } + + ep2->idx = idx; + ep2->data = priv; + + endpoint_set_handler(ep2, handle_io, c); + + iomgr_monitor(iomgr, ep2); + return 0; +} + +static int controller_init(controller_t* c, cluster_t* cluster) +{ + c->ep = NULL; + + return 0; +} + +static void endpoint_killer(void* p) +{ + endpoint_t* ep = (endpoint_t*)p; + cluster_t* c = (cluster_t*)ep->owner; + + if (ep->idx >= 0) { + zarray_clear(&(c->endpoints), ep->idx); + ep->idx = -1; + } + endpoint_deref(ep); + + return; +} + +int cluster_build_config(cluster_t* c) +{ + char valbuf[128]; + int i; + + snprintf(valbuf, sizeof(valbuf) - 1, + "%d", c->myconf->cluster_config->heartbeat_interval); + add_config(c, "heartbeat_interval", valbuf); + snprintf(valbuf, sizeof(valbuf) - 1, + "%d", c->myconf->cluster_config->inactive_heartbeat_count); + add_config(c, "inactive_heartbeat_count", valbuf); + snprintf(valbuf, sizeof(valbuf) - 1, + "%d", c->myconf->cluster_config->ncandidates); + add_config(c, "ncandidates", valbuf); + + for (i = 0; i < c->myconf->cluster_config->ncandidates; i++) { + char keybuf[128]; + + sprintf(keybuf, "candidate_%d_name", i); + sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->name); + + add_config(c, keybuf, valbuf); + + sprintf(keybuf, "candidate_%d_host", i); + sprintf(valbuf, "%s", c->myconf->cluster_config->candidates[i]->host); + + add_config(c, keybuf, valbuf); + sprintf(keybuf, "candidate_%d_port", i); + sprintf(valbuf, "%d", c->myconf->cluster_config->candidates[i]->port); + + add_config(c, keybuf, valbuf); + } + + return 0; +} + +int cluster_build_resources(cluster_t* c) +{ + int i; + + if (c->myconf == NULL) { + return -EINVAL; + } + + for (i = 0; i < c->myconf->cluster_config->nres; i++) { + xce_resource_t* res; + + res = create_obj_by_typename(&(c->objmgr), "xresource", + c->myconf->cluster_config->resources[i]); + if (res == NULL) { + return -errno; + } + } + + return 0; +} + +int cluster_load_jobs(cluster_t* c) +{ + load_jobs(c); + + return 0; +} + +int cluster_build_queues(cluster_t* c) +{ + build_queues(c); + return 0; +} + +int cluster_build_groups(cluster_t* c) +{ + return 0; +} + +static int xce_config_init(int type, + int situation, + void* data, + va_list ap) +{ + int rc = 0; + cluster_t* c = va_arg(ap, cluster_t*); + + if (situation == HOOK_BEFORE_EVENT) { + c->myconf->cluster_config = get_default_conf(c); + if (c->myconf->cluster_config == NULL) { + return -EINVAL; + } + return 0; + } + + xce_use(c); + + rc = cluster_build_config(c); + if (rc < 0) { + return rc; + } + rc = cluster_build_resources(c); + if (rc < 0) { + return rc; + } + + rc = cluster_build_queues(c); + if (rc < 0) { + return rc; + } + rc = cluster_build_groups(c); + if (rc < 0) { + return rc; + } + + rc = cluster_load_jobs(c); + if (rc < 0) { + return rc; + } + + tlog_info("xce controller is running"); + + return rc; +} + +static int cluster_hook_fun(int type, + int situation, + int detail, + void* data, + va_list ap) +{ + if (detail == HOOK_DETAIL_CLUSTER_INIT) { + return xce_config_init(type, situation, data, ap); + } + + return 0; +} + +static int xce_install_hooks(cluster_t* c) +{ + xce_add_hook(&(c->hooksys), + XCE_CONTROLLER_HOOK_CLUSTER, + (hook_fn_t)cluster_hook_fun, + NULL); + return 0; +} + +int cluster_init(cluster_t* c) +{ + int i, rc; + + if (c == NULL) { + return -EINVAL; + } + + c->myhost = &from_controller; + /* FIXME: */ + c->ncontrollers = 1; + c->myidx = 0; + + xce_hook_init(&(c->hooksys), XCE_CONTROLLER_HOOK_MAX); + + xce_install_hooks(c); + + rc = xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_CLUSTER, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_CLUSTER_INIT, c); + if (rc < 0) { + return rc; + } + + mylock_init(&(c->sched.lock)); + c->sched.ep = NULL; + + kvmgr_init(&(c->configmgr), + string_hash_key, + 128, 128, + (zarray_item_killer_t)config_obj_killer); + + objmgr_init(&(c->objmgr)); + jobmgr_init(&(c->jobmgr), + c->myconf->cluster_config->jobmgr->nhash_slots, + c->myconf->cluster_config->jobmgr->delta); + resmgr_init(&(c->resmgr), + c->myconf->cluster_config->resmgr->nhash_slots, + c->myconf->cluster_config->resmgr->delta, + c->myconf->cluster_config->resmgr->inst_delta); + workermgr_init(&(c->workermgr), + c->myconf->cluster_config->workermgr->nhash_slots, + c->myconf->cluster_config->workermgr->delta); + qmgr_init(&(c->qmgr), + c->myconf->cluster_config->qmgr->nhash_slots, + c->myconf->cluster_config->qmgr->delta); + iomgr_init(&(c->iomgr)); + register_xstring(&(c->objmgr), + c->myconf->cluster_config->xstrings->nhash_slots, + c->myconf->cluster_config->xstrings->delta); + register_types(c); + + zarray_init(&(c->endpoints), 16, endpoint_killer); + + rc = endpoint_init(&(c->ep), + AF_INET, + SOCK_STREAM, + c->myconf->cluster_config->port, + 5); + if (rc < 0) { + return rc; + } + + for (i = 0; i < c->ncontrollers; i++) { + controller_init(&(c->controllers[i]), c); + } + + cbuf_init(&(c->wbuf), 1, sizeof(int)); + + zarray_init(&(c->configs), 64, NULL /* FIXME */); + + vtaskmgr_init(&(c->vtaskmgr), c->myconf->cluster_config->vcpus); + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_CLUSTER, + HOOK_AFTER_EVENT, + HOOK_DETAIL_CLUSTER_INIT, c); + + return 0; +} + +int cluster_start(cluster_t* c) +{ + int rc; + + endpoint_set_handler(&(c->ep), handle_connection, c); + rc = iomgr_monitor(&(c->iomgr), &(c->ep)); + + if (rc < 0) { + return rc; + } + + iomgr_start(&(c->iomgr)); + vtaskmgr_start(&(c->vtaskmgr)); + + c->running = 1; + return 0; +} + +int cluster_run(cluster_t* c) +{ + for (;;) { + int stop; + + cbuf_get(&(c->wbuf), &stop); + if (stop) { + break; + } + } + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_CLUSTER, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_CLUSTER_SHUTDOWN, c); + c->running = 0; + iomgr_unmonitor(&(c->iomgr), &(c->ep)); + iomgr_stop(&(c->iomgr)); + + return 0; +} + +void cluster_fini(cluster_t* c) +{ + vtaskmgr_fini(&(c->vtaskmgr)); + endpoint_deref(&(c->ep)); + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_CLUSTER, + HOOK_AFTER_EVENT, + HOOK_DETAIL_CLUSTER_SHUTDOWN, c); + cbuf_fini(&(c->wbuf)); + iomgr_fini(&(c->iomgr)); + + jobmgr_fini(&(c->jobmgr)); + workermgr_fini(&(c->workermgr)); + qmgr_fini(&(c->qmgr)); + resmgr_fini(&(c->resmgr)); + kvmgr_fini(&(c->configmgr)); + mylock_fini(&(c->sched.lock)); + + zarray_fini(&(c->endpoints)); + zarray_fini(&(c->configs)); + + objmgr_fini(&(c->objmgr)); + + xce_hook_fini(&(c->hooksys)); + + return; +} diff --git a/src/controller/dataex.c b/src/controller/dataex.c new file mode 100644 index 0000000..8ce3468 --- /dev/null +++ b/src/controller/dataex.c @@ -0,0 +1,537 @@ +#include + +#include "cluster.h" +#include "request.h" +#include "xce/cluster_def.h" +#include "xce/types.h" + +typedef int (*check_fn_t)(void*); + +int check_controller(void* arg) +{ + cluster_t* c = (cluster_t*)arg; + + if (!c->running) { + return -EINVAL; + } + + return 1; +} + +struct intern_array_ { + int count; + int idx; + void* data; + check_fn_t checkfn; +}; + +int not_null_check(void* arg) +{ + if (arg == NULL) { + return 0; + } + + return 1; +} + +static int collect_resval_with_idx_(zarray_t* array, + int* nres, + xce_val_with_idx_t*** resvals) +{ + int nc; + int i, j; + xce_val_with_idx_t** vals; + + nc = 0; + + for (i = 0; i < array->caps; i++) { + if (array->data[i] == NULL) { + continue; + } + + nc++; + } + + if (nc == 0) { + *nres = 0; + *resvals = NULL; + return 0; + } + + vals = (xce_val_with_idx_t**)calloc(nc, sizeof(xce_val_with_idx_t*)); + if (vals == NULL) { + return -ENOMEM; + } + + j = 0; + + for (i = 0; i < array->caps; i++) { + xce_val_with_idx_t* val; + xce_val_t* src = array->data[i]; + + if (array->data[i] == NULL) { + continue; + } + + val = (xce_val_with_idx_t*)calloc(1, sizeof(*val)); + if (val == NULL) { + free_resval_with_idx_list(&vals, j); + return -ENOMEM; + } + + vals[j] = val; + j++; + + val->idx = i; + val->val = (xce_val_t*)calloc(1, sizeof(xce_val_t)); + if (val->val == NULL) { + free_resval_with_idx_list(&vals, j); + return -ENOMEM; + } + + if (src->which == Value_u_svalue) { + val->val->which = Value_u_svalue; + if (src->u.svalue != NULL) { + val->val->u.svalue = strdup(src->u.svalue); + } else { + val->val->u.svalue = strdup("-"); + } + } else { + memcpy(val->val, array->data[i], + sizeof(xce_val_t)); + } + } + + *nres = j; + *resvals = vals; + return j; +} + +int collect_resval_with_idx(zarray_t* array, + int* nres, + xce_val_with_idx_t*** resvals) +{ + int rc = 0; + + mylock_lock(&(array->lock)); + + rc = collect_resval_with_idx_(array, nres, resvals); + + mylock_unlock(&(array->lock)); + + return rc; +} +static int count_(void* arg1, + void* arg2) +{ + struct intern_array_* arg = (struct intern_array_*)arg1; + + if ((arg == NULL) || (arg->checkfn == NULL)) { + return -EINVAL; + } + + if (!(arg->checkfn(arg2))) { + return 0; + } + + arg->count++; + + return 0; +} + +static int add_confvar_(void* arg1, + void* arg2) +{ + struct intern_array_* arg = (struct intern_array_*)arg1; + xce_config_t** p = NULL; + + if ((arg == NULL) || (arg->count < 0)) { + return -EINVAL; + } + + if (arg->count == 0) { + return 0; + } + + if (!(arg->checkfn(arg2))) { + return 0; + } + + p = (xce_config_t**)arg->data; + if (p == NULL) { + p = (xce_config_t**)calloc(arg->count, + sizeof(xce_config_t*)); + if (p == NULL) { + return -ENOMEM; + } + + arg->data = p; + } + + p[arg->idx] = (xce_config_t*)obj_ref(arg2, OBJREF_READ); + arg->idx++; + + return 1; +} + +void collect_confvars(cluster_t* c, + collect_check_fn_t fn, + int* nconfig, + xce_config_t*** configs) +{ + xlist_t lst; + struct intern_array_ args = { + .count = 0, + .idx = 0, + .data = NULL, + .checkfn = (fn == NULL) ? not_null_check : fn + }; + int rc; + + *nconfig = 0; + *configs = NULL; + + xlist_init(&lst, 3); + xlist_store(&lst, count_); + xlist_store(&lst, add_confvar_); + + rc = kvmgr_iter(&(c->configmgr), &lst, &args); + + xlist_fini(&lst); + + if (rc >= 0) { + *nconfig = args.count; + *configs = args.data; + } +} + +static int add_queue_(void* arg1, + void* arg2) +{ + struct intern_array_* arg = (struct intern_array_*)arg1; + xce_queue_t** p = NULL; + + if (arg == NULL) { + return -EINVAL; + } + + if (arg->count < 0) { + return -EINVAL; + } + + if (arg->count == 0) { + return 0; + } + + if (!(arg->checkfn(arg2))) { + return 0; + } + + p = (xce_queue_t**)arg->data; + if (p == NULL) { + p = (xce_queue_t**)calloc(arg->count, + sizeof(xce_queue_t*)); + if (p == NULL) { + return -ENOMEM; + } + + arg->data = p; + } + + p[arg->idx] = (xce_queue_t*)obj_ref(arg2, OBJREF_READ); + arg->idx++; + + return 1; +} + +void collect_queues(cluster_t* c, + collect_check_fn_t fn, + int* nqueues, + xce_queue_t*** queues) +{ + xlist_t lst; + struct intern_array_ args = { + .count = 0, + .idx = 0, + .data = NULL, + .checkfn = (fn == NULL) ? not_null_check : fn + }; + int rc; + + *nqueues = 0; + *queues = NULL; + + xlist_init(&lst, 3); + xlist_store(&lst, count_); + xlist_store(&lst, add_queue_); + + rc = qmgr_iter(&(c->qmgr), &lst, &args); + + xlist_fini(&lst); + + if (rc >= 0) { + *nqueues = args.count; + *queues = args.data; + } +} + +static int add_resource_(void* arg1, + void* arg2) +{ + struct intern_array_* arg = (struct intern_array_*)arg1; + xce_resource_t** p = NULL; + + if (arg == NULL) { + return -EINVAL; + } + + if (arg->count < 0) { + return -EINVAL; + } + + if (arg->count == 0) { + return 0; + } + + if (!(arg->checkfn(arg2))) { + return 0; + } + + p = (xce_resource_t**)arg->data; + if (p == NULL) { + p = (xce_resource_t**)calloc(arg->count, + sizeof(xce_resource_t*)); + if (p == NULL) { + return -ENOMEM; + } + + arg->data = p; + } + + p[arg->idx] = (xce_resource_t*)obj_ref(arg2, OBJREF_READ); + + arg->idx++; + + return 1; +} + +void collect_resources(cluster_t* c, + collect_check_fn_t fn, + int* nresources, + xce_resource_t*** resources) +{ + xlist_t lst; + struct intern_array_ args = { + .count = 0, + .idx = 0, + .data = NULL, + .checkfn = (fn == NULL) ? not_null_check : fn + }; + int rc; + + *nresources = 0; + *resources = NULL; + + xlist_init(&lst, 3); + xlist_store(&lst, count_); + xlist_store(&lst, add_resource_); + + rc = resmgr_iter(&(c->resmgr), &lst, &args); + + xlist_fini(&lst); + + if (rc >= 0) { + *nresources = args.count; + *resources = args.data; + } +} + +static int add_job_(void* arg1, + void* arg2) +{ + struct intern_array_* arg = (struct intern_array_*)arg1; + xce_job_t** p = NULL; + + if (arg == NULL) { + return -EINVAL; + } + + if (arg->count < 0) { + return -EINVAL; + } + + if (arg->count == 0) { + return 0; + } + + if (!(arg->checkfn(arg2))) { + return 0; + } + + p = (xce_job_t**)arg->data; + if (p == NULL) { + p = (xce_job_t**)calloc(arg->count, sizeof(xce_job_t*)); + if (p == NULL) { + return -ENOMEM; + } + + arg->data = p; + } + + p[arg->idx] = (xce_job_t*)obj_ref(arg2, OBJREF_READ); + arg->idx++; + + return 1; +} + +void collect_jobs(cluster_t* c, + collect_check_fn_t fn, + int* njobs, + xce_job_t*** jobs) +{ + xlist_t lst; + struct intern_array_ args = { + .count = 0, + .idx = 0, + .data = NULL, + .checkfn = (check_fn_t)fn + }; + int rc = 0; + + *njobs = 0; + *jobs = NULL; + + xlist_init(&lst, 3); + xlist_store(&lst, count_); + xlist_store(&lst, add_job_); + + rc = jobmgr_iter(&(c->jobmgr), &lst, &args); + xlist_fini(&lst); + + if (rc >= 0) { + *njobs = args.count; + *jobs = args.data; + } + + return; +} + +static int add_worker_(void* arg1, + void* arg2) +{ + struct intern_array_* arg = (struct intern_array_*)arg1; + xce_worker_t** p = NULL; + xce_worker_t* w; + + if (arg == NULL) { + return -EINVAL; + } + + if (arg->count < 0) { + return -EINVAL; + } + + if (arg->count == 0) { + return 0; + } + + if (!(arg->checkfn(arg2))) { + return 0; + } + + p = (xce_worker_t**)arg->data; + if (p == NULL) { + p = (xce_worker_t**)calloc(arg->count, sizeof(xce_worker_t*)); + if (p == NULL) { + return -ENOMEM; + } + + arg->data = p; + } + + w = (xce_worker_t*)obj_ref(arg2, OBJREF_READ); + + collect_resval_with_idx(&(w->resvals), + &(w->nresvals_to_sched), + &(w->resvals_to_sched)); + + p[arg->idx] = w; + arg->idx++; + + return 1; +} + +void collect_workers(cluster_t* c, + collect_check_fn_t fn, + int* nworkers, + xce_worker_t*** workers) +{ + xlist_t lst; + struct intern_array_ args = { + .count = 0, + .idx = 0, + .data = NULL, + .checkfn = (check_fn_t)fn + }; + int rc = 0; + + *nworkers = 0; + *workers = NULL; + + xlist_init(&lst, 3); + xlist_store(&lst, count_); + xlist_store(&lst, add_worker_); + + rc = workermgr_iter(&(c->workermgr), &lst, &args); + + xlist_fini(&lst); + + if (rc >= 0) { + *nworkers = args.count; + *workers = args.data; + } + + return; +} + +void collect_shared_resvals(cluster_t* c, + collect_check_fn_t fn, + int* nres, + xce_val_with_idx_t*** resvals) +{ + + collect_resval_with_idx(&(c->resmgr.sharedinst), + nres, resvals); + + return; +} + +void deref_objs(int count, + void** objs) +{ + int i; + + for (i = 0; i < count; i++) { + void* obj = objs[i]; + + if (obj == NULL) { + continue; + } + + DEREF_OBJ(obj); + } +} + +int notify(cluster_t* c, + endpoint_t* ep, + xce_notify_data_t* data) +{ + ioctx_t* ctx = NULL; + + ctx = ioctx_alloc(ep, c->myhost); + if (ctx == NULL) { + return -ENOMEM; + } + + return xce_notify_req(ctx, data); +} diff --git a/src/controller/include/cluster.h b/src/controller/include/cluster.h new file mode 100644 index 0000000..b8afc0c --- /dev/null +++ b/src/controller/include/cluster.h @@ -0,0 +1,172 @@ +#if !defined(_CLUSTER_H_) + +#define _CLUSTER_H_ 1 +#include "config.h" + +#include +#include + +#include "hook.h" +#include "htab.h" +#include "kvmgr.h" +#include "myconf.h" +#include "mylock.h" +#include "netio.h" +#include "objmgr.h" +#include "vtaskmgr.h" +#include "worker.h" +#include "xce/cluster_def.h" +#include "xce/types.h" +#include "xlist.h" +#include "zarray.h" + +typedef int (*collect_check_fn_t)(void*); + +extern int check_controller(void*); +extern int cluster_init(cluster_t*); +extern int cluster_start(cluster_t*); +extern int cluster_run(cluster_t*); +extern void cluster_fini(cluster_t*); +extern int register_types(cluster_t*); +extern int jobmgr_init(jobmgr_t*, int, int); +extern void jobmgr_fini(jobmgr_t*); +extern int workermgr_init(workermgr_t*, int, int); +extern void workermgr_fini(workermgr_t*); +extern int workermgr_add(workermgr_t*, + worker_t*); +extern worker_t* workermgr_lock(workermgr_t*, + char*, int); +extern worker_t* workermgr_lock_byid(workermgr_t*, + int, int); +extern void workermgr_del(workermgr_t*, worker_t*); +extern int worker_ctrl(cluster_t*, xce_hostctrl_t*); +extern void qmgr_fini(qmgr_t*); +extern int qmgr_init(qmgr_t*, int, int); +extern int qmgr_add(qmgr_t*, queue_t*); +extern queue_t* qmgr_lockq(qmgr_t*, char*, int); +extern void qmgr_delq(qmgr_t*, queue_t*); +extern int qmgr_iter(qmgr_t*, xlist_t*, void*); + +extern xce_resource_t* get_resource(cluster_t*, + char*); +extern int add_to_xlist(xlist_t*, void*); +extern int add_worker_to_xlist(xlist_t*, void*); +extern int add_queue_to_xlist(xlist_t*, void*); +extern int add_job_to_xlist(xlist_t*, void*); + +extern void resource_killer(void*); +extern int add_resource(void*, char*, char*, + int, xce_order_t, + int, int); +extern int resmgr_init(resmgr_t*, int, int, int); +extern void resmgr_fini(resmgr_t*); +extern int32_t resmgr_add_res(resmgr_t*, + xce_resource_t*); +extern xce_resource_t* resmgr_lockres(resmgr_t* mgr, + char*, int); +extern void res_obj_killer(xce_resource_t*); +extern void resmgr_del_res(resmgr_t*, + xce_resource_t*); +extern int resmgr_iter(resmgr_t*, xlist_t*, void*); + +extern void config_obj_killer(xce_config_t*); + +extern int update_resvals(cluster_t*, worker_t*, + int, xce_resval_t*); + +extern int load_jobs(void*); +extern int update_job(void*, uint64_t, xce_kv_list_t*); +extern int do_job_started(void*, uint64_t, int, int*, int, int*); + +extern queue_t* create_queue(cluster_t*, xce_queue_t*); +extern int build_queues(void*); +extern void queue_fini(queue_t*); +extern int queue_ctrl(cluster_t*, xce_queuectrl_t*); +extern queue_t* qmgr_lockq(qmgr_t*, char*, int); + +extern int xce_update_data(endpoint_t*, cluster_t*, + int, int, int*, int, xce_kv_t*); +extern int on_scheduler_join(cluster_t*); + +extern int xce_push_reslist(endpoint_t*, cluster_t*, + int, xce_resource_t**); +extern int xce_push_confvars(endpoint_t*, cluster_t*, + int, xce_config_t**); +extern int xce_push_workerlist(endpoint_t*, cluster_t*, + int, xce_worker_t**); +extern int xce_push_joblist(endpoint_t*, cluster_t*, + int, xce_job_t**); +extern int xce_push_queuelist(endpoint_t*, cluster_t*, + int, xce_queue_t**); +extern int xcesched_push_job(cluster_t*, uint64_t); +extern int on_worker_join(void*, endpoint_t*); +extern int get_resval(cluster_t*, char*, + char*, xce_val_t*); +extern job_t* jobmgr_lockjob(jobmgr_t*, uint64_t, int); +extern int32_t jobmgr_add_job(jobmgr_t*, job_t*); +extern void jobmgr_del_job(jobmgr_t*, job_t*); +extern void job_obj_killer(job_t*); +extern int handle_push(struct capn*, cluster_t*, + endpoint_t*, struct Request*); +extern int do_get_op(cluster_t*, endpoint_t*, xce_request_t*); +extern int do_update_op(cluster_t*, endpoint_t*, xce_request_t*); +extern int handle_join(cluster_t*, + endpoint_t*, + xce_request_t*); +extern int handle_reply(endpoint_t*, struct Request*, + cluster_t*); +extern int handle_jobdispatch(cluster_t*, endpoint_t*, + xce_request_t*); +extern int handle_jobsubmit(cluster_t*, endpoint_t*, xce_request_t*); +extern int handle_worker_leave(cluster_t*, int); +extern int handle_scheduler_leave(cluster_t*); +extern int handle_clusterconf(cluster_t*, endpoint_t*, xce_request_t*); + +extern void collect_confvars(cluster_t*, collect_check_fn_t, int*, xce_config_t***); +extern void collect_queues(cluster_t*, collect_check_fn_t, int*, xce_queue_t***); +extern void collect_resources(cluster_t*, collect_check_fn_t, int*, xce_resource_t***); +extern void collect_jobs(cluster_t*, collect_check_fn_t, int*, xce_job_t***); +extern void collect_workers(cluster_t*, collect_check_fn_t, int*, xce_worker_t***); +extern void deref_objs(int, void**); +extern int get_queue_exec(cluster_t*, char*, char**, char**); +extern int jobmgr_iter(jobmgr_t*, xlist_t*, void*); +extern int workermgr_iter(workermgr_t*, xlist_t*, void*); + +extern int on_job_start(cluster_t*, job_pidinfo_t*); +extern int on_job_finish(cluster_t*, xce_jobfinish_t*); +extern int on_job_fail(cluster_t*, job_failinfo_t*); +extern int on_job_stop(cluster_t*, uint64_t); +extern int on_job_resume(cluster_t*, uint64_t); +extern int on_job_postdone(cluster_t*, endpoint_t*, uint64_t); +extern int on_job_ckpt(cluster_t*, endpoint_t*, xce_ckptinfo_t*); +extern int notify(cluster_t*, endpoint_t*, xce_notify_data_t*); +extern int not_null_check(void*); +extern int check_worker_(xce_worker_t*); +extern int update_resval_list(cluster_t*, worker_t*, xce_resval_list_t*); +extern int do_jobcontrol_op(cluster_t*, endpoint_t*, xce_request_t*); +extern int job_ctrl(cluster_t*, xce_jobctrl_t*); +extern int update_job_rusages(cluster_t*, endpoint_t*, xce_request_t*); +extern int update_host_runusage(xce_job_runusage_t**, char*, + int, xce_rusage_t**); +extern int update_pids(job_pidinfo_t**, int, int*); +extern int update_pgids(job_pidinfo_t**, int, int*); +extern int get_ep_role(endpoint_t*); +extern worker_t* get_ep_worker(cluster_t*, endpoint_t*, int); + +extern int not_null_check(void*); +extern int check_worker_(xce_worker_t*); +extern int update_resval_list(cluster_t*, worker_t*, xce_resval_list_t*); +extern int do_jobcontrol_op(cluster_t*, endpoint_t*, xce_request_t*); +extern int job_ctrl(cluster_t*, xce_jobctrl_t*); +extern int do_queuecontrol_op(cluster_t*, endpoint_t*, xce_request_t*); +extern int do_runtasks(cluster_t*, endpoint_t*, xce_request_t*); +extern int on_task_start(cluster_t*, xce_task_pidinfo_t*); +extern int on_task_fail(cluster_t*, xce_task_failinfo_t*); +extern int on_task_finish(cluster_t*, xce_task_finish_t*); + +extern int xce_controller_res_init(cluster_t*); +extern int map_resource_to_host(char*, cluster_t*); +extern int release_resource_from_worker(char*, cluster_t*); +extern int collect_resval_with_idx(zarray_t*, int*, xce_val_with_idx_t***); + +#endif diff --git a/src/controller/include/worker.h b/src/controller/include/worker.h new file mode 100644 index 0000000..cd56d55 --- /dev/null +++ b/src/controller/include/worker.h @@ -0,0 +1,27 @@ +#if !defined(_WORKER_H_) +#define _WORKER_H_ 1 + +#include "config.h" +#include "mylock.h" +#include "netio.h" +#include "xce/types.h" +#include "zarray.h" + +typedef struct { + uint64_t jobid; + xce_alloc_t alloc; + double used; +} res_usage_t; + +typedef struct { + xce_worker_t host; + char hostkey[XCE_MAXKEY_LEN]; + endpoint_t* ep; + int idx; + zarray_t usages; +} worker_t; + +extern int add_worker(void*, endpoint_t*, const char*, + hostattrs_t*); +extern int restore_worker(void*, xce_worker_t); +#endif diff --git a/src/controller/job.c b/src/controller/job.c new file mode 100644 index 0000000..802791a --- /dev/null +++ b/src/controller/job.c @@ -0,0 +1,922 @@ +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" +#include "xce/types.h" + +static int mark_job_start(cluster_t*, job_t*, job_pidinfo_t*); +static int mark_job_finish(cluster_t*, job_t*, xce_jobfinish_t*); +static int mark_job_fail(cluster_t*, job_t*, job_failinfo_t*); +static int mark_job_update(cluster_t*, job_t*, int); + +#define NJOBS (100) + +uint64_t jobid = 100; + +static job_t* create_job(cluster_t* c, + uint64_t jobid, + char* queue, + char* user, + char* exechome, + char* execcwd, + char* execuser, + char* cmdline, + char* exdata, + uint32_t mintasks, + uint32_t maxtasks) +{ + job_t* job = NULL; + return job; +} + +uint64_t job_submit(void* c, + char* user, + char* queue, + char* resreq, + char* cmdline, + char* exdata, + uint32_t mintasks, + uint32_t maxtasks) +{ + job_t* job = NULL; + + job = create_job(c, jobid++, queue, + user, "/tmp", + "/tmp", user, cmdline, + exdata, mintasks, maxtasks); + if (job == NULL) { + return -errno; + } + + return job->job.jobid; +} + +int load_jobs(void* c) +{ + job_t* job; + int i; + char cmdline[1024]; + + for (i = 1; i < NJOBS; i++) { + sprintf(cmdline, "echo normal; sleep %d", i); + job = create_job(c, jobid++, "normal", + "rshen", "/home/rshen", + "/tmp", "rshen", (char*)cmdline, + NULL, 1, 1); + if (job == NULL) { + return -errno; + } + } + + for (i = 1; i < NJOBS; i++) { + job = create_job(c, jobid++, "xce", + "rshen", "/home/rshen", + "/tmp", "rshen", "echo xce; sleep 30", + NULL, 1, 1); + if (job == NULL) { + return -errno; + } + } + + return 0; +} + +static int get_nslots_in_alloc(xce_host_alloc_t* alloc) +{ + int i; + + for (i = 0; i < alloc->nallocs; i++) { + if (strcmp(alloc->allocs[i]->resname, "slots") == 0) { + return (int)alloc->allocs[i]->value; + } + } + + return 1; +} + +static void free_launchinfo(xce_launch_t* launch) +{ + int i; + + for (i = 0; i < launch->nhosts; i++) { + FREEUP(launch->hosts[i]); + } + + FREEUP(launch->hosts); +} + +static int mk_launchinfo(xce_launch_t* launch, + xce_job_alloc_t* alloc) +{ + int i; + int rc = 0; + + memset(launch, 0x0, sizeof(*launch)); + + launch->hosts = (xce_host_launch_t**)calloc(alloc->nhosts, + sizeof(xce_host_launch_t*)); + if (launch->hosts == NULL) { + return -ENOMEM; + } + + launch->nhosts = alloc->nhosts; + for (i = 0; i < alloc->nhosts; i++) { + launch->hosts[i] = (xce_host_launch_t*)calloc(1, sizeof(xce_host_launch_t)); + if (launch->hosts[i] == NULL) { + rc = -ENOMEM; + break; + } + + launch->hosts[i]->host = alloc->allocs[i]->host; + launch->hosts[i]->nranks = get_nslots_in_alloc(alloc->allocs[i]); + } + + if (rc < 0) { + free_launchinfo(launch); + } + + return rc; +} + +static int dispatch_a_job_(cluster_t* c, + endpoint_t* ep, + xce_job_alloc_t* alloc) +{ + int rc = 0; + job_t* j; + worker_t* w; + ioctx_t* ioctx; + xce_host_alloc_t** allocs; + int nc; + xce_job_exec_t einfo; + xce_launch_t launchinfo; + + TRACE("%s jobid %ld", __func__, alloc->jobid); + + j = jobmgr_lockjob(&(c->jobmgr), alloc->jobid, OBJREF_UPDATE); + + if (j == NULL) { + PERROR("jobmgr_lockjob"); + + return -errno; + } + + nc = alloc->nhosts; + if (nc <= 0) { + obj_deref(j); + return -EINVAL; + } + + /* FIXME: change case of nc > 1 just hard coded mpi as + * the real job type later + */ + mk_launchinfo(&launchinfo, alloc); + + allocs = (xce_host_alloc_t**)alloc->allocs; + /* using head node of job */ + w = workermgr_lock(&(c->workermgr), + allocs[0]->host, + OBJREF_READ); + if (w == NULL) { + PERROR("workermgr_lock"); + free_launchinfo(&launchinfo); + obj_deref(j); + return -errno; + } + + if ((w->ep == NULL) || (w->host.status != 0)) { + ERROR("worker %s is not in correct state, status %x ep %p", + w->host.name, + w->host.status, + w->ep); + free_launchinfo(&launchinfo); + + obj_deref(w); + obj_deref(j); + + SET_ERROR(E_XCE_BAD_WORKER); + return -errno; + } + + /* FIXME: we may consider to optimize when a job has enter a queue? */ + if (1) { + char* q_pre_exec; + char* q_post_exec; + + rc = get_queue_exec(c, j->job.queue, + &q_pre_exec, &q_post_exec); + if (rc < 0) { + obj_deref(j); + obj_deref(w); + return rc; + } + + if (j->job.start_info->q_pre_exec != NULL) { + obj_deref(j->job.start_info->q_pre_exec); + } + + if (j->job.start_info->q_post_exec != NULL) { + obj_deref(j->job.start_info->q_post_exec); + } + + j->job.start_info->q_pre_exec = q_pre_exec; + j->job.start_info->q_post_exec = q_post_exec; + } + + ioctx = ioctx_alloc(w->ep, c->myhost); + if (ioctx == NULL) { + ERROR("memory allocate failed"); + free_launchinfo(&launchinfo); + obj_deref(j); + DEREF_OBJ(w); + + return -ENOMEM; + } + + einfo.jobid = alloc->jobid; + einfo.start_info = j->job.start_info; + einfo.launchinfo = &launchinfo; + + rc = xce_startjob_req(ioctx, &einfo); + + free_launchinfo(&launchinfo); + + if (j->job.allocs != NULL) { + free_JobAlloc_ptr(&(j->job.allocs)); + } + j->job.allocs = (xce_job_alloc_t*)calloc(1, sizeof(xce_job_alloc_t)); + j->job.allocs->jobid = alloc->jobid; + j->job.allocs->nhosts = nc; + MOVE_PTR(j->job.allocs->allocs, allocs); + alloc->allocs = NULL; + + DEREF_OBJ(w); + DEREF_OBJ(j); + + if (rc < 0) { + SET_ERROR(-rc); + PERROR("xce_startjob_req"); + return rc; + } + + return rc; +} + +int job_ctrl(cluster_t* c, + xce_jobctrl_t* ctrinfo) +{ + uint64_t jobid = ctrinfo->jobid; + job_t* j; + int rc = 0; + + TRACE("%s job_ctrl job %ld", __func__, jobid); + j = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE); + if (j == NULL) { + PERROR("jobmgr_lockjob"); + return -ENOENT; + } + + if (j->job.allocs != NULL) { + xce_host_alloc_t** allocs = j->job.allocs->allocs; + worker_t* w; + ioctx_t* ioctx; + + /* job control command will be sent to first node + * for parallel jobs + */ + w = workermgr_lock(&(c->workermgr), + allocs[0]->host, + OBJREF_READ); + if (w != NULL) { + /* FIXME: better method to check host status */ + if ((w->ep != NULL) && (w->host.status == 0)) { + ioctx = ioctx_alloc(w->ep, c->myhost); + if (ioctx != NULL) { + rc = xce_jobctrl_req(ioctx, ctrinfo); + if (rc < 0) { + ERROR("job ctrl %ld on host %s returns %d", + jobid, w->host.name, rc); + } + } else { + ERROR("memory allocate failed"); + rc = -ENOMEM; + } + } else { + ERROR("worker %s is not in correct state, status %x ep %p", + w->host.name, + w->host.status, + w->ep); + rc = -errno; + } + obj_deref(w); + } else { + PERROR("workermgr_lock"); + rc = -errno; + } + } else { + /* job has not been started yet */ + job_failinfo_t failinfo = { + .jobid = jobid, + }; + + xce_use(&failinfo); + + switch (ctrinfo->action) { + case JobControl_u_siginfo: + switch (ctrinfo->u.siginfo->signo) { + case SIGCONT: + if (j->job.state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP)) { + int state = j->job.state; + + /* resume suspend job? */ + state &= ~(JOB_STAT_PSUSP | JOB_STAT_USUSP); + mark_job_update(c, j, state); + } + break; + case SIGSTOP: + case SIGTSTP: + if (!(j->job.state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP))) { + int state = j->job.state; + + state |= JOB_STAT_PSUSP; + mark_job_update(c, j, state); + } + break; + case SIGKILL: + case SIGTERM: + default: + if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) { + break; + } + /* FIXME: */ + failinfo.code = 128 + ctrinfo->u.siginfo->signo; + failinfo.detail = "Job killed by signal"; + + mark_job_fail(c, j, &failinfo); + break; + } + break; + case JobControl_u_ckptoption: + if (j->job.state & (JOB_STAT_DONE | JOB_STAT_EXIT | JOB_STAT_ERR)) { + ERROR("job <%ld> has been finished, no checkpoint can be done.", + jobid); + } else { + ERROR("job %ld has not been started, no checkpoint can be done", + jobid); + } + rc = -EINVAL; + break; + default: + rc = -EINVAL; + break; + } + } + + obj_deref(j); + return rc; +} + +static void notify_dispatch(cluster_t* c, + endpoint_t* ep, + uint64_t jobid, + int code) +{ + /* clang-format off */ + job_failinfo_t info = { + .jobid = jobid, + .code = code + }; + xce_notify_data_t data = { + .event = NotifyData_data_jobDispatchFail, + .data = { + .dispatchfail = &info + } + }; + /* clang-format on */ + + notify(c, ep, &data); +} + +int handle_jobdispatch(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + int i; + xce_job_dispatch_t* dispatchmsg = request->body.dispatchmsg; + xce_reply_t reply; + + /* reply job dispatch that we'll handle request */ + init_reply_by_req(&reply, request); + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + + for (i = 0; i < dispatchmsg->njobs; i++) { + int rc1; + xce_job_alloc_t* alloc = NULL; + + MOVE_PTR(alloc, dispatchmsg->allocs[i]); + + rc1 = dispatch_a_job_(c, ep, alloc); + if (rc1 < 0) { + /* notify dispatch fail */ + notify_dispatch(c, ep, alloc->jobid, rc1); + } else { + /* notify dispatch ok */ + notify_dispatch(c, ep, alloc->jobid, 0); + free_JobAlloc_ptr(&(alloc)); + } + } + return rc; +} + +int handle_jobsubmit(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + uint64_t jobid; + xce_job_submit_t* submit; + xce_reply_t reply; + xlist_t jlist; + job_t* j = NULL; + + submit = request->body.jobsubmsg; + + init_reply_by_req(&reply, request); + + if (NULLSTR(submit->user) || NULLSTR(submit->queue) || NULLSTR(submit->cmdline)) { + jobid = 0; + errno = EINVAL; + } else { + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_NEW, + c, submit); + + jobid = job_submit(c, (char*)submit->user, + (char*)submit->queue, + (char*)submit->resreq, + (char*)submit->cmdline, + (char*)submit->exdata, + submit->mintasks, + submit->maxtasks); + } + if (jobid == 0) { + reply.status = -errno; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + return -errno; + } + + reply.status = 0; + reply.data_which = RequestReply_data_jobid; + reply.data.jobid = jobid; + + xce_reply(ep, c->myhost, &reply); + + /* Run hooks to tell scheduler */ + j = jobmgr_lockjob(&(c->jobmgr), + jobid, OBJREF_READ); + if (j == NULL) { + ERROR("%s fail to lock job %lu", + __func__, jobid); + } else { + xlist_init(&jlist, 2); + if (xlist_store(&jlist, j) >= 0) { + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_NEW, + c, &jlist); + } else { + PERROR("xlist_store"); + } + + obj_deref(j); + xlist_fini(&jlist); + } + + return 0; +} + +static int mark_job_start(cluster_t* c, + job_t* job, + job_pidinfo_t* info) +{ + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_START, + c, info, job); + /* update job information */ + job->job.state = JOB_STAT_RUN; + job->job.start_time = time(NULL); + if (job->job.pidinfo == NULL) { + job->job.pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t)); + } + + job->job.pidinfo->npids = copy_int_list((int**)&(job->job.pidinfo->pids), + (int*)info->pids, info->npids); + job->job.pidinfo->npgids = copy_int_list((int**)&(job->job.pidinfo->pgids), + (int*)info->pgids, info->npgids); + if (info->exdata != NULL) { + job->job.pidinfo->exdata = create_obj_by_typename(&(c->objmgr), "xstring", + info->exdata); + } else { + job->job.pidinfo->exdata = NULL; + } + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_START, + c, info, job); + + return 0; +} + +int on_job_start(cluster_t* c, + job_pidinfo_t* info) +{ + job_t* job = NULL; + + TRACE("%s jobid %ld", __func__, info->jobid); + + job = jobmgr_lockjob(&(c->jobmgr), info->jobid, OBJREF_UPDATE); + if (job == NULL) { + ERROR("can not find job %ld", info->jobid); + return -errno; + } + + mark_job_start(c, job, info); + + /* unlock the job */ + DEREF_OBJ(job); + + return 0; +} + +static int mark_job_finish(cluster_t* c, + job_t* job, + xce_jobfinish_t* finish) +{ + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_EXIT, + c, finish, job); + /* update job information */ + if (finish->exit_code == 0) { + job->job.state = JOB_STAT_DONE; + } else { + job->job.state = JOB_STAT_EXIT; + } + job->job.finish_time = time(NULL); + + /* fill job run time/rusage information */ + + if (job->job.exit_info == NULL) { + job->job.exit_info = (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t)); + } + + if (job->job.exit_info != NULL) { + job->job.exit_info->exitcode = finish->exit_code; + } + + job->job.run_time = job->job.finish_time - job->job.start_time; + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_EXIT, + c, finish, job); + + return 0; +} + +int on_job_finish(cluster_t* c, + xce_jobfinish_t* finish) +{ + job_t* job = NULL; + + TRACE("%s job finish jobid %ld exit code %d", + __func__, finish->jobid, finish->exit_code); + + job = jobmgr_lockjob(&(c->jobmgr), finish->jobid, OBJREF_UPDATE); + if (job == NULL) { + ERROR("can not find job %ld", finish->jobid); + return -errno; + } + + mark_job_finish(c, job, finish); + + /* unlock the job */ + DEREF_OBJ(job); + + return 0; +} + +static int mark_job_fail(cluster_t* c, + job_t* job, + job_failinfo_t* failinfo) +{ + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_FAIL, + c, failinfo, job); + /* update job information */ + + switch (failinfo->stage) { + case JOB_EXEC_STAGE_Q_POST_EXEC: + case JOB_EXEC_STAGE_POST_EXEC: + if ((job->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) { + WARN("job %ld got post exec fail notification but job's exit status has not been set", + job->job.jobid); + job->job.state |= JOB_STAT_EXIT; + } + job->job.state |= JOB_STAT_PERR; + break; + default: + job->job.state = JOB_STAT_EXIT; + break; + } + + job->job.finish_time = time(NULL); + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_FAIL, + c, failinfo, job); + + return 0; +} + +static int mark_job_update(cluster_t* c, + job_t* job, + int state) +{ + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_UPDATE, + c, job); + /* update job state */ + job->job.state = state; + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_UPDATE, + c, job); + + return 0; +} + +int on_job_fail(cluster_t* c, + job_failinfo_t* failinfo) +{ + job_t* job = NULL; + + TRACE("%s job fail jobid %ld code %d", + __func__, failinfo->jobid, failinfo->code); + + job = jobmgr_lockjob(&(c->jobmgr), failinfo->jobid, OBJREF_UPDATE); + if (job == NULL) { + ERROR("can not find job %ld", failinfo->jobid); + return -errno; + } + + mark_job_fail(c, job, failinfo); + + /* unlock the job */ + DEREF_OBJ(job); + + return 0; +} + +int on_job_stop(cluster_t* c, + uint64_t jobid) +{ + job_t* job = NULL; + int state; + + TRACE("%s job stop jobid %ld", + __func__, jobid); + + job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE); + if (job == NULL) { + ERROR("can not find job %ld", jobid); + return -errno; + } + + state = job->job.state; + + state |= JOB_STAT_USUSP; + mark_job_update(c, job, state); + + /* unlock the job */ + DEREF_OBJ(job); + + return 0; +} + +int on_job_resume(cluster_t* c, + uint64_t jobid) +{ + job_t* job = NULL; + int state; + + TRACE("%s job resume jobid %ld", + __func__, jobid); + + job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE); + if (job == NULL) { + ERROR("can not find job %ld", jobid); + return -errno; + } + + state = job->job.state; + state &= ~JOB_STAT_USUSP; + mark_job_update(c, job, state); + + /* unlock the job */ + DEREF_OBJ(job); + + return 0; +} + +int on_job_postdone(cluster_t* c, + endpoint_t* ep, + uint64_t jobid) +{ + job_t* job = NULL; + int state; + + TRACE("%s job post done jobid %ld", + __func__, jobid); + + job = jobmgr_lockjob(&(c->jobmgr), jobid, OBJREF_UPDATE); + if (job == NULL) { + ERROR("can not find job %ld", jobid); + return -errno; + } + + state = job->job.state; + if ((state & (JOB_STAT_EXIT | JOB_STAT_DONE)) == 0) { + WARN("job %ld got post exec fail notification but job's exit status has not been set", + job->job.jobid); + state |= JOB_STAT_EXIT; + /* ensure that job finish time has been set */ + job->job.finish_time = time(NULL); + } + + state |= JOB_STAT_PDONE; + mark_job_update(c, job, state); + + /* unlock the job */ + DEREF_OBJ(job); + + return 0; +} + +int on_job_ckpt(cluster_t* c, + endpoint_t* ep, + xce_ckptinfo_t* ckptinfo) +{ + uint64_t jobid; + char* info = NULL; + job_t* j; + + if (ckptinfo == NULL) { + ERROR("%s ckptinfo should not be NULL", + __func__); + return -EINVAL; + } + + jobid = ckptinfo->jobid; + info = ckptinfo->ckptinfo; + + TRACE("%s job %ld ckptinfo %s", + __func__, jobid, info); + + j = jobmgr_lockjob(&(c->jobmgr), jobid, + OBJREF_UPDATE); + if (j == NULL) { + ERROR("%s can not find job %ld", + __func__, jobid); + return -errno; + } + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_CKPT, + c, j); + if (j->job.ckptinfo != NULL) { + obj_deref(j->job.ckptinfo); + } + + j->job.ckptinfo = create_obj_by_typename(&(c->objmgr), "xstring", + info); + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_CKPT, + c, j); + + obj_deref(j); + + return 0; +} + +int do_jobcontrol_op(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + xce_reply_t reply; + + init_reply_by_req(&reply, request); + + rc = job_ctrl(c, request->body.jobctrlmsg); + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + return rc; +} + +int update_job_rusages(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_job_rusages_t* rusages = request->body.updatemsg->data.jobrusages; + job_t* j; + worker_t* w; + char* host = NULL; + + TRACE("%s trying to update job(%ld) rusage", + __func__, rusages->jobid); + + j = jobmgr_lockjob(&(c->jobmgr), rusages->jobid, OBJREF_UPDATE); + if (j == NULL) { + PERROR("jobmgr_lockjob"); + return -errno; + } + + w = get_ep_worker(c, ep, OBJREF_READ); + if (w == NULL) { + obj_deref(j); + return errno; + } + + host = create_obj_by_typename(&(c->objmgr), "xstring", + w->host.name); + obj_deref(w); + + if (host == NULL) { + return -ENOMEM; + } + + TRACE("%s update job (%ld) rusage for host %s", + __func__, rusages->jobid, host); + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_JOB_UPDATE, + c, j); + update_host_runusage(&(j->job.run_usage), + host, + rusages->count, + rusages->usages); + + update_pids(&(j->job.pidinfo), + rusages->npids, + rusages->pids); + + update_pgids(&(j->job.pidinfo), + rusages->npgids, + rusages->pgids); + + /* FIXME + * update other fields of job rusage later + */ + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + HOOK_AFTER_EVENT, + HOOK_DETAIL_JOB_UPDATE, + c, j); + + obj_deref(j); + return 0; +} diff --git a/src/controller/jobmgr.c b/src/controller/jobmgr.c new file mode 100644 index 0000000..979dcba --- /dev/null +++ b/src/controller/jobmgr.c @@ -0,0 +1,172 @@ +#include +#include +#include +#include + +#include "cluster.h" +#include "htab.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" +#include "xstring.h" + +int jobmgr_init(jobmgr_t* mgr, + int nhslots, + int delta) +{ + ht_init(&(mgr->htab), nhslots, + int_hash_key); + zarray_init(&(mgr->jobs), delta, + (zarray_item_killer_t)job_obj_killer); + mylock_init(&(mgr->lock)); + return 0; +} + +void jobmgr_fini(jobmgr_t* mgr) +{ + ht_fini(&(mgr->htab)); + zarray_fini(&(mgr->jobs)); + mylock_fini(&(mgr->lock)); +} + +job_t* jobmgr_lockjob_(jobmgr_t* mgr, + uint64_t jobid, + int op) +{ + hent_t* e = NULL; + int idx; + job_t* job = NULL; + + e = ht_get(&(mgr->htab), (void*)jobid); + if (e == NULL) { + SET_ERROR(E_XCE_BAD_JOB); + return NULL; + } + + idx = e->data; + + if (idx < 0) { + SET_ERROR(E_XCE_BAD_JOB); + return NULL; + } + + job = (job_t*)zarray_get(&(mgr->jobs), idx); + + return obj_ref(job, op); +} + +job_t* jobmgr_lockjob(jobmgr_t* mgr, + uint64_t jobid, + int op) +{ + job_t* job = NULL; + + if (mgr == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + /* clang-format off */ + if ((op != OBJREF_NOP) && + (op != OBJREF_READ) && + (op != OBJREF_UPDATE)) { + SET_ERROR(EINVAL); + return NULL; + } + /* clang-format on */ + + mylock_lock(&(mgr->lock)); + job = jobmgr_lockjob_(mgr, jobid, op); + mylock_unlock(&(mgr->lock)); + + return job; +} + +static int32_t jobmgr_add_job_(jobmgr_t* mgr, + job_t* job) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), + (void*)job->job.jobid); + if (e != NULL) { + return -EEXIST; + } + + idx = zarray_store(&(mgr->jobs), job); + if (idx < 0) { + return idx; + } + + e = ht_add(&(mgr->htab), + (void*)job->job.jobid); + if (e == NULL) { + zarray_del(&(mgr->jobs), idx); + return -errno; + } + + e->data = idx; + return idx; +} + +int32_t jobmgr_add_job(jobmgr_t* mgr, + job_t* job) +{ + int32_t idx; + + if ((mgr == NULL) || (job == NULL)) { + return -EINVAL; + } + + mylock_lock(&(mgr->lock)); + idx = jobmgr_add_job_(mgr, job); + mylock_unlock(&(mgr->lock)); + + return idx; +} + +static void jobmgr_del_job_(jobmgr_t* mgr, + job_t* job) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), + (void*)job->job.jobid); + if (e == NULL) { + return; + } + + idx = e->data; + if (idx < 0) { + return; + } + + zarray_del(&(mgr->jobs), idx); + + return; +} + +void jobmgr_del_job(jobmgr_t* mgr, + job_t* job) +{ + if ((mgr == NULL) || (job == NULL)) { + return; + } + mylock_lock(&(mgr->lock)); + jobmgr_del_job_(mgr, job); + mylock_unlock(&(mgr->lock)); +} + +int jobmgr_iter(jobmgr_t* mgr, + xlist_t* l, + void* arg) +{ + int rc = 0; + + mylock_lock(&(mgr->lock)); + rc = zarray_iter_ex(&(mgr->jobs), l, arg); + mylock_unlock(&(mgr->lock)); + + return rc; +} diff --git a/src/controller/join.c b/src/controller/join.c new file mode 100644 index 0000000..e80761a --- /dev/null +++ b/src/controller/join.c @@ -0,0 +1,202 @@ +#include "config.h" + +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" + +static int validate_host(cluster_t* c, + const char* host, + const char* key) +{ + return 1; +} + +static int handle_client_join(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + + init_reply_by_req(&reply, request); + + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + + return 0; +} + +static int handle_worker_join(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int idx; + ep_private_t* priv = (ep_private_t*)ep->data; + xce_reply_t reply; + xce_peerinfo_t* from = request->from; + int rc = 0; + xce_joindata_t* joinmsg = NULL; + + TRACE("%s worker %s joining...", + __func__, from->name); + + init_reply_by_req(&reply, request); + + if (!validate_host(c, from->name, from->key)) { + ERROR("%s reject join of host %s", + __func__, from->name); + rc = -EPERM; + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + return rc; + } + + joinmsg = request->body.joinmsg; + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_WORKER, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_WORKER_JOIN, + c, from->name, 0); + idx = add_worker(c, ep, from->name, joinmsg->data.attrs); + + reply.status = idx; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_WORKER, + HOOK_AFTER_EVENT, + HOOK_DETAIL_WORKER_JOIN, + c, from->name, idx); + if (idx < 0) { + ERROR("worker %s join failed, rc %d", + from->name, idx); + + return idx; + } + + priv->u.idx = idx; + on_worker_join(c, ep); + + return 0; +} + +static int handle_scheduler_join(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + ep_private_t* priv = (ep_private_t*)ep->data; + + TRACE("%s scheduler join, ep %p", + __func__, ep); + + init_reply_by_req(&reply, request); + + c->sched.ep = ep; + priv->u.idx = 0; + + reply.status = 0; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + + on_scheduler_join(c); + + return 0; +} + +static int handle_controller_join(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + int rc = -ENOSYS; + + ERROR("we still have not support join of controller"); + + init_reply_by_req(&reply, request); + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + + return rc; +} + +static int handle_join_(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + ep_private_t* priv; + int rc = -EINVAL; + int role; + xce_reply_t reply; + xce_peerinfo_t* from = request->from; + + init_reply_by_req(&reply, request); + + priv = (ep_private_t*)ep->data; + if (priv == NULL) { + rc = -E_XCE_INTERNAL; + + ERROR("priv data of endpoint should not be NULL"); + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + + return rc; + } + + role = request->body.joinmsg->role; + priv->role = role; + + TRACE("%s role %d join from %s", + __func__, role, from->name); + + switch (role) { + case HostRole_client: + rc = handle_client_join(c, ep, request); + break; + case HostRole_worker: + rc = handle_worker_join(c, ep, request); + break; + case HostRole_scheduler: + rc = handle_scheduler_join(c, ep, request); + break; + case HostRole_controller: + rc = handle_controller_join(c, ep, request); + break; + default: + rc = -E_XCE_BAD_ROLE; + ERROR("unknown role %d from host %s", + role, from->name); + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + break; + } + + return rc; +} + +int handle_join(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + + rc = handle_join_(c, ep, request); + + return rc; +} diff --git a/src/controller/main.c b/src/controller/main.c new file mode 100644 index 0000000..eb176af --- /dev/null +++ b/src/controller/main.c @@ -0,0 +1,162 @@ +#include +#include +#include +#include +#include + +#include "myconf.h" +#include "utils.h" + +extern int myapp_run(myconf_t*); + +static char doc[] = "xce_controller"; +static char args_doc[] = ""; +const char* argp_program_version = MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID)); +const char* argp_program_bug_address = BUG_EMAIL; +static char* default_config = "/tmp/xce/xce.yaml"; + +struct arguments { + char* config; + char* logdir; + char* clusterconf; + int verbose; +}; + +static struct argp_option options[] = { + { "config", 'c', "FILE", 0, "Load initial configuration from FILE" }, + { "logdir", 'd', "DIR", 0, "Directory for log file" }, + { "verbose", 'v', 0, 0, "Produce verbose logs" }, + { "clusterconf", 'f', "FILE", 0, "Load Cluster Configuration from FILE" }, + { 0 } +}; + +static error_t parse_opt(int key, + char* arg, + struct argp_state* state) +{ + struct arguments* arguments = state->input; + + switch (key) { + case 'c': + arguments->config = arg; + break; + case 'd': + arguments->logdir = arg; + break; + case 'v': + arguments->verbose++; + break; + case 'f': + arguments->clusterconf = arg; + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +static struct argp argp = { + options, + parse_opt, + args_doc, + doc +}; + +static char* verbose_to_loglevel(int verbose) +{ + char* loglevel = "info"; + + switch (verbose) { + case 1: + loglevel = "debug"; + break; + case 2: + loglevel = "debug2"; + break; + case 3: + loglevel = "debug3"; + break; + default: + if (verbose > 3) { + loglevel = "debug3"; + } + break; + } + + return loglevel; +} + +int main(int argc, char* argv[]) +{ + struct arguments arguments; + initial_config_t* init_config; + myconf_t myconf; + char* logdir = "/tmp"; + char* loglevel = "info"; + int logsize = 64 * 1024 * 1024; + int logcount = 10; + char* myname = "xce_controller"; + + memset(&arguments, 0x0, sizeof(arguments)); + + argp_parse(&argp, argc, argv, 0, 0, &arguments); + + if (arguments.clusterconf == NULL) { + tlog_error("no cluster configuration specified. please use -f to specify cluster configuration file"); + return -1; + } + + if (arguments.config == NULL) { + arguments.config = default_config; + } + + if (arguments.logdir != NULL) { + logdir = arguments.logdir; + } + + if (arguments.verbose != 0) { + loglevel = verbose_to_loglevel(arguments.verbose); + } + + mylog_init(logdir, myname, loglevel, logsize, logcount); + + load_initial_config(arguments.config, + &init_config, + arguments.verbose); + + if (init_config == NULL) { + tlog_error("Fail to read initial log"); + return -1; + } + myconf.init_config = init_config; + myconf.cluster_config = NULL; + myconf.conffile = arguments.clusterconf; + + if (init_config != NULL) { + if (init_config->logging != NULL) { + if ((init_config->logging->logdir != NULL) && (arguments.logdir == NULL)) { + char logfile[XCE_MAXPATH_LEN + 1]; + + logdir = init_config->logging->logdir; + snprintf(logfile, sizeof(logfile) - 1, + "%s/%s.log", logdir, myname); + + tlog_set_logfile(logfile); + } + + if ((init_config->logging->loglevel != NULL) && (arguments.verbose == 0)) { + loglevel = init_config->logging->loglevel; + + mylog_setlevel(loglevel); + } + } + } + + myapp_run(&myconf); + + free_initial_config(init_config); + mylog_fini(); + + return 0; +} diff --git a/src/controller/meson.build b/src/controller/meson.build new file mode 100644 index 0000000..22e2194 --- /dev/null +++ b/src/controller/meson.build @@ -0,0 +1,52 @@ +controller_src = [ + 'cluster.c', + 'dataex.c', + 'job.c', + 'jobmgr.c', + 'join.c', + 'main.c', + 'misc.c', + 'myapp.c', + 'opget.c', + 'opupd.c', + 'qmgr.c', + 'queue.c', + 'reply.c', + 'resmgr.c', + 'resources.c', + 'sched.c', + 'task.c', + 'worker.c', + 'workermgr.c', + 'wrap.c', + 'xceobjs.c', + 'myconf.c' +] + +incdir = include_directories('.', '../include', '../lib', + 'include') + +cc = meson.get_compiler('c') +math_dep = cc.find_library('m', required : false) + +controller_deps = [libxce_dep, math_dep] + +controller = executable('xce_controller', + controller_src, + include_directories: incdir, + dependencies: controller_deps, + install: true) + +xce_test_src = [ 'testsuite/test.c'] + +readline_dep = dependency('readline') +ncurses_dep = dependency('ncurses') +history_dep = dependency('history') + +xce_test_deps = [libxce_dep, readline_dep, history_dep, ncurses_dep] +xce_test = executable('xce_test', + xce_test_src, + include_directories: [incdir], + dependencies: xce_test_deps, + install: true) + diff --git a/src/controller/misc.c b/src/controller/misc.c new file mode 100644 index 0000000..fce3f57 --- /dev/null +++ b/src/controller/misc.c @@ -0,0 +1,215 @@ +#include +#include + +#include "cluster.h" +#include "objmgr.h" +#include "worker.h" +#include "xce/types.h" +#include "xlist.h" + +int add_to_xlist(xlist_t* l, void* data) +{ + return xlist_store(l, data); +} + +int add_worker_to_xlist(xlist_t* l, void* data) +{ + worker_t* w = (worker_t*)data; + + return xlist_store(l, &(w->host)); +} + +int add_queue_to_xlist(xlist_t* l, void* data) +{ + queue_t* q = (queue_t*)data; + + return xlist_store(l, &(q->queue)); +} + +int add_job_to_xlist(xlist_t* l, void* data) +{ + job_t* j = (job_t*)data; + + return xlist_store(l, &(j->job)); +} + +int get_ep_role(endpoint_t* ep) +{ + ep_private_t* priv; + + if (ep == NULL) { + return -EINVAL; + } + + priv = (ep_private_t*)ep->data; + if (priv == NULL) { + ERROR("priv data of endpoint %p is NULL", + ep); + return -EINVAL; + } + + return priv->role; +} + +worker_t* get_ep_worker(cluster_t* c, + endpoint_t* ep, + int op) +{ + int role, idx; + worker_t* w = NULL; + + role = get_ep_role(ep); + if (role != HostRole_worker) { + ERROR("endpoint %p is not a worker", ep); + return NULL; + } + + idx = ((ep_private_t*)ep->data)->u.idx; + if (idx < 0) { + ERROR("invalid idx (%d) for endpoint %p", idx, ep); + return NULL; + } + + w = workermgr_lock_byid(&(c->workermgr), idx, op); + if (w == NULL) { + ERROR("can not find worker with id (%d)", idx); + } + + return w; +} + +int update_host_runusage(xce_job_runusage_t** runusage, + char* host, + int count, + xce_rusage_t** usages) +{ + xce_job_runusage_t* rusages = NULL; + int i, j; + + if ((*runusage) == NULL) { + *runusage = (xce_job_runusage_t*)calloc(1, sizeof(xce_job_runusage_t)); + if ((*runusage) == NULL) { + return -ENOMEM; + } + } + + rusages = *runusage; + j = -1; + + for (i = 0; i < rusages->num_hosts; i++) { + if (strcmp(host, rusages->host_runusage[i]->host_name) == 0) { + j = i; + break; + } + } + + if (j < 0) { + int cnt = rusages->num_hosts + 1; + xce_host_runusage_t** hru = (xce_host_runusage_t**)realloc(rusages->host_runusage, + cnt * sizeof(xce_host_runusage_t*)); + if (hru == NULL) { + return -ENOMEM; + } + + rusages->host_runusage = hru; + j = cnt - 1; + + hru[j] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t)); + if (hru[j] == NULL) { + return -ENOMEM; + } + + rusages->num_hosts++; + hru[j]->host_name = host; + } + + for (i = 0; i < count; i++) { + if (strcmp(usages[i]->resname, "cputime") == 0) { + rusages->host_runusage[j]->cpu_time = usages[i]->value; + continue; + } + + if (strcmp(usages[i]->resname, "mem") == 0) { + rusages->host_runusage[j]->mem_usage = usages[i]->value; + continue; + } + } + + return 0; +} + +int update_pids(job_pidinfo_t** pidinfo, + int npids, + int* pids) +{ + job_pidinfo_t* info; + int i; + + if ((*pidinfo) == NULL) { + *pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t)); + if ((*pidinfo) == NULL) { + return -ENOMEM; + } + } + + info = *pidinfo; + + if (info->pids == NULL) { + info->pids = (unsigned int*)calloc(npids, sizeof(unsigned int)); + } else if (info->npids < npids) { + unsigned int* ptr; + ptr = (unsigned int*)realloc(info->pids, + npids * sizeof(unsigned int)); + if (ptr == NULL) { + return -ENOMEM; + } + + info->pids = ptr; + } + + for (i = 0; i < npids; i++) { + info->pids[i] = pids[i]; + } + + info->npids = npids; + + return 0; +} + +int update_pgids(job_pidinfo_t** pidinfo, + int npgids, + int* pgids) +{ + job_pidinfo_t* info; + int i; + + if ((*pidinfo) == NULL) { + *pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t)); + if ((*pidinfo) == NULL) { + return -ENOMEM; + } + } + + info = *pidinfo; + + if (info->pgids == NULL) { + info->pgids = (unsigned int*)calloc(npgids, sizeof(unsigned int)); + } else if (info->npgids < npgids) { + unsigned int* ptr; + ptr = (unsigned int*)realloc(info->pgids, + npgids * sizeof(unsigned int)); + if (ptr == NULL) { + return -ENOMEM; + } + + info->pgids = ptr; + } + + for (i = 0; i < npgids; i++) { + info->pgids[i] = pgids[i]; + } + + info->npgids = npgids; + + return 0; +} diff --git a/src/controller/myapp.c b/src/controller/myapp.c new file mode 100644 index 0000000..43586b7 --- /dev/null +++ b/src/controller/myapp.c @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include +#include + +#include "cluster.h" +#include "myconf.h" +#include "netio.h" +#include "xce/cluster_def.h" + +#include "request.h" +#include "utils.h" + +time_t last_print_time = 0; +unsigned long npkts = 0; +unsigned long npkts_old = 0; + +int myapp_run(myconf_t* myconf) +{ + cluster_t c; + int rc; + + c.myconf = myconf; + + rc = cluster_init(&c); + if (rc < 0) { + SET_ERROR(-rc); + + ERROR("cluster_init: %s", strerror(-rc)); + + cluster_fini(&c); + + return rc; + } + + rc = cluster_start(&c); + if (rc < 0) { + SET_ERROR(-rc); + perror("cluster_start"); + cluster_fini(&c); + + return rc; + } + + rc = cluster_run(&c); + + cluster_fini(&c); + + return rc; +} diff --git a/src/controller/myconf.c b/src/controller/myconf.c new file mode 100644 index 0000000..0081112 --- /dev/null +++ b/src/controller/myconf.c @@ -0,0 +1,173 @@ +#include +#include +#include +#include +#include + +#include "controller.capnp.h" +#include "myconf.h" +#include "objmgr.h" +#include "utils.h" +#include "xce/cluster_def.h" +#include "xce/types.h" + +int add_config(void* c, + char* name, + char* value) +{ + cluster_t* cls = (cluster_t*)c; + xce_config_t var = { + .name = name, + .value = value + }; + xce_config_t* cv = NULL; + + cv = create_obj_by_typename(&(cls->objmgr), "xconfig", + &var); + if (cv == NULL) { + return -errno; + } + + return 0; +} + +struct cluster_configuration { + cluster_config_t* cluster; +}; + +static int config_fini(int type, + int situation, + void* data, + va_list ap) +{ + struct cluster_configuration* cf = (struct cluster_configuration*)data; + + switch (situation) { + case HOOK_AFTER_EVENT: + if (cf->cluster != NULL) { + free_ClusterConfig(cf->cluster); + FREEUP(cf->cluster); + } + break; + default: + break; + } + return 0; +} + +static int cluster_hook_fun(int type, + int situation, + int detail, + void* data, + va_list ap) +{ + if (detail == HOOK_DETAIL_CLUSTER_SHUTDOWN) { + return config_fini(type, situation, data, ap); + } + + return 0; +} + +#define SET_DEFAULT_IF_UNSET(var, unset, val) \ + do { \ + if ((var) == (unset)) { \ + (var) = (val); \ + } \ + } while (0) + +static void fill_default_cluster_config(cluster_config_t* c) +{ + SET_DEFAULT_IF_UNSET(c->port, 0, 9990); + SET_DEFAULT_IF_UNSET(c->heartbeat_interval, 0, 5); + SET_DEFAULT_IF_UNSET(c->vcpus, 0, 8); + SET_DEFAULT_IF_UNSET(c->fetch_job_interval, 0, 10); +} + +static int load_cluster_config_from_file(char* file, + cluster_config_t** config) +{ + struct capn c; + ClusterConfig_ptr p; + FILE* fp; + int rc = 0; + + fp = fopen(file, "rb"); + if (fp == NULL) { + rc = -errno; + perror("fopen"); + return rc; + } + + capn_init_fp(&c, fp, 0); + p.p = capn_getp(capn_root(&c), 0, 1); + + decode_ClusterConfig_ptr(config, p); + + capn_free(&c); + + fclose(fp); + + return 0; +} + +int load_cluster_config(cluster_t* c, + struct cluster_configuration** cluster_config) +{ + cluster_config_t* cfg; + int rc; + + *cluster_config = (struct cluster_configuration*)calloc(1, sizeof(*cluster_config)); + if ((*cluster_config) == NULL) { + return -ENOMEM; + } + + rc = load_cluster_config_from_file(c->myconf->conffile, &((*cluster_config)->cluster)); + + if (rc < 0) { + return rc; + } + + if ((*cluster_config)->cluster == NULL) { + return -errno; + } + + cfg = (*cluster_config)->cluster; + + fill_default_cluster_config(cfg); + + xce_add_hook(&(c->hooksys), + XCE_CONTROLLER_HOOK_CLUSTER, + (hook_fn_t)cluster_hook_fun, + (*cluster_config)); + + return 0; +} + +static struct cluster_configuration* cf = NULL; + +cluster_config_t* get_default_conf(void* cl) +{ + cluster_config_t* c; + cluster_t* cls = (cluster_t*)cl; + + load_cluster_config(cls, &cf); + if (cf == NULL) { + PERROR("load_cluster_config"); + return NULL; + } + + c = cf->cluster; + + return c; +} + +int load_initial_config(char* file, + initial_config_t** conf, + int verbose) +{ + return 0; +} + +void free_initial_config(initial_config_t* conf) +{ +} diff --git a/src/controller/opget.c b/src/controller/opget.c new file mode 100644 index 0000000..10d8365 --- /dev/null +++ b/src/controller/opget.c @@ -0,0 +1,225 @@ +#include "config.h" + +#include +#include +#include + +#include "cluster.h" +#include "controller.capnp.h" +#include "utils.h" +#include "xce/cluster_def.h" + +static int +do_reply_reslist(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + xce_resource_data_t resdata; + xce_data_t getdata = { + .table = request->body.getmsg->table, + .data = { + .resources = &resdata } + }; + int rc = 0; + + init_reply_by_req(&reply, request); + collect_resources(c, NULL, &(resdata.n_total), + &(resdata.resources)); + + reply.status = 0; + reply.data_which = RequestReply_data_getdata; + reply.data.getdata = &getdata; + + rc = xce_reply_get(ep, c->myhost, &reply, NULL, NULL, -1); + + deref_objs(resdata.n_total, (void**)resdata.resources); + FREEUP(resdata.resources); + + return rc; +} + +static int +do_reply_resvals(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + xce_resval_data_t resvaldata; + xce_get_t* getmsg = request->body.getmsg; + xce_data_t getdata = { + .data = { + .resvals = &resvaldata } + }; + int rc = 0; + xce_val_t val; + xce_resval_t resval; + xce_resval_t* vals[] = { + &resval + }; + + init_reply_by_req(&reply, request); + + resvaldata.n_total = 0; + resvaldata.vals = NULL; + + if (getmsg != NULL) { + xce_match_t* condition = getmsg->condition; + + if (condition != NULL) { + getdata.table = getmsg->table; + rc = get_resval(c, condition->condition.reskey->resname, + condition->condition.reskey->hostname, + &val); + if (rc >= 0) { + resval.val = &val; + resval.name = condition->condition.reskey->resname; + resval.host = condition->condition.reskey->hostname; + resvaldata.n_total = 1; + resvaldata.vals = vals; + } + } else { + rc = -EINVAL; + } + } else { + rc = -EINVAL; + } + + reply.status = rc; + reply.data_which = RequestReply_data_getdata; + reply.data.getdata = &getdata; + + rc = xce_reply_get(ep, c->myhost, &reply, NULL, NULL, -1); + + return rc; +} + +int check_worker_(xce_worker_t* w) +{ + if (w == NULL) { + return 0; + } + + return 1; +} + +static int +do_reply_workerlist(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + xce_workers_data_t workerdata; + /* clang-format off */ + xce_data_t getdata = { + .table = request->body.getmsg->table, + .data = { + .workers = &workerdata } + }; + /* clang-format on */ + int rc = 0; + int i; + + init_reply_by_req(&reply, request); + collect_workers(c, (collect_check_fn_t)check_worker_, + &(workerdata.n_total), &(workerdata.workers)); + + reply.status = 0; + reply.data_which = RequestReply_data_getdata; + reply.data.getdata = &getdata; + + rc = xce_reply_get(ep, c->myhost, &reply, + check_controller, c, 100); + + for (i = 0; i < workerdata.n_total; i++) { + xce_worker_t* w = workerdata.workers[i]; + + free_resval_with_idx_list(&(w->resvals_to_sched), + w->nresvals_to_sched); + } + + deref_objs(workerdata.n_total, (void**)workerdata.workers); + FREEUP(workerdata.workers); + + return rc; +} + +static int check_job_(xce_job_t* j) +{ + if (j == NULL) { + return 0; + } + + return 1; +} + +static int +do_reply_joblist(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + xce_reply_t reply; + xce_jobs_data_t jobdata; + xce_data_t getdata = { + .table = request->body.getmsg->table, + .data = { + .jobs = &jobdata } + }; + int rc = 0; + + init_reply_by_req(&reply, request); + collect_jobs(c, (collect_check_fn_t)check_job_, + &(jobdata.n_total), &(jobdata.jobs)); + + reply.status = 0; + reply.data_which = RequestReply_data_getdata; + reply.data.getdata = &getdata; + + rc = xce_reply_get(ep, c->myhost, &reply, + check_controller, c, 100); + + deref_objs(jobdata.n_total, (void**)jobdata.jobs); + FREEUP(jobdata.jobs); + + return rc; +} + +int do_get_op(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + xce_reply_t reply; + + TRACE("%s table %d", + __func__, request->body.getmsg->table); + + init_reply_by_req(&reply, request); + + switch (request->body.getmsg->table) { + case Data_data_resources: + rc = do_reply_reslist(c, ep, request); + break; + case Data_data_workers: + case Data_data_schedworkers: + rc = do_reply_workerlist(c, ep, request); + break; + case Data_data_schedjobs: + case Data_data_jobs: + rc = do_reply_joblist(c, ep, request); + break; + case Data_data_resvals: + rc = do_reply_resvals(c, ep, request); + break; + default: + rc = -ENOENT; + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + break; + } + + return rc; +} diff --git a/src/controller/opupd.c b/src/controller/opupd.c new file mode 100644 index 0000000..c90dda9 --- /dev/null +++ b/src/controller/opupd.c @@ -0,0 +1,103 @@ +#include "config.h" + +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" + +static int +do_update_resvals(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + ep_private_t* priv = NULL; + int rc = -EPERM; + xce_reply_t reply; + + priv = (ep_private_t*)ep->data; + TRACE("endpoint %p priv data %p", ep, priv); + + init_reply_by_req(&reply, request); + + if (priv != NULL) { + int idx = priv->u.idx; + + if ((priv->role == HostRole_worker) && (idx >= 0)) { + worker_t* w; + + w = workermgr_lock_byid(&(c->workermgr), + idx, OBJREF_UPDATE); + if (w == NULL) { + rc = -E_XCE_BAD_WORKER; + } else { + xce_update_data_t* updatemsg = request->body.updatemsg; + xce_resval_list_t* resvals = updatemsg->data.resvals; + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_RESOURCE, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_RESOURCE_UPDATE, + c, resvals); + rc = update_resval_list(c, w, resvals); + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_RESOURCE, + HOOK_AFTER_EVENT, + HOOK_DETAIL_RESOURCE_UPDATE, + c, resvals); + DEREF_OBJ(w); + } + } + } + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + + return 0; +} + +static int do_update_jobrusage(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + xce_reply_t reply; + + init_reply_by_req(&reply, request); + rc = update_job_rusages(c, ep, request); + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + + return rc; +} + +int do_update_op(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + xce_reply_t reply; + + init_reply_by_req(&reply, request); + + switch (request->body.updatemsg->table) { + case Update_data_resvals: + rc = do_update_resvals(c, ep, request); + break; + case Update_data_jobrusages: + rc = do_update_jobrusage(c, ep, request); + break; + default: + rc = -ENOENT; + + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + xce_reply(ep, c->myhost, &reply); + break; + } + return rc; +} diff --git a/src/controller/qmgr.c b/src/controller/qmgr.c new file mode 100644 index 0000000..105b12c --- /dev/null +++ b/src/controller/qmgr.c @@ -0,0 +1,171 @@ +#include +#include +#include + +#include "cluster.h" +#include "htab.h" +#include "xce/cluster_def.h" +#include "xstring.h" + +extern void queue_obj_killer(queue_t*); + +int qmgr_init(qmgr_t* mgr, + int nslots, + int delta) +{ + mylock_init(&(mgr->lock)); + ht_init(&(mgr->htab), nslots, + string_hash_key); + zarray_init(&(mgr->queues), delta, + (zarray_item_killer_t)queue_obj_killer); + return 0; +} + +void qmgr_fini(qmgr_t* mgr) +{ + zarray_fini(&(mgr->queues)); + ht_fini(&(mgr->htab)); + mylock_fini(&(mgr->lock)); +} + +static int qmgr_add_(qmgr_t* mgr, + queue_t* q) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), (void*)q->queue.queue); + if (e != NULL) { + return -EEXIST; + } + + idx = zarray_store(&(mgr->queues), q); + if (idx < 0) { + return idx; + } + + e = ht_add(&(mgr->htab), (void*)q->queue.queue); + if (e == NULL) { + zarray_del(&(mgr->queues), idx); + return -errno; + } + + e->data = idx; + return idx; +} + +int qmgr_add(qmgr_t* mgr, + queue_t* q) +{ + int idx; + + if ((mgr == NULL) || (q == NULL)) { + return -EINVAL; + } + + mylock_lock(&(mgr->lock)); + idx = qmgr_add_(mgr, q); + mylock_unlock(&(mgr->lock)); + + return idx; +} +static queue_t* qmgr_lockq_(qmgr_t* mgr, + char* name, + int op) +{ + hent_t* e = NULL; + int idx; + queue_t* q = NULL; + + e = ht_get(&(mgr->htab), (void*)name); + if (e == NULL) { + SET_ERROR(ENOENT); + return NULL; + } + + idx = e->data; + if (idx < 0) { + SET_ERROR(ENOENT); + return NULL; + } + + q = (queue_t*)zarray_get(&(mgr->queues), idx); + + return obj_ref(q, op); +} +queue_t* qmgr_lockq(qmgr_t* mgr, + char* name, + int op) +{ + queue_t* q = NULL; + + if (mgr == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + if (name == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + /* clang-format off */ + if ((op != OBJREF_NOP) && + (op != OBJREF_READ) && + (op != OBJREF_UPDATE)) { + SET_ERROR(EINVAL); + return NULL; + } + /* clang-format on */ + + mylock_lock(&(mgr->lock)); + q = qmgr_lockq_(mgr, name, op); + mylock_unlock(&(mgr->lock)); + + return q; +} + +static void qmgr_delq_(qmgr_t* mgr, + queue_t* q) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), + (void*)q->queue.queue); + if (e == NULL) { + return; + } + + idx = e->data; + if (idx < 0) { + return; + } + + zarray_del(&(mgr->queues), idx); +} + +void qmgr_delq(qmgr_t* mgr, + queue_t* q) +{ + if ((mgr == NULL) || (q == NULL)) { + return; + } + + mylock_lock(&(mgr->lock)); + qmgr_delq_(mgr, q); + mylock_unlock(&(mgr->lock)); +} + +int qmgr_iter(qmgr_t* mgr, + xlist_t* l, + void* arg) +{ + int rc = 0; + + mylock_lock(&(mgr->lock)); + rc = zarray_iter_ex(&(mgr->queues), l, arg); + mylock_unlock(&(mgr->lock)); + + return rc; +} diff --git a/src/controller/queue.c b/src/controller/queue.c new file mode 100644 index 0000000..7f694e1 --- /dev/null +++ b/src/controller/queue.c @@ -0,0 +1,163 @@ +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" +#include "xce/types.h" + +void queue_fini(queue_t* q) +{ + if (q != NULL) { + free(q); + } +} + +queue_t* create_queue(cluster_t* c, + xce_queue_t* p) +{ + queue_t q1; + queue_t* q; + + memcpy(&(q1.queue), p, sizeof(q1.queue)); + + q = create_obj_by_typename(&(c->objmgr), "xqueue", + &q1); + return q; +} + +int build_queues(void* c) +{ + cluster_t* cls = (cluster_t*)c; + int i; + + if (cls == NULL) { + return -EINVAL; + } + + if (cls->myconf == NULL) { + return -EINVAL; + } + + for (i = 0; i < cls->myconf->cluster_config->nqueues; i++) { + xce_queue_t* q = cls->myconf->cluster_config->queues[i]; + + create_queue(cls, q); + } + + return 0; +} + +int queue_ctrl(cluster_t* c, xce_queuectrl_t* ctrl) +{ + queue_t* q = NULL; + int rc = 0; + + if (!c || !ctrl || !ctrl->queue || *ctrl->queue == '\0') { + TRACE("%s invalid arguments.", __func__); + return -EINVAL; + } + TRACE("%s queue_ctrl queue %s", __func__, ctrl->queue); + q = qmgr_lockq(&(c->qmgr), ctrl->queue, OBJREF_UPDATE); + if (q == NULL) { + PERROR("qmgr_lockq"); + return -ENOENT; + } + + switch (ctrl->action) { + case QUEUE_OPEN: + if (!(q->queue.status & QUEUE_STAT_CLOSED)) { + goto clean; + } + q->queue.status &= ~QUEUE_STAT_CLOSED; + break; + case QUEUE_CLOSE: + if (q->queue.status & QUEUE_STAT_CLOSED) { + goto clean; + } + q->queue.status |= QUEUE_STAT_CLOSED; + break; + case QUEUE_INACTIVE: + if (q->queue.status & QUEUE_STAT_INACT_ADMIN) { + goto clean; + } + q->queue.status |= QUEUE_STAT_INACT_ADMIN; + break; + case QUEUE_ACTIVE: + if (!(q->queue.status & QUEUE_STAT_INACT_ADMIN)) { + goto clean; + } + q->queue.status &= ~QUEUE_STAT_INACT_ADMIN; + break; + default: + rc = -1; + break; + } + if (rc == 0) { + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_QUEUE, + HOOK_AFTER_EVENT, + HOOK_DETAIL_QUEUE_UPDATE, + c, q); + } +clean: + obj_deref(q); + return rc; +} + +int do_queuecontrol_op(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + int rc = 0; + xce_reply_t reply; + + init_reply_by_req(&reply, request); + + rc = queue_ctrl(c, request->body.queuectrlmsg); + reply.status = rc; + reply.data_which = RequestReply_data_nodata; + + xce_reply(ep, c->myhost, &reply); + return rc; +} + +int get_queue_exec(cluster_t* c, + char* queue, + char** pre_exec, + char** post_exec) +{ + queue_t* q = NULL; + + TRACE("entering %s", __func__); + + if ((c == NULL) || (queue == NULL) || (pre_exec == NULL) || (post_exec == NULL)) { + return -EINVAL; + } + + TRACE("%s queue %s", __func__, queue); + + q = qmgr_lockq(&(c->qmgr), queue, OBJREF_READ); + if (q == NULL) { + PERROR("qmgr_lockq"); + return -ENOENT; + } + + if (q->queue.pre_exec != NULL) { + *pre_exec = create_obj_by_typename(&(c->objmgr), "xstring", + q->queue.pre_exec); + } else { + *pre_exec = NULL; + } + + if (q->queue.post_exec != NULL) { + *post_exec = create_obj_by_typename(&(c->objmgr), "xstring", + q->queue.post_exec); + } else { + *post_exec = NULL; + } + + obj_deref(q); + + return 0; +} diff --git a/src/controller/reply.c b/src/controller/reply.c new file mode 100644 index 0000000..989f794 --- /dev/null +++ b/src/controller/reply.c @@ -0,0 +1,75 @@ +#include "config.h" + +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" + +int handle_clusterconf(cluster_t* c, + endpoint_t* ep, + xce_request_t* request) +{ + /* clang-format off */ + xce_reply_t reply; + int rc = 0; + xce_clusterconf_t clusterconf = { + .name = c->myconf->cluster_config->clustername, + .wlist = { + .count = 0, + .workers = NULL + }, + .uglist = { + .count = 0, + .groups = NULL + }, + .hglist = { + .count = 0, + .groups = NULL + } + }; + /* clang-format on */ + int i; + + init_reply_by_req(&reply, request); + + reply.status = 0; + reply.data_which = RequestReply_data_clusterconf; + reply.data.clusterconf = &clusterconf; + + collect_confvars(c, NULL, &(clusterconf.cfglist.count), + &(clusterconf.cfglist.configs)); + collect_queues(c, NULL, &(clusterconf.qlist.count), + &(clusterconf.qlist.queues)); + collect_resources(c, NULL, &(clusterconf.reslist.count), + &(clusterconf.reslist.res)); + collect_workers(c, (collect_check_fn_t)check_worker_, + &(clusterconf.wlist.count), + &(clusterconf.wlist.workers)); + + xce_reply(ep, c->myhost, &reply); + + for (i = 0; i < clusterconf.wlist.count; i++) { + xce_worker_t* w = clusterconf.wlist.workers[i]; + + free_resval_with_idx_list(&(w->resvals_to_sched), + w->nresvals_to_sched); + } + + deref_objs(clusterconf.cfglist.count, + (void**)clusterconf.cfglist.configs); + deref_objs(clusterconf.qlist.count, + (void**)clusterconf.qlist.queues); + deref_objs(clusterconf.reslist.count, + (void**)clusterconf.reslist.res); + deref_objs(clusterconf.wlist.count, + (void**)clusterconf.wlist.workers); + + FREEUP(clusterconf.cfglist.configs); + FREEUP(clusterconf.qlist.queues); + FREEUP(clusterconf.reslist.res); + FREEUP(clusterconf.wlist.workers); + + return rc; +} diff --git a/src/controller/resmgr.c b/src/controller/resmgr.c new file mode 100644 index 0000000..e959172 --- /dev/null +++ b/src/controller/resmgr.c @@ -0,0 +1,209 @@ +#include +#include +#include +#include + +#include "cluster.h" +#include "htab.h" +#include "utils.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" +#include "xstring.h" + +extern void res_obj_killer(xce_resource_t*); +extern void resval_obj_killer(xce_resval_t*); + +int resmgr_init(resmgr_t* mgr, + int nslots, + int delta, + int inst_delta) +{ + TRACE("%s nslots %d delta %d inst_delta %d", + __func__, nslots, delta, + inst_delta); + + mylock_init(&(mgr->lock)); + + ht_init(&(mgr->htab), nslots, + string_hash_key); + zarray_init(&(mgr->defs), delta, + (zarray_item_killer_t)res_obj_killer); + zarray_init(&(mgr->sharedinst), inst_delta, + (zarray_item_killer_t)resval_obj_killer); + mgr->nshared = 0; + mgr->nunshared = 0; + + return 0; +} + +void resmgr_fini(resmgr_t* mgr) +{ + + zarray_fini(&(mgr->sharedinst)); + zarray_fini(&(mgr->defs)); + ht_fini(&(mgr->htab)); + mylock_fini(&(mgr->lock)); + + return; +} + +static int32_t resmgr_add_res_(resmgr_t* mgr, + xce_resource_t* res) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), (void*)res->name); + if (e != NULL) { + return -EEXIST; + } + + idx = zarray_store(&(mgr->defs), res); + if (idx < 0) { + return idx; + } + + e = ht_add(&(mgr->htab), + (void*)res->name); + if (e == NULL) { + zarray_del(&(mgr->defs), idx); + return -errno; + } + + e->data = idx; + if (res->flags & XCE_RESF_SHARED) { + res->idx = mgr->nshared; + mgr->nshared++; + } else { + res->idx = mgr->nunshared; + mgr->nunshared++; + } + return idx; +} + +int32_t resmgr_add_res(resmgr_t* mgr, + xce_resource_t* res) +{ + int32_t idx; + + if ((mgr == NULL) || (res == NULL)) { + return -EINVAL; + } + + mylock_lock(&(mgr->lock)); + idx = resmgr_add_res_(mgr, res); + mylock_unlock(&(mgr->lock)); + + return idx; +} + +static xce_resource_t* resmgr_lockres_(resmgr_t* mgr, + char* resname, + int op) +{ + hent_t* e = NULL; + int idx; + xce_resource_t* r = NULL; + + e = ht_get(&(mgr->htab), (void*)resname); + if (e == NULL) { + SET_ERROR(E_XCE_BAD_RES); + return NULL; + } + + idx = e->data; + if (idx < 0) { + SET_ERROR(E_XCE_INTERNAL); + return NULL; + } + + r = (xce_resource_t*)zarray_get(&(mgr->defs), idx); + if (r == NULL) { + SET_ERROR(E_XCE_INTERNAL); + return NULL; + } + + return obj_ref(r, op); +} +xce_resource_t* resmgr_lockres(resmgr_t* mgr, + char* resname, + int op) +{ + xce_resource_t* r = NULL; + + if (mgr == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + if (resname == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + /* clang-format off */ + if ((op != OBJREF_NOP) && + (op != OBJREF_READ) && + (op != OBJREF_UPDATE)) { + SET_ERROR(EINVAL); + return NULL; + } + /* clang-format on */ + + mylock_lock(&(mgr->lock)); + r = resmgr_lockres_(mgr, resname, op); + mylock_unlock(&(mgr->lock)); + + return r; +} + +static void resmgr_del_res_(resmgr_t* mgr, + xce_resource_t* r) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), + (void*)r->name); + if (e == NULL) { + SET_ERROR(E_XCE_BAD_RES); + + return; + } + + idx = e->data; + if (idx < 0) { + SET_ERROR(E_XCE_INTERNAL); + + return; + } + + zarray_del(&(mgr->defs), idx); +} + +void resmgr_del_res(resmgr_t* mgr, + xce_resource_t* r) +{ + if ((mgr == NULL) || (r == NULL)) { + SET_ERROR(EINVAL); + + return; + } + + mylock_lock(&(mgr->lock)); + resmgr_del_res_(mgr, r); + mylock_unlock(&(mgr->lock)); +} + +int resmgr_iter(resmgr_t* mgr, + xlist_t* l, + void* arg) +{ + int rc = 0; + + mylock_lock(&(mgr->lock)); + rc = zarray_iter_ex(&(mgr->defs), l, arg); + mylock_unlock(&(mgr->lock)); + + return rc; +} diff --git a/src/controller/resources.c b/src/controller/resources.c new file mode 100644 index 0000000..80adf7b --- /dev/null +++ b/src/controller/resources.c @@ -0,0 +1,10 @@ +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" + +#include "htab.h" +#include "worker.h" +#include "xce/types.h" diff --git a/src/controller/sched.c b/src/controller/sched.c new file mode 100644 index 0000000..0dab9aa --- /dev/null +++ b/src/controller/sched.c @@ -0,0 +1,620 @@ +#include +#include + +#include "cluster.h" +#include "controller.capnp.h" +#include "objmgr.h" +#include "request.h" +#include "worker.h" +#include "xce/cluster_def.h" +#include "xce/types.h" + +extern void collect_shared_resvals(cluster_t*, collect_check_fn_t, + int*, xce_val_with_idx_t***); + +static int xcesched_push_joblist(cluster_t* c, + xlist_t* l) +{ + xce_data_t data; + xce_jobs_data_t jobs = { + .n_total = l->count, + .pos = 0, + .count = l->count, + .jobs = (xce_job_t**)l->data + }; + int rc = 0; + + memset(&data, 0x0, sizeof(data)); + + data.table = Data_data_schedjobs; + data.data.jobs = &jobs; + + rc = xce_push_data_req(c->sched.ep, c->myhost, &data, + check_controller, c, 100); + return rc; +} + +static int xcesched_push_workerlist(cluster_t* c, + xlist_t* l) +{ + xce_data_t data; + xce_workers_data_t workers = { + .n_total = l->count, + .pos = 0, + .count = l->count, + .workers = (xce_worker_t**)l->data + }; + int rc = 0; + + memset(&data, 0x0, sizeof(data)); + + data.table = Data_data_schedworkers; + data.data.workers = &workers; + + rc = xce_push_data_req(c->sched.ep, c->myhost, &data, + check_controller, c, 100); + return rc; +} + +static int xcesched_push_configs(cluster_t* c) +{ + xce_configdata_t configdata; + int rc = 0; + + memset(&configdata, 0x0, sizeof(configdata)); + + configdata.data.conf4sched = (xce_conf4sched_t*)calloc(1, sizeof(xce_conf4sched_t)); + if (configdata.data.conf4sched == NULL) { + return -ENOMEM; + } + + collect_confvars(c, NULL, &(configdata.data.conf4sched->n_confvars), + &(configdata.data.conf4sched->confvars)); + collect_queues(c, NULL, &(configdata.data.conf4sched->n_queues), + &(configdata.data.conf4sched->queues)); + collect_resources(c, NULL, &(configdata.data.conf4sched->n_resources), + &(configdata.data.conf4sched->resources)); + collect_shared_resvals(c, NULL, &(configdata.data.conf4sched->nresvals), + &(configdata.data.conf4sched->resvals)); + + configdata.data.conf4sched->n_ugroups = c->myconf->cluster_config->nugroups; + configdata.data.conf4sched->ugroups = c->myconf->cluster_config->ugroups; + configdata.data.conf4sched->n_hgroups = c->myconf->cluster_config->nhgroups; + configdata.data.conf4sched->hgroups = c->myconf->cluster_config->hgroups; + + /* add resources */ + configdata.data_which = ConfigData_data_schedconfig; + configdata.data.conf4sched->clustername = c->myconf->cluster_config->clustername; + + rc = xce_pushconfig_req(c->sched.ep, c->myhost, + &configdata); + + free_resval_with_idx_list(&(configdata.data.conf4sched->resvals), + configdata.data.conf4sched->nresvals); + + deref_objs(configdata.data.conf4sched->n_confvars, + (void**)configdata.data.conf4sched->confvars); + deref_objs(configdata.data.conf4sched->n_queues, + (void**)configdata.data.conf4sched->queues); + deref_objs(configdata.data.conf4sched->n_resources, + (void**)configdata.data.conf4sched->resources); + + FREEUP(configdata.data.conf4sched->confvars); + FREEUP(configdata.data.conf4sched->queues); + FREEUP(configdata.data.conf4sched->resources); + + FREEUP(configdata.data.conf4sched); + + return rc; +} + +static int check_job_(xce_job_t* j) +{ + if (j == NULL) { + return 0; + } + + return 1; +} + +static int xcesched_push_jobs(cluster_t* c) +{ + xce_data_t data; + xce_jobs_data_t jobs = { + .n_total = 0, + .pos = 0, + .count = 0, + .jobs = NULL + }; + int rc = 0; + + memset(&data, 0x0, sizeof(data)); + + data.table = Data_data_schedjobs; + data.data.jobs = &jobs; + + collect_jobs(c, (collect_check_fn_t)check_job_, + &(jobs.n_total), &(jobs.jobs)); + jobs.count = jobs.n_total; + + rc = xce_push_data_req(c->sched.ep, c->myhost, &data, + check_controller, c, 100); + deref_objs(jobs.n_total, (void**)jobs.jobs); + FREEUP(jobs.jobs); + + return rc; +} + +static int xcesched_push_workers(cluster_t* c) +{ + xce_data_t data; + xce_workers_data_t workers = { + .n_total = 0, + .pos = 0, + .count = 0, + .workers = NULL + }; + int rc = 0; + int i; + + memset(&data, 0x0, sizeof(data)); + + data.table = Data_data_schedworkers; + data.data.workers = &workers; + + collect_workers(c, (collect_check_fn_t)check_worker_, + &(workers.n_total), &(workers.workers)); + workers.count = workers.n_total; + + rc = xce_push_data_req(c->sched.ep, c->myhost, &data, + check_controller, c, 100); + + /* free up collected resource value which used + passed to scheduler + */ + for (i = 0; i < workers.n_total; i++) { + xce_worker_t* w = workers.workers[i]; + + free_resval_with_idx_list(&(w->resvals_to_sched), + w->nresvals_to_sched); + } + + deref_objs(workers.n_total, (void**)workers.workers); + FREEUP(workers.workers); + + return rc; +} + +static int sched_job_new_fun(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + xlist_t* l = va_arg(ap, xlist_t*); + int rc = 0; + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = -EINVAL; + if (l != NULL) { + rc = xcesched_push_joblist(c, l); + } + break; + default: + break; + } + } + mylock_unlock(&(c->sched.lock)); + + return rc; +} + +static int sched_job_start_fun(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + job_pidinfo_t* info = va_arg(ap, job_pidinfo_t*); + int rc = 0; + /* clang-format off */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedStartJob, + .data = { + .sched_start_job = info->jobid + } + }; + /* clang-format on */ + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + + mylock_unlock(&(c->sched.lock)); + + return rc; +} + +static int sched_job_exit_fun(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + xce_jobfinish_t* info = va_arg(ap, xce_jobfinish_t*); + int rc = 0; + /* clang-format off */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedExitJob, + .data = { + .sched_exit_job = info->jobid + } + }; + /* clang-format on */ + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + + mylock_unlock(&(c->sched.lock)); + + return rc; +} + +static int sched_job_fail_fun(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + job_failinfo_t* info = va_arg(ap, job_failinfo_t*); + int rc = 0; + /* clang-format off */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedFailJob, + .data = { + .sched_fail_job = info->jobid + } + }; + /* clang-format on */ + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + + mylock_unlock(&(c->sched.lock)); + + return rc; +} + +static int sched_job_update_fun(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + job_t* job = va_arg(ap, job_t*); + int rc = 0; + /* clang-format off */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedUpdateJob, + .data = { + .sched_update_job = &(job->job) + } + }; + /* clang-format on */ + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + + mylock_unlock(&(c->sched.lock)); + + return rc; +} + +static int sched_job_hook_fun(int type, + int situation, + int detail, + void* data, + va_list ap) +{ + switch (detail) { + case HOOK_DETAIL_JOB_NEW: + return sched_job_new_fun(type, situation, + data, ap); + case HOOK_DETAIL_JOB_START: + return sched_job_start_fun(type, situation, + data, ap); + case HOOK_DETAIL_JOB_EXIT: + return sched_job_exit_fun(type, situation, + data, ap); + case HOOK_DETAIL_JOB_UPDATE: + return sched_job_update_fun(type, situation, + data, ap); + case HOOK_DETAIL_JOB_FAIL: + return sched_job_fail_fun(type, situation, + data, ap); + } + + return 0; +} +static int sched_worker_change(int type, + int situation, + void* data, + cluster_t* c, + worker_t* w) +{ + int rc = 0; + xlist_t wlist; + + TRACE("%s worker %s idx %d situation %d", + __func__, w->host.name, + w->idx, situation); + + collect_resval_with_idx(&(w->host.resvals), + &(w->host.nresvals_to_sched), + &(w->host.resvals_to_sched)); + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + xlist_init(&wlist, 2); + xlist_store(&wlist, w); + rc = xcesched_push_workerlist(c, &wlist); + xlist_fini(&wlist); + break; + default: + break; + } + } + mylock_unlock(&(c->sched.lock)); + + free_resval_with_idx_list(&(w->host.resvals_to_sched), + w->host.nresvals_to_sched); + + return rc; +} + +static int sched_worker_join(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + char* name = va_arg(ap, char*); + worker_t* w = NULL; + int rc; + + if (situation == HOOK_BEFORE_EVENT) { + /* nothing to do */ + return 0; + } + + w = workermgr_lock(&(c->workermgr), name, + OBJREF_READ); + + rc = sched_worker_change(type, situation, data, + c, w); + + /* FIXME: need to mark the jobs in this worker */ + obj_deref(w); + + return rc; +} + +static int sched_worker_dead(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + worker_t* w = va_arg(ap, worker_t*); + + /* FIXME: need to handle job status in this worker */ + if (situation == HOOK_BEFORE_EVENT) { + return 0; + } + + return sched_worker_change(type, situation, data, + c, w); +} + +static int sched_worker_update(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + worker_t* w = va_arg(ap, worker_t*); + + if (situation == HOOK_BEFORE_EVENT) { + return 0; + } + + return sched_worker_change(type, situation, data, + c, w); +} +static int sched_worker_hook_fun(int type, + int situation, + int detail, + void* data, + va_list ap) +{ + switch (detail) { + case HOOK_DETAIL_WORKER_JOIN: + return sched_worker_join(type, situation, + data, ap); + case HOOK_DETAIL_WORKER_DEAD: + return sched_worker_dead(type, situation, + data, ap); + case HOOK_DETAIL_WORKER_UPDATE: + return sched_worker_update(type, situation, + data, ap); + } + + return 0; +} + +static int sched_resource_hook_fun(int type, + int situation, + int detail, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + xce_resval_list_t* resvals = va_arg(ap, xce_resval_list_t*); + /* clang-format on */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedUpdateResvals, + .data = { + .sched_update_resvals = resvals } + }; + /* clang-format off */ + int rc = 0; + + if (detail != HOOK_DETAIL_RESOURCE_UPDATE) { + return 0; + } + + xce_use(resvals); + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + mylock_unlock(&(c->sched.lock)); + + return rc; +} + +static int sched_queue_update(int type, + int situation, + void* data, + va_list ap) +{ + cluster_t* c = va_arg(ap, cluster_t*); + queue_t* q = va_arg(ap, queue_t*); + int rc = 0; + /* clang-format off */ + xce_notify_data_t xdata = { + .event = NotifyData_data_schedUpdateQueue, + .data = { + .sched_update_queue = &(q->queue) + } + }; + /* clang-format on */ + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + switch (situation) { + case HOOK_AFTER_EVENT: + rc = notify(c, c->sched.ep, &xdata); + break; + default: + break; + } + } + mylock_unlock(&(c->sched.lock)); + + return rc; +} +static int sched_queue_hook_fun(int type, + int situation, + int detail, + void* data, + va_list ap) +{ + switch (detail) { + case HOOK_DETAIL_QUEUE_UPDATE: + return sched_queue_update(type, situation, data, ap); + default: + break; + } + + return 0; +} + +int on_scheduler_join(cluster_t* c) +{ + static int hook_installed = 0; + + mylock_lock(&(c->sched.lock)); + if (c->sched.ep != NULL) { + xcesched_push_configs(c); + xcesched_push_workers(c); + xcesched_push_jobs(c); + } + mylock_unlock(&(c->sched.lock)); + + if (!hook_installed) { + hook_installed = 1; + + xce_add_hook(&(c->hooksys), + XCE_CONTROLLER_HOOK_JOB, + (hook_fn_t)sched_job_hook_fun, + NULL); + xce_add_hook(&(c->hooksys), + XCE_CONTROLLER_HOOK_WORKER, + (hook_fn_t)sched_worker_hook_fun, + NULL); + xce_add_hook(&(c->hooksys), + XCE_CONTROLLER_HOOK_RESOURCE, + (hook_fn_t)sched_resource_hook_fun, + NULL); + xce_add_hook(&(c->hooksys), + XCE_CONTROLLER_HOOK_QUEUE, + (hook_fn_t)sched_queue_hook_fun, + NULL); + } + return 0; +} + +int handle_scheduler_leave(cluster_t* c) +{ + /* we need to mark the cluster status */ + TRACE("%s scheduler leave", + __func__); + mylock_lock(&(c->sched.lock)); + c->sched.ep = NULL; + mylock_unlock(&(c->sched.lock)); + return 0; +} diff --git a/src/controller/task.c b/src/controller/task.c new file mode 100644 index 0000000..438c7f5 --- /dev/null +++ b/src/controller/task.c @@ -0,0 +1,203 @@ +#include +#include +#include + +#include "cluster.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" +#include "xce/types.h" + +static int run_tasks_on_host(cluster_t* c, + endpoint_t* ep, + uint64_t jobid, + xce_taskhost_t* info) +{ + worker_t* w; + ioctx_t* ioctx; + int rc; + xce_taskrun_t runmsg = { + .nhosts = 1, + .jobid = jobid, + .clientidx = ep->idx, + .taskhosts = (xce_taskhost_t*[]) { info } + }; + + w = workermgr_lock(&(c->workermgr), info->host, OBJREF_READ); + if (w == NULL) { + return -ENOENT; + } + + if ((w->ep == NULL) || (IS_WORKER_UNAVAIL(w->host.status))) { + obj_deref(w); + return -EINVAL; + } + + ioctx = ioctx_alloc(w->ep, c->myhost); + if (ioctx == NULL) { + obj_deref(w); + return -ENOMEM; + } + + rc = xce_runtask_req(ioctx, &runmsg); + + obj_deref(w); + + return rc; +} + +static int check_task_hosts(cluster_t* c, + xce_taskrun_t* runmsg) +{ + int i; + + for (i = 0; i < runmsg->nhosts; i++) { + worker_t* w; + + w = workermgr_lock(&(c->workermgr), + runmsg->taskhosts[i]->host, + OBJREF_READ); + if (w == NULL) { + return -ENOENT; + } + + if ((w->ep == NULL) || IS_WORKER_UNAVAIL(w->host.status)) { + obj_deref(w); + return -ENOENT; + } + } + + return 0; +} + +int do_runtasks(cluster_t* c, + endpoint_t* ep, + xce_request_t* req) +{ + int rc = 0; + xce_reply_t reply; + int i; + xce_taskrun_t* runmsg = req->body.runtaskmsg; + + init_reply_by_req(&reply, req); + reply.data_which = RequestReply_data_nodata; + rc = check_task_hosts(c, runmsg); + + reply.status = rc; + xce_reply(ep, c->myhost, &reply); + + if (rc < 0) { + return rc; + } + + rc = 0; + + for (i = 0; i < runmsg->nhosts; i++) { + int rc1; + + rc1 = run_tasks_on_host(c, ep, + runmsg->jobid, + runmsg->taskhosts[i]); + if (rc1 < 0) { + xce_task_failinfo_t failinfo = { + .jobid = runmsg->jobid, + .host = runmsg->taskhosts[i]->host, + .taskidx = -1, /* all tasks failed */ + .code = rc1, + .detail = NULL + }; + xce_notify_data_t data = { + .event = NotifyData_data_taskFailInfo, + .data = { + .taskfailinfo = &failinfo } + }; + + /* notify client that run tasks on host fail */ + notify(c, ep, &data); + + break; + } + } + + return rc; +} + +int on_task_start(cluster_t* c, + xce_task_pidinfo_t* info) +{ + endpoint_t* ep; + xce_notify_data_t data = { + .event = NotifyData_data_taskPidInfo, + .data = { + .taskinfo = info } + }; + int clientidx = info->clientidx; + + if (clientidx < 0) { + ERROR("incorrect client idx %d", clientidx); + return -EINVAL; + } + + ep = (endpoint_t*)zarray_get(&(c->endpoints), clientidx); + if (ep == NULL) { + ERROR("incorrect client idx %d", clientidx); + return -EINVAL; + } + + notify(c, ep, &data); + + return 0; +} + +int on_task_fail(cluster_t* c, + xce_task_failinfo_t* info) +{ + endpoint_t* ep; + xce_notify_data_t data = { + .event = NotifyData_data_taskFailInfo, + .data = { + .taskfailinfo = info } + }; + int clientidx = info->clientidx; + + if (clientidx < 0) { + ERROR("incorrect client idx %d", clientidx); + return -EINVAL; + } + + ep = (endpoint_t*)zarray_get(&(c->endpoints), clientidx); + if (ep == NULL) { + ERROR("incorrect client idx %d", clientidx); + return -EINVAL; + } + + notify(c, ep, &data); + + return 0; +} + +int on_task_finish(cluster_t* c, + xce_task_finish_t* info) +{ + endpoint_t* ep; + xce_notify_data_t data = { + .event = NotifyData_data_taskFinish, + .data = { + .taskfinish = info } + }; + int clientidx = info->clientidx; + + if (clientidx < 0) { + ERROR("incorrect client idx %d", clientidx); + return -EINVAL; + } + + ep = (endpoint_t*)zarray_get(&(c->endpoints), clientidx); + if (ep == NULL) { + ERROR("incorrect client idx %d", clientidx); + return -EINVAL; + } + + notify(c, ep, &data); + + return 0; +} diff --git a/src/controller/testsuite/test.c b/src/controller/testsuite/test.c new file mode 100644 index 0000000..0a56cff --- /dev/null +++ b/src/controller/testsuite/test.c @@ -0,0 +1,2364 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "client.h" +#include "controller.capnp.h" +#include "xce/types.h" + +typedef int (*cmd_func_t)(int, char*[]); + +typedef struct { + char* name; + cmd_func_t func; + char* doc; +} command_t; + +typedef struct { + char* key; + char* value; + char* defval; +} setting_t; + +typedef struct _command_ctx { + struct _command_ctx* parent; + char* prompt; + setting_t* settings; + command_t* commands; + xcecli_ctx_t ctx; +} command_ctx_t; + +typedef struct { + uint64_t jobid; + int pid; + time_t ts; +} run_job_t; + +typedef struct { + uint64_t jobid; + uint32_t taskid; + int clientidx; + int pid; + time_t ts; +} run_task_t; + +typedef struct { + char* host; + int ntasks; +} host_task_t; + +#define MAX_RUN_JOBS (20) +#define MAX_TASK_HOSTS (5) +#define MAX_RUN_TASKS (50) +#define RUSAGE_UPDATE_INTERVAL (60) + +static run_job_t run_jobs[MAX_RUN_JOBS]; +static run_task_t run_tasks[MAX_RUN_TASKS]; + +static int myreq_handler(xce_request_t*); + +extern int cmd_help(int, char*[]); +extern int cmd_up(int, char*[]); +extern int cmd_worker(int, char*[]); +extern int cmd_scheduler(int, char*[]); +extern int cmd_client(int, char*[]); +extern int cmd_quit(int, char*[]); +extern int cmd_hello(int, char*[]); +extern int cmd_shutdown(int, char*[]); +extern int cmd_set(int, char*[]); +extern int cmd_reslist(int, char*[]); +extern int cmd_join_worker(int, char*[]); +extern int cmd_join_sched(int, char*[]); +extern int cmd_workerlist(int, char*[]); +extern int cmd_joblist(int, char*[]); +extern int cmd_jobrun(int, char*[]); +extern int cmd_jobsub(int, char*[]); +extern int cmd_getconf(int, char*[]); +extern int cmd_jobfinish(int, char*[]); +extern int cmd_resupdate(int, char*[]); +extern int cmd_resval(int, char*[]); +extern int cmd_jobkill(int, char*[]); +extern int cmd_jobckpt(int, char*[]); +extern int cmd_qctrl(int, char*[]); +extern int cmd_runtasks(int, char*[]); + +static void handler(int, siginfo_t*, void*); + +#define CLOCKID CLOCK_REALTIME +#define SIG SIGRTMIN + +static timer_t timerid; + +static setting_t settings[] = { + { (char*)"host", (char*)NULL, (char*)"localhost" }, + { (char*)"port", (char*)NULL, (char*)"9990" }, + { (char*)"secret", (char*)NULL, (char*)"keepSecret" }, + { (char*)"myname", (char*)NULL, (char*)"localhost" }, + { (char*)NULL, (char*)NULL, (char*)NULL } +}; + +static command_t top_commands[] = { + { "help", cmd_help, "Display this text" }, + { "worker", cmd_worker, "In worker context" }, + { "scheduler", cmd_scheduler, "In scheduler context" }, + { "client", cmd_client, "In client context" }, + { "set", cmd_set, "Show/Set settings" }, + { "quit", cmd_quit, "Quit xce test" }, + { (char*)NULL, (cmd_func_t)NULL, (char*)NULL } +}; + +static command_ctx_t topctx = { + .parent = NULL, + .prompt = "xce", + .settings = settings, + .commands = top_commands +}; + +static command_t worker_commands[] = { + { "..", cmd_up, "Goto upper context" }, + { "help", cmd_help, "Display this text" }, + { "quit", cmd_quit, "Quit xce test" }, + { "join", cmd_join_worker, "Join the cluster" }, + { "jfinish", cmd_jobfinish, "tell controller jobfinish" }, + { "resupdate", cmd_resupdate, "update res vals" }, + { (char*)NULL, (cmd_func_t)NULL, (char*)NULL } +}; + +static command_ctx_t workerctx = { + .parent = &topctx, + .prompt = "worker", + .settings = settings, + .commands = worker_commands +}; + +static command_t client_commands[] = { + { "..", cmd_up, "Goto upper context" }, + { "hello", cmd_hello, "Say hello to xce controller" }, + { "reslist", cmd_reslist, "Display cluster resources" }, + { "workerlist", cmd_workerlist, "list workers" }, + { "qctrl", cmd_qctrl, "queue control" }, + { "joblist", cmd_joblist, "list jobs" }, + { "shutdown", cmd_shutdown, "shutdown xce controller" }, + { "jobrun", cmd_jobrun, "run a job" }, + { "jobsub", cmd_jobsub, "submit a job" }, + { "jobkill", cmd_jobkill, "signal a job" }, + { "jobckpt", cmd_jobckpt, "checkpoint a job" }, + { "resval", cmd_resval, "get value of res" }, + { "taskrun", cmd_runtasks, "run tasks" }, + { "help", cmd_help, "Display this text" }, + { "quit", cmd_quit, "Quit xce test" }, + { (char*)NULL, (cmd_func_t)NULL, (char*)NULL } +}; + +static command_ctx_t clientctx = { + .parent = &topctx, + .prompt = "client", + .settings = settings, + .commands = client_commands +}; + +static command_t scheduler_commands[] = { + { "..", cmd_up, "Goto upper context" }, + { "help", cmd_help, "Display this text" }, + { "quit", cmd_quit, "Quit xce test" }, + { "join", cmd_join_sched, "Join the cluster" }, + { "getconf", cmd_getconf, "Get conf of cluster" }, + { (char*)NULL, (cmd_func_t)NULL, (char*)NULL } +}; + +static command_ctx_t schedulerctx = { + .parent = &topctx, + .prompt = "scheduler", + .settings = settings, + .commands = scheduler_commands +}; + +static command_ctx_t* active_ctx = &topctx; + +static char prompt[256]; +static int break_out = 0; + +static char doc[] = "xce_test"; +static char args_doc[] = ""; + +const char* argp_program_version = MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID)); +const char* argp_program_bug_address = BUG_EMAIL; + +struct arguments { + int verbose; +}; + +static struct argp_option options[] = { + { "verbose", 'v', 0, 0, "Produce verbose logs" }, + { 0 } +}; + +static error_t parse_opt(int key, + char* arg, + struct argp_state* state) +{ + struct arguments* arguments = state->input; + + switch (key) { + case 'v': + arguments->verbose++; + break; + default: + return ARGP_ERR_UNKNOWN; + } + + return 0; +} + +static struct argp argp = { + options, + parse_opt, + args_doc, + doc +}; + +static char* verbose_to_loglevel(int verbose) +{ + char* loglevel = "info"; + + switch (verbose) { + case 1: + loglevel = "debug"; + break; + case 2: + loglevel = "debug2"; + break; + case 3: + loglevel = "debug3"; + break; + default: + if (verbose > 3) { + loglevel = "debug3"; + } + break; + } + + return loglevel; +} + +void sigbreak(int signo) +{ + break_out = 1; +} + +static char* get_setting(char* key) +{ + int i; + + for (i = 0; active_ctx->settings[i].key; i++) { + if (strcmp(key, active_ctx->settings[i].key) != 0) { + continue; + } + + if (active_ctx->settings[i].value != NULL) { + return active_ctx->settings[i].value; + } + + return active_ctx->settings[i].defval; + } + + printf("Key `%s' is not in settings\n", + key); + return NULL; +} + +static xce_client_config_t config; + +int enter_ctx(command_ctx_t* ctx) +{ + config.nmasters = 1; + config.masters[0] = get_setting("host"); + config.master_ports[0] = atoi(get_setting("port")); + config.from = get_setting("myname"); + config.key = get_setting("secret"); + + xcecli_ctx_init(&(ctx->ctx), &config); + xcecli_set_handler(ctx->ctx, &myreq_handler); + xcecli_start(ctx->ctx); + + return 0; +} + +int leave_ctx(command_ctx_t* ctx) +{ + if (ctx == &topctx) { + return 0; + } + + if (ctx == &workerctx) { + timer_delete(timerid); + } + + xcecli_wakeup(active_ctx->ctx, (int)-EINTR); + + xcecli_stop(ctx->ctx); + xcecli_ctx_fini(ctx->ctx); + + return 0; +} + +static int fill_prompt(command_ctx_t* ctx, + char* buf) +{ + if (ctx->parent != NULL) { + fill_prompt(ctx->parent, buf); + strcat(buf, "/"); + } + + strcat(buf, ctx->prompt); + + return strlen(buf); +} + +command_t* find_command(char* name) +{ + int i; + + for (i = 0; active_ctx->commands[i].name; i++) { + if (strcmp(active_ctx->commands[i].name, name) == 0) { + return &(active_ctx->commands[i]); + } + } + + return NULL; +} + +/* Execute a command line. */ +int execute_line(char* line) +{ + int rc; + command_t* command; + int argc; + char** argv; + + argc = parse_argv(line, &argv); + if (argc < 0) { + SET_ERROR(-argc); + perror("execute_line"); + return -1; + } + + if ((argv == NULL) || (argv[0] == NULL)) { + return 0; + } + + command = find_command(argv[0]); + + if (!command) { + fprintf(stderr, "%s: No such command for xce test.\n", + argv[0]); + free_argv(argv, argc); + return (-1); + } + + rc = (command->func)(argc, argv); + + free_argv(argv, argc); + return rc; +} + +int cmd_help(int argc, char* argv[]) +{ + int i; + int rc = 0; + + if (argc > 1) { + char* cmd = argv[1]; + + for (i = 0; active_ctx->commands[i].name != NULL; i++) { + if (strcmp(argv[1], active_ctx->commands[i].name) == 0) { + printf("%s\t\t%s.\n", + active_ctx->commands[i].name, + active_ctx->commands[i].doc); + return 0; + } + } + + printf("No commands matchs `%s'. Possibilities are:\n", cmd); + rc = -1; + } + + for (i = 0; active_ctx->commands[i].name != NULL; i++) { + printf("\t%-12s\t%s\n", + active_ctx->commands[i].name, + active_ctx->commands[i].doc); + } + + return rc; +} + +int cmd_up(int argc, char* argv[]) +{ + if (active_ctx->parent != NULL) { + int rc; + + if ((rc = leave_ctx(active_ctx)) < 0) { + return rc; + } + + active_ctx = active_ctx->parent; + } + + return 0; +} + +int cmd_worker(int argc, char* argv[]) +{ + int rc; + struct sigevent sev; + struct sigaction sa; + struct itimerspec its; + + if ((rc = enter_ctx(&workerctx)) < 0) { + return rc; + } + + active_ctx = &workerctx; + + sa.sa_flags = SA_SIGINFO; + sa.sa_sigaction = handler; + sigemptyset(&(sa.sa_mask)); + sigaction(SIG, &sa, NULL); + + sev.sigev_notify = SIGEV_SIGNAL; + sev.sigev_signo = SIG; + sev.sigev_value.sival_ptr = &timerid; + + timer_create(CLOCKID, &sev, &timerid); + + memset(&its, 0x0, sizeof(its)); + its.it_value.tv_sec = 1; + + timer_settime(timerid, 0, &its, NULL); + + return 0; +} + +int cmd_scheduler(int argc, char* argv[]) +{ + int rc; + + if ((rc = enter_ctx(&schedulerctx)) < 0) { + return rc; + } + + active_ctx = &schedulerctx; + return 0; +} + +int cmd_client(int argc, char* argv[]) +{ + int rc; + + if ((rc = enter_ctx(&clientctx)) < 0) { + return rc; + } + + active_ctx = &clientctx; + return 0; +} + +static int done = 0; + +int cmd_quit(int argc, char* argv[]) +{ + leave_ctx(active_ctx); + + done = 1; + return 0; +} + +int cmd_hello(int argc, char* argv[]) +{ + int rc; + int opt; + char* msg = "Greeting from xce client"; + int count = 1; + + optind = 1; + + while ((opt = getopt(argc, argv, "c:m:")) != -1) { + switch (opt) { + case 'c': + count = atoi(optarg); + break; + case 'm': + msg = optarg; + break; + default: + fprintf(stderr, " hello [-c count] [-w wait_usec] [-m message]\n"); + return -1; + break; + } + } + + break_out = 0; + while ((!break_out) && (count > 0)) { + rc = xcecli_hello(active_ctx->ctx, msg); + + count--; + } + + return rc; +} + +int cmd_shutdown(int argc, char* argv[]) +{ + int rc; + rc = xcecli_shutdown(active_ctx->ctx); + /* leave client context */ + if (rc >= 0) { + execute_line(".."); + } + + return rc; +} + +int cmd_reslist(int argc, char* argv[]) +{ + int rc, i; + xce_resource_t** res; + + rc = xcecli_reslist(active_ctx->ctx, &res); + + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_reslist"); + return rc; + } + + printf("%10s%10s%10s%64s\n", + "Name", "Flags", + "Interval", "Description"); + + for (i = 0; i < rc; i++) { + printf("%10s%10x%10d%64s\n", + res[i]->name, + res[i]->flags, res[i]->interval, + res[i]->desc); + } + + free(res); + + return 0; +} + +static char* worker_status_str(uint32_t status) +{ + if ((status & XCE_WORKER_UNAVAIL) != 0) { + return "UNAVAIL"; + } else if ((status & XCE_WORKER_LOCKEDU) != 0) { + return "LOCKU"; + } else if ((status & XCE_WORKER_LOCKEDW) != 0) { + return "LOCKW"; + } else if ((status & XCE_WORKER_BUSY) != 0) { + return "BUSY"; + } else if (status == 0) { + return "OK"; + } + + return "UNKOWN"; +} + +int cmd_workerlist(int argc, char* argv[]) +{ + int rc, i; + xce_worker_t** workers; + + rc = xcecli_workerlist(active_ctx->ctx, &workers); + + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_workerlist"); + return rc; + } + + printf("%-20s%-8s%-15s%-15s%12s%12s%12s%12s%12s%12s\n", + "HostName", "Status", "HostType", "HostModel", + "CPUfactor", "MaxCPU", "MaxMem", + "MaxSwap", "MaxTmp", "NDisks"); + + for (i = 0; i < rc; i++) { + printf("%-20s%-8s%-15s%-15s%12f%12d%12f%12f%12f%12d\n", + workers[i]->name, + worker_status_str(workers[i]->status), + workers[i]->attrs->hosttype, + workers[i]->attrs->hostmodel, + workers[i]->attrs->cpufactor, + workers[i]->attrs->maxcpus, + workers[i]->attrs->maxmem, + workers[i]->attrs->maxswap, + workers[i]->attrs->maxtmp, + workers[i]->attrs->ndisks); + } + + free(workers); + + return 0; +} + +static char* job_state_str(int state) +{ + if ((state & JOB_STAT_PEND) != 0) { + if ((state & JOB_STAT_PSUSP) != 0) { + return "PSUSP"; + } + + return "PEND"; + } else if ((state & JOB_STAT_RUN) != 0) { + if ((state & JOB_STAT_USUSP) != 0) { + return "USUSP"; + } else if ((state & JOB_STAT_SSUSP) != 0) { + return "SSUSP"; + } + return "RUN"; + } else if ((state & JOB_STAT_EXIT) != 0) { + return "EXIT"; + } else if ((state & JOB_STAT_DONE) != 0) { + return "DONE"; + } else if ((state & JOB_STAT_ERR) != 0) { + return "ERROR"; + } + + return "UNKNOWN"; +} + +int cmd_joblist(int argc, char* argv[]) +{ + int rc, i; + xce_job_t** jobs; + + rc = xcecli_joblist(active_ctx->ctx, &jobs); + + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_joblist"); + return rc; + } + + printf("%-16s%-10s%-10s%-8s%-16s%-16s%-16s%10s%30s\n", + "JobId", "User", "Queue", "State", + "SubmitTime", "StartTime", "FinishTime", + "NTasks", "Command"); + + for (i = 0; i < rc; i++) { + printf("%-16ld%-10s%-10s%-8s%-16ld%-16ld%-16ld%4d,%-5d%30s\n", + jobs[i]->jobid, + jobs[i]->user, + jobs[i]->queue, + job_state_str(jobs[i]->state), + jobs[i]->queue_time, + jobs[i]->start_time, + jobs[i]->finish_time, + jobs[i]->mintasks, + jobs[i]->maxtasks, + jobs[i]->start_info->cmdline); + } + + free(jobs); + + return 0; +} + +static int fill_job_alloc(xce_job_alloc_t* jalloc, + int nhosts, + host_task_t* hosts) +{ + xce_host_alloc_t** halloc = NULL; + int i; + + halloc = (xce_host_alloc_t**)calloc(nhosts, sizeof(xce_host_alloc_t*)); + if (halloc == NULL) { + return -ENOMEM; + } + + for (i = 0; i < nhosts; i++) { + xce_alloc_t* alloc; + xce_alloc_t** allocs; + + alloc = (xce_alloc_t*)calloc(1, sizeof(*alloc)); + allocs = (xce_alloc_t**)calloc(1, sizeof(*allocs)); + halloc[i] = (xce_host_alloc_t*)calloc(1, sizeof(xce_host_alloc_t)); + if ((halloc[i] == NULL) || (alloc == NULL) || (allocs == NULL)) { + /* FIXME: handle error */ + return -ENOMEM; + } + + alloc->resname = "slots"; + alloc->reserve = ""; + alloc->instid = -1; + alloc->value = (float)hosts[i].ntasks; + alloc->from = time(NULL); + alloc->duration = -1; + + allocs[0] = alloc; + + halloc[i]->nallocs = 1; + halloc[i]->allocs = allocs; + halloc[i]->host = hosts[i].host; + } + + jalloc->nhosts = nhosts; + jalloc->allocs = halloc; + + return 0; +} +int cmd_jobrun(int argc, char* argv[]) +{ + int rc; + int opt; + uint64_t jobid; + host_task_t hosts[MAX_TASK_HOSTS]; + xce_job_alloc_t jalloc; + xce_job_dispatch_t jdispatch = { + .njobs = 1, + .allocs = (xce_job_alloc_t*[]) { &jalloc } + }; + int nhosts = 0; + char* host; + int count; + + optind = 1; + + while ((opt = getopt(argc, argv, "m:j:")) != -1) { + switch (opt) { + case 'm': + if (nhosts >= MAX_TASK_HOSTS) { + fprintf(stderr, "too many run hosts, we only support max %d hosts\n", + MAX_TASK_HOSTS); + return -1; + } + + if (strchr(optarg, (int)':') != NULL) { + char* p; + char buf[256]; + + p = strchr(optarg, (int)':'); + count = atoi(p + 1); + memset(buf, 0x0, sizeof(buf)); + + strncpy(buf, optarg, (int)(p - optarg)); + host = strdup(buf); + } else { + host = strdup(optarg); + count = 1; + } + + hosts[nhosts].host = host; + hosts[nhosts].ntasks = count; + nhosts++; + break; + case 'j': + jobid = atol(optarg); + break; + } + } + + fill_job_alloc(&jalloc, nhosts, &hosts[0]); + + jalloc.jobid = jobid; + + rc = xcecli_jobdispatch(active_ctx->ctx, &jdispatch); + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_jobdispatch"); + return rc; + } + +#if 0 + printf("job %s will be started on host: %s\n", + argv[1], halloc.host); +#endif + + return 0; +} + +int cmd_jobsub(int argc, char* argv[]) +{ + int rc = 0; + xce_job_submit_t submit; + int opt; + uint64_t jobid; + uint32_t min = 1, max = 1; + + optind = 1; + + memset(&submit, 0x0, sizeof(submit)); + + while ((opt = getopt(argc, argv, "u:q:R:c:x:n:")) != -1) { + switch (opt) { + case 'u': + submit.user = strdup(optarg); + break; + case 'q': + submit.queue = strdup(optarg); + break; + case 'R': + submit.resreq = strdup(optarg); + break; + case 'c': + submit.cmdline = strdup(optarg); + break; + case 'x': + submit.exdata = strdup(optarg); + break; + case 'n': + if (strchr(optarg, (int)',') != NULL) { + char buf[64]; + char* p = strchr(optarg, (int)','); + + max = atoi(p + 1); + memset(buf, 0x0, sizeof(buf)); + strncpy(buf, optarg, (int)(p - optarg)); + min = atoi(buf); + + if (max == 0) { + printf("Incorrect setting of max tasks count, set to same as min task value\n"); + max = min; + } + } else { + min = atoi(optarg); + max = min; + } + break; + default: + printf("jobsub -u -q -R -c \n"); + return -EINVAL; + } + } + + if (min == 0) { + printf("Incorrect setting of min task count, set to 1 as default\n"); + min = 1; + } + + if (NULLSTR(submit.user)) { + printf("you need to specify job user with -u\n"); + return -EINVAL; + } + + if (NULLSTR(submit.queue)) { + printf("you need to specify job queue with -q\n"); + return -EINVAL; + } + + if (NULLSTR(submit.cmdline)) { + printf("you need to specify job command with -c\n"); + return -EINVAL; + } + + if (NULLSTR(submit.resreq)) { + submit.resreq = strdup("type == any"); + } + + submit.mintasks = min; + submit.maxtasks = max; + + rc = xcecli_jobsubmit(active_ctx->ctx, &submit, &jobid); + if (rc < 0) { + SET_ERROR(-rc); + perror("xcecli_jobsubmit"); + return rc; + } + + FREEUP(submit.user); + FREEUP(submit.queue); + FREEUP(submit.resreq); + FREEUP(submit.cmdline); + FREEUP(submit.exdata); + + printf("jobsubmit ok: JobId: %lu\n", jobid); + + return 0; +} + +int cmd_resval(int argc, char* argv[]) +{ + char* host = NULL; + char* res = NULL; + xce_resval_t** vals = NULL; + int opt; + int rc; + + optind = 1; + + while ((opt = getopt(argc, argv, "n:m:")) != -1) { + switch (opt) { + case 'n': + res = strdup(optarg); + break; + case 'm': + host = strdup(optarg); + break; + default: + fprintf(stderr, "resval -n resname [-m host]\n"); + FREEUP(host); + FREEUP(res); + return -1; + } + } + + rc = xcecli_resval(active_ctx->ctx, host, + res, &vals); + if (rc > 0) { + int i; + + for (i = 0; i < rc; i++) { + xce_resval_t* val = vals[i]; + + printf("resource %s@%s:\n", + val->name, (val->host == NULL) ? "cluster" : val->host); + printf("value: "); + switch (val->val->which) { + case Value_u_svalue: + printf("%s\n", val->val->u.svalue); + break; + case Value_u_fvalue: + printf("%f\n", val->val->u.fvalue); + break; + case Value_u_bvalue: + printf("%s\n", + (val->val->u.bvalue == 0) ? "false" : "true"); + break; + default: + printf(" \n"); + break; + } + + free_ResourceVal_ptr(&val); + } + + FREEUP(vals); + } else { + printf("no resource value found in given host\n"); + } + + FREEUP(host); + FREEUP(res); + + return 0; +} + +int cmd_jobkill(int argc, char* argv[]) +{ + int signo; + uint64_t jobid = 0; + int opt; + int rc; + xce_jobctrl_t jobctrl; + xce_signal_t siginfo; + + optind = 1; + signo = SIGTERM; + + while ((opt = getopt(argc, argv, "s:j:")) != -1) { + switch (opt) { + case 's': + signo = atoi(optarg); + break; + case 'j': + jobid = atol(optarg); + break; + default: + break; + } + } + + if (jobid == 0) { + fprintf(stderr, "you need to specify jobid with -j\n"); + return -1; + } + + jobctrl.action = JobControl_u_siginfo; + jobctrl.jobid = jobid; + jobctrl.u.siginfo = &siginfo; + siginfo.signo = signo; + siginfo.force = 1; + + rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl); + if (rc < 0) { + perror("xcecli_jobctrl"); + return rc; + } + + printf("Job %lu signaled\n", jobid); + + return 0; +} + +int cmd_jobckpt(int argc, char* argv[]) +{ + uint64_t jobid = 0; + int opt; + int rc; + xce_jobctrl_t jobctrl; + xce_ckpt_options_t ckptoption; + + optind = 1; + ckptoption.kill = 0; + + while ((opt = getopt(argc, argv, "kj:m:")) != -1) { + switch (opt) { + case 'k': + ckptoption.kill = 1; + break; + case 'j': + jobid = atol(optarg); + break; + case 'm': + ckptoption.method = optarg; + break; + default: + break; + } + } + + if (jobid == 0) { + fprintf(stderr, "you need to specify jobid with -j\n"); + return -1; + } + + if (ckptoption.method == NULL) { + ckptoption.method = "default"; + } + jobctrl.action = JobControl_u_ckptoption; + jobctrl.jobid = jobid; + jobctrl.u.ckptoption = &ckptoption; + + rc = xcecli_jobctrl(active_ctx->ctx, &jobctrl); + if (rc < 0) { + perror("xcecli_jobctrl"); + return rc; + } + + printf("Job %lu checkpointed\n", jobid); + + return 0; +} + +int cmd_qctrl(int argc, char* argv[]) +{ + int opt; + int rc; + xce_queuectrl_t queuectrl; + int action; + char* queue = NULL; + + optind = 1; + + while ((opt = getopt(argc, argv, "aiociq:")) != -1) { + switch (opt) { + case 'a': + action = QUEUE_ACTIVE; + break; + case 'i': + action = QUEUE_INACTIVE; + break; + case 'o': + action = QUEUE_OPEN; + break; + case 'c': + action = QUEUE_CLOSE; + break; + case 'q': + queue = strdup(optarg); + break; + default: + break; + } + } + + if (queue == NULL) { + fprintf(stderr, "you need to specify queue name with -q\n"); + return -1; + } + + queuectrl.action = action; + queuectrl.queue = queue; + + rc = xcecli_queuectrl(active_ctx->ctx, &queuectrl); + if (rc < 0) { + perror("xcecli_queuectrl"); + return rc; + } + + printf("Operation on queue %s done\n", queue); + + free(queue); + + return 0; +} + +int cmd_runtasks(int argc, char* argv[]) +{ + xce_taskhost_t hosts[MAX_TASK_HOSTS]; + int nhosts = 0; + xce_taskrun_t taskrun; + int opt; + int rc; + int count; + int totaltasks = 0; + char* user = NULL; + char* cmdline = NULL; + char* dir = NULL; + char* exdata = NULL; + char* host = NULL; + char* launch_type = NULL; + int i; + uint64_t jobid = 0; + + optind = 1; + + while ((opt = getopt(argc, argv, "m:u:c:x:d:j:t:")) != -1) { + switch (opt) { + case 'm': + if (nhosts >= MAX_TASK_HOSTS) { + fprintf(stderr, "too many run hosts, just support %d run hosts\n", + MAX_TASK_HOSTS); + return -1; + } + + if (strchr(optarg, (int)':') != NULL) { + char* p; + char buf[256]; + + p = strchr(optarg, (int)':'); + count = atoi(p + 1); + memset(buf, 0x0, sizeof(buf)); + strncpy(buf, optarg, (int)(p - optarg)); + host = strdup(buf); + } else { + host = strdup(optarg); + count = 1; + } + + hosts[nhosts].host = host; + hosts[nhosts].ntasks = count; + nhosts++; + totaltasks += count; + break; + case 'u': + if (user != NULL) { + free(user); + } + user = strdup(optarg); + break; + case 'c': + if (cmdline != NULL) { + free(cmdline); + } + cmdline = strdup(optarg); + break; + case 'x': + if (exdata != NULL) { + free(exdata); + } + exdata = strdup(optarg); + break; + case 'd': + if (dir != NULL) { + free(dir); + } + dir = strdup(optarg); + break; + case 'j': + jobid = atol(optarg); + break; + case 't': + launch_type = strdup(optarg); + break; + default: + break; + } + } + + for (i = 0; i < nhosts; i++) { + hosts[i].exechome = dir; + hosts[i].execcwd = dir; + hosts[i].execuser = user; + hosts[i].exdata = exdata; + hosts[i].cmdline = cmdline; + hosts[i].launch_type = launch_type; + if (hosts[i].launch_type == NULL) { + if (totaltasks > 1) { + hosts[i].launch_type = strdup("mpi"); + } else { + hosts[i].launch_type = strdup("sequentail"); + } + } + } + + taskrun.taskhosts = (xce_taskhost_t**)calloc(nhosts, sizeof(xce_taskhost_t*)); + taskrun.clientidx = -1; + + if (taskrun.taskhosts != NULL) { + for (i = 0; i < nhosts; i++) { + taskrun.taskhosts[i] = &hosts[i]; + } + + taskrun.nhosts = nhosts; + taskrun.jobid = jobid; + rc = xcecli_runtask(active_ctx->ctx, &taskrun); + } else { + rc = -ENOMEM; + } + + if (rc < 0) { + errno = -rc; + perror("xcecli_runtask"); + } else { + printf("run task ok\n"); + } + + return rc; +} + +int cmd_join_worker(int argc, char* argv[]) +{ + int rc; + int role; + int opt; + hostattrs_t attrs = { + .cpufactor = 1.0, + .maxcpus = 4, + .maxmem = 64, + .maxswap = 1024, + .maxtmp = 2048, + .ndisks = 1 + }; + + role = HostRole_worker; + + optind = 1; + + attrs.hostmodel = "Intel64"; + attrs.hosttype = "LINUX64"; + + while ((opt = getopt(argc, argv, "f:c:m:s:t:d:")) != -1) { + switch (opt) { + case 'f': + attrs.cpufactor = atof(optarg); + break; + case 'c': + attrs.maxcpus = atoi(optarg); + break; + case 'm': + attrs.maxmem = atoi(optarg); + break; + case 's': + attrs.maxswap = atoi(optarg); + break; + case 't': + attrs.maxtmp = atoi(optarg); + break; + case 'd': + attrs.ndisks = atoi(optarg); + break; + default: + printf(" join [-f cpufactor] [-c maxcpus] [-m maxmem] [-s maxswap] [-t maxtmp] [-d ndisks]\n"); + return -EINVAL; + } + } + + rc = xcecli_join(active_ctx->ctx, role, &attrs); + + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_join"); + return rc; + } + + printf("host join ok\n"); + + return 0; +} + +int cmd_jobfinish(int argc, char* argv[]) +{ + int rc; + int opt; + uint64_t jobid = 0; + int code = 0; + xce_jobfinish_t finish; + + while ((opt = getopt(argc, argv, "j:c:")) != -1) { + switch (opt) { + case 'j': + jobid = atol(optarg); + break; + case 'c': + code = atoi(optarg); + break; + default: + printf(" jfinish -j jobid [-c exitcode]\n"); + return -EINVAL; + } + } + + if (jobid == 0) { + printf(" you need specify a positive jobid\n"); + return -EINVAL; + } + + finish.jobid = jobid; + finish.exit_code = code; + + rc = xcecli_notify_jobfinish(active_ctx->ctx, &finish); + if (rc < 0) { + SET_ERROR(-rc); + perror("xcecli_jobfinish"); + return rc; + } + + printf("ok\n"); + + return 0; +} + +static int set_resval(xce_resval_t* vals, + int idx, + char* name, + char* val) +{ + static struct { + char* name; + int valtype; + } resinfo[] = { + { "r15s", Value_u_fvalue }, + { "r1m", Value_u_fvalue }, + { "slots", Value_u_fvalue }, + { "ut", Value_u_fvalue }, + { "mem", Value_u_fvalue }, + { "maxcpus", Value_u_fvalue }, + { "ndisks", Value_u_fvalue }, + { "cpuf", Value_u_fvalue }, + { "maxswap", Value_u_fvalue }, + { "maxtmp", Value_u_fvalue }, + { "type", Value_u_svalue }, + { "model", Value_u_svalue }, + { "verilog", Value_u_fvalue }, + { "dyna", Value_u_fvalue }, + { NULL, 0 } + }; + int i; + int rc = -1; + + if (name == NULL) { + return 0; + } + + for (i = 0; resinfo[i].name != NULL; i++) { + if (strcmp(resinfo[i].name, name) == 0) { + rc = 1; + vals[idx].val = (xce_val_t*)calloc(1, sizeof(xce_val_t)); + + switch (resinfo[i].valtype) { + case Value_u_fvalue: + vals[idx].name = name; + vals[idx].val->which = resinfo[i].valtype; + if (val == NULL) { + vals[idx].val->u.fvalue = 0.0; + } else { + vals[idx].val->u.fvalue = atof(val); + } + break; + case Value_u_ivalue32: + vals[idx].name = name; + vals[idx].val->which = resinfo[i].valtype; + if (val == NULL) { + vals[idx].val->u.ivalue32 = 0; + } else { + vals[idx].val->u.ivalue32 = atoi(val); + } + break; + case Value_u_ivalue64: + vals[idx].name = name; + vals[idx].val->which = resinfo[i].valtype; + if (val == NULL) { + vals[idx].val->u.ivalue64 = 0; + } else { + vals[idx].val->u.ivalue64 = atol(val); + } + break; + case Value_u_bvalue: + vals[idx].name = name; + vals[idx].val->which = resinfo[i].valtype; + if (val == NULL) { + vals[idx].val->u.bvalue = 0; + } else { + if (strcasecmp(val, "t") == 0) { + vals[idx].val->u.bvalue = 1; + } else { + vals[idx].val->u.bvalue = 0; + } + } + break; + } + + vals[idx].host = get_setting("myname"); + return rc; + } + } + + return rc; +} +int cmd_resupdate(int argc, char* argv[]) +{ +#define MAX_N_RES (16) + int nres; + xce_resval_t vals[MAX_N_RES]; + xce_resval_t* valptrs[MAX_N_RES]; + char* resname = NULL; + int opt; + int rc; + + optind = 1; + nres = 0; + while ((opt = getopt(argc, argv, "n:v:")) != -1) { + switch (opt) { + case 'n': + if (resname != NULL) { + printf("res `%s' has no value specified, default will be used\n", + resname); + rc = set_resval(vals, nres, + resname, NULL); + if (rc >= 0) { + valptrs[nres] = &vals[nres]; + nres++; + } + resname = NULL; + } + + resname = strdup(optarg); + break; + case 'v': + if (resname != NULL) { + rc = set_resval(vals, nres, + resname, optarg); + if (rc >= 0) { + valptrs[nres] = &vals[nres]; + nres++; + } + resname = NULL; + } else { + printf("ignore res value `%s' as no res name found\n", + optarg); + } + break; + } + } + + rc = xcecli_update_resvals(active_ctx->ctx, + nres, &valptrs[0]); + if (rc >= 0) { + printf("resource update ok!\n"); + } else { + SET_ERROR(-rc); + perror("xcecli_update_resval"); + } + return 0; +} + +int cmd_join_sched(int argc, char* argv[]) +{ + int role = HostRole_scheduler; + int rc = -1; + + rc = xcecli_join(active_ctx->ctx, role, NULL); + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_join"); + return rc; + } + + printf("scheduler join ok\n"); + return 0; +} + +static int print_reslist(xce_reslist_t reslist) +{ + int i; + + for (i = 0; i < reslist.count; i++) { + printf("name: %s\nflags: %d\ninterval: %d\ndesc: %s\n\n", + reslist.res[i]->name, + reslist.res[i]->flags, + reslist.res[i]->interval, + reslist.res[i]->desc); + } + return 0; +} + +static int print_queuelist(xce_queuelist_t qlist) +{ + int i; + + for (i = 0; i < qlist.count; i++) { + printf("name: %s\nstatus: %d\npriority: %d\n\n", + qlist.queues[i]->queue, + qlist.queues[i]->status, + qlist.queues[i]->priority); + } + return 0; +} + +static int print_workerlist(xce_workerlist_t wlist) +{ + int i; + + for (i = 0; i < wlist.count; i++) { + printf("name: %s\nstatus: %d\n\n", + wlist.workers[i]->name, + wlist.workers[i]->status); + } + return 0; +} + +static int print_grouplist(xce_grouplist_t glist) +{ + return 0; +} + +static int print_configlist(xce_configlist_t clist) +{ + return 0; +} + +int cmd_getconf(int argc, char* argv[]) +{ + xce_clusterconf_t* conf; + int rc; + + rc = xcecli_clusterconf(active_ctx->ctx, &conf); + if (rc < 0) { + SET_ERROR(-rc); + perror("xce_clusterconf"); + return rc; + } + + printf("Cluster Name: %s\n", conf->name); + + printf("\nResources\n===========\n"); + print_reslist(conf->reslist); + + printf("\nQueues\n========\n"); + print_queuelist(conf->qlist); + + printf("\nWorkers\n========\n"); + print_workerlist(conf->wlist); + + printf("\nUser Groups\n===========\n"); + print_grouplist(conf->uglist); + + printf("\nHost Groups\n============\n"); + print_grouplist(conf->hglist); + + printf("\nConfigs\n=======\n"); + print_configlist(conf->cfglist); + + free_ClusterConf(conf); + + return 0; +} +#define KVAL(v) (((v) == NULL) ? "" : (v)) + +int cmd_set(int argc, char* argv[]) +{ + int i; + + if (argc == 1) { + for (i = 0; active_ctx->settings[i].key; i++) { + printf("%-12s:%s default: %s\n", + active_ctx->settings[i].key, + KVAL(active_ctx->settings[i].value), + KVAL(active_ctx->settings[i].defval)); + } + return 0; + } + + if (argc != 3) { + printf("set \n"); + return -EINVAL; + } + + for (i = 0; active_ctx->settings[i].key; i++) { + if (strcmp(active_ctx->settings[i].key, argv[1]) == 0) { + if (active_ctx->settings[i].value != NULL) { + free(active_ctx->settings[i].value); + } + + active_ctx->settings[i].value = strdup(argv[2]); + return 0; + } + } + + printf("Can not find setting key: %s\n", argv[1]); + return -EINVAL; +} + +static int run_command(char* cmdline, pid_t* pid) +{ + pid_t child; + + child = fork(); + + *pid = child; + if (child < 0) { + *pid = -errno; + + return -errno; + } + + if (child == 0) { + execlp("/bin/sh", "/bin/sh", "-c", cmdline, NULL); + perror("execlp"); + exit(-1); + } + + return 0; +} + +static int run_task(char* cmdline, pid_t* pid) +{ + return run_command(cmdline, pid); +} + +static int get_avail_run_job_slots() +{ + int i; + + for (i = 0; i < MAX_RUN_JOBS; i++) { + if (run_jobs[i].jobid == 0) { + return i; + } + } + + return -1; +} + +static int get_avail_run_task_slots() +{ + int i; + + for (i = 0; i < MAX_RUN_TASKS; i++) { + if (run_tasks[i].pid <= 0) { + return i; + } + } + + return -1; +} + +static int sched_on_jobs(int njobs, + xce_job_t** jobs) +{ + printf("%s %d jobs gotten\n", + __func__, njobs); + + return 0; +} + +static int sched_on_workers(int nworkers, + xce_worker_t** workers) +{ + printf("%s %d workers gotten\n", + __func__, nworkers); + + return 0; +} + +static void notify_job_fail(uint64_t jobid, + int code) +{ + /* clang-format off */ + job_failinfo_t failinfo = { + .jobid = jobid, + .stage = JOB_EXEC_STAGE_MAIN, + .code = code, + .detail = "no detail information" + }; + /*clang-format on*/ + + xcecli_notify_jobfail(active_ctx->ctx, &failinfo); + return; +} + +static void notify_task_fail(uint64_t jobid, + int clientidx, + int taskidx, + int code) { + xce_task_failinfo_t failinfo = { + .jobid = jobid, + .host = get_setting("myname"), + .clientidx = clientidx, + .taskidx = taskidx, + .code = code, + .detail = NULL + }; + + xcecli_notify_taskfail(active_ctx->ctx, &failinfo); +} + +static void notify_jobstart(uint64_t jobid, + int pid, + int pgid) { + /* clang-format off */ + job_pidinfo_t pidinfo = { + .jobid = jobid, + .npids = 1, + .pids = (unsigned int []){(unsigned int)pid}, + .npgids = 0, + .pgids = (unsigned int []){(unsigned int)pgid}, + .exdata = NULL + }; + /* clang-format on */ + + xcecli_notify_jobstart(active_ctx->ctx, &pidinfo); + + return; +} + +static void notify_task_start(uint64_t jobid, + int clientidx, + int taskidx, + int pid, + int pgid) +{ + xce_task_pidinfo_t pidinfo = { + .jobid = jobid, + .host = get_setting("myname"), + .clientidx = clientidx, + .taskidx = taskidx, + .npids = 1, + .pids = (unsigned int[]) { (unsigned int)pid }, + .npgids = 1, + .pgids = (unsigned int[]) { (unsigned int)pgid }, + .exdata = NULL + }; + + xcecli_notify_taskstart(active_ctx->ctx, &pidinfo); + + return; +} + +static int run_task_remote(uint64_t jobid, + job_startinfo_t* startinfo, + xce_launch_t* launchinfo) +{ + xce_taskhost_t hosts[launchinfo->nhosts - 1]; + xce_taskrun_t taskrun; + int i; + int rc; + + taskrun.taskhosts = (xce_taskhost_t**)calloc(launchinfo->nhosts - 1, + sizeof(xce_taskhost_t*)); + if (taskrun.taskhosts == NULL) { + return -ENOMEM; + } + + taskrun.clientidx = -1; + + for (i = 1; i < launchinfo->nhosts; i++) { + hosts[i - 1].exechome = startinfo->exechome; + hosts[i - 1].execcwd = startinfo->execcwd; + hosts[i - 1].cmdline = startinfo->cmdline; + hosts[i - 1].exdata = startinfo->exdata; + hosts[i - 1].execuser = startinfo->execuser; + hosts[i - 1].launch_type = startinfo->launch_type; + hosts[i - 1].host = launchinfo->hosts[i]->host; + hosts[i - 1].ntasks = launchinfo->hosts[i]->nranks; + + if (hosts[i - 1].launch_type == NULL) { + /* use mpi as default */ + hosts[i - 1].launch_type = "mpi"; + } + taskrun.taskhosts[i - 1] = &hosts[i - 1]; + } + + taskrun.nhosts = launchinfo->nhosts - 1; + taskrun.jobid = jobid; + + xcecli_unblock_tasks(active_ctx->ctx); + + rc = xcecli_runtask(active_ctx->ctx, &taskrun); + if (rc < 0) { + fprintf(stderr, "run remote task failed\n"); + } + + FREEUP(taskrun.taskhosts); + + return rc; +} + +static int run_task_local(uint64_t jobid, + job_startinfo_t* startinfo, + int ntasks) +{ + pid_t pid; + + pid = fork(); + if (pid < 0) { + return pid; + } + + if (pid == 0) { + int i; + pid_t pids[ntasks]; + int rc = 0; + int nlive = 0; + + /* disable detection of address sanitizer as + * the request structure will not be freed in + * child and we will exit directly + */ + setenv("ASAN_OPTIONS", "detect_leaks=false", 1); + + for (i = 0; i < ntasks; i++) { + rc = run_command(startinfo->cmdline, &pids[i]); + if (rc >= 0) { + nlive++; + } + } + + if (rc >= 0) { + rc = 0; + } + + while (nlive > 0) { + sleep(1); + for (i = 0; i < ntasks; i++) { + int wstatus; + + if (pids[i] <= 0) { + continue; + } + + if (waitpid(pids[i], &wstatus, WNOHANG) <= 0) { + continue; + } + + if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) { + nlive--; + + if (WIFSIGNALED(wstatus)) { + rc = 128 + WTERMSIG(wstatus); + } else { + if (WEXITSTATUS(wstatus) != 0) { + rc = WEXITSTATUS(wstatus); + } + } + } + } + } + + exit(rc); + } + + printf("Job %lu started, PID: %d\n", + jobid, pid); + + return pid; +} + +int run_job(xce_job_exec_t* startmsg) +{ + int rc = 0; + uint64_t jobid = startmsg->jobid; + job_startinfo_t* startinfo = startmsg->start_info; + xce_launch_t* launchinfo = startmsg->launchinfo; + + printf("%s: exechome= %s\n", __func__, + (startinfo->exechome == NULL) ? "NULL" : startinfo->exechome); + printf("%s: execcwd= %s\n", __func__, + (startinfo->execcwd == NULL) ? "NULL" : startinfo->execcwd); + printf("%s: execuser= %s\n", __func__, + (startinfo->execuser == NULL) ? "NULL" : startinfo->execuser); + printf("%s: cmdline= %s\n", __func__, + (startinfo->cmdline == NULL) ? "NULL" : startinfo->cmdline); + printf("%s: exdata= %s\n", __func__, + (startinfo->exdata == NULL) ? "NULL" : startinfo->exdata); + printf("%s: launch_type= %s\n", __func__, + (startinfo->launch_type == NULL) ? "NULL" : startinfo->launch_type); + printf("%s: q_pre_exec= %s\n", __func__, + (startinfo->q_pre_exec == NULL) ? "NULL" : startinfo->q_pre_exec); + printf("%s: q_post_exec= %s\n", __func__, + (startinfo->q_post_exec == NULL) ? "NULL" : startinfo->q_post_exec); + printf("%s: job_pre_exec= %s\n", __func__, + (startinfo->job_pre_exec == NULL) ? "NULL" : startinfo->job_pre_exec); + printf("%s: job_post_exec= %s\n", __func__, + (startinfo->job_post_exec == NULL) ? "NULL" : startinfo->job_post_exec); + + if (launchinfo->nhosts > 1) { + rc = run_task_remote(jobid, startinfo, launchinfo); + if (rc < 0) { + fprintf(stderr, "run_task_remote() fail, rc %d\n", rc); + return rc; + } + } + + rc = run_task_local(jobid, startinfo, + launchinfo->hosts[0]->nranks); + if (rc < 0) { + notify_job_fail(jobid, rc); + } else { + int idx; + int pid; + + pid = rc; + idx = get_avail_run_job_slots(); + if (idx < 0) { + printf("too many jobs run in this worker\n"); + } else { + run_jobs[idx].jobid = jobid; + run_jobs[idx].pid = pid; + run_jobs[idx].ts = 0; + } + + notify_jobstart(jobid, pid, pid); + } + + return rc; +} + +static int handle_startjob(xce_request_t* request) +{ + return run_job(request->body.jobstartmsg); +} + +static int handle_notify(xce_request_t* request) +{ + xce_notify_data_t* data = request->body.notifymsg; + + switch (data->event) { + case NotifyData_data_taskPidInfo: + printf("job %lu host %s taskidx %d pid %d\n", + data->data.taskinfo->jobid, + data->data.taskinfo->host, + data->data.taskinfo->taskidx, + data->data.taskinfo->pids[0]); + break; + case NotifyData_data_taskFailInfo: + printf("job %lu host %s task %d error code %d\n", + data->data.taskfailinfo->jobid, + data->data.taskfailinfo->host, + data->data.taskfailinfo->taskidx, + data->data.taskfailinfo->code); + break; + case NotifyData_data_taskFinish: + printf("job %lu host %s task %d exit code %d\n", + data->data.taskfinish->jobid, + data->data.taskfinish->host, + data->data.taskfinish->taskidx, + data->data.taskfinish->exit_code); + break; + } + return 0; +} + +static int job_kill(uint64_t jobid, + int signo) +{ + int i; + int killed = 0; + + for (i = 0; i < MAX_RUN_JOBS; i++) { + if ((run_jobs[i].jobid == 0) || (run_jobs[i].jobid != jobid)) { + continue; + } + + if (run_jobs[i].pid > 1) { + kill(run_jobs[i].pid, signo); + killed++; + } + } + + return killed; +} + +static int job_chkpnt(uint64_t jobid, + char* method, + int kill) +{ + char buf[128]; + char info[256]; + + printf("checkpoint job %ld by method %s\n", + jobid, method); + sprintf(buf, "/tmp/%ld/ckpt_dir", jobid); + + /* do some actions to checkpoint jobs */ + + sprintf(info, "{\nhost_select=\"criu\"; \nrestart_command=\"cd %s; criu restore -vvv\"}\n", + buf); + xcecli_notify_jobchkpt(active_ctx->ctx, + jobid, info); + + if (kill) { + job_kill(jobid, SIGTERM); + } + + return 0; +} + +static int handle_jobcontrol(xce_request_t* request) +{ + uint64_t jobid; + int signo; + xce_jobctrl_t* jobctrl = request->body.jobctrlmsg; + int rc = 0; + + switch (jobctrl->action) { + case JobControl_u_siginfo: + jobid = jobctrl->jobid; + signo = jobctrl->u.siginfo->signo; + rc = job_kill(jobid, signo); + break; + case JobControl_u_ckptoption: + rc = job_chkpnt(jobctrl->jobid, + jobctrl->u.ckptoption->method, + jobctrl->u.ckptoption->kill); + break; + default: + rc = -EINVAL; + ERROR("unknown job control action %d", + jobctrl->action); + break; + } + + return rc; +} + +static int handle_runtasks(xce_request_t* request) +{ + xce_taskrun_t* runmsg = request->body.runtaskmsg; + int rc = 0; + int i = 0; +#if 0 + job_startinfo_t startinfo; +#endif + + if (runmsg->nhosts != 1) { + /* something wrong */ + rc = -EINVAL; + + notify_task_fail(runmsg->jobid, + runmsg->clientidx, + -1, rc); + return rc; + } + + if (strcmp(runmsg->taskhosts[0]->host, get_setting("myname")) != 0) { + /* something wrong */ + rc = -EINVAL; + + notify_task_fail(runmsg->jobid, + runmsg->clientidx, + -1, rc); + return rc; + } + + printf("run task: jobid = %lu\n", runmsg->jobid); + printf("run task: user = %s\n", runmsg->taskhosts[0]->execuser); + printf("run task: cmdline = %s\n", runmsg->taskhosts[0]->cmdline); + printf("run task: exdata = %s\n", runmsg->taskhosts[0]->exdata); + printf("run task: ntasks =%d\n", runmsg->taskhosts[0]->ntasks); + +#if 0 + startinfo.exechome = runmsg->taskhosts[0]->exechome; + startinfo.execcwd = runmsg->taskhosts[0]->execcwd; + startinfo.execuser = runmsg->taskhosts[0]->execuser; + startinfo.cmdline = runmsg->taskhosts[0]->cmdline; + startinfo.exdata = runmsg->taskhosts[0]->exdata; + startinfo.launch_type = runmsg->taskhosts[0]->launch_type; +#endif + + for (i = 0; i < runmsg->taskhosts[0]->ntasks; i++) { + int pid = 0; + int pgid = 0; + + rc = run_task(runmsg->taskhosts[0]->cmdline, &pid); + if (rc < 0) { + notify_task_fail(runmsg->jobid, + runmsg->clientidx, + i, rc); + } else { + int idx; + + printf("job %lu task %d started, PID: %d\n", + runmsg->jobid, i, pid); + + idx = get_avail_run_task_slots(); + if (idx < 0) { + printf("too many running tasks\n"); + } else { + run_tasks[idx].jobid = runmsg->jobid; + run_tasks[idx].taskid = i; + run_tasks[idx].clientidx = runmsg->clientidx; + run_tasks[idx].pid = pid; + run_tasks[idx].ts = 0; + } + notify_task_start(runmsg->jobid, + runmsg->clientidx, + i, pid, pgid); + } + } + + return 0; +} + +static int myreq_handler(xce_request_t* request) +{ + int rc = 0; + + switch (request->opcode) { + case OPCode_push: + if (request->body_which == Request_body_datamsg) { + dataset_t* datamsg = request->body.datamsg; + + switch (datamsg->table) { + case Data_data_schedjobs: + case Data_data_jobs: + sched_on_jobs(datamsg->nrows, + (xce_job_t**)datamsg->rows); + break; + case Data_data_schedworkers: + case Data_data_workers: + sched_on_workers(datamsg->nrows, + (xce_worker_t**)datamsg->rows); + break; + default: + break; + } + } + break; + case OPCode_startjob: + rc = handle_startjob(request); + break; + case OPCode_notify: + rc = handle_notify(request); + break; + case OPCode_jobcontrol: + rc = handle_jobcontrol(request); + break; + case OPCode_runtasks: + rc = handle_runtasks(request); + break; + default: + break; + } + + return rc; +} + +static void check_run_jobs() +{ + int i; + time_t now = time(NULL); + + for (i = 0; i < MAX_RUN_JOBS; i++) { + int wstatus; + + if (run_jobs[i].jobid == 0) { + continue; + } + + if (run_jobs[i].pid <= 0) { + continue; + } + + if (waitpid(run_jobs[i].pid, &wstatus, WNOHANG | WUNTRACED | WCONTINUED) <= 0) { + /* fake job rusage */ + int npids = 1; + int npgids = 0; + int nusage = 2; + xce_rusage_t data[] = { + { "cputime", 100.0 }, + { "mem", 200.0 } + }; + xce_rusage_t* usages[] = { + &(data[0]), + &(data[1]) + }; + + if (now - run_jobs[i].ts > RUSAGE_UPDATE_INTERVAL) { + xcecli_update_jusage(active_ctx->ctx, run_jobs[i].jobid, + npids, &(run_jobs[i].pid), npgids, NULL, + nusage, &usages[0]); + run_jobs[i].ts = now; + } + + continue; + } + + if (WIFSTOPPED(wstatus)) { + xcecli_notify_jobstop(active_ctx->ctx, run_jobs[i].jobid); + } else if (WIFCONTINUED(wstatus)) { + xcecli_notify_jobresume(active_ctx->ctx, run_jobs[i].jobid); + } else if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) { + xce_jobfinish_t finish; + + finish.jobid = run_jobs[i].jobid; + + if (WIFEXITED(wstatus)) { + finish.exit_code = WEXITSTATUS(wstatus); + } else { + finish.exit_code = 128 + WTERMSIG(wstatus); + } + + printf("job %lu exit code %d\n", + finish.jobid, finish.exit_code); + xcecli_notify_jobfinish(active_ctx->ctx, &finish); + run_jobs[i].jobid = 0; + run_jobs[i].pid = 0; + } + } +} + +static void check_run_tasks() +{ + int i; + time_t now = time(NULL); + + for (i = 0; i < MAX_RUN_TASKS; i++) { + int wstatus; + + if (run_tasks[i].jobid == 0) { + continue; + } + + if (run_tasks[i].pid <= 0) { + continue; + } + + if (waitpid(run_tasks[i].pid, &wstatus, WNOHANG | WUNTRACED | WCONTINUED) <= 0) { + /* fake job rusage */ + int npids = 1; + int npgids = 0; + int nusage = 2; + xce_rusage_t data[] = { + { "cputime", 100.0 }, + { "mem", 200.0 } + }; + xce_rusage_t* usages[] = { + &(data[0]), + &(data[1]) + }; + + if (now - run_tasks[i].ts > RUSAGE_UPDATE_INTERVAL) { + xcecli_update_jusage(active_ctx->ctx, run_tasks[i].jobid, + npids, &(run_tasks[i].pid), npgids, NULL, + nusage, &usages[0]); + run_tasks[i].ts = now; + } + continue; + } + + if (WIFEXITED(wstatus) || WIFSIGNALED(wstatus)) { + xce_task_finish_t finish = { + .jobid = run_tasks[i].jobid, + .host = get_setting("myname"), + .taskidx = run_tasks[i].taskid, + .clientidx = run_tasks[i].clientidx, + .exit_code = 0 + }; + + if (WIFEXITED(wstatus)) { + finish.exit_code = WEXITSTATUS(wstatus); + } else { + finish.exit_code = 128 + WTERMSIG(wstatus); + } + + printf("job %lu task %d exit code %d\n", + finish.jobid, finish.taskidx, + finish.exit_code); + xcecli_notify_taskfinish(active_ctx->ctx, &finish); + run_tasks[i].jobid = 0; + run_tasks[i].pid = 0; + } + } +} + +static void handler(int sig, siginfo_t* si, void* uc) +{ + struct itimerspec its; + + check_run_jobs(); + check_run_tasks(); + + memset(&its, 0x0, sizeof(its)); + + its.it_value.tv_sec = 1; + + timer_settime(timerid, 0, &its, NULL); +} + +int main(int argc, char* argv[]) +{ + struct arguments arguments; + char* buf; + char* logdir = "/tmp"; + char* loglevel = "info"; + int logsize = 64 * 1024 * 1024; + int logcount = 10; + + memset(&arguments, 0x0, sizeof(arguments)); + argp_parse(&argp, argc, argv, 0, 0, &arguments); + + if (arguments.verbose != 0) { + loglevel = verbose_to_loglevel(arguments.verbose); + } + + mylog_init(logdir, "xce_test", loglevel, logsize, logcount); + + signal(SIGINT, sigbreak); + + memset(&run_jobs, 0x0, sizeof(run_jobs)); + memset(&run_tasks, 0x0, sizeof(run_tasks)); + + while (!done) { + char* s; + int rc; + + prompt[0] = 0x0; + fill_prompt(active_ctx, prompt); + strcat(prompt, ">"); + buf = readline(prompt); + + if (buf == NULL) { + execute_line("quit"); + continue; + } + + if (strlen(buf) == 0) { + free(buf); + continue; + } + + s = buf; + + rc = execute_line(s); + if (rc < 0) { + SET_ERROR(-rc); + } else { + add_history(s); + } + free(buf); + } + + return 0; +} diff --git a/src/controller/worker.c b/src/controller/worker.c new file mode 100644 index 0000000..6d93e43 --- /dev/null +++ b/src/controller/worker.c @@ -0,0 +1,599 @@ +#include +#include + +#include "cluster.h" +#include "worker.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" + +static int hostmatched(const char* pattern, + const char* name) +{ + if (strcmp(pattern, name) == 0) { + return 1; + } + + return 0; +} + +static xce_hostconfig_t* find_matched_hostconfig(cluster_t* c, + const char* name) +{ + xce_hostconfig_t* defaultconfig = NULL; + xce_hostconfig_t** hconfigs = NULL; + int i, nhconfigs; + + if (c->myconf == NULL) { + errno = EINVAL; + return NULL; + } + + if (c->myconf->cluster_config == NULL) { + errno = EINVAL; + return NULL; + } + + if (c->myconf->cluster_config->hconfigs == NULL) { + return NULL; + } + + nhconfigs = c->myconf->cluster_config->nhconfigs; + hconfigs = c->myconf->cluster_config->hconfigs; + + for (i = 0; i < nhconfigs; i++) { + xce_hostconfig_t* config = hconfigs[i]; + + if (strcmp(config->name, "default") == 0) { + defaultconfig = config; + continue; + } + + if (hostmatched(config->name, name)) { + return config; + } + } + + return defaultconfig; +} + +static int update_val_res(cluster_t* c, + worker_t* w, + xce_resource_t* r, + xce_val_t* val) +{ + int idx; + zarray_t* array = NULL; + xce_val_t* v; + + idx = r->idx; + + if (r->flags & XCE_RESF_SHARED) { + /* update shared resource */ + array = &(c->resmgr.sharedinst); + } else { + /* update host based resource */ + array = &(w->host.resvals); + } + + if (idx < array->caps) { + v = (xce_val_t*)array->data[idx]; + } else { + v = NULL; + } + + if (v == NULL) { + int rc; + + v = (xce_val_t*)calloc(1, sizeof(*v)); + rc = zarray_put(array, idx, v); + if (rc < 0) { + FREEUP(v); + return rc; + } + } + + memcpy(v, val, sizeof(*v)); + + return 0; +} + +static int update_resval(cluster_t* c, + worker_t* w, + xce_resval_t* val) +{ + xce_resource_t* r; + xce_val_t* v = NULL; + int rc; + + r = resmgr_lockres(&(c->resmgr), + val->name, + OBJREF_READ); + if (r == NULL) { + ERROR("resource `%s' can not be found", + val->name); + return -errno; + } + + if (r->idx < 0) { + ERROR("invalid resource definition for %s", + val->name); + return -EINVAL; + } + + v = val->val; + + rc = update_val_res(c, w, r, v); + + obj_deref(r); + + return rc; +} + +int update_resvals(cluster_t* c, + worker_t* w, + int nres, + xce_resval_t* vals) +{ + int rc = 0; + int i; + + for (i = 0; i < nres; i++) { + rc = update_resval(c, w, &(vals[i])); + if (rc < 0) { + PERROR("update_resval"); + } + } + + return 0; +} + +int update_resval_list(cluster_t* c, + worker_t* w, + xce_resval_list_t* l) +{ + int rc = 0; + int i; + + for (i = 0; i < l->count; i++) { + rc = update_resval(c, w, l->vals[i]); + if (rc < 0) { + PERROR("update_resval"); + } + } + + return 0; +} + +static int mark_host_res(cluster_t* c, + worker_t* w, + xce_config_t* config) +{ + xce_resource_t* r; + char* n = config->name; + int rc; + /* clang-format-off */ + xce_val_t val; + /* clang-format-on */ + + r = resmgr_lockres(&(c->resmgr), n, OBJREF_READ); + if (r == NULL) { + ERROR("resource `%s' can not be found", + n); + return -ENOENT; + } + + if (r->idx < 0) { + ERROR("invalid resource definition for `%s'", + n); + return -EINVAL; + } + + if (r->flags & XCE_RESF_SHARED) { + ERROR("resource `%s' should not be a shared resource", n); + return -EINVAL; + } + + switch (r->valtype) { + case XCE_VAL_STRING: + val.which = Value_u_svalue; + if (config->value == NULL) { + val.u.svalue = ""; + } else { + val.u.svalue = config->value; + } + + break; + case XCE_VAL_NUMERIC: + val.which = Value_u_fvalue; + if (config->value == NULL) { + val.u.fvalue = 0.0; + } else { + val.u.fvalue = atof(config->value); + } + break; + case XCE_VAL_BOOLEAN: + val.which = Value_u_bvalue; + val.u.bvalue = 1; + break; + } + rc = update_val_res(c, w, r, &val); + + obj_deref(r); + return rc; +} + +static int add_hostattrs_resvals(cluster_t* c, + worker_t* w) +{ +#define N_HOST_ATTRS (8) + xce_val_t values[N_HOST_ATTRS] = { + { .which = Value_u_svalue, .u = { .svalue = w->host.attrs->hosttype } }, + { .which = Value_u_svalue, .u = { .svalue = w->host.attrs->hostmodel } }, + { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->cpufactor } }, + { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxcpus } }, + { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxmem } }, + { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxswap } }, + { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->maxtmp } }, + { .which = Value_u_fvalue, .u = { .fvalue = w->host.attrs->ndisks } } + }; + xce_resval_t resvals[N_HOST_ATTRS] = { + { .name = "type", .val = &values[0] }, + { .name = "model", .val = &values[1] }, + { .name = "cpuf", .val = &values[2] }, + { .name = "maxcpus", .val = &values[3] }, + { .name = "maxmem", .val = &values[4] }, + { .name = "maxswap", .val = &values[5] }, + { .name = "maxtmp", .val = &values[6] }, + { .name = "ndisks", .val = &values[7] } + }; + + return update_resvals(c, w, N_HOST_ATTRS, + &(resvals[0])); +} + +static int set_host_slots(cluster_t* c, + worker_t* w, + int nslots) +{ + /* clang-format off */ + xce_val_t values[1] = { + { + .which = Value_u_fvalue, + .u = { + .fvalue = (float)nslots + } + } + }; + xce_resval_t resvals[1] = { + { + .name = "slots", + .val = &values[0] + } + }; + /* clang-format on */ + + return update_resvals(c, w, 1, &(resvals[0])); +} + +int add_worker(void* cls, + endpoint_t* ep, + const char* host, + hostattrs_t* attrs) +{ + cluster_t* c = (cluster_t*)cls; + /* clang-format off */ + worker_t worker = { + .host = { + .name = (char*)host, + } + }; + /* clang-format on */ + worker_t* w = NULL; + int idx, i; + xce_hostconfig_t* hconfig = NULL; + int nslots; + + if ((cls == NULL) || (host == NULL)) { + return -EINVAL; + } + + hconfig = find_matched_hostconfig(c, host); + if (hconfig == NULL) { + return -errno; + } + + worker.host.attrs = attrs; + + w = create_obj_by_typename(&(c->objmgr), "xworker", &worker); + if (w == NULL) { + return -errno; + } + + /* we should hold the update lock */ + /* If admin makes the host in locked or closed, we need keep this */ + w->host.status &= ~XCE_WORKER_UNAVAIL; /* status OK */ + w->ep = ep; + + /* override host type? */ + if (strcmp(hconfig->hosttype, "!") != 0) { + obj_deref(w->host.attrs->hosttype); + w->host.attrs->hosttype = create_obj_by_typename(&(c->objmgr), + "xstring", + hconfig->hosttype); + } + + /* override host model */ + if (strcmp(hconfig->hostmodel, "!") != 0) { + obj_deref(w->host.attrs->hostmodel); + w->host.attrs->hostmodel = create_obj_by_typename(&(c->objmgr), + "xstring", + hconfig->hostmodel); + } + add_hostattrs_resvals(c, w); + idx = w->idx; + + /* using auto detected or pre configured MXJ? */ + nslots = hconfig->mxj; + if (nslots < 0) { + nslots = w->host.attrs->maxcpus; + } + + set_host_slots(c, w, nslots); + + if (w->host.dispatch_window_config != NULL) { + free_timewindow_config(w->host.dispatch_window_config); + w->host.dispatch_window_config = NULL; + } + + if (hconfig->dispatchwindow != NULL) { + w->host.dispatch_window_config = dup_timewindow_config(hconfig->dispatchwindow); + } + + if (w->host.thresholds != NULL) { + free_resource_thresholds(w->host.thresholds, + w->host.nthresholds); + w->host.nthresholds = 0; + w->host.thresholds = NULL; + } + + if (hconfig->thresholds != NULL) { + w->host.thresholds = dup_resource_thresholds(hconfig->thresholds, + hconfig->nthresholds); + w->host.nthresholds = hconfig->nthresholds; + } + + /* add bool resources to host */ + for (i = 0; i < hconfig->nresources; i++) { + mark_host_res(c, w, hconfig->resources[i]); + } + + DEREF_OBJ(w); /* release update lock */ + + return idx; +} + +int restore_worker(void* cls, + xce_worker_t host) +{ + cluster_t* c = (cluster_t*)cls; + int idx = 0; + worker_t* w = NULL; + /* do work in add_worker() doing for normal worker join */ + /* clang-format off */ + worker_t worker = { + .host = { + .name = (char*)host.name, + .status = host.status, + }, + }; + /* clang-format on */ + + w = create_obj_by_typename(&(c->objmgr), "xworker", &worker); + if (w == NULL) { + return -errno; + } + add_hostattrs_resvals(c, w); + idx = w->idx; + w->host.status |= XCE_WORKER_UNAVAIL; + + DEREF_OBJ(w); + + return idx; +} + +struct worker_config_publish_args { + cluster_t* c; + endpoint_t* ep; +}; + +static int check_conf4worker_(xce_config_t* c) +{ + if (c != NULL) { + return 1; + } + + return 0; +} + +static int worker_push_config(cluster_t* c, + endpoint_t* ep) +{ + xce_configdata_t configdata; + int rc = 0; + + memset(&configdata, 0x0, sizeof(configdata)); + + configdata.data.conf4worker = (xce_conf4worker_t*)calloc(1, sizeof(xce_conf4worker_t)); + if (configdata.data.conf4worker == NULL) { + return -ENOMEM; + } + + collect_confvars(c, (collect_check_fn_t)check_conf4worker_, + &(configdata.data.conf4worker->n_confvars), + &(configdata.data.conf4worker->confvars)); + configdata.data_which = ConfigData_data_workerconfig; + configdata.data.conf4worker->clustername = c->myconf->cluster_config->clustername; + + rc = xce_pushconfig_req(ep, c->myhost, &configdata); + + deref_objs(configdata.data.conf4worker->n_confvars, + (void**)configdata.data.conf4worker->confvars); + + FREEUP(configdata.data.conf4worker->confvars); + FREEUP(configdata.data.conf4worker); + + return rc; +} + +/* consider to push jobs on hosts to */ +int on_worker_join(void* cls, + endpoint_t* ep) +{ + cluster_t* c = (cluster_t*)cls; + int rc = 0; + + rc = worker_push_config(c, ep); + + return rc; +} + +int get_resval(cluster_t* c, + char* resname, + char* host, + xce_val_t* val) +{ + xce_resource_t* r; + zarray_t* array = NULL; + xce_val_t* v = NULL; + worker_t* w = NULL; + + r = resmgr_lockres(&(c->resmgr), + resname, + OBJREF_READ); + if (r == NULL) { + return -ENOENT; + } + + if ((r->flags & XCE_RESF_SHARED) == 0) { + if ((host == NULL) || (strlen(host) == 0)) { + obj_deref(r); + + return -EINVAL; + } + + w = workermgr_lock(&(c->workermgr), + host, OBJREF_READ); + if (w == NULL) { + DEREF_OBJ(r); + return -ENOENT; + } + + array = &(w->host.resvals); + } else { + array = &(c->resmgr.sharedinst); + } + + if (r->idx < array->caps) { + v = (xce_val_t*)array->data[r->idx]; + } + + obj_deref(r); + + if (v == NULL) { + DEREF_OBJ(w); + return -ENOENT; + } + + memcpy(val, v, sizeof(*val)); + DEREF_OBJ(w); + return 1; +} + +int handle_worker_leave(cluster_t* c, + int idx) +{ + worker_t* w = NULL; + + TRACE("%s worker %d broken connection", + __func__, idx); + + if (c == NULL) { + SET_ERROR(EINVAL); + return -errno; + } + + if (idx < 0) { + SET_ERROR(EINVAL); + return -errno; + } + + w = workermgr_lock_byid(&(c->workermgr), + idx, OBJREF_UPDATE); + if (w == NULL) { + ERROR("fail to find worker by id %d", idx); + SET_ERROR(E_XCE_BAD_WORKER); + return -errno; + } + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_WORKER, + HOOK_BEFORE_EVENT, + HOOK_DETAIL_WORKER_DEAD, + c, w); + w->host.status |= XCE_WORKER_UNAVAIL; + w->ep = NULL; + + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_WORKER, + HOOK_AFTER_EVENT, + HOOK_DETAIL_WORKER_DEAD, + c, w); + obj_deref(w); + return 0; +} + +int worker_ctrl(cluster_t* c, xce_hostctrl_t* ctrl) +{ + worker_t* w = NULL; + int rc = 0; + + if (!c || !ctrl || !ctrl->hostname || *ctrl->hostname == '\0') { + TRACE("%s invalid arguments.", __func__); + return -EINVAL; + } + TRACE("%s worker_ctrl worker %s", __func__, ctrl->hostname); + w = workermgr_lock(&(c->workermgr), ctrl->hostname, OBJREF_UPDATE); + if (w == NULL) { + PERROR("workermgr_lock"); + return -ENOENT; + } + + switch (ctrl->action) { + case HOST_OPEN: + if (!(w->host.status & XCE_WORKER_CLOSED)) { + goto clean; + } + w->host.status &= ~XCE_WORKER_CLOSED; + break; + case HOST_CLOSE: + if (w->host.status & XCE_WORKER_CLOSED) { + goto clean; + } + w->host.status |= XCE_WORKER_CLOSED; + break; + break; + default: + rc = -1; + break; + } + if (rc == 0) { + xce_run_hooks(&(c->hooksys), + XCE_CONTROLLER_HOOK_WORKER, + HOOK_AFTER_EVENT, + HOOK_DETAIL_WORKER_UPDATE, + c, w); + } +clean: + obj_deref(w); + return rc; +} diff --git a/src/controller/workermgr.c b/src/controller/workermgr.c new file mode 100644 index 0000000..c433bfa --- /dev/null +++ b/src/controller/workermgr.c @@ -0,0 +1,216 @@ +#include +#include +#include + +#include "cluster.h" +#include "htab.h" +#include "mylock.h" +#include "xce/cluster_def.h" +#include "xce/errnox.h" +#include "xstring.h" + +extern void worker_obj_killer(worker_t*); + +int workermgr_init(workermgr_t* mgr, + int nslots, + int delta) +{ + TRACE("%s nslots %d delta %d", + __func__, nslots, delta); + + mylock_init(&(mgr->lock)); + ht_init(&(mgr->htab), nslots, + string_hash_key); + zarray_init(&(mgr->workers), delta, + (zarray_item_killer_t)worker_obj_killer); + return 0; +} + +void workermgr_fini(workermgr_t* mgr) +{ + zarray_fini(&(mgr->workers)); + ht_fini(&(mgr->htab)); + mylock_fini(&(mgr->lock)); +} + +static int32_t workermgr_add_(workermgr_t* mgr, + worker_t* w) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), (void*)w->host.name); + if (e != NULL) { + return (int)e->data; + } + + idx = zarray_store(&(mgr->workers), w); + if (idx < 0) { + return idx; + } + + e = ht_add(&(mgr->htab), + (void*)w->host.name); + if (e == NULL) { + zarray_del(&(mgr->workers), idx); + return -errno; + } + + e->data = idx; + + return idx; +} + +int32_t workermgr_add(workermgr_t* mgr, + worker_t* w) +{ + int idx; + + if ((mgr == NULL) || (w == NULL)) { + return -EINVAL; + } + + mylock_lock(&(mgr->lock)); + idx = workermgr_add_(mgr, w); + mylock_unlock(&(mgr->lock)); + + return idx; +} + +static worker_t* workermgr_lock_(workermgr_t* mgr, + char* name, + int op) +{ + hent_t* e = NULL; + int idx; + worker_t* w = NULL; + + e = ht_get(&(mgr->htab), (void*)name); + if (e == NULL) { + SET_ERROR(E_XCE_BAD_WORKER); + return NULL; + } + + idx = e->data; + if (idx < 0) { + SET_ERROR(E_XCE_BAD_WORKER); + return NULL; + } + + w = zarray_get(&(mgr->workers), idx); + + return obj_ref(w, op); +} + +worker_t* workermgr_lock(workermgr_t* mgr, + char* name, + int op) +{ + worker_t* w = NULL; + + if (mgr == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + if (name == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + /* clang-format off */ + if ((op != OBJREF_NOP) && + (op != OBJREF_READ) && + (op != OBJREF_UPDATE)) { + SET_ERROR(EINVAL); + return NULL; + } + /* clang-format on */ + + mylock_lock(&(mgr->lock)); + w = workermgr_lock_(mgr, name, op); + mylock_unlock(&(mgr->lock)); + + return w; +} + +worker_t* workermgr_lock_byid(workermgr_t* mgr, + int idx, + int op) +{ + worker_t* w = NULL; + + if (mgr == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + if ((idx < 0) || (idx >= mgr->workers.caps)) { + SET_ERROR(EINVAL); + return NULL; + } + + /* clang-format off */ + if ((op != OBJREF_NOP) && + (op != OBJREF_READ) && + (op != OBJREF_UPDATE)) { + SET_ERROR(EINVAL); + return NULL; + } + /* clang-format on */ + + mylock_lock(&(mgr->lock)); + w = (worker_t*)zarray_get(&(mgr->workers), idx); + if (w != NULL) { + w = obj_ref(w, op); + } + mylock_unlock(&(mgr->lock)); + + return w; +} + +static void workermgr_del_(workermgr_t* mgr, + worker_t* w) +{ + hent_t* e = NULL; + int idx; + + e = ht_get(&(mgr->htab), + (void*)w->host.name); + if (e == NULL) { + return; + } + + idx = e->data; + if (idx < 0) { + return; + } + + zarray_del(&(mgr->workers), idx); +} + +void workermgr_del(workermgr_t* mgr, + worker_t* w) +{ + if ((mgr == NULL) || (w == NULL)) { + SET_ERROR(EINVAL); + return; + } + + mylock_lock(&(mgr->lock)); + workermgr_del_(mgr, w); + mylock_unlock(&(mgr->lock)); +} + +int workermgr_iter(workermgr_t* mgr, + xlist_t* l, + void* arg) +{ + int rc = 0; + + mylock_lock(&(mgr->lock)); + rc = zarray_iter_ex(&(mgr->workers), l, arg); + mylock_unlock(&(mgr->lock)); + + return rc; +} diff --git a/src/controller/wrap.c b/src/controller/wrap.c new file mode 100644 index 0000000..f6dc1b3 --- /dev/null +++ b/src/controller/wrap.c @@ -0,0 +1,59 @@ +#include + +#include "utils.h" +#include "xce/errnox.h" + +int __wrap___xpg_strerror_r(int errnum, + char* buf, + size_t bufsize) +{ + return xce_strerror_r(errnum, buf, bufsize); +} + +static void xce_perror_(const char* s, int errnum) +{ + char buf[XCE_MAX_ERRMSG_LEN]; + ectx_t* ectx = get_ectx(); + + xce_strerror_r(errnum, buf, sizeof(buf)); + + if (tlog_get_root() == NULL) { + if ((ectx->file != NULL) && (ectx->func != NULL) && (ectx->line > 0) && (ectx->code == errnum)) { + fprintf(stderr, "%s: %s at %s:%d %s\n", + s, buf, ectx->file, + ectx->line, ectx->func); + } else { + fprintf(stderr, "%s: %s", s, buf); + } + } else { + if ((ectx->file != NULL) && (ectx->func != NULL) && (ectx->line > 0) && (ectx->code == errnum)) { + tlog_error("%s: %s at %s:%d %s\n", + s, buf, ectx->file, + ectx->line, ectx->func); + } else { + tlog_error("%s: %s", s, buf); + } + } +} + +void xce_perror(const char* s) +{ + xce_perror_(s, errno); +} + +void __wrap_perror(const char* s) +{ + xce_perror(s); +} + +int __wrap_printf(const char* fmt, ...) +{ + va_list ap; + int rc; + + va_start(ap, fmt); + rc = xce_vprintf(fmt, ap); + va_end(ap); + + return rc; +} diff --git a/src/controller/xceobjs.c b/src/controller/xceobjs.c new file mode 100644 index 0000000..e75761a --- /dev/null +++ b/src/controller/xceobjs.c @@ -0,0 +1,1044 @@ +#include +#include +#include +#include + +#include "cluster.h" +#include "objmgr.h" +#include "utils.h" +#include "xce/cluster_def.h" +#include "xstring.h" + +static void* xjob_create(objmgr_t*, objtype_t*, va_list); +static void xjob_fini(void*); +static void xjob_type_fini(objtype_t*); +static void* xres_create(objmgr_t*, objtype_t*, va_list); +static void xres_fini(void*); +static void xres_type_fini(objtype_t*); +static void* xworker_create(objmgr_t*, objtype_t*, va_list); +static void xworker_fini(void*); +static void xworker_type_fini(objtype_t*); +static void* xqueue_create(objmgr_t*, objtype_t*, va_list); +static void xqueue_fini(void*); +static void xqueue_type_fini(objtype_t*); +static void* xconfig_create(objmgr_t*, objtype_t*, va_list); +static void xconfig_fini(void*); +static void xconfig_type_fini(objtype_t*); +static void* xresval_create(objmgr_t*, objtype_t*, va_list); +static void xresval_fini(void*); +static void xresval_type_fini(objtype_t*); +static void* xgroup_create(objmgr_t*, objtype_t*, va_list); +static void xgroup_fini(void*); +static void xgroup_type_fini(objtype_t*); + +/* clang-format off */ +static objtype_t types[] = { + { + .type = "xjob", + .fn_create = (obj_create_fn_t)xjob_create, + .fn_fini = xjob_fini, + .fn_type_fini = (objtype_fini_fn_t)xjob_type_fini, + .typedata = NULL + }, + { + .type = "xresource", + .fn_create = (obj_create_fn_t)xres_create, + .fn_fini = xres_fini, + .fn_type_fini = (objtype_fini_fn_t)xres_type_fini, + .typedata = NULL, + }, + { + .type = "xworker", + .fn_create = (obj_create_fn_t)xworker_create, + .fn_fini = xworker_fini, + .fn_type_fini = (objtype_fini_fn_t)xworker_type_fini, + .typedata = NULL + }, + { + .type = "xqueue", + .fn_create = (obj_create_fn_t)xqueue_create, + .fn_fini = xqueue_fini, + .fn_type_fini = (objtype_fini_fn_t)xqueue_type_fini, + .typedata = NULL + }, + { + .type = "xconfig", + .fn_create = (obj_create_fn_t)xconfig_create, + .fn_fini = xconfig_fini, + .fn_type_fini = (objtype_fini_fn_t)xconfig_type_fini, + .typedata = NULL + }, + { + .type = "xresval", + .fn_create = (obj_create_fn_t)xresval_create, + .fn_fini = xresval_fini, + .fn_type_fini = (objtype_fini_fn_t)xresval_type_fini, + .typedata = NULL + + }, + { + .type = "xgroup", + .fn_create = (obj_create_fn_t)xgroup_create, + .fn_fini = xgroup_fini, + .fn_type_fini = (objtype_fini_fn_t)xgroup_type_fini, + .typedata = NULL + } +}; +/* clang-format on */ + +static void* xjob_create(objmgr_t* mgr, + objtype_t* type, + va_list ap) +{ + jobmgr_t* jobmgr = &(((cluster_t*)type->typedata)->jobmgr); + xce_job_t* submit = va_arg(ap, xce_job_t*); + job_t* job = NULL; + myobj_t* obj; + int32_t idx; + + if (submit == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + job = jobmgr_lockjob(jobmgr, submit->jobid, OBJREF_READ); + if (job != NULL) { + SET_ERROR(EEXIST); + obj_deref(job); + + return NULL; + } + + obj = obj_alloc(sizeof(*job)); + if (obj == NULL) { + return NULL; + } + + job = MYOBJ_TO_OBJ(job_t*, obj); + /* fill job data */ + job->job.jobid = submit->jobid; + job->job.queue = create_obj_by_typename(mgr, "xstring", + submit->queue); + job->job.sub_time = submit->sub_time; + job->job.queue_time = submit->queue_time; + if (job->job.queue_time == 0) { + job->job.queue_time = job->job.sub_time; + } + job->job.state = submit->state; + job->job.user = create_obj_by_typename(mgr, "xstring", + submit->user); + job->job.start_time = submit->start_time; + job->job.finish_time = submit->finish_time; + job->job.job_name = create_obj_by_typename(mgr, "xstring", + submit->job_name); + job->job.project_name = create_obj_by_typename(mgr, "xstring", + submit->project_name); + job->job.user_group = create_obj_by_typename(mgr, "xstring", + submit->user_group); + job->job.job_description = create_obj_by_typename(mgr, "xstring", + submit->job_description); + job->job.resreq = create_obj_by_typename(mgr, "xstring", + submit->resreq); + job->job.run_time = submit->run_time; + job->job.mintasks = submit->mintasks; + job->job.maxtasks = submit->maxtasks; + job->job.pending_ctrl = submit->pending_ctrl; + + job->job.pidinfo = NULL; + + if (submit->pidinfo != NULL) { + job->job.pidinfo = (job_pidinfo_t*)calloc(1, sizeof(job_pidinfo_t)); + if (job->job.pidinfo == NULL) { + SET_ERROR(ENOMEM); + goto clean_job; + } + job->job.pidinfo->npids = copy_int_list((int**)&(job->job.pidinfo->pids), + (int*)submit->pidinfo->pids, submit->pidinfo->npids); + job->job.pidinfo->npgids = copy_int_list((int**)&(job->job.pidinfo->pgids), + (int*)submit->pidinfo->pgids, submit->pidinfo->npgids); + if (submit->pidinfo->exdata) { + job->job.pidinfo->exdata = create_obj_by_typename(mgr, "xstring", + submit->pidinfo->exdata); + } + } + + job->job.start_info = NULL; + + if (submit->start_info != NULL) { + job->job.start_info = (job_startinfo_t*)calloc(1, sizeof(job_startinfo_t)); + if (job->job.start_info == NULL) { + SET_ERROR(ENOMEM); + goto clean_job; + } + job->job.start_info->cmdline = create_obj_by_typename(mgr, "xstring", + submit->start_info->cmdline); + job->job.start_info->exdata = create_obj_by_typename(mgr, "xstring", + submit->start_info->exdata); + job->job.start_info->execcwd = create_obj_by_typename(mgr, "xstring", + submit->start_info->execcwd); + job->job.start_info->exechome = create_obj_by_typename(mgr, "xstring", + submit->start_info->exechome); + job->job.start_info->execuser = create_obj_by_typename(mgr, "xstring", + submit->start_info->execuser); + job->job.start_info->launch_type = create_obj_by_typename(mgr, "xstring", + submit->start_info->launch_type); + if (submit->start_info->job_pre_exec != NULL) { + job->job.start_info->job_pre_exec = create_obj_by_typename(mgr, "xstring", + submit->start_info->job_pre_exec); + } + + if (submit->start_info->job_post_exec != NULL) { + job->job.start_info->job_post_exec = create_obj_by_typename(mgr, "xstring", + submit->start_info->job_post_exec); + } + } + + job->job.allocs = NULL; + + if (submit->allocs) { + MOVE_PTR(job->job.allocs, submit->allocs); + } + + job->job.run_usage = NULL; + + if (submit->run_usage != NULL) { + MOVE_PTR(job->job.run_usage, submit->run_usage); + } + + /* clang-format off */ + if ((job->job.queue == NULL) || + (job->job.user == NULL) || + (job->job.job_name == NULL) || + (job->job.start_info->cmdline == NULL) || + (job->job.project_name == NULL) || + (job->job.job_description == NULL) || + (job->job.resreq == NULL)) { + /* clang-format on */ + + FREEUP(job->job.start_info); + + job_obj_killer(job); + + SET_ERROR(ENOMEM); + return NULL; + } + idx = jobmgr_add_job(jobmgr, job); + + obj_init(mgr, type, idx, obj); + + return job; + +clean_job: + job_obj_killer(job); + return NULL; +} + +void job_obj_killer(job_t* job) +{ + myobj_t* obj; + + if (job == NULL) { + return; + } + + obj = TO_MYOBJ(job); + DEREF_OBJ(job->job.queue); + DEREF_OBJ(job->job.user); + DEREF_OBJ(job->job.job_name); + DEREF_OBJ(job->job.project_name); + DEREF_OBJ(job->job.job_description); + DEREF_OBJ(job->job.resreq); + + if (job->job.pidinfo != NULL) { + FREEUP(job->job.pidinfo->pids); + FREEUP(job->job.pidinfo->pgids); + if (job->job.pidinfo->exdata != NULL) { + DEREF_OBJ(job->job.pidinfo->exdata); + } + + FREEUP(job->job.pidinfo); + } + + if (job->job.allocs != NULL) { + for (int i = 0; i < job->job.allocs->nhosts; i++) { + FREEUP(job->job.allocs->allocs[i]->host); + for (int k = 0; k < job->job.allocs->allocs[i]->nallocs; k++) { + xce_alloc_t* alloc = (xce_alloc_t*)job->job.allocs->allocs[i]->allocs[k]; + if (alloc == NULL) { + continue; + } + FREEUP(alloc->resname); + FREEUP(alloc->reserve); + free(alloc); + } + FREEUP(job->job.allocs->allocs[i]); + } + FREEUP(job->job.allocs->allocs); + FREEUP(job->job.allocs); + } + + if (job->job.run_usage != NULL) { + for (int i = 0; i < job->job.run_usage->num_hosts; i++) { + DEREF_OBJ(job->job.run_usage->host_runusage[i]->host_name); + FREEUP(job->job.run_usage->host_runusage[i]); + } + FREEUP(job->job.run_usage->host_runusage); + FREEUP(job->job.run_usage); + } + + if (job->job.start_info != NULL) { + DEREF_OBJ(job->job.start_info->cmdline); + DEREF_OBJ(job->job.start_info->exechome); + DEREF_OBJ(job->job.start_info->execcwd); + DEREF_OBJ(job->job.start_info->execuser); + DEREF_OBJ(job->job.start_info->exdata); + DEREF_OBJ(job->job.start_info->launch_type); + DEREF_OBJ(job->job.start_info->job_pre_exec); + DEREF_OBJ(job->job.start_info->job_post_exec); + FREEUP(job->job.start_info); + } + + FREEUP(job->job.exit_info); + + memset(obj, 0x0, sizeof(*obj)); + + FREEUP(obj); +} + +static void xjob_fini(void* j) +{ + myobj_t* myobj; + objtype_t* typeval; + job_t* job; + jobmgr_t* jobmgr; + + if (j == NULL) { + return; + } + + job = (job_t*)j; + myobj = TO_MYOBJ(j); + typeval = myobj->type; + jobmgr = &(((cluster_t*)typeval->typedata)->jobmgr); + if (jobmgr == NULL) { + return; + } + + jobmgr_del_job(jobmgr, job); + + return; +} + +static void xjob_type_fini(objtype_t* typeval) +{ + return; +} + +int register_types(cluster_t* c) +{ + int i; + + for (i = 0; i < N_EL_ARRAY(types); i++) { + int rc; + + types[i].typedata = c; + rc = objmgr_register(&(c->objmgr), &(types[i])); + if (rc < 0) { + return rc; + } + } + + return 0; +} + +void resval_obj_killer(xce_resval_t* v) +{ + return; +} +static void* xres_create(objmgr_t* mgr, + objtype_t* type, + va_list ap) +{ + resmgr_t* resmgr = &(((cluster_t*)type->typedata)->resmgr); + xce_resource_t* r = va_arg(ap, xce_resource_t*); + xce_resource_t* r1 = NULL; + myobj_t* obj; + int32_t idx; + + xce_use(resmgr); + + if (r == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + r1 = resmgr_lockres(resmgr, r->name, OBJREF_READ); + if (r1 != NULL) { + SET_ERROR(EEXIST); + obj_deref(r1); + return NULL; + } + + obj = obj_alloc(sizeof(*r)); + if (obj == NULL) { + return NULL; + } + + r1 = MYOBJ_TO_OBJ(xce_resource_t*, obj); + + r1->name = create_obj_by_typename(mgr, "xstring", r->name); + r1->desc = create_obj_by_typename(mgr, "xstring", r->desc); + if ((r1->name == NULL) || (r->desc != NULL && r1->desc == NULL)) { + res_obj_killer(r1); + SET_ERROR(ENOMEM); + return NULL; + } + r1->valtype = r->valtype; + r1->ordtype = r->ordtype; + r1->flags = r->flags; + r1->interval = r->interval; + r1->last_report = r->last_report; + + r1->ndistributions = r->ndistributions; + if (r->distributions) { + MOVE_PTR(r1->distributions, r->distributions); + } + MOVE_PTR(r1->hbased_defined, r->hbased_defined); + idx = resmgr_add_res(resmgr, r1); + + obj_init(mgr, type, idx, obj); + + return r1; +} + +void res_obj_killer(xce_resource_t* r) +{ + myobj_t* obj; + + if (r == NULL) { + return; + } + + obj = TO_MYOBJ(r); + DEREF_OBJ(r->name); + DEREF_OBJ(r->desc); + if (r->hbased_defined) { + if (r->hbased_defined->which == Value_u_svalue) { + free(r->hbased_defined->u.svalue); + } + free(r->hbased_defined); + } + + if (r->distributions != NULL) { + for (int i = 0; i < r->ndistributions; i++) { + if (r->distributions[i] != NULL) { + res_distribution_t* dist = r->distributions[i]; + if (dist->hosts != NULL) { + for (int j = 0; j < dist->nhosts; j++) { + FREEUP(dist->hosts[j]); + } + FREEUP(dist->hosts); + } + if (dist->value != NULL) { + if (dist->value->which == Value_u_svalue) { + FREEUP(dist->value->u.svalue); + } + free(dist->value); + } + FREEUP(dist); + } + } + FREEUP(r->distributions); + } + memset(obj, 0x0, sizeof(*obj)); + FREEUP(obj); +} + +static void xres_fini(void* p) +{ + myobj_t* myobj; + objtype_t* typeval; + xce_resource_t* r; + resmgr_t* resmgr; + + if (p == NULL) { + return; + } + + r = (xce_resource_t*)p; + myobj = TO_MYOBJ(p); + typeval = myobj->type; + resmgr = &(((cluster_t*)typeval->typedata)->resmgr); + if (resmgr == NULL) { + return; + } + + resmgr_del_res(resmgr, r); + + return; +} + +static void xres_type_fini(objtype_t* typeval) +{ + return; +} + +void worker_obj_killer(worker_t* w) +{ + myobj_t* obj; + + if (w == NULL) { + return; + } + + obj = TO_MYOBJ(w); + + DEREF_OBJ(w->host.name); + + if ((w->host.thresholds != NULL) && (w->host.nthresholds > 0)) { + free_resource_thresholds(w->host.thresholds, + w->host.nthresholds); + w->host.thresholds = NULL; + w->host.nthresholds = 0; + } + + if (w->host.dispatch_window_config != NULL) { + free_timewindow_config(w->host.dispatch_window_config); + w->host.dispatch_window_config = NULL; + } + + if (w->host.attrs != NULL) { + DEREF_OBJ(w->host.attrs->hosttype); + DEREF_OBJ(w->host.attrs->hostmodel); + FREEUP(w->host.attrs); + } + + zarray_fini(&(w->host.resvals)); + + memset(obj, 0x0, sizeof(*obj)); + FREEUP(obj); +} + +static void kill_resval(void* p) +{ + if (p == NULL) { + return; + } + + /* FIXME: improve resval free */ + free(p); +} + +static void* xworker_create(objmgr_t* mgr, + objtype_t* typeval, + va_list ap) +{ + workermgr_t* workermgr = &(((cluster_t*)typeval->typedata)->workermgr); + worker_t* w = va_arg(ap, worker_t*); + worker_t* w1 = NULL; + myobj_t* obj; + int32_t idx; + + if (w == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + w1 = workermgr_lock(workermgr, w->host.name, OBJREF_UPDATE); + if (w1 == NULL) { + obj = obj_alloc(sizeof(*w1)); + if (obj == NULL) { + return NULL; + } + + w1 = MYOBJ_TO_OBJ(worker_t*, obj); + memcpy(w1, w, sizeof(*w1)); + w1->host.attrs = (hostattrs_t*)calloc(1, sizeof(hostattrs_t)); + if (w1->host.attrs == NULL) { + free(obj); + + return NULL; + } + + w1->host.name = create_obj_by_typename(mgr, "xstring", + w->host.name); + + idx = workermgr_add(workermgr, w1); + obj_init(mgr, typeval, idx, obj); + w1->idx = idx; + + /* gain update lock */ + w1 = obj_ref(w1, OBJREF_UPDATE); + w1->host.attrs->hosttype = NULL; + w1->host.attrs->hostmodel = NULL; + zarray_init(&(w1->host.resvals), 64, + kill_resval); + } + + DEREF_OBJ(w1->host.attrs->hosttype); + DEREF_OBJ(w1->host.attrs->hostmodel); + + if (w->host.attrs) { + memcpy(w1->host.attrs, w->host.attrs, sizeof(*(w1->host.attrs))); + + w1->host.attrs->hosttype = create_obj_by_typename(mgr, "xstring", + w->host.attrs->hosttype); + w1->host.attrs->hostmodel = create_obj_by_typename(mgr, "xstring", + w->host.attrs->hostmodel); + } else { + w1->host.attrs->hosttype = create_obj_by_typename(mgr, "xstring", "Unknown"); + w1->host.attrs->hostmodel = create_obj_by_typename(mgr, "xstring", "Unknown"); + } + + /* clang-format off */ + if ((w1->host.name == NULL) || + (w1->host.attrs->hosttype == NULL) || + (w1->host.attrs->hostmodel == NULL)) { + /* clang-format on */ + + worker_obj_killer(w1); + SET_ERROR(ENOMEM); + return NULL; + } + + return w1; +} +static void xworker_fini(void* p) +{ + myobj_t* myobj; + objtype_t* typeval; + worker_t* w; + workermgr_t* mgr; + + if (p == NULL) { + return; + } + + w = (worker_t*)p; + myobj = TO_MYOBJ(p); + typeval = myobj->type; + mgr = &(((cluster_t*)typeval->typedata)->workermgr); + + if (mgr == NULL) { + return; + } + + workermgr_del(mgr, w); + return; +} + +static void xworker_type_fini(objtype_t* typeval) +{ + return; +} + +void queue_obj_killer(queue_t* q) +{ + myobj_t* obj; + + if (q == NULL) { + return; + } + + obj = TO_MYOBJ(q); + DEREF_OBJ(q->queue.queue); + if (q->queue.thresholds) { + for (int i = 0; i < q->queue.nthresholds; i++) { + if (q->queue.thresholds[i]) { + FREEUP(q->queue.thresholds[i]->name); + free(q->queue.thresholds[i]); + } + } + free(q->queue.thresholds); + } + q->queue.nthresholds = 0; + if (q->queue.runwindow) { + FREEUP(q->queue.runwindow->ranges); + q->queue.runwindow->nsegs = 0; + free(q->queue.runwindow); + } + + if (q->queue.dispatchwindow) { + FREEUP(q->queue.dispatchwindow->ranges); + q->queue.dispatchwindow->nsegs = 0; + free(q->queue.dispatchwindow); + } + DEREF_OBJ(q->queue.pre_exec); + DEREF_OBJ(q->queue.post_exec); + if (q->queue.users) { + DEREF_OBJ(q->queue.users->name); + for (int i = 0; i < q->queue.users->members.count; i++) { + DEREF_OBJ(q->queue.users->members.data[i]); + } + + FREEUP(q->queue.users->members.data); + + for (int i = 0; i < q->queue.users->ex_members.count; i++) { + DEREF_OBJ(q->queue.users->ex_members.data[i]); + } + + FREEUP(q->queue.users->ex_members.data); + + free(q->queue.users); + } + if (q->queue.hosts) { + DEREF_OBJ(q->queue.hosts->name); + for (int i = 0; i < q->queue.hosts->members.count; i++) { + DEREF_OBJ(q->queue.hosts->members.data[i]); + } + + FREEUP(q->queue.hosts->members.data); + + for (int i = 0; i < q->queue.hosts->ex_members.count; i++) { + DEREF_OBJ(q->queue.hosts->ex_members.data[i]); + } + + FREEUP(q->queue.hosts->ex_members.data); + + free(q->queue.hosts); + } + + if (q->queue.runwindow_config != NULL) { + free_timewindow_config(q->queue.runwindow_config); + q->queue.runwindow_config = NULL; + } + + if (q->queue.dispatchwindow_config != NULL) { + free_timewindow_config(q->queue.dispatchwindow_config); + q->queue.dispatchwindow_config = NULL; + } + + memset(obj, 0x0, sizeof(*obj)); + FREEUP(obj); +} + +static void* xqueue_create(objmgr_t* mgr, + objtype_t* typeval, + va_list ap) +{ + qmgr_t* qmgr = &(((cluster_t*)typeval->typedata)->qmgr); + queue_t* q = va_arg(ap, queue_t*); + queue_t* q1 = NULL; + myobj_t* obj; + int32_t idx; + + if (q == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + q1 = qmgr_lockq(qmgr, q->queue.queue, OBJREF_READ); + if (q1 != NULL) { + SET_ERROR(EEXIST); + DEREF_OBJ(q1); + + return NULL; + } + + obj = obj_alloc(sizeof(*q1)); + if (obj == NULL) { + SET_ERROR(ENOMEM); + return NULL; + } + + q1 = MYOBJ_TO_OBJ(queue_t*, obj); + q1->queue.queue = create_obj_by_typename(mgr, "xstring", + q->queue.queue); + if (q1->queue.queue == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + q1->queue.attrs = q->queue.attrs; + q1->queue.status = q->queue.status; + q1->queue.priority = q->queue.priority; + q1->queue.accept_interval = q->queue.accept_interval; + q1->queue.sched_delay = q->queue.sched_delay; + q1->queue.resreq = NULL; + q1->queue.pre_exec = NULL; + q1->queue.post_exec = NULL; + + if (q->queue.resreq != NULL) { + q1->queue.resreq = create_obj_by_typename(mgr, "xstring", + q->queue.resreq); + if (q1->queue.resreq == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + } + + if (q->queue.pre_exec != NULL) { + q1->queue.pre_exec = create_obj_by_typename(mgr, "xstring", + q->queue.pre_exec); + if (q1->queue.pre_exec == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + } + + if (q->queue.post_exec != NULL) { + q1->queue.post_exec = create_obj_by_typename(mgr, "xstring", + q->queue.post_exec); + if (q1->queue.post_exec == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + } + q1->queue.pending_ctrl = q->queue.pending_ctrl; + + q1->queue.nthresholds = 0; + q1->queue.thresholds = NULL; + if ((q->queue.nthresholds > 0) && (q->queue.thresholds != NULL)) { + q1->queue.thresholds = dup_resource_thresholds(q->queue.thresholds, + q->queue.nthresholds); + if (q1->queue.thresholds != NULL) { + q1->queue.nthresholds = q->queue.nthresholds; + } + } + + if (q->queue.users != NULL) { + q1->queue.users = (xce_group_t*)calloc(1, sizeof(xce_group_t)); + if (q1->queue.users == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + if (q->queue.users->name) { + q1->queue.users->name = create_obj_by_typename(mgr, "xstring", + q->queue.users->name); + } + q1->queue.users->members.count = q->queue.users->members.count; + q1->queue.users->members.data = (char**)calloc(q1->queue.users->members.count, sizeof(char*)); + if (q1->queue.users->members.data == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + for (int i = 0; i < q->queue.users->members.count; i++) { + if (q->queue.users->members.data[i] == NULL) { + continue; + } + q1->queue.users->members.data[i] = create_obj_by_typename(mgr, "xstring", + q->queue.users->members.data[i]); + } + q1->queue.users->ex_members.count = q->queue.users->ex_members.count; + q1->queue.users->ex_members.data = (char**)calloc(q1->queue.users->ex_members.count, sizeof(char*)); + if (q1->queue.users->ex_members.data == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + for (int i = 0; i < q->queue.users->ex_members.count; i++) { + if (q->queue.users->ex_members.data[i] == NULL) { + continue; + } + q1->queue.users->ex_members.data[i] = create_obj_by_typename(mgr, "xstring", + q->queue.users->ex_members.data[i]); + } + } + if (q->queue.hosts != NULL) { + q1->queue.hosts = (xce_group_t*)calloc(1, sizeof(xce_group_t)); + if (q1->queue.hosts == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + if (q->queue.hosts->name) { + q1->queue.hosts->name = create_obj_by_typename(mgr, "xstring", + q->queue.hosts->name); + } + q1->queue.hosts->members.count = q->queue.hosts->members.count; + q1->queue.hosts->members.data = (char**)calloc(q1->queue.hosts->members.count, sizeof(char*)); + if (q1->queue.hosts->members.data == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + for (int i = 0; i < q->queue.hosts->members.count; i++) { + if (q->queue.hosts->members.data[i] == NULL) { + continue; + } + q1->queue.hosts->members.data[i] = create_obj_by_typename(mgr, "xstring", + q->queue.hosts->members.data[i]); + } + q1->queue.hosts->ex_members.count = q->queue.hosts->ex_members.count; + q1->queue.hosts->ex_members.data = (char**)calloc(q1->queue.hosts->ex_members.count, sizeof(char*)); + if (q1->queue.hosts->ex_members.data == NULL) { + queue_obj_killer(q1); + SET_ERROR(ENOMEM); + return NULL; + } + for (int i = 0; i < q->queue.hosts->ex_members.count; i++) { + if (q->queue.hosts->ex_members.data[i] == NULL) { + continue; + } + q1->queue.hosts->ex_members.data[i] = create_obj_by_typename(mgr, "xstring", + q->queue.hosts->ex_members.data[i]); + } + } + + q1->queue.runwindow_config = NULL; + if (q->queue.runwindow_config != NULL) { + q1->queue.runwindow_config = dup_timewindow_config(q->queue.runwindow_config); + } + + q1->queue.dispatchwindow_config = NULL; + if (q->queue.dispatchwindow_config != NULL) { + q1->queue.dispatchwindow_config = dup_timewindow_config(q->queue.dispatchwindow_config); + } + + /* leave run windows parse & dispatch parse later */ + idx = qmgr_add(qmgr, q1); + obj_init(mgr, typeval, idx, obj); + + return q1; +} +static void xqueue_fini(void* p) +{ + myobj_t* myobj; + objtype_t* typeval; + queue_t* q; + qmgr_t* mgr; + + if (p == NULL) { + return; + } + + q = (queue_t*)p; + myobj = TO_MYOBJ(q); + typeval = myobj->type; + mgr = &(((cluster_t*)typeval->typedata)->qmgr); + + if (mgr == NULL) { + return; + } + + qmgr_delq(mgr, q); + + return; +} +static void xqueue_type_fini(objtype_t* typeval) +{ + return; +} + +static void* xresval_create(objmgr_t* mgr, + objtype_t* typeval, + va_list ap) +{ + return NULL; +} +static void xresval_fini(void* r) +{ + return; +} +static void xresval_type_fini(objtype_t* typeval) +{ + return; +} + +void config_obj_killer(xce_config_t* c) +{ + myobj_t* obj; + + if (c == NULL) { + return; + } + + obj = TO_MYOBJ(c); + DEREF_OBJ(c->name); + DEREF_OBJ(c->value); + memset(obj, 0x0, sizeof(*obj)); + FREEUP(obj); +} + +static void* xconfig_create(objmgr_t* mgr, + objtype_t* typeval, + va_list ap) +{ + cluster_t* c = (cluster_t*)typeval->typedata; + xce_config_t* kv = va_arg(ap, xce_config_t*); + myobj_t* obj; + xce_config_t* kv1 = NULL; + kvmgr_t* kvmgr = NULL; + uint32_t idx; + + if (kv == NULL) { + SET_ERROR(EINVAL); + return NULL; + } + + kvmgr = &(c->configmgr); + + kv1 = kvmgr_lock(kvmgr, kv->name, OBJREF_READ); + if (kv1 != NULL) { + SET_ERROR(EEXIST); + DEREF_OBJ(kv1); + + return NULL; + } + + obj = obj_alloc(sizeof(*kv1)); + if (obj == NULL) { + SET_ERROR(ENOMEM); + return NULL; + } + + kv1 = MYOBJ_TO_OBJ(xce_config_t*, obj); + kv1->name = create_obj_by_typename(mgr, "xstring", + kv->name); + kv1->value = create_obj_by_typename(mgr, "xstring", + kv->value); + if ((kv1->name == NULL) || (kv1->value == NULL)) { + config_obj_killer(kv1); + SET_ERROR(ENOMEM); + return NULL; + } + + idx = kvmgr_add(kvmgr, kv1->name, kv1); + obj_init(mgr, typeval, idx, obj); + + return kv1; +} +static void xconfig_fini(void* p) +{ + myobj_t* myobj; + objtype_t* typeval; + xce_config_t* config; + kvmgr_t* kvmgr; + + if (p == NULL) { + return; + } + + config = (xce_config_t*)p; + myobj = TO_MYOBJ(p); + typeval = myobj->type; + kvmgr = &(((cluster_t*)typeval->typedata)->configmgr); + + kvmgr_del(kvmgr, config->name); + + return; +} +static void xconfig_type_fini(objtype_t* typeval) +{ + return; +} + +static void* xgroup_create(objmgr_t* mgr, + objtype_t* typeval, + va_list ap) +{ + return NULL; +} +static void xgroup_fini(void* g) +{ + return; +} +static void xgroup_type_fini(objtype_t* typeval) +{ + return; +} diff --git a/src/flake.lock b/src/flake.lock index ab60de4..3cbdb13 100644 --- a/src/flake.lock +++ b/src/flake.lock @@ -77,11 +77,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1762246108, - "narHash": "sha256-99qtmUXV4UZcUc8klD+n0MCBU52DwxpqHf/KjXcd4vQ=", + "lastModified": 1762312034, + "narHash": "sha256-ocI3qyR1GSIS3cOZihGTrlOOllo8fHZx9sC1izpbN2A=", "owner": "shen390s", "repo": "my-nix-channel", - "rev": "de44a16949de901790ae712c1a1cc39ccc960fae", + "rev": "f7244c7e451310b6440b4aedcf84b5f239939d1d", "type": "github" }, "original": { diff --git a/src/include/myconf.h b/src/include/myconf.h new file mode 100644 index 0000000..84b5874 --- /dev/null +++ b/src/include/myconf.h @@ -0,0 +1,31 @@ +#if !defined(_MYCONF_H_) + +#define _MYCONF_H_ 1 + +#define MAX_MASTER_CANDIDATES (64) + +#include + +#include "clusterconfig.h" + +struct logging { + char* logdir; + char* loglevel; +}; + +typedef struct { + struct logging* logging; +} initial_config_t; + +typedef struct { + initial_config_t* init_config; + cluster_config_t* cluster_config; + char* conffile; +} myconf_t; + +extern cluster_config_t* get_default_conf(void*); +extern int add_config(void*, char*, char*); +extern int load_initial_config(char*, initial_config_t**, int); +extern void free_initial_config(initial_config_t*); + +#endif diff --git a/src/include/objs.h b/src/include/objs.h index 0bb13a1..288914e 100644 --- a/src/include/objs.h +++ b/src/include/objs.h @@ -4,6 +4,6 @@ #include "xce/types.h" -job_pidinfo_t *new_job_pidinfo(job_pidinfo_t *); +job_pidinfo_t* new_job_pidinfo(job_pidinfo_t*); #endif diff --git a/src/include/request.h b/src/include/request.h index 0c4316f..482ff61 100644 --- a/src/include/request.h +++ b/src/include/request.h @@ -77,6 +77,6 @@ extern int extract_data(dataset_t*, xce_data_t*); extern int xce_update_req(ioctx_t*, xce_update_data_t*); extern int xce_jobctrl_req(ioctx_t*, xce_jobctrl_t*); extern int xce_queuectrl_req(ioctx_t*, xce_queuectrl_t*); -extern int xce_runtask_req(ioctx_t* , xce_taskrun_t *) ; +extern int xce_runtask_req(ioctx_t*, xce_taskrun_t*); #endif diff --git a/src/include/utils.h b/src/include/utils.h index 395b54e..e40df14 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -6,8 +6,8 @@ #include #include -#include #include +#include #include #include "controller.capnp.h" diff --git a/src/include/vtaskmgr.h b/src/include/vtaskmgr.h index d530be0..8bd1623 100644 --- a/src/include/vtaskmgr.h +++ b/src/include/vtaskmgr.h @@ -55,7 +55,7 @@ extern int vtask_enq(vtaskmgr_t*, vtask_t*); extern int vtaskmgr_init(vtaskmgr_t*, int); extern int vtaskmgr_start(vtaskmgr_t*); extern void vtaskmgr_fini(vtaskmgr_t*); -extern void vtaskmgr_wait(vtaskmgr_t *); -extern void vtaskmgr_wakeup(vtaskmgr_t *); +extern void vtaskmgr_wait(vtaskmgr_t*); +extern void vtaskmgr_wakeup(vtaskmgr_t*); #endif diff --git a/src/include/xce/cluster_def.h b/src/include/xce/cluster_def.h index 770fde4..d46121d 100644 --- a/src/include/xce/cluster_def.h +++ b/src/include/xce/cluster_def.h @@ -26,7 +26,6 @@ enum { XCE_CONTROLLER_HOOK_SCHEDULER, XCE_CONTROLLER_HOOK_CLUSTER, XCE_CONTROLLER_HOOK_RESOURCE, - XCE_CONTROLLER_HOOK_CTRL_REQ, XCE_CONTROLLER_HOOK_QUEUE, XCE_CONTROLLER_HOOK_USERGROUP, XCE_CONTROLLER_HOOK_HOSTGROUP, @@ -46,12 +45,9 @@ enum { HOOK_DETAIL_SCHEDULER_JOIN, HOOK_DETAIL_SCHEDULER_DEAD, HOOK_DETAIL_CLUSTER_INIT, - HOOK_DETAIL_DB_INIT, HOOK_DETAIL_CLUSTER_SHUTDOWN, - HOOK_DETAIL_DB_SHUTDOWN, HOOK_DETAIL_RESOURCE_UPDATE, HOOK_DETAIL_RESOURCE_LOC_UPDATE, - HOOK_DETAIL_CTRL_REQ_FINISH, HOOK_DETAIL_QUEUE_UPDATE, HOOK_DETAIL_USERGROUP_UPDATE, HOOK_DETAIL_HOSTGROUP_UPDATE @@ -116,27 +112,6 @@ typedef struct { endpoint_t* ep; } scheduler_t; -typedef struct { - DBPool* db_pool; - DBHandle* db_handle; - pthread_t fetch_thread; - int64_t fetch_job_pos; - int fetch_job_interval; - int running; - mylock_t db_lock; -} db_ctx_t; - -typedef struct { - pthread_t thread; - int runing; - int interval; -} ctrlreq_ctx_t; - -typedef struct { - mylock_t lock; - ctrlreq_ctx_t ctrlreq_ctx; -} ctrl_req_mgr_t; - typedef struct { myconf_t* myconf; xce_peerinfo_t* myhost; @@ -161,8 +136,6 @@ typedef struct { vtaskmgr_t vtaskmgr; cbuf_t wbuf; int running; - db_ctx_t dbctx; - ctrl_req_mgr_t ctrl_req_mgr; } cluster_t; #endif /* XCE_CLUSTER_DEF_H */ diff --git a/src/include/xce/errnox.h b/src/include/xce/errnox.h index eadee39..ceecd41 100644 --- a/src/include/xce/errnox.h +++ b/src/include/xce/errnox.h @@ -3,10 +3,10 @@ #define __XCE_ERRNOX_H__ 1 #include -#include #include +#include -#define XCE_ERRNO_BASE (10001) +#define XCE_ERRNO_BASE (10001) enum { E_XCE_BAD_JOB = XCE_ERRNO_BASE, @@ -21,6 +21,5 @@ enum { E_XCE_INTERNAL, E_XCE_MAX }; - #endif diff --git a/src/include/xstring.h b/src/include/xstring.h index 524482a..4bf83f7 100644 --- a/src/include/xstring.h +++ b/src/include/xstring.h @@ -10,16 +10,16 @@ typedef struct { } xstring_typedata_t; #define XSTRING_OBJ_TYPE_HTAB(obj) \ - (((xstring_typedata_t *)obj->type->typedata)->tab) + (((xstring_typedata_t*)obj->type->typedata)->tab) #define XSTRING_OBJ_TYPE_STRINGS(obj) \ - (((xstring_typedata_t *)obj->type->typedata)->strings) + (((xstring_typedata_t*)obj->type->typedata)->strings) #define XSTRING_TYPE_HTAB(type) \ - (((xstring_typedata_t *)type->typedata)->tab) + (((xstring_typedata_t*)type->typedata)->tab) #define XSTRING_TYPE_STRINGS(type) \ - (((xstring_typedata_t *)type->typedata)->strings) + (((xstring_typedata_t*)type->typedata)->strings) -extern int register_xstring(objmgr_t*, int,int); +extern int register_xstring(objmgr_t*, int, int); #endif diff --git a/src/lib/dataq.c b/src/lib/dataq.c index 0b3a26f..b6b5034 100644 --- a/src/lib/dataq.c +++ b/src/lib/dataq.c @@ -106,14 +106,14 @@ void dataq_fini(dataq_t* q) { data_item_t* it; while ((it = dataq_deq_head(q)) != NULL) { - if (q->killer != NULL) { + if (q->killer != NULL) { (q->killer)(it); } // Note: dataq_deq_head already isolates 'it' (it->next = NULL) } q->head = q->tail = NULL; // Explicitly set just in case. - q->killer = NULL; - + q->killer = NULL; + return; } diff --git a/src/lib/hook.c b/src/lib/hook.c index 7d7f5d1..9c7f5b3 100644 --- a/src/lib/hook.c +++ b/src/lib/hook.c @@ -65,7 +65,7 @@ int xce_add_hook(hooksys_t* s, int xce_run_hooks(hooksys_t* s, int type, int situation, - int detail, + int detail, ...) { xlist_t* l; @@ -84,17 +84,17 @@ int xce_run_hooks(hooksys_t* s, for (i = 0; i < l->count; i++) { hook_item_t* it = l->data[i]; - va_list ap; + va_list ap; if (it == NULL) { continue; } - va_start(ap, detail); + va_start(ap, detail); rc = (*(it->fn))(type, situation, detail, it->data, ap); - va_end(ap); - + va_end(ap); + if ((situation == HOOK_BEFORE_EVENT) && (rc < 0)) { break; } @@ -103,10 +103,11 @@ int xce_run_hooks(hooksys_t* s, return rc; } -static int free_hook_item(void *data1, - void *data2) { +static int free_hook_item(void* data1, + void* data2) +{ if (data1 == NULL) { - return -EINVAL; + return -EINVAL; } free(data1); @@ -126,9 +127,9 @@ void xce_hook_fini(hooksys_t* s) } for (i = 0; i < s->maxtypes; i++) { - xlist_iter(&(s->hooks[i]), - (xlist_iter_fn_t)free_hook_item, - NULL); + xlist_iter(&(s->hooks[i]), + (xlist_iter_fn_t)free_hook_item, + NULL); xlist_fini(&(s->hooks[i])); } diff --git a/src/lib/mylock.c b/src/lib/mylock.c index 76fa676..b636ee0 100644 --- a/src/lib/mylock.c +++ b/src/lib/mylock.c @@ -171,41 +171,46 @@ int cbuf_get(cbuf_t* b, return 0; } -int my_rwlock_init(my_rwlock_t *l) { - if ( l == NULL) { - return -EINVAL; +int my_rwlock_init(my_rwlock_t* l) +{ + if (l == NULL) { + return -EINVAL; } return pthread_rwlock_init(l, NULL); } -void my_rwlock_fini(my_rwlock_t *l) { +void my_rwlock_fini(my_rwlock_t* l) +{ if (l == NULL) { - return; + return; } pthread_rwlock_destroy(l); } -int my_rwlock_unlock(my_rwlock_t *l) { - if (l == NULL) { - return -EINVAL; +int my_rwlock_unlock(my_rwlock_t* l) +{ + if (l == NULL) { + return -EINVAL; } return pthread_rwlock_unlock(l); } -int my_rwlock_rlock(my_rwlock_t *l) { +int my_rwlock_rlock(my_rwlock_t* l) +{ if (l == NULL) { - return -EINVAL; + return -EINVAL; } return pthread_rwlock_rdlock(l); } -int my_rwlock_wlock(my_rwlock_t *l) { - if (l==NULL) { - return -EINVAL; +int my_rwlock_wlock(my_rwlock_t* l) +{ + if (l == NULL) { + return -EINVAL; } return pthread_rwlock_wrlock(l); diff --git a/src/lib/netio.c b/src/lib/netio.c index b2420a0..e46a0aa 100644 --- a/src/lib/netio.c +++ b/src/lib/netio.c @@ -663,7 +663,7 @@ int iomgr_stop(iomgr_t* iomgr) if (iomgr->worker == 0) { return 0; } - + rc = pthread_cancel(iomgr->worker); if (rc != 0) { SET_ERROR(errno); diff --git a/src/lib/objmgr.c b/src/lib/objmgr.c index 6f319d1..f6d0fec 100644 --- a/src/lib/objmgr.c +++ b/src/lib/objmgr.c @@ -309,14 +309,15 @@ int objmgr_set_default(objmgr_t* mgr) return 0; } -myobj_t *obj_alloc(size_t size) { - myobj_t *obj; - +myobj_t* obj_alloc(size_t size) +{ + myobj_t* obj; + if (size <= 0) { - SET_ERROR(EINVAL); - return NULL; + SET_ERROR(EINVAL); + return NULL; } - obj = (myobj_t *)calloc(1, sizeof(*obj) + size); + obj = (myobj_t*)calloc(1, sizeof(*obj) + size); return obj; } diff --git a/src/lib/request.c b/src/lib/request.c index 908d98c..2bf869b 100644 --- a/src/lib/request.c +++ b/src/lib/request.c @@ -163,7 +163,7 @@ int xce_reply_with_ctx(ioctx_t* ctx, TRACE("%s seq %lu opcode %d status %d data which %d", __func__, reply->seq, reply->opcode, reply->status, reply->data_which); - + request.from = from; request.body.replymsg = reply; @@ -629,7 +629,7 @@ int extract_data(dataset_t* dataset, int count = 0; TRACE("%s ...", __func__); - + switch (getdata->table) { case Data_data_schedjobs: case Data_data_jobs: diff --git a/src/lib/testsuite/test_error.c b/src/lib/testsuite/test_error.c index db71cfc..10a7d54 100644 --- a/src/lib/testsuite/test_error.c +++ b/src/lib/testsuite/test_error.c @@ -2,12 +2,13 @@ #include #include -#include "xce/errnox.h" #include "utils.h" +#include "xce/errnox.h" -static void test_error_setget(void) { - int myerrno =1; - ectx_t *e; +static void test_error_setget(void) +{ + int myerrno = 1; + ectx_t* e; SET_ERROR(myerrno); e = get_ectx(); @@ -34,7 +35,8 @@ static void test_strerror_r(void) { } } #endif -void test_error(void) { +void test_error(void) +{ printf("\n>> error test\n\n"); RUN_TEST(test_error_setget); diff --git a/src/lib/testsuite/test_htab.c b/src/lib/testsuite/test_htab.c index b49f038..fe0dcab 100644 --- a/src/lib/testsuite/test_htab.c +++ b/src/lib/testsuite/test_htab.c @@ -2,48 +2,52 @@ #include #include -#include #include "htab.h" +#include static htab_t htab; -static char *s = "Hello,world"; +static char* s = "Hello,world"; -static void test_ht_add_string(void) { - hent_t *e = NULL; +static void test_ht_add_string(void) +{ + hent_t* e = NULL; - e = ht_add(&(htab), (void *)s); + e = ht_add(&(htab), (void*)s); TEST_ASSERT_NOT_NULL(e); - + return; } -static void test_ht_get_string(void) { - hent_t *e = NULL; +static void test_ht_get_string(void) +{ + hent_t* e = NULL; - e = ht_get(&htab, (void *)s); + e = ht_get(&htab, (void*)s); TEST_ASSERT_NOT_NULL(e); TEST_ASSERT_TRUE(0 == strcmp(e->key, s)); - + return; } -static void test_ht_del_string(void) { - hent_t *e = NULL; - - ht_del(&htab, (void *)s); +static void test_ht_del_string(void) +{ + hent_t* e = NULL; + + ht_del(&htab, (void*)s); + + e = ht_get(&htab, (void*)s); + TEST_ASSERT_TRUE(e == NULL); - e = ht_get(&htab, (void *)s); - TEST_ASSERT_TRUE( e == NULL); - return; } -void test_ht_string(void) { +void test_ht_string(void) +{ printf("\n>> htab test string\n\n"); ht_init(&htab, 13, string_hash_key); - + RUN_TEST(test_ht_add_string); RUN_TEST(test_ht_get_string); RUN_TEST(test_ht_del_string); diff --git a/src/lib/testsuite/test_htab_int.c b/src/lib/testsuite/test_htab_int.c index ff86850..06c2631 100644 --- a/src/lib/testsuite/test_htab_int.c +++ b/src/lib/testsuite/test_htab_int.c @@ -1,7 +1,7 @@ +#include #include #include #include -#include #include #include "htab.h" @@ -10,42 +10,46 @@ static htab_t htab; uint64_t key = (uint64_t)0xfabcde6666666666; -static void test_ht_add_string(void) { - hent_t *e = NULL; +static void test_ht_add_string(void) +{ + hent_t* e = NULL; - e = ht_add(&(htab), (void *)key); + e = ht_add(&(htab), (void*)key); TEST_ASSERT_NOT_NULL(e); - + return; } -static void test_ht_get_string(void) { - hent_t *e = NULL; +static void test_ht_get_string(void) +{ + hent_t* e = NULL; - e = ht_get(&htab, (void *)key); + e = ht_get(&htab, (void*)key); TEST_ASSERT_NOT_NULL(e); TEST_ASSERT_TRUE((uint64_t)e->key == (uint64_t)key); - + return; } -static void test_ht_del_string(void) { - hent_t *e = NULL; - - ht_del(&htab, (void *)key); +static void test_ht_del_string(void) +{ + hent_t* e = NULL; + + ht_del(&htab, (void*)key); + + e = ht_get(&htab, (void*)key); + TEST_ASSERT_TRUE(e == NULL); - e = ht_get(&htab, (void *)key); - TEST_ASSERT_TRUE( e == NULL); - return; } -void test_ht_int(void) { +void test_ht_int(void) +{ printf("\n>> htab test int\n\n"); ht_init(&htab, 13, int_hash_key); - + RUN_TEST(test_ht_add_string); RUN_TEST(test_ht_get_string); RUN_TEST(test_ht_del_string); diff --git a/src/lib/testsuite/test_xstring.c b/src/lib/testsuite/test_xstring.c index d75db0e..07c574e 100644 --- a/src/lib/testsuite/test_xstring.c +++ b/src/lib/testsuite/test_xstring.c @@ -15,7 +15,7 @@ void test_xstring_create(void) myobj_t* obj; objmgr_init(&mgr); - register_xstring(&mgr,4,4); + register_xstring(&mgr, 4, 4); TEST_ASSERT_EQUAL(1, mgr.types.count); s2 = create_obj_by_typename(&mgr, "xstring", s1); @@ -37,7 +37,7 @@ void test_xstring_ref(void) myobj_t* obj; objmgr_init(&mgr); - register_xstring(&mgr,4,4); + register_xstring(&mgr, 4, 4); TEST_ASSERT_EQUAL(1, mgr.types.count); s1 = create_obj_by_typename(&mgr, "xstring", @@ -63,7 +63,7 @@ void test_xstring_deref(void) myobj_t* obj; objmgr_init(&mgr); - register_xstring(&mgr,4,4); + register_xstring(&mgr, 4, 4); TEST_ASSERT_EQUAL(1, mgr.types.count); s2 = create_obj_by_typename(&mgr, "xstring", s1); @@ -73,12 +73,12 @@ void test_xstring_deref(void) TEST_ASSERT_EQUAL(0, rc); if (rc == 0) { - objtype_t *typeval; + objtype_t* typeval; - obj = TO_MYOBJ(s2); - typeval = obj->type; + obj = TO_MYOBJ(s2); + typeval = obj->type; - obj_deref(s2); + obj_deref(s2); TEST_ASSERT_EQUAL(0, XSTRING_TYPE_STRINGS(typeval).count); } @@ -95,7 +95,7 @@ void test_xstring_share(void) int rc = 0; objmgr_init(&mgr); - register_xstring(&mgr,4,4); + register_xstring(&mgr, 4, 4); TEST_ASSERT_EQUAL(1, mgr.types.count); s2 = create_obj_by_typename(&mgr, "xstring", s1); diff --git a/src/lib/testsuite/test_zarray.c b/src/lib/testsuite/test_zarray.c index 5b7788d..1348622 100644 --- a/src/lib/testsuite/test_zarray.c +++ b/src/lib/testsuite/test_zarray.c @@ -5,38 +5,41 @@ #include "zarray.h" -static void test_zarray_create(void) { +static void test_zarray_create(void) +{ zarray_t array; zarray_init(&array, 8, NULL); - TEST_ASSERT_EQUAL(0,array.count); + TEST_ASSERT_EQUAL(0, array.count); TEST_ASSERT_NULL(array.data); } -static void test_zarray_store(void) { +static void test_zarray_store(void) +{ zarray_t array; - int idx; - char *s1 = "hello"; - char *s2 = NULL; + int idx; + char* s1 = "hello"; + char* s2 = NULL; zarray_init(&array, 8, NULL); - idx = zarray_store(&array, (void *)s1); + idx = zarray_store(&array, (void*)s1); TEST_ASSERT_TRUE(idx >= 0); - s2 = (char *)zarray_get(&array, idx); + s2 = (char*)zarray_get(&array, idx); TEST_ASSERT_NOT_NULL(s2); TEST_ASSERT_EQUAL(0, strcmp(s1, s2)); zarray_fini(&(array)); } -static void test_zarray_del(void) { +static void test_zarray_del(void) +{ zarray_t array; - int idx; - char *s1 = "hello"; + int idx; + char* s1 = "hello"; zarray_init(&array, 8, NULL); - idx = zarray_store(&array, (void *)s1); + idx = zarray_store(&array, (void*)s1); TEST_ASSERT_TRUE(idx >= 0); TEST_ASSERT_TRUE(array.count == 1); zarray_del(&array, idx); @@ -46,52 +49,56 @@ static void test_zarray_del(void) { zarray_fini(&array); } -static void test_zarray_put(void) { +static void test_zarray_put(void) +{ zarray_t array; - int idx; - char *s1 = "hello"; - char *s2 = "test"; + int idx; + char* s1 = "hello"; + char* s2 = "test"; zarray_init(&array, 8, NULL); - idx = zarray_store(&array, (void *)s1); + idx = zarray_store(&array, (void*)s1); TEST_ASSERT_TRUE(idx >= 0); TEST_ASSERT_TRUE(array.count == 1); TEST_ASSERT_TRUE(zarray_get(&array, idx) != NULL); - - TEST_ASSERT_TRUE(strcmp((char *)zarray_get(&array, idx), s1) == 0); - zarray_put(&array, idx, (void *)s2); + TEST_ASSERT_TRUE(strcmp((char*)zarray_get(&array, idx), s1) == 0); + + zarray_put(&array, idx, (void*)s2); TEST_ASSERT_TRUE(zarray_get(&array, idx) != NULL); - TEST_ASSERT_TRUE(strcmp((char *)zarray_get(&array, idx), s2) == 0); + TEST_ASSERT_TRUE(strcmp((char*)zarray_get(&array, idx), s2) == 0); zarray_fini(&array); } -static int sum_(void *arg1, void *arg2) { - int *pv = (int *)arg1; - int x = (int)((long)arg2); +static int sum_(void* arg1, void* arg2) +{ + int* pv = (int*)arg1; + int x = (int)((long)arg2); (*pv) += x; return 0; } -static void test_zarray_iter(void) { +static void test_zarray_iter(void) +{ zarray_t array; - int i; - int val = 0; + int i; + int val = 0; zarray_init(&array, 10, NULL); - for(i = 0; i < 10; i ++) { - zarray_store(&array, (void *)(long)i); + for (i = 0; i < 10; i++) { + zarray_store(&array, (void*)(long)i); } zarray_iter(&array, (zarray_iter_fn_t)sum_, &val); TEST_ASSERT_TRUE(val == 45); - + zarray_fini(&(array)); } -void test_zarray(void) { +void test_zarray(void) +{ printf("\n>> zarray test\n\n"); RUN_TEST(test_zarray_create); diff --git a/src/lib/utils.c b/src/lib/utils.c old mode 100755 new mode 100644 index 2c2bbf4..8a3091c --- a/src/lib/utils.c +++ b/src/lib/utils.c @@ -456,7 +456,7 @@ void free_resval_with_idx_list(xce_val_with_idx_t*** vals, if (val->val->which == Value_u_svalue) { FREEUP(val->val->u.svalue); } - + FREEUP(val->val); FREEUP(val); } @@ -465,10 +465,11 @@ void free_resval_with_idx_list(xce_val_with_idx_t*** vals, *vals = NULL; } -void free_strlist(xce_strlist_t *l) { +void free_strlist(xce_strlist_t* l) +{ int i; - - if ( l == NULL) { + + if (l == NULL) { return; } @@ -476,7 +477,7 @@ void free_strlist(xce_strlist_t *l) { return; } - for(i = 0; i < l->count; i ++) { + for (i = 0; i < l->count; i++) { FREEUP(l->data[i]); } diff --git a/src/lib/vtaskmgr.c b/src/lib/vtaskmgr.c index 6020525..356e98e 100644 --- a/src/lib/vtaskmgr.c +++ b/src/lib/vtaskmgr.c @@ -63,14 +63,14 @@ void vcpu_idle(vcpu_t* vcpu) void run_task(vtask_t* task) { - vtaskmgr_t *vtaskmgr; - + vtaskmgr_t* vtaskmgr; + if (task->entry != NULL) { (task->entry)(task); } endpoint_unblock_tasks(task->ctx.ep); - vtaskmgr = (vtaskmgr_t *)task->owner; + vtaskmgr = (vtaskmgr_t*)task->owner; if (vtaskmgr != NULL) { /* wake up dispatcher */ vtaskmgr_wakeup(vtaskmgr); @@ -209,14 +209,14 @@ static vcpu_t* pick_idle_vcpu(vtaskmgr_t* vtaskmgr) TRACE("%s pick idle cpu #%d", __func__, i); - + return &(vtaskmgr->vcpus[i]); } static int vtask_eligible(data_item_t* data) { - vtask_t* task = ITPTR_TO_VTASK(data); - int eligible = 0; + vtask_t* task = ITPTR_TO_VTASK(data); + int eligible = 0; if (task->ctx.ep == NULL) { WARN("no endpoint for task 0x%p", task); @@ -226,7 +226,7 @@ static int vtask_eligible(data_item_t* data) eligible = !(endpoint_is_task_blocked(task->ctx.ep)); TRACE("%s vtask %p is %s", __func__, task, - eligible?"eligible":"not eligible"); + eligible ? "eligible" : "not eligible"); return eligible; } @@ -283,8 +283,7 @@ static void* vdispatcher_entry(void* arg) bind_vtask(task, vcpu); vcpu = NULL; task = NULL; - } - else { + } else { vtaskmgr_wait(vtaskmgr); } } @@ -308,13 +307,15 @@ int vtaskmgr_start(vtaskmgr_t* vtaskmgr) return 0; } -void vtaskmgr_wait(vtaskmgr_t *vtaskmgr) { +void vtaskmgr_wait(vtaskmgr_t* vtaskmgr) +{ int dummy = 0; - + cbuf_get(&(vtaskmgr->wtbuf), &dummy); } -void vtaskmgr_wakeup(vtaskmgr_t *vtaskmgr) { +void vtaskmgr_wakeup(vtaskmgr_t* vtaskmgr) +{ int dummy = 0; cbuf_tryput(&(vtaskmgr->wtbuf), &dummy); @@ -351,7 +352,7 @@ int vtask_enq(vtaskmgr_t* vtaskmgr, ioctx_ref(&(task->ctx)); rc = dataq_enq(&(vtaskmgr->taskq), - &(task->ctx.link)); + &(task->ctx.link)); if (rc >= 0) { /* wake up task dispatcher */ task->owner = vtaskmgr; diff --git a/src/lib/xceapi.c b/src/lib/xceapi.c index 70c0f80..690c30b 100644 --- a/src/lib/xceapi.c +++ b/src/lib/xceapi.c @@ -633,8 +633,8 @@ int xce_queuectrl(xce_ctx_t* ctx, return status; } -int xce_runtask(xce_ctx_t* ctx, - xce_taskrun_t* runinfo) +int xce_runtask(xce_ctx_t* ctx, + xce_taskrun_t* runinfo) { ioctx_t* ioctx; int rc; diff --git a/src/lib/xlist.c b/src/lib/xlist.c index 0a60b72..d438083 100644 --- a/src/lib/xlist.c +++ b/src/lib/xlist.c @@ -35,10 +35,10 @@ static int xlist_ensure(xlist_t* l) } l->data = p; - for(i = l->caps; i < nc; i ++) { - l->data[i] = NULL; + for (i = l->caps; i < nc; i++) { + l->data[i] = NULL; } - + l->caps = nc; return 1; @@ -64,25 +64,27 @@ int xlist_store(xlist_t* l, return 1; } -void xlist_clear(xlist_t *l) { +void xlist_clear(xlist_t* l) +{ if (l == NULL) { - return; + return; } l->count = 0; } -int xlist_iter(xlist_t *l, - xlist_iter_fn_t fn, - void *data) { +int xlist_iter(xlist_t* l, + xlist_iter_fn_t fn, + void* data) +{ int i; - + if ((l == NULL) || (fn == NULL)) { - return -EINVAL; + return -EINVAL; } - for(i = 0; i < l->count; i ++) { - (*fn)((void *)l->data[i], data); + for (i = 0; i < l->count; i++) { + (*fn)((void*)l->data[i], data); } return 0; diff --git a/src/meson.build b/src/meson.build index bfebe34..2c6f337 100644 --- a/src/meson.build +++ b/src/meson.build @@ -14,15 +14,11 @@ if get_option('with_debug') with_debug=true endif -if get_option('without_db') - add_project_arguments('-DWITHOUT_DB=1', language: 'c') -endif - add_project_arguments('-DVERSION_COMMIT_ID='+get_option('commit_id'), language: 'c') include_dir = include_directories('include', 'include/xce') subdir('lib') -# subdir('controller') +subdir('controller') # subdir('worker') # subdir('scheduler') diff --git a/src/meson.options b/src/meson.options index 12dd673..3152c57 100644 --- a/src/meson.options +++ b/src/meson.options @@ -1,3 +1,2 @@ option('with_debug', type: 'boolean', value: false, yield: true) -option('without_db', type: 'boolean', value: false, yield: true) option('commit_id', type: 'string', value:'', yield: true) diff --git a/src/nixSupport/default.nix b/src/nixSupport/default.nix index 01266e4..5f67b8b 100644 --- a/src/nixSupport/default.nix +++ b/src/nixSupport/default.nix @@ -8,30 +8,14 @@ let inherit pkgs myCommitId; withDebug = false; withAsan = false; - withoutDb = false; - }; - xce_withoutdb = pkgs.callPackage ./xce.nix { - inherit pkgs myCommitId; - withDebug = false; - withAsan = false; - withoutDb = true; - }; - xce_withoutdb_debug = pkgs.callPackage ./xce.nix { - inherit pkgs myCommitId; - withDebug = true; - withAsan = true; - withoutDb = true; }; xce_debug = pkgs.callPackage ./xce.nix { inherit pkgs myCommitId; withDebug = true; withAsan = true; - withoutDb = false; }; in { xce = xce; xce_debug = xce_debug; - xce_withoutdb = xce_withoutdb; - xce_withoutdb_debug = xce_withoutdb_debug; }