security communication with openssl and simplify netio
This commit is contained in:
parent
2a7b7c39bb
commit
8f796b3109
20 changed files with 1176 additions and 922 deletions
|
|
@ -48,8 +48,6 @@ static void handle_endpoint_error(endpoint_t* ep)
|
|||
return;
|
||||
}
|
||||
|
||||
iomgr_unmonitor(&(c->iomgr), ep);
|
||||
|
||||
priv = (ep_private_t*)ep->data;
|
||||
|
||||
TRACE("endpoint %p priv data %p", ep, priv);
|
||||
|
|
@ -79,11 +77,13 @@ static void handle_endpoint_error(endpoint_t* ep)
|
|||
}
|
||||
|
||||
if (ep->idx >= 0) {
|
||||
/* zarray_del will finalize ep */
|
||||
zarray_del(&(c->endpoints), ep->idx);
|
||||
return;
|
||||
}
|
||||
|
||||
endpoint_deref(ep);
|
||||
endpoint_fini(ep);
|
||||
FREEUP(ep);
|
||||
|
||||
return;
|
||||
}
|
||||
|
|
@ -130,13 +130,15 @@ static int do_notify(cluster_t* c, endpoint_t* ep, xce_request_t* req)
|
|||
}
|
||||
static int myreq_handler(vtask_t* task)
|
||||
{
|
||||
cluster_t* c = (cluster_t*)task->ctx.ep->owner;
|
||||
vtask_data_t* data = (vtask_data_t*)task->data;
|
||||
endpoint_t* ep = data->u.iodata.ep;
|
||||
cluster_t* c = (cluster_t*)ep->owner;
|
||||
int stop = 0;
|
||||
int rc = 0;
|
||||
xce_request_t request;
|
||||
xce_reply_t reply;
|
||||
|
||||
rc = ioctx_decode(&(task->ctx), &request);
|
||||
rc = xce_decode(data->u.iodata.pkt, &request);
|
||||
|
||||
TRACE("%s cpu#%d opcode %d", __func__, task->vcpu, request.opcode);
|
||||
|
||||
|
|
@ -146,7 +148,7 @@ static int myreq_handler(vtask_t* task)
|
|||
case OPCode_shutdown:
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(task->ctx.ep, c->myhost, &reply);
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
stop = 1;
|
||||
cbuf_put(&(c->wbuf), &stop);
|
||||
break;
|
||||
|
|
@ -154,44 +156,44 @@ static int myreq_handler(vtask_t* task)
|
|||
case OPCode_nop:
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(task->ctx.ep, c->myhost, &reply);
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
break;
|
||||
case OPCode_join:
|
||||
rc = handle_join(c, task->ctx.ep, &request);
|
||||
rc = handle_join(c, ep, &request);
|
||||
break;
|
||||
case OPCode_jobdispatch:
|
||||
rc = handle_jobdispatch(c, task->ctx.ep, &request);
|
||||
rc = handle_jobdispatch(c, ep, &request);
|
||||
break;
|
||||
case OPCode_jobsub:
|
||||
rc = handle_jobsubmit(c, task->ctx.ep, &request);
|
||||
rc = handle_jobsubmit(c, ep, &request);
|
||||
break;
|
||||
case OPCode_get:
|
||||
rc = do_get_op(c, task->ctx.ep, &request);
|
||||
rc = do_get_op(c, ep, &request);
|
||||
break;
|
||||
case OPCode_notify:
|
||||
rc = do_notify(c, task->ctx.ep, &request);
|
||||
rc = do_notify(c, ep, &request);
|
||||
break;
|
||||
case OPCode_clusterconf:
|
||||
rc = handle_clusterconf(c, task->ctx.ep, &request);
|
||||
rc = handle_clusterconf(c, ep, &request);
|
||||
break;
|
||||
case OPCode_update:
|
||||
rc = do_update_op(c, task->ctx.ep, &request);
|
||||
rc = do_update_op(c, ep, &request);
|
||||
break;
|
||||
case OPCode_jobcontrol:
|
||||
rc = do_jobcontrol_op(c, task->ctx.ep, &request);
|
||||
rc = do_jobcontrol_op(c, ep, &request);
|
||||
break;
|
||||
case OPCode_queuecontrol:
|
||||
rc = do_queuecontrol_op(c, task->ctx.ep, &request);
|
||||
rc = do_queuecontrol_op(c, ep, &request);
|
||||
break;
|
||||
case OPCode_runtasks:
|
||||
rc = do_runtasks(c, task->ctx.ep, &request);
|
||||
rc = do_runtasks(c, ep, &request);
|
||||
break;
|
||||
default:
|
||||
rc = -E_XCE_BAD_OPCODE;
|
||||
ERROR("Unknown opcode: %d", request.opcode);
|
||||
reply.status = rc;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(task->ctx.ep, c->myhost, &reply);
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
|
||||
break;
|
||||
}
|
||||
|
|
@ -210,93 +212,21 @@ static int vtask_entry(vtask_t* task)
|
|||
return rc;
|
||||
}
|
||||
|
||||
static int handle_input(endpoint_t* ep)
|
||||
static int handle_msg(endpoint_t* ep)
|
||||
{
|
||||
cluster_t* c = (cluster_t*)ep->owner;
|
||||
time_t now;
|
||||
vtask_t* task;
|
||||
int rc = 0;
|
||||
cluster_t* c = NULL;
|
||||
|
||||
task = vtask_alloc();
|
||||
if (task == NULL) {
|
||||
PERROR("vtask_alloc");
|
||||
|
||||
return -ENOMEM;
|
||||
if (ep == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rc = vtask_init(task, (vtask_func_t)&vtask_entry, ep);
|
||||
if (rc < 0) {
|
||||
SET_ERROR(-rc);
|
||||
|
||||
INFO("vtask_init failed, maybe peer side closed the connection");
|
||||
|
||||
if (rc == -EPIPE) {
|
||||
/* connection broken */
|
||||
TRACE("broken pipe, ep %p", ep);
|
||||
|
||||
handle_endpoint_error(ep);
|
||||
}
|
||||
|
||||
FREEUP(task);
|
||||
return rc;
|
||||
c = (cluster_t*)ep->owner;
|
||||
if (c == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rc = vtask_enq(&(c->vtaskmgr), task);
|
||||
if (rc < 0) {
|
||||
PERROR("vtask_enq");
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
now = time(NULL);
|
||||
npkts++;
|
||||
|
||||
if (now - last_print_time > 10) {
|
||||
TRACE("%ld npkts:%ld iops: %f\n", now, npkts,
|
||||
(npkts - npkts_old) * 1.0 / (now - last_print_time));
|
||||
npkts_old = npkts;
|
||||
last_print_time = now;
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int handle_output(endpoint_t* ep)
|
||||
{
|
||||
data_item_t* it;
|
||||
ioctx_t* ctx;
|
||||
int rc;
|
||||
|
||||
it = endpoint_deq(ep);
|
||||
|
||||
if (it == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
ctx = ITPTR_TO_IOCTX(it);
|
||||
|
||||
rc = ioctx_flush(ctx, ep->fd);
|
||||
ioctx_deref(ctx, 1);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int handle_io(struct epoll_event* p)
|
||||
{
|
||||
endpoint_t* ep = (endpoint_t*)p->data.ptr;
|
||||
int rc = 0;
|
||||
|
||||
if ((p->events & EPOLLIN) != 0) {
|
||||
rc = handle_input(ep);
|
||||
}
|
||||
|
||||
if (rc >= 0) {
|
||||
if ((p->events & EPOLLOUT) != 0) {
|
||||
rc = handle_output(ep);
|
||||
}
|
||||
}
|
||||
|
||||
return rc;
|
||||
return vtask_handle_msg(&(c->vtaskmgr), (vtask_func_t)vtask_entry,
|
||||
(void*)ep);
|
||||
}
|
||||
|
||||
static int handle_connection(struct epoll_event* p)
|
||||
|
|
@ -327,7 +257,9 @@ static int handle_connection(struct epoll_event* p)
|
|||
if (priv == NULL) {
|
||||
PERROR("ep_get_private");
|
||||
|
||||
endpoint_deref(ep2);
|
||||
endpoint_fini(ep2);
|
||||
FREEUP(ep2);
|
||||
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
|
|
@ -342,17 +274,21 @@ static int handle_connection(struct epoll_event* p)
|
|||
PERROR("zarray_store");
|
||||
|
||||
FREEUP(priv);
|
||||
endpoint_deref(ep2);
|
||||
endpoint_fini(ep2);
|
||||
FREEUP(ep2);
|
||||
|
||||
return idx;
|
||||
}
|
||||
|
||||
ep2->idx = idx;
|
||||
ep2->data = priv;
|
||||
ep2->idx = idx;
|
||||
ep2->data = priv;
|
||||
ep2->iomgr = iomgr;
|
||||
|
||||
endpoint_set_handler(ep2, handle_io, c);
|
||||
endpoint_set_owner(ep2, c);
|
||||
endpoint_set_msg_handler(ep2, (msg_handler_t)handle_msg);
|
||||
endpoint_set_err_handler(ep2, (err_handler_t)handle_endpoint_error);
|
||||
|
||||
iomgr_monitor(iomgr, ep2);
|
||||
iomgr_monitor(ep2);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -372,7 +308,8 @@ static void endpoint_killer(void* p)
|
|||
zarray_clear(&(c->endpoints), ep->idx);
|
||||
ep->idx = -1;
|
||||
}
|
||||
endpoint_deref(ep);
|
||||
endpoint_fini(ep);
|
||||
FREEUP(ep);
|
||||
|
||||
return;
|
||||
}
|
||||
|
|
@ -511,7 +448,8 @@ static int xce_install_hooks(cluster_t* c)
|
|||
|
||||
int cluster_init(cluster_t* c)
|
||||
{
|
||||
int i, rc;
|
||||
int i, rc;
|
||||
SSL_CTX* ssl_ctx = NULL;
|
||||
|
||||
if (c == NULL) {
|
||||
return -EINVAL;
|
||||
|
|
@ -535,6 +473,12 @@ int cluster_init(cluster_t* c)
|
|||
mylock_init(&(c->sched.lock));
|
||||
c->sched.ep = NULL;
|
||||
|
||||
initialize_openssl();
|
||||
ssl_ctx = create_ssl_context(c->myconf->certfile, c->myconf->keyfile);
|
||||
if (ssl_ctx == NULL) {
|
||||
return -errno;
|
||||
}
|
||||
|
||||
kvmgr_init(&(c->configmgr), string_hash_key, 128, 128,
|
||||
(zarray_item_killer_t)config_obj_killer);
|
||||
|
||||
|
|
@ -550,6 +494,9 @@ int cluster_init(cluster_t* c)
|
|||
qmgr_init(&(c->qmgr), c->myconf->cluster_config->qmgr->nhash_slots,
|
||||
c->myconf->cluster_config->qmgr->delta);
|
||||
iomgr_init(&(c->iomgr));
|
||||
|
||||
iomgr_set_ssl_ctx(&(c->iomgr), ssl_ctx);
|
||||
|
||||
register_xstring(&(c->objmgr),
|
||||
c->myconf->cluster_config->xstrings->nhash_slots,
|
||||
c->myconf->cluster_config->xstrings->delta);
|
||||
|
|
@ -560,9 +507,12 @@ int cluster_init(cluster_t* c)
|
|||
rc = endpoint_init(&(c->ep), AF_INET, SOCK_STREAM,
|
||||
c->myconf->cluster_config->port, 5);
|
||||
if (rc < 0) {
|
||||
cleanup_openssl();
|
||||
return rc;
|
||||
}
|
||||
|
||||
endpoint_set_peer(&(c->ep), c->myhost);
|
||||
|
||||
for (i = 0; i < c->ncontrollers; i++) {
|
||||
controller_init(&(c->controllers[i]), c);
|
||||
}
|
||||
|
|
@ -583,8 +533,11 @@ int cluster_start(cluster_t* c)
|
|||
{
|
||||
int rc;
|
||||
|
||||
endpoint_set_handler(&(c->ep), handle_connection, c);
|
||||
rc = iomgr_monitor(&(c->iomgr), &(c->ep));
|
||||
endpoint_set_owner(&(c->ep), c);
|
||||
endpoint_set_conn_handler(&(c->ep), (conn_handler_t)handle_connection);
|
||||
c->ep.iomgr = &(c->iomgr);
|
||||
|
||||
rc = iomgr_monitor(&(c->ep));
|
||||
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
|
|
@ -620,7 +573,7 @@ int cluster_run(cluster_t* c)
|
|||
void cluster_fini(cluster_t* c)
|
||||
{
|
||||
vtaskmgr_fini(&(c->vtaskmgr));
|
||||
endpoint_deref(&(c->ep));
|
||||
endpoint_fini(&(c->ep));
|
||||
|
||||
xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
|
||||
|
|
@ -641,5 +594,7 @@ void cluster_fini(cluster_t* c)
|
|||
|
||||
xce_hook_fini(&(c->hooksys));
|
||||
|
||||
cleanup_openssl();
|
||||
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue