forked from lei.chen/myxce
519 lines
12 KiB
C
519 lines
12 KiB
C
#include <errno.h>
|
|
#include <rtti.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
|
|
#include "kvmgr.h"
|
|
#include "plugin.h"
|
|
#include "sched/types.h"
|
|
#include "scheduler.h"
|
|
#include "utils.h"
|
|
#include "xcelimit.h"
|
|
#include "xlist.h"
|
|
#include "xstring.h"
|
|
|
|
typedef struct {
|
|
var limits;
|
|
var instances;
|
|
} limit_plugin_data_t;
|
|
|
|
static int limit_priv_data_init(limit_plugin_data_t* data)
|
|
{
|
|
kvmgr_params_t limits_params = {
|
|
.key_type = KVMGR_KEY_STRING,
|
|
.nslots = 1021,
|
|
.delta = 64,
|
|
.datakiller = (destroy_fn_t)kill_limit,
|
|
};
|
|
kvmgr_params_t instances_params = {
|
|
.key_type = KVMGR_KEY_STRING,
|
|
.nslots = 1021,
|
|
.delta = 64,
|
|
.datakiller = (destroy_fn_t)kill_limit,
|
|
};
|
|
|
|
data->limits = makeInstance(&xkvmgr_type, &limits_params);
|
|
if (data->limits == NULL) {
|
|
return -errno;
|
|
}
|
|
|
|
data->instances = makeInstance(&xkvmgr_type, &instances_params);
|
|
if (data->instances == NULL) {
|
|
return -errno;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int limit_update_config(xce_sched_plugin_t* plugin,
|
|
xce_conf4sched_t* config)
|
|
{
|
|
limit_plugin_data_t* privdata = NULL;
|
|
xce_scheduler_t* sched = (xce_scheduler_t*)plugin->sched;
|
|
int i;
|
|
xce_limit_config_t** limits = config->limits;
|
|
int nlimits = config->nlimits;
|
|
|
|
TRACE("%s ...", __func__);
|
|
privdata = (limit_plugin_data_t*)plugin->data;
|
|
|
|
for (i = 0; i < nlimits; i++) {
|
|
int rc;
|
|
xce_limit_t* limit;
|
|
|
|
rc = build_limit(&limit, (sched->ugmgr), (sched->qgmgr), (sched->pgmgr),
|
|
(sched->hgmgr), limits[i]);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
lock(privdata->limits, RTTI_LOCK_WRITE);
|
|
rc = key_add(privdata->limits, (void*)limit->name, (void*)limit);
|
|
unlock(privdata->limits);
|
|
|
|
if (rc < 0) {
|
|
kill_limit(limit);
|
|
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
typedef struct {
|
|
xce_sched_plugin_t* plugin;
|
|
xce_sched_ctx_t* ctx;
|
|
} match_limit_args_t;
|
|
|
|
static int update_limit_res_value(zarray_t* array, xce_vres_t* vres)
|
|
{
|
|
float val = 0.0;
|
|
int rc = 0;
|
|
|
|
if ((array == NULL) || (vres == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (vres->limit->val.percent == 0) {
|
|
val = vres->limit->val.val;
|
|
} else {
|
|
rc = get_resvals(array, 1, &(vres->res->idx), &val);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
val = val * vres->limit->val.val;
|
|
}
|
|
|
|
rc = put_resvals(array, 1, &(vres->vres->idx), &val);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static float get_vres_value(xce_sched_ctx_t* ctx, xce_vres_t* vres)
|
|
{
|
|
int i;
|
|
xce_sched_obj_t* schedobj = ctx->schedobj;
|
|
|
|
#if 0
|
|
if (resreq->op != RESREQ_OP_SIMPLE) {
|
|
return 0.0;
|
|
}
|
|
|
|
for (i = 0; i < resreq->u.simple.res_shared.count; i++) {
|
|
request_res_t* rq
|
|
= (request_res_t*)xlist_get(&(resreq->u.simple.res_shared), i);
|
|
if (strcmp(vres->res->name, rq->res) == 0) {
|
|
return rq->value;
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < resreq->u.simple.res_host.count; i++) {
|
|
request_res_t* rq
|
|
= (request_res_t*)xlist_get(&(resreq->u.simple.res_host), i);
|
|
|
|
if (strcmp(vres->res->name, rq->res) == 0) {
|
|
return rq->value;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
return 0.0;
|
|
}
|
|
|
|
static int match_job_global_limit_(xce_sched_plugin_t* plugin,
|
|
xce_sched_ctx_t* ctx, xce_limit_t* limit)
|
|
{
|
|
xce_sched_obj_t* obj = NULL;
|
|
int rc = 0;
|
|
char* vresname = NULL;
|
|
xce_vres_t* vres = NULL;
|
|
limit_plugin_data_t* privdata = NULL;
|
|
xce_scheduler_t* sched = NULL;
|
|
|
|
if ((limit == NULL) || (plugin == NULL) || (ctx == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
obj = ctx->schedobj;
|
|
if (obj == NULL) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
sched = plugin->sched;
|
|
privdata = (limit_plugin_data_t*)plugin->data;
|
|
if ((privdata == NULL) || (sched == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (is_host_based_limit(&(sched->resmgr), limit) > 0) {
|
|
return 0;
|
|
}
|
|
|
|
rc = match_limit(limit, 1, obj->user, obj->queue, obj->project);
|
|
TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
|
|
limit->name, obj->user, obj->queue, obj->project, rc);
|
|
if (rc <= 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = build_vres_name(&vresname, limit, obj->user, obj->queue, obj->project);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
lock(privdata->instances, RTTI_LOCK_READ);
|
|
rc = key_find(privdata->instances, (void*)vresname, (void*)&vres);
|
|
unlock(privdata->instances);
|
|
|
|
if (rc <= 0) {
|
|
return rc;
|
|
}
|
|
|
|
FREEUP(vresname);
|
|
|
|
if (vres == NULL) {
|
|
rc = instantiate_limit(&vres, &(sched->resmgr), limit, obj->user,
|
|
obj->queue, obj->project);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
lock(privdata->instances, RTTI_LOCK_WRITE);
|
|
rc = key_add(privdata->instances, (void*)vres->vres->name, vres);
|
|
unlock(privdata->instances);
|
|
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
rc = update_limit_res_value(&(sched->sharedInst), vres);
|
|
if (rc >= 0) {
|
|
rc = xlist_store(&(obj->glimits), vres);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int limit_job_election(void* arg, xce_sched_job_t* job)
|
|
{
|
|
xce_sched_plugin_t* plugin = (xce_sched_plugin_t*)arg;
|
|
limit_plugin_data_t* privdata = (limit_plugin_data_t*)plugin->data;
|
|
xce_scheduler_t* sched = (xce_scheduler_t*)plugin->sched;
|
|
xce_sched_obj_t* obj = job->schedobj;
|
|
int rc = 0;
|
|
iterator_t it;
|
|
xce_sched_ctx_t ctx = { 0 };
|
|
char jobidString[64];
|
|
|
|
fmt_jobid(jobidString, sizeof(jobidString), job->jobid);
|
|
TRACE("%s job %s plugin %s", __func__, jobidString, plugin->name);
|
|
|
|
if (obj->configver >= sched->configver) {
|
|
return 0;
|
|
}
|
|
|
|
reset_sched_obj(obj);
|
|
|
|
ctx.sched = plugin->sched;
|
|
ctx.worker = NULL;
|
|
ctx.schedobj = job->schedobj;
|
|
|
|
rc = iter_make(&it, privdata->limits);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
for (;;) {
|
|
xce_limit_t* limit = NULL;
|
|
|
|
rc = iter_next(&it, (void**)&limit);
|
|
if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) {
|
|
break;
|
|
}
|
|
|
|
rc = match_job_global_limit_(plugin, &ctx, limit);
|
|
if (rc < 0) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (rc == RTTI_ITER_NO_DATA) {
|
|
rc = 0;
|
|
}
|
|
|
|
iter_destroy(&it);
|
|
|
|
obj->configver = sched->configver;
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int match_job_host_limit_(xce_sched_plugin_t* plugin,
|
|
xce_sched_ctx_t* ctx, xce_limit_t* limit)
|
|
{
|
|
xce_sched_obj_t* obj = NULL;
|
|
int rc = 0;
|
|
char* vresname = NULL;
|
|
xce_vres_t* vres = NULL;
|
|
limit_plugin_data_t* privdata = NULL;
|
|
xce_scheduler_t* sched = NULL;
|
|
xce_sched_worker_t* worker = NULL;
|
|
xlist_t* hlimits = NULL;
|
|
|
|
if ((limit == NULL) || (ctx == NULL) || (limit == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
obj = ctx->schedobj;
|
|
if (obj == NULL) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
sched = plugin->sched;
|
|
privdata = (limit_plugin_data_t*)plugin->data;
|
|
worker = ctx->worker;
|
|
|
|
if ((sched == NULL) || (privdata == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (worker == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
if (is_host_based_limit(&(sched->resmgr), limit) <= 0) {
|
|
return 0;
|
|
}
|
|
|
|
hlimits = (xlist_t*)zarray_get(&(obj->hlimits), worker->idx);
|
|
if (hlimits == NULL) {
|
|
hlimits = (xlist_t*)calloc(1, sizeof(*hlimits));
|
|
if (hlimits == NULL) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
xlist_init(hlimits, 8);
|
|
|
|
zarray_put(&(obj->hlimits), worker->idx, hlimits);
|
|
}
|
|
|
|
rc = match_limit(limit, 0, obj->user, obj->queue, obj->project,
|
|
worker->name);
|
|
TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
|
|
limit->name, obj->user, obj->queue, obj->project, rc);
|
|
if (rc <= 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = build_vres_name(&vresname, limit, obj->user, obj->queue, obj->project);
|
|
if (rc <= 0) {
|
|
return rc;
|
|
}
|
|
|
|
lock(privdata->instances, RTTI_LOCK_READ);
|
|
rc = key_find(privdata->instances, (void*)vresname, (void*)&vres);
|
|
unlock(privdata->instances);
|
|
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
FREEUP(vresname);
|
|
|
|
if (vres == NULL) {
|
|
rc = instantiate_limit(&vres, &(sched->resmgr), limit, obj->user,
|
|
obj->queue, obj->project);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
lock(privdata->instances, RTTI_LOCK_WRITE);
|
|
rc = key_add(privdata->instances, vres->vres->name, vres);
|
|
unlock(privdata->instances);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
rc = update_limit_res_value(&(worker->resvals), vres);
|
|
if (rc >= 0) {
|
|
rc = xlist_store(hlimits, vres);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
#if 0
|
|
static int build_limit_vres_requests(xce_scheduler_t* sched,
|
|
xce_sched_ctx_t* ctx)
|
|
{
|
|
int i;
|
|
|
|
if ((sched == NULL) || (ctx == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
for (i = 0; i < requests->count; i++) {
|
|
request_res_t* rq = (request_res_t*)xlist_get(requests, i);
|
|
if (rq == NULL) {
|
|
continue;
|
|
}
|
|
|
|
FREEUP(rq->res);
|
|
FREEUP(rq);
|
|
}
|
|
|
|
xlist_clear(requests);
|
|
|
|
for (i = 0; i < lst->count; i++) {
|
|
xce_vres_t* vres = (xce_vres_t*)xlist_get(lst, i);
|
|
request_res_t* rq = NULL;
|
|
|
|
if (vres == NULL) {
|
|
continue;
|
|
}
|
|
|
|
rq = (request_res_t*)calloc(1, sizeof(*rq));
|
|
if (rq == NULL) {
|
|
continue;
|
|
}
|
|
|
|
rq->res = makeString(vres->vres->name);
|
|
rq->value = get_vres_value(ctx, vres);
|
|
|
|
xlist_store(requests, rq);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
static int limit_prepare_allocation(void* arg, xce_sched_ctx_t* ctx)
|
|
{
|
|
xce_sched_plugin_t* plugin = (xce_sched_plugin_t*)arg;
|
|
xce_sched_worker_t* worker = NULL;
|
|
int rc = 0;
|
|
limit_plugin_data_t* privdata = NULL;
|
|
iterator_t it;
|
|
|
|
if ((plugin == NULL) || (ctx == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
worker = ctx->worker;
|
|
if (worker == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
privdata = (limit_plugin_data_t*)plugin->data;
|
|
|
|
rc = iter_make(&it, privdata->limits);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
for (;;) {
|
|
xce_limit_t* limit = NULL;
|
|
|
|
rc = iter_next(&it, (void**)&limit);
|
|
if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) {
|
|
break;
|
|
}
|
|
|
|
rc = match_job_host_limit_(plugin, ctx, limit);
|
|
if (rc < 0) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (rc == RTTI_ITER_NO_DATA) {
|
|
rc = 0;
|
|
}
|
|
|
|
iter_destroy(&it);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int limit_init(xce_sched_plugin_t* plugin)
|
|
{
|
|
limit_plugin_data_t* privdata = NULL;
|
|
|
|
privdata = (limit_plugin_data_t*)calloc(1, sizeof(*privdata));
|
|
if (privdata == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
limit_priv_data_init(privdata);
|
|
|
|
plugin->data = privdata;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void limit_fini(xce_sched_plugin_t* plugin)
|
|
{
|
|
limit_plugin_data_t* privdata = NULL;
|
|
|
|
if (plugin == NULL) {
|
|
return;
|
|
}
|
|
|
|
privdata = (limit_plugin_data_t*)plugin->data;
|
|
if (privdata != NULL) {
|
|
destroy(privdata->limits);
|
|
destroy(privdata->instances);
|
|
|
|
FREEUP(privdata);
|
|
}
|
|
|
|
FREEUP(plugin);
|
|
}
|
|
|
|
int register_limit(void* sched)
|
|
{
|
|
static xce_sched_plugin_t limit_plugin = {
|
|
.name = "xce_sched_limit",
|
|
.hooks
|
|
= { .job_election_hook = (plugin_job_election_hook_t)limit_job_election,
|
|
.config_hook = (plugin_update_config_hook_t)limit_update_config,
|
|
.prepare_allocation_hook
|
|
= (plugin_prepare_allocation_hook_t)limit_prepare_allocation },
|
|
.init_fn = (plugin_init_fn_t)limit_init,
|
|
.fini_fn = (plugin_fini_fn_t)limit_fini,
|
|
.data = NULL
|
|
};
|
|
|
|
xce_sched_plugin_t* plugin
|
|
= (xce_sched_plugin_t*)calloc(1, sizeof(*plugin));
|
|
if (plugin == NULL) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
memcpy(plugin, &limit_plugin, sizeof(*plugin));
|
|
|
|
return plugin_register(sched, plugin);
|
|
}
|