forked from lei.chen/myxce
1055 lines
25 KiB
C
1055 lines
25 KiB
C
#include <errno.h>
|
|
#include <pthread.h>
|
|
#include <rtti.h>
|
|
#include <signal.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "client.h"
|
|
#include "controller.capnp.h"
|
|
#include "decision.h"
|
|
#include "kvmgr.h"
|
|
#include "misc.h"
|
|
#include "plugin.h"
|
|
#include "preason.h"
|
|
#include "scheduler.h"
|
|
#include "utils.h"
|
|
#include "xcealloc.h"
|
|
#include "xstring.h"
|
|
|
|
static void* sched_job_picker(void* arg)
|
|
{
|
|
xce_sched_plugin_t* job_picker = NULL;
|
|
xce_scheduler_t* sched = (xce_scheduler_t*)arg;
|
|
char* job_pick_plugin = "xce_sched_fcfs";
|
|
int rc = 0;
|
|
|
|
lock(sched->plugins, RTTI_LOCK_READ);
|
|
rc = key_find(sched->plugins, job_pick_plugin, (void**)&job_picker);
|
|
unlock(sched->plugins);
|
|
|
|
if (rc < 0) {
|
|
errno = -rc;
|
|
|
|
return NULL;
|
|
}
|
|
|
|
if ((job_picker == NULL) || (job_picker->job_elect_fn == NULL)) {
|
|
return NULL;
|
|
}
|
|
|
|
return job_picker->job_elect_fn(job_picker);
|
|
}
|
|
|
|
static int sched_make_seqjob_allocation(xce_scheduler_t* sched,
|
|
xce_sched_ctx_t* ctx)
|
|
{
|
|
xce_sched_plugin_t* plugin = NULL;
|
|
|
|
plugin = sched->seqjob_allocator;
|
|
if (plugin->job_alloc_fn == NULL) {
|
|
ERROR("%s plugin %s has no allocation function", __func__,
|
|
plugin->name);
|
|
return -EINVAL;
|
|
}
|
|
|
|
return plugin->job_alloc_fn(plugin, ctx);
|
|
}
|
|
|
|
static int sched_make_parjob_allocation(xce_scheduler_t* sched,
|
|
xce_sched_ctx_t* ctx)
|
|
{
|
|
xce_sched_plugin_t* plugin = NULL;
|
|
|
|
plugin = sched->parjob_allocator;
|
|
if (plugin->job_alloc_fn == NULL) {
|
|
ERROR("%s plugin %s has no allocation function", __func__,
|
|
plugin->name);
|
|
return -EINVAL;
|
|
}
|
|
|
|
return plugin->job_alloc_fn(plugin, ctx);
|
|
}
|
|
|
|
static int sched_make_primary_alloc_(xce_scheduler_t* sched,
|
|
xce_sched_ctx_t* ctx)
|
|
{
|
|
xce_primary_resreq_t* primary = ctx->primary;
|
|
|
|
if (primary->maxTasks == 1) {
|
|
return sched_make_seqjob_allocation(sched, ctx);
|
|
}
|
|
|
|
return sched_make_parjob_allocation(sched, ctx);
|
|
}
|
|
|
|
static int sched_make_primary_allocation(xce_scheduler_t* sched,
|
|
xce_sched_obj_t* obj)
|
|
{
|
|
xce_primary_resreq_t* primary = obj->u.primary->resreq;
|
|
xce_sched_ctx_t ctx;
|
|
int rc = 0;
|
|
|
|
sched_ctx_init(&ctx, sched, obj);
|
|
sched_ctx_set_primary(&ctx, primary);
|
|
rc = sched_make_primary_alloc_(sched, &ctx);
|
|
if (ctx.alloc != NULL) {
|
|
MOVE_PTR(obj->alloc, ctx.alloc);
|
|
}
|
|
|
|
if (ctx.reason != NULL) {
|
|
MOVE_PTR(obj->reason, ctx.reason);
|
|
}
|
|
|
|
sched_ctx_fini(&ctx);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int sched_make_compound_allocation(xce_scheduler_t* sched,
|
|
xce_sched_obj_t* obj)
|
|
{
|
|
int i;
|
|
xce_compound_resreq_t* resreq = obj->u.compound->resreq;
|
|
xce_sched_ctx_t ctx;
|
|
int rc = 0;
|
|
|
|
sched_ctx_init(&ctx, sched, obj);
|
|
|
|
for (i = 0; i < resreq->primaries.count; i++) {
|
|
xce_primary_resreq_t* primary
|
|
= (xce_primary_resreq_t*)xlist_get(&(resreq->primaries), i);
|
|
|
|
sched_ctx_set_primary(&ctx, primary);
|
|
rc = sched_make_primary_alloc_(sched, &ctx);
|
|
if (rc <= 0) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (ctx.alloc != NULL) {
|
|
MOVE_PTR(obj->alloc, ctx.alloc);
|
|
}
|
|
|
|
if (ctx.reason != NULL) {
|
|
MOVE_PTR(obj->reason, ctx.reason);
|
|
}
|
|
|
|
sched_ctx_fini(&ctx);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int sched_make_alter_allocation(xce_scheduler_t* sched,
|
|
xce_sched_obj_t* obj)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < obj->u.alters.count; i++) {
|
|
xce_sched_obj_t* obj1
|
|
= (xce_sched_obj_t*)xlist_get(&(obj->u.alters), i);
|
|
int rc;
|
|
|
|
if (obj1 == NULL) {
|
|
continue;
|
|
}
|
|
|
|
rc = sched_make_compound_allocation(sched, obj1);
|
|
if (rc > 0) {
|
|
if (obj1->alloc != NULL) {
|
|
MOVE_PTR(obj->alloc, obj1->alloc);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
/* FIXME: merge pend reason from all children */
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_make_allocation(xce_scheduler_t* sched, xce_sched_obj_t* obj)
|
|
{
|
|
int rc = 0;
|
|
|
|
switch (obj->tag) {
|
|
case XCE_RESREQ_PRIMARY:
|
|
rc = sched_make_primary_allocation(sched, obj);
|
|
break;
|
|
case XCE_RESREQ_COMPOUND:
|
|
rc = sched_make_compound_allocation(sched, obj);
|
|
break;
|
|
case XCE_RESREQ_ALTER:
|
|
rc = sched_make_alter_allocation(sched, obj);
|
|
break;
|
|
default:
|
|
rc = -EINVAL;
|
|
break;
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static int commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
|
|
{
|
|
return sched_commit_decision(sched, decision);
|
|
}
|
|
|
|
static void* sched_allocator(void* arg)
|
|
{
|
|
xce_scheduler_t* sched = (xce_scheduler_t*)arg;
|
|
|
|
xce_use(sched);
|
|
while (1) {
|
|
uint64_t jobid;
|
|
xce_decision_t* decision = NULL;
|
|
xce_sched_job_t* job = NULL;
|
|
int rc = 0;
|
|
|
|
pthread_testcancel();
|
|
|
|
cbuf_get(&(sched->jbuf), &jobid);
|
|
if (jobid == 0) {
|
|
/* end of scheduling session */
|
|
decision = NULL;
|
|
cbuf_put(&(sched->decisions), &decision);
|
|
continue;
|
|
}
|
|
|
|
job = sched_get_job(sched, jobid);
|
|
if (job == NULL) {
|
|
TRACE("%s job %lu can not be found", __func__, jobid);
|
|
continue;
|
|
}
|
|
|
|
plugin_job_election(sched, job);
|
|
|
|
/* calculate decision for jobid */
|
|
rc = sched_make_allocation(sched, job->schedobj);
|
|
if (rc < 0) {
|
|
/* something wrong */
|
|
continue;
|
|
} else if (rc == 0) {
|
|
/* job is pend */
|
|
xce_pendreason_t* reason = job->schedobj->reason;
|
|
|
|
reason->jobid = job->jobid;
|
|
decision = make_pend_decision(reason);
|
|
} else {
|
|
MOVE_PTR(job->allocs, job->schedobj->alloc);
|
|
job->allocs->jobid = job->jobid;
|
|
|
|
decision = make_dispatch_decision(job->allocs);
|
|
}
|
|
|
|
if (decision != NULL) {
|
|
commit_decision(sched, decision);
|
|
cbuf_put(&(sched->decisions), &decision);
|
|
}
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static void sched_pend_update_killer(data_item_t* it)
|
|
{
|
|
xce_sched_update_request_t* req;
|
|
int i;
|
|
|
|
if (it == NULL) {
|
|
return;
|
|
}
|
|
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
switch (req->op) {
|
|
case XCE_SCHED_OP_UPDATE_CONFIG:
|
|
free_Conf4Sched_ptr(&(req->u.config));
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_JOB:
|
|
for (i = 0; i < req->u.jobupdate.njobs; i++) {
|
|
free_Job_ptr(&(req->u.jobupdate.jobs[i]));
|
|
}
|
|
|
|
free(req->u.jobupdate.jobs);
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_WORKER:
|
|
for (i = 0; i < req->u.workerupdate.nworkers; i++) {
|
|
free_Worker_ptr(&(req->u.workerupdate.workers[i]));
|
|
}
|
|
|
|
free(req->u.workerupdate.workers);
|
|
break;
|
|
case XCE_SCHED_OP_JSTATE_CHANGE:
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
free(req);
|
|
}
|
|
|
|
static int sched_register_plugins(xce_scheduler_t* sched)
|
|
{
|
|
register_fcfs(sched);
|
|
register_limit(sched);
|
|
register_seqjob_allocator(sched);
|
|
register_parjob_allocator(sched);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_plugin_init(xce_scheduler_t* sched)
|
|
{
|
|
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 16,
|
|
.delta = 8,
|
|
.datakiller = plugin_unregister };
|
|
|
|
sched->plugins = makeInstance(&xkvmgr_type, ¶ms);
|
|
if (sched->plugins == NULL) {
|
|
return -errno;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|
xce_request_handler_t handler)
|
|
{
|
|
int rc = 0;
|
|
int ndecisions = 5; /* FIXME: make this configurable */
|
|
int njobs = 1; /* FIXME: make this configurable */
|
|
kvmgr_params_t jobs_params = { .key_type = KVMGR_KEY_INT,
|
|
.nslots = 5381, /* FIXME */
|
|
.delta = 1024,
|
|
.datakiller = kill_sched_job };
|
|
kvmgr_params_t queues_params
|
|
= { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 59,
|
|
.delta = 16,
|
|
.datakiller = (destroy_fn_t)kill_sched_queue };
|
|
kvmgr_params_t workers_params
|
|
= { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 5381,
|
|
.delta = 64,
|
|
.datakiller = (destroy_fn_t)kill_sched_worker };
|
|
kvmgr_params_t users_params
|
|
= { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 5381,
|
|
.delta = 64,
|
|
.datakiller = (destroy_fn_t)kill_sched_user };
|
|
kvmgr_params_t projects_params
|
|
= { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 31,
|
|
.delta = 8,
|
|
.datakiller = (destroy_fn_t)kill_sched_project };
|
|
kvmgr_params_t resmgr_params = { .key_type = KVMGR_KEY_STRING,
|
|
.nslots = 64,
|
|
.delta = 8,
|
|
.datakiller = NULL };
|
|
|
|
if ((sched == NULL) || (config == NULL) || (handler == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
memset(sched, 0x0, sizeof(*sched));
|
|
|
|
sched->nextid = 1;
|
|
sched->configver = 0;
|
|
|
|
xstring_table_init(1024*1024);
|
|
|
|
mylock_init(&(sched->lock));
|
|
cbuf_init(&(sched->wbuf), 1, sizeof(int));
|
|
cbuf_init(&(sched->jbuf), njobs, sizeof(uint64_t));
|
|
cbuf_init(&(sched->decisions), ndecisions, sizeof(xce_decision_t*));
|
|
|
|
sched->pend_updates[0] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->pend_updates[0], sched_pend_update_killer);
|
|
|
|
sched->pend_updates[1] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->pend_updates[1], sched_pend_update_killer);
|
|
|
|
sched->resmgr = makeInstance(&xkvmgr_type, &resmgr_params);
|
|
|
|
groupmgr_init(&(sched->ugmgr), 64, 64);
|
|
groupmgr_init(&(sched->qgmgr), 16, 16);
|
|
groupmgr_init(&(sched->pgmgr), 16, 16);
|
|
groupmgr_init(&(sched->hgmgr), 64, 64);
|
|
|
|
sched_plugin_init(sched);
|
|
|
|
sched->jobs = makeInstance(&xkvmgr_type, &jobs_params);
|
|
sched->queues = makeInstance(&xkvmgr_type, &queues_params);
|
|
sched->workers = makeInstance(&xkvmgr_type, &workers_params);
|
|
sched->users = makeInstance(&xkvmgr_type, &users_params);
|
|
sched->projects = makeInstance(&xkvmgr_type, &projects_params);
|
|
|
|
zarray_init(&(sched->sharedInst), 128,
|
|
(zarray_item_killer_t)kill_sched_resval);
|
|
zarray_init(&(sched->sharedInUse), 128, NULL);
|
|
|
|
sched_register_plugins(sched);
|
|
|
|
rc = xcecli_ctx_init(&(sched->apictx), config);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
xcecli_set_handler(sched->apictx, handler, sched);
|
|
|
|
rc = xcecli_start(sched->apictx);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = pthread_create(&(sched->jobpicker), NULL, &sched_job_picker, sched);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = pthread_create(&(sched->allocator), NULL, &sched_allocator, sched);
|
|
|
|
return rc;
|
|
}
|
|
|
|
void sched_fini(xce_scheduler_t* sched)
|
|
{
|
|
void* res;
|
|
uint64_t jobid;
|
|
|
|
if (sched == NULL) {
|
|
return;
|
|
}
|
|
|
|
pthread_cancel(sched->jobpicker);
|
|
pthread_join(sched->jobpicker, &res);
|
|
|
|
pthread_cancel(sched->allocator);
|
|
/* try to wake up allocator */
|
|
cbuf_tryput(&(sched->jbuf), &jobid);
|
|
pthread_join(sched->allocator, &res);
|
|
|
|
xcecli_wakeup(sched->apictx, (int)-EINTR);
|
|
xcecli_stop(sched->apictx);
|
|
xcecli_ctx_fini(sched->apictx);
|
|
|
|
cbuf_fini(&(sched->decisions));
|
|
cbuf_fini(&(sched->jbuf));
|
|
cbuf_fini(&(sched->wbuf));
|
|
|
|
dataq_fini(sched->pend_updates[0]);
|
|
dataq_fini(sched->pend_updates[1]);
|
|
|
|
destroy(sched->resmgr);
|
|
|
|
destroy(sched->ugmgr);
|
|
destroy(sched->qgmgr);
|
|
destroy(sched->pgmgr);
|
|
destroy(sched->hgmgr);
|
|
|
|
mylock_fini(&(sched->lock));
|
|
|
|
return;
|
|
}
|
|
|
|
static int sched_init_resources(xce_scheduler_t* sched)
|
|
{
|
|
int i;
|
|
xce_conf4sched_t* config = sched->config;
|
|
|
|
for (i = 0; i < config->n_resources; i++) {
|
|
xce_resource_t* r = config->resources[i];
|
|
|
|
lock(sched->resmgr, RTTI_LOCK_WRITE);
|
|
key_add(sched->resmgr, (void*)r->name, (void*)r);
|
|
unlock(sched->resmgr);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int sched_init_sharedInst(xce_scheduler_t* sched)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < sched->config->nresvals; i++) {
|
|
int idx = sched->config->resvals[i]->idx;
|
|
xce_val_t* val = NULL;
|
|
|
|
MOVE_PTR(val, sched->config->resvals[i]->val);
|
|
zarray_put(&(sched->sharedInst), idx, val);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_init_queues(xce_scheduler_t* sched)
|
|
{
|
|
int i;
|
|
xce_conf4sched_t* config = sched->config;
|
|
|
|
for (i = 0; i < config->n_queues; i++) {
|
|
xce_sched_queue_t* q = make_sched_queue(sched, config->queues[i]);
|
|
|
|
plugin_queue_update(sched, q);
|
|
sched_add_queue(sched, q);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
int sched_init_config(xce_scheduler_t* sched, xce_conf4sched_t* config)
|
|
{
|
|
plugin_config_update(sched, config);
|
|
sched->config = config;
|
|
|
|
sched_init_resources(sched);
|
|
sched_init_sharedInst(sched);
|
|
sched_init_queues(sched);
|
|
|
|
sched->configver++; /* increase the version of configuration */
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_update_jobs(xce_scheduler_t* sched, int njobs,
|
|
xce_job_t** jobs)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < njobs; i++) {
|
|
xce_sched_job_t* j = make_sched_job(sched, jobs[i]);
|
|
|
|
plugin_job_update(sched, j);
|
|
sched_add_job(sched, j);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_job_state_change(xce_scheduler_t* sched, uint64_t jobid,
|
|
uint32_t state)
|
|
{
|
|
xce_sched_job_t* job = sched_get_job(sched, jobid);
|
|
int rc = 0;
|
|
if (job == NULL) {
|
|
return -ENOENT;
|
|
}
|
|
|
|
rc = plugin_job_state_change(sched, job, state);
|
|
|
|
if (!JOB_IS_RUN(state)) {
|
|
if (job->accounts != NULL) {
|
|
free_job_account(job->accounts);
|
|
job->accounts = NULL;
|
|
}
|
|
|
|
if (job->allocs != NULL) {
|
|
free_JobAlloc_ptr(&(job->allocs));
|
|
}
|
|
}
|
|
|
|
job->state = state;
|
|
|
|
return rc;
|
|
}
|
|
static int sched_update_workers(xce_scheduler_t* sched, int nworkers,
|
|
xce_worker_t** workers)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < nworkers; i++) {
|
|
xce_sched_worker_t* w = make_sched_worker(sched, workers[i]);
|
|
|
|
plugin_worker_update(sched, w);
|
|
sched_add_worker(sched, w);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sched_update(xce_scheduler_t* sched)
|
|
{
|
|
dataq_t* q = NULL;
|
|
|
|
/* swap dataq */
|
|
mylock_lock(&(sched->lock));
|
|
q = sched->pend_updates[0];
|
|
sched->pend_updates[0] = sched->pend_updates[1];
|
|
sched->pend_updates[1] = q;
|
|
mylock_unlock(&(sched->lock));
|
|
|
|
while (1) {
|
|
data_item_t* it;
|
|
xce_sched_update_request_t* req;
|
|
int rc;
|
|
|
|
it = dataq_deq(q, NULL);
|
|
if (it == NULL) {
|
|
break;
|
|
}
|
|
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
switch (req->op) {
|
|
case XCE_SCHED_OP_UPDATE_CONFIG:
|
|
/* this will only be handled when scheduler has been started, so
|
|
* should never get here */
|
|
rc = -EINVAL;
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_JOB:
|
|
rc = sched_update_jobs(sched, req->u.jobupdate.njobs,
|
|
req->u.jobupdate.jobs);
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_WORKER:
|
|
rc = sched_update_workers(sched, req->u.workerupdate.nworkers,
|
|
req->u.workerupdate.workers);
|
|
break;
|
|
case XCE_SCHED_OP_JSTATE_CHANGE:
|
|
rc = sched_job_state_change(sched, req->u.job_state_change.jobid,
|
|
req->u.job_state_change.state);
|
|
break;
|
|
default:
|
|
rc = -EINVAL;
|
|
break;
|
|
}
|
|
|
|
sched_pend_update_killer(it);
|
|
|
|
if (rc < 0) {
|
|
/* something wrong? */
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void sched_go(xce_scheduler_t* sched)
|
|
{
|
|
int go = 1;
|
|
|
|
cbuf_put(&(sched->wbuf), &go);
|
|
|
|
return;
|
|
}
|
|
|
|
static int my_reset_resval_(xce_scheduler_t* sched, xce_val_t* val)
|
|
{
|
|
return reset_resval(val);
|
|
}
|
|
|
|
static int sched_reset_shared_resval(xce_scheduler_t* sched)
|
|
{
|
|
return zarray_iter(&(sched->sharedInUse),
|
|
(zarray_iter_fn_t)my_reset_resval_, sched);
|
|
}
|
|
|
|
static int sched_reset_worker_(xce_scheduler_t* sched, xce_sched_worker_t* w)
|
|
{
|
|
if (w == NULL) {
|
|
return 0;
|
|
}
|
|
|
|
if (IS_WORKER_UNAVAIL(w->status)) {
|
|
return 0;
|
|
}
|
|
|
|
return zarray_iter(&(w->resInUse), (zarray_iter_fn_t)my_reset_resval_,
|
|
sched);
|
|
}
|
|
|
|
static int sched_reset_workers(xce_scheduler_t* sched)
|
|
{
|
|
iterator_t it;
|
|
int rc = 0;
|
|
|
|
rc = iter_make(&it, sched->workers);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
for (;;) {
|
|
xce_sched_worker_t* w = NULL;
|
|
|
|
rc = iter_next(&it, (void**)&w);
|
|
if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) {
|
|
break;
|
|
}
|
|
|
|
sched_reset_worker_(sched, w);
|
|
}
|
|
|
|
if (rc == RTTI_ITER_NO_DATA) {
|
|
rc = 0;
|
|
}
|
|
|
|
iter_destroy(&it);
|
|
|
|
return rc;
|
|
}
|
|
|
|
static int sched_account_jobs(xce_scheduler_t* sched)
|
|
{
|
|
iterator_t it;
|
|
int rc = 0;
|
|
|
|
rc = iter_make(&it, sched->jobs);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
for (;;) {
|
|
xce_sched_job_t* job = NULL;
|
|
|
|
rc = iter_next(&it, (void**)&job);
|
|
if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) {
|
|
break;
|
|
}
|
|
|
|
if (!JOB_IS_RUN(job->state)) {
|
|
continue;
|
|
}
|
|
|
|
if (job->accounts != NULL) {
|
|
sched_charge_account(sched, job->accounts);
|
|
}
|
|
}
|
|
|
|
if (rc == RTTI_ITER_NO_DATA) {
|
|
rc = 0;
|
|
}
|
|
|
|
iter_destroy(&it);
|
|
|
|
return rc;
|
|
}
|
|
|
|
void sched_begin_session(xce_scheduler_t* sched)
|
|
{
|
|
plugin_begin_session(sched);
|
|
|
|
sched_reset_shared_resval(sched);
|
|
sched_reset_workers(sched);
|
|
|
|
sched_account_jobs(sched);
|
|
|
|
return;
|
|
}
|
|
|
|
void sched_end_session(xce_scheduler_t* sched)
|
|
{
|
|
plugin_end_session(sched);
|
|
|
|
return;
|
|
}
|
|
|
|
int sched_add_job(xce_scheduler_t* sched, xce_sched_job_t* j)
|
|
{
|
|
int rc;
|
|
|
|
lock(sched->jobs, RTTI_LOCK_WRITE);
|
|
rc = key_add(sched->jobs, (void*)j->jobid, j);
|
|
unlock(sched->jobs);
|
|
|
|
return rc;
|
|
}
|
|
|
|
xce_sched_job_t* sched_get_job(xce_scheduler_t* sched, uint64_t jobid)
|
|
{
|
|
int rc = 0;
|
|
xce_sched_job_t* job = NULL;
|
|
|
|
lock(sched->jobs, RTTI_LOCK_READ);
|
|
rc = key_find(sched->jobs, (void*)jobid, (void**)&job);
|
|
unlock(sched->jobs);
|
|
|
|
if (rc <= 0) {
|
|
return NULL;
|
|
}
|
|
|
|
return job;
|
|
}
|
|
|
|
int sched_add_queue(xce_scheduler_t* sched, xce_sched_queue_t* q)
|
|
{
|
|
int rc;
|
|
|
|
lock(sched->queues, RTTI_LOCK_WRITE);
|
|
rc = key_add(sched->queues, (void*)q->name, q);
|
|
unlock(sched->queues);
|
|
|
|
return rc;
|
|
}
|
|
|
|
xce_sched_queue_t* sched_get_queue(xce_scheduler_t* sched, char* name)
|
|
{
|
|
xce_sched_queue_t* q = NULL;
|
|
int rc = 0;
|
|
|
|
lock(sched->queues, RTTI_LOCK_READ);
|
|
rc = key_find(sched->queues, (void*)name, (void**)&q);
|
|
unlock(sched->queues);
|
|
|
|
if (rc <= 0) {
|
|
return NULL;
|
|
}
|
|
|
|
return q;
|
|
}
|
|
|
|
int sched_add_worker(xce_scheduler_t* sched, xce_sched_worker_t* w)
|
|
{
|
|
int rc;
|
|
|
|
lock(sched->workers, RTTI_LOCK_WRITE);
|
|
rc = key_add(sched->workers, (void*)w->name, (void*)w);
|
|
unlock(sched->workers);
|
|
if (rc >= 0) {
|
|
w->idx = rc;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
xce_sched_worker_t* sched_get_worker(xce_scheduler_t* sched, char* name)
|
|
{
|
|
xce_sched_worker_t* w = NULL;
|
|
int rc = 0;
|
|
|
|
lock(sched->workers, RTTI_LOCK_READ);
|
|
rc = key_find(sched->workers, (void*)name, (void**)&w);
|
|
unlock(sched->workers);
|
|
|
|
if (rc <= 0) {
|
|
return NULL;
|
|
}
|
|
|
|
return w;
|
|
}
|
|
|
|
int sched_add_user(xce_scheduler_t* sched, xce_sched_user_t* u)
|
|
{
|
|
int rc = 0;
|
|
|
|
lock(sched->users, RTTI_LOCK_WRITE);
|
|
rc = key_add(sched->users, (void*)u->name, (void*)u);
|
|
unlock(sched->users);
|
|
|
|
return rc;
|
|
}
|
|
|
|
xce_sched_user_t* sched_get_user(xce_scheduler_t* sched, char* name)
|
|
{
|
|
xce_sched_user_t* user = NULL;
|
|
int rc = 0;
|
|
|
|
lock(sched->users, RTTI_LOCK_READ);
|
|
rc = key_find(sched->users, (void*)name, (void**)&user);
|
|
unlock(sched->users);
|
|
if (rc < 0) {
|
|
return NULL;
|
|
}
|
|
if (user != NULL) {
|
|
return user;
|
|
}
|
|
|
|
return make_sched_user(sched, name);
|
|
}
|
|
|
|
int sched_add_project(xce_scheduler_t* sched, xce_sched_project_t* p)
|
|
{
|
|
int rc;
|
|
|
|
lock(sched->projects, RTTI_LOCK_WRITE);
|
|
rc = key_add(sched->projects, (void*)p->name, (void*)p);
|
|
unlock(sched->projects);
|
|
|
|
return rc;
|
|
}
|
|
|
|
xce_sched_project_t* sched_get_project(xce_scheduler_t* sched, char* name)
|
|
{
|
|
xce_sched_project_t* project = NULL;
|
|
int rc = 0;
|
|
|
|
if ((name == NULL) || (strlen(name) == 0)) {
|
|
name = "default";
|
|
}
|
|
|
|
lock(sched->projects, RTTI_LOCK_READ);
|
|
rc = key_find(sched->projects, (void*)name, (void**)&project);
|
|
unlock(sched->projects);
|
|
if (rc < 0) {
|
|
return NULL;
|
|
}
|
|
|
|
if (rc > 0) {
|
|
return project;
|
|
}
|
|
|
|
return make_sched_project(sched, name);
|
|
}
|
|
|
|
/* FIXME this is just stub, we need to add more */
|
|
int sched_job_is_ready(xce_sched_job_t* j) { return 1; }
|
|
|
|
/* FIXME this is just stub, we need to add more */
|
|
int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
|
|
|
|
int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
|
|
{
|
|
int rc = 0;
|
|
xce_sched_job_t* job = NULL;
|
|
|
|
rc = plugin_commit_decision(sched, decision);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
if (decision->op == XCE_DECISION_DISPATCH) {
|
|
job = sched_get_job(sched, decision->u.alloc->jobid);
|
|
job->allocs = decision->u.alloc;
|
|
job->accounts = make_job_account(job->allocs);
|
|
|
|
return sched_commit_job_alloc(sched, job);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sched_publish_pend_reason(xce_scheduler_t* sched,
|
|
xce_preason_set_t* reasons)
|
|
{
|
|
return xcecli_set_pend_reason(sched->apictx, reasons);
|
|
}
|
|
|
|
int sched_publish_dispatch(xce_scheduler_t* sched, xce_job_dispatch_t* dispatch)
|
|
{
|
|
return xcecli_jobdispatch(sched->apictx, dispatch);
|
|
}
|
|
|
|
typedef struct {
|
|
xlist_t* lst;
|
|
xce_scheduler_t* sched;
|
|
sched_group_worker_fn_t groupfn;
|
|
} add_worker_args_t;
|
|
|
|
static void* default_worker_groupfn_(void* arg, xce_sched_worker_t* w)
|
|
{
|
|
return (void*)w->name;
|
|
}
|
|
|
|
static int wgroup_matched(xce_worker_group_t* wg, va_list ap)
|
|
{
|
|
void* id = va_arg(ap, void*);
|
|
|
|
return (wg->id == id);
|
|
}
|
|
|
|
static xce_worker_group_t* find_match_worker_group(xlist_t* groups, void* id)
|
|
{
|
|
xce_worker_group_t* wg = NULL;
|
|
|
|
wg = xlist_find(groups, (xlist_find_fn_t)wgroup_matched, id);
|
|
|
|
return wg;
|
|
}
|
|
|
|
int sched_collect_workers(xce_scheduler_t* sched, xlist_t* lst,
|
|
zarray_iter_skip_fn_t skipfn,
|
|
sched_group_worker_fn_t groupfn)
|
|
{
|
|
iterator_t it;
|
|
int rc = 0;
|
|
|
|
if (groupfn == NULL) {
|
|
groupfn = default_worker_groupfn_;
|
|
}
|
|
|
|
rc = iter_make(&it, sched->workers);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
for (;;) {
|
|
xce_sched_worker_t* w = NULL;
|
|
void* gid = NULL;
|
|
xce_worker_group_t* wg = NULL;
|
|
|
|
rc = iter_next(&it, (void**)&w);
|
|
if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) {
|
|
break;
|
|
}
|
|
|
|
gid = groupfn(sched, w);
|
|
wg = find_match_worker_group(lst, gid);
|
|
if (wg == NULL) {
|
|
wg = (xce_worker_group_t*)calloc(1, sizeof(*wg));
|
|
wg->id = gid;
|
|
|
|
xlist_init(&(wg->members), 8);
|
|
xlist_store(lst, wg);
|
|
}
|
|
|
|
xlist_store(&(wg->members), w);
|
|
}
|
|
|
|
iter_destroy(&it);
|
|
|
|
if (rc == RTTI_ITER_NO_DATA) {
|
|
rc = 0;
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
uint64_t sched_get_id(xce_scheduler_t* sched)
|
|
{
|
|
uint64_t id;
|
|
|
|
mylock_lock(&(sched->lock));
|
|
id = sched->nextid++;
|
|
mylock_unlock(&(sched->lock));
|
|
|
|
return id;
|
|
}
|
|
|
|
xce_alloc_t** sched_make_allocs(xce_scheduler_t* sched, char* reserve, int nres,
|
|
int* idxs, float* vals, int* duration)
|
|
{
|
|
xce_alloc_t** allocs = NULL;
|
|
int i;
|
|
|
|
if ((sched == NULL) || (nres <= 0) || (idxs == NULL) || (vals == NULL)) {
|
|
errno = EINVAL;
|
|
return NULL;
|
|
}
|
|
|
|
allocs = (xce_alloc_t**)calloc(nres, sizeof(*allocs));
|
|
if (allocs == NULL) {
|
|
return NULL;
|
|
}
|
|
|
|
for (i = 0; i < nres; i++) {
|
|
xce_resource_t* r = NULL;
|
|
int rc = 0;
|
|
int d;
|
|
|
|
rc = vget(sched->resmgr, idxs[i], (void**)&r);
|
|
d = (duration != NULL) ? duration[i] : -1;
|
|
|
|
allocs[i] = NULL;
|
|
if (r == NULL) {
|
|
continue;
|
|
}
|
|
|
|
allocs[i]
|
|
= make_alloc(r->name, reserve, sched_get_id(sched), vals[i], d);
|
|
}
|
|
|
|
return allocs;
|
|
}
|