427 lines
11 KiB
C
427 lines
11 KiB
C
#include <errno.h>
|
|
#include <pthread.h>
|
|
#include <signal.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "client.h"
|
|
#include "controller.capnp.h"
|
|
#include "decision.h"
|
|
#include "kvmgr.h"
|
|
#include "objmgr.h"
|
|
#include "plugin.h"
|
|
#include "scheduler.h"
|
|
#include "utils.h"
|
|
|
|
static void* sched_job_picker(void* arg)
|
|
{
|
|
xce_sched_plugin_t* job_picker = NULL;
|
|
xce_scheduler_t* sched = (xce_scheduler_t*)arg;
|
|
char* job_pick_plugin = "xce_sched_fcfs";
|
|
|
|
job_picker = kvmgr_lock(&(sched->plugins), job_pick_plugin, OBJREF_NOP);
|
|
if ((job_picker == NULL) || (job_picker->job_elect_fn == NULL)) {
|
|
return NULL;
|
|
}
|
|
|
|
return job_picker->job_elect_fn(job_picker);
|
|
}
|
|
|
|
static xce_decision_t* calc_decision(xce_scheduler_t* sched, uint64_t jobid)
|
|
{
|
|
xce_sched_job_t* job = sched_get_job(sched, jobid);
|
|
xce_sched_plugin_t* plugin = NULL;
|
|
xce_job_alloc_t* alloc = NULL;
|
|
xce_pendreason_t* reason = NULL;
|
|
|
|
if (job == NULL) {
|
|
ERROR("%s job %lu can not be found", __func__, jobid);
|
|
return NULL;
|
|
}
|
|
|
|
if (job->allocator == NULL) {
|
|
ERROR("%s job %lu has invalid allocator", __func__, jobid);
|
|
return NULL;
|
|
}
|
|
|
|
plugin = (xce_sched_plugin_t*)job->allocator;
|
|
if (plugin->job_alloc_fn == NULL) {
|
|
ERROR("%s plugin %s has no allocation function", __func__,
|
|
plugin->name);
|
|
return NULL;
|
|
}
|
|
|
|
alloc = plugin->job_alloc_fn(job->allocator, jobid, &reason);
|
|
if (alloc == NULL) {
|
|
return make_pend_decision(reason);
|
|
}
|
|
|
|
return make_dispatch_decision(alloc);
|
|
}
|
|
|
|
static int commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
|
|
{
|
|
return sched_commit_decision(sched, decision);
|
|
}
|
|
|
|
static void* sched_allocator(void* arg)
|
|
{
|
|
xce_scheduler_t* sched = (xce_scheduler_t*)arg;
|
|
|
|
xce_use(sched);
|
|
while (1) {
|
|
uint64_t jobid;
|
|
xce_decision_t* decision = NULL;
|
|
|
|
pthread_testcancel();
|
|
|
|
cbuf_get(&(sched->jbuf), &jobid);
|
|
if (jobid == 0) {
|
|
/* end of scheduling session */
|
|
decision = NULL;
|
|
cbuf_put(&(sched->decisions), &decision);
|
|
continue;
|
|
}
|
|
|
|
/* calculate decision for jobid */
|
|
decision = calc_decision(sched, jobid);
|
|
|
|
commit_decision(sched, decision);
|
|
|
|
cbuf_put(&(sched->decisions), &decision);
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
static void sched_pend_update_killer(data_item_t* it)
|
|
{
|
|
xce_sched_update_request_t* req;
|
|
int i;
|
|
|
|
if (it == NULL) {
|
|
return;
|
|
}
|
|
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
switch (req->op) {
|
|
case XCE_SCHED_OP_UPDATE_CONFIG:
|
|
free_Conf4Sched_ptr(&(req->u.config));
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_JOB:
|
|
for (i = 0; i < req->u.jobupdate.njobs; i++) {
|
|
free_Job_ptr(&(req->u.jobupdate.jobs[i]));
|
|
}
|
|
|
|
free(req->u.jobupdate.jobs);
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_WORKER:
|
|
for (i = 0; i < req->u.workerupdate.nworkers; i++) {
|
|
free_Worker_ptr(&(req->u.workerupdate.workers[i]));
|
|
}
|
|
|
|
free(req->u.workerupdate.workers);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
free(req);
|
|
}
|
|
|
|
static int sched_register_plugins(xce_scheduler_t* sched)
|
|
{
|
|
register_fcfs(sched);
|
|
register_default_allocator(sched);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|
xce_request_handler_t handler)
|
|
{
|
|
int rc = 0;
|
|
int ndecisions = 5; /* FIXME: make this configurable */
|
|
int njobs = 1; /* FIXME: make this configurable */
|
|
|
|
if ((sched == NULL) || (config == NULL) || (handler == NULL)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
memset(sched, 0x0, sizeof(*sched));
|
|
|
|
mylock_init(&(sched->lock));
|
|
cbuf_init(&(sched->wbuf), 1, sizeof(int));
|
|
cbuf_init(&(sched->jbuf), njobs, sizeof(uint64_t));
|
|
cbuf_init(&(sched->decisions), ndecisions, sizeof(xce_decision_t*));
|
|
|
|
sched->pend_updates[0] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->pend_updates[0], sched_pend_update_killer);
|
|
|
|
sched->pend_updates[1] = (dataq_t*)calloc(1, sizeof(dataq_t));
|
|
dataq_init(sched->pend_updates[1], sched_pend_update_killer);
|
|
|
|
kvmgr_init(&(sched->plugins), string_hash_key, 8, 8,
|
|
(zarray_item_killer_t)plugin_unregister);
|
|
kvmgr_init(&(sched->jobs), int_hash_key, 1000, 1000,
|
|
(zarray_item_killer_t)kill_sched_job);
|
|
kvmgr_init(&(sched->queues), string_hash_key, 8, 8,
|
|
(zarray_item_killer_t)kill_sched_queue);
|
|
kvmgr_init(&(sched->workers), string_hash_key, 256, 256,
|
|
(zarray_item_killer_t)kill_sched_worker);
|
|
kvmgr_init(&(sched->users), string_hash_key, 256, 256,
|
|
(zarray_item_killer_t)kill_sched_user);
|
|
kvmgr_init(&(sched->projects), string_hash_key, 8, 8,
|
|
(zarray_item_killer_t)kill_sched_project);
|
|
|
|
sched_register_plugins(sched);
|
|
|
|
rc = xcecli_ctx_init(&(sched->apictx), config);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
xcecli_set_handler(sched->apictx, handler, sched);
|
|
|
|
rc = xcecli_start(sched->apictx);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = pthread_create(&(sched->jobpicker), NULL, &sched_job_picker, sched);
|
|
if (rc < 0) {
|
|
return rc;
|
|
}
|
|
|
|
rc = pthread_create(&(sched->allocator), NULL, &sched_allocator, sched);
|
|
|
|
return rc;
|
|
}
|
|
|
|
void sched_fini(xce_scheduler_t* sched)
|
|
{
|
|
void* res;
|
|
uint64_t jobid;
|
|
|
|
if (sched == NULL) {
|
|
return;
|
|
}
|
|
|
|
pthread_cancel(sched->jobpicker);
|
|
pthread_join(sched->jobpicker, &res);
|
|
|
|
pthread_cancel(sched->allocator);
|
|
/* try to wake up allocator */
|
|
cbuf_tryput(&(sched->jbuf), &jobid);
|
|
pthread_join(sched->allocator, &res);
|
|
|
|
xcecli_wakeup(sched->apictx, (int)-EINTR);
|
|
xcecli_stop(sched->apictx);
|
|
xcecli_ctx_fini(sched->apictx);
|
|
|
|
cbuf_fini(&(sched->decisions));
|
|
cbuf_fini(&(sched->jbuf));
|
|
cbuf_fini(&(sched->wbuf));
|
|
|
|
dataq_fini(sched->pend_updates[0]);
|
|
dataq_fini(sched->pend_updates[1]);
|
|
|
|
mylock_fini(&(sched->lock));
|
|
|
|
return;
|
|
}
|
|
|
|
static int sched_update_config(xce_scheduler_t* sched, xce_conf4sched_t* config)
|
|
{
|
|
int i;
|
|
|
|
plugin_config_update(sched, config);
|
|
|
|
for (i = 0; i < config->n_queues; i++) {
|
|
xce_sched_queue_t* qold
|
|
= sched_get_queue(sched, config->queues[i]->queue);
|
|
xce_sched_queue_t* q = make_sched_queue(sched, config->queues[i]);
|
|
|
|
plugin_queue_update(sched, qold, q);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_update_jobs(xce_scheduler_t* sched, int njobs,
|
|
xce_job_t** jobs)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < njobs; i++) {
|
|
xce_sched_job_t* jold = sched_get_job(sched, jobs[i]->jobid);
|
|
xce_sched_job_t* j = make_sched_job(sched, jobs[i]);
|
|
|
|
plugin_job_update(sched, jold, j);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int sched_update_workers(xce_scheduler_t* sched, int nworkers,
|
|
xce_worker_t** workers)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < nworkers; i++) {
|
|
xce_sched_worker_t* wold = sched_get_worker(sched, workers[i]->name);
|
|
xce_sched_worker_t* w = make_sched_worker(sched, workers[i]);
|
|
|
|
plugin_worker_update(sched, wold,w);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int sched_update(xce_scheduler_t* sched)
|
|
{
|
|
dataq_t* q = NULL;
|
|
|
|
/* swap dataq */
|
|
mylock_lock(&(sched->lock));
|
|
q = sched->pend_updates[0];
|
|
sched->pend_updates[0] = sched->pend_updates[1];
|
|
sched->pend_updates[1] = q;
|
|
mylock_unlock(&(sched->lock));
|
|
|
|
while (1) {
|
|
data_item_t* it;
|
|
xce_sched_update_request_t* req;
|
|
int rc;
|
|
|
|
it = dataq_deq(q, NULL);
|
|
if (it == NULL) {
|
|
break;
|
|
}
|
|
|
|
req = ITPTR_TO_UPDATE_REQ(it);
|
|
switch (req->op) {
|
|
case XCE_SCHED_OP_UPDATE_CONFIG:
|
|
rc = sched_update_config(sched, req->u.config);
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_JOB:
|
|
rc = sched_update_jobs(sched, req->u.jobupdate.njobs,
|
|
req->u.jobupdate.jobs);
|
|
break;
|
|
case XCE_SCHED_OP_UPDATE_WORKER:
|
|
rc = sched_update_workers(sched, req->u.workerupdate.nworkers,
|
|
req->u.workerupdate.workers);
|
|
break;
|
|
default:
|
|
rc = -EINVAL;
|
|
break;
|
|
}
|
|
|
|
sched_pend_update_killer(it);
|
|
|
|
if (rc < 0) {
|
|
/* something wrong? */
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void sched_go(xce_scheduler_t* sched)
|
|
{
|
|
int go = 1;
|
|
|
|
cbuf_put(&(sched->wbuf), &go);
|
|
|
|
return;
|
|
}
|
|
|
|
void sched_begin_session(xce_scheduler_t* sched)
|
|
{
|
|
plugin_begin_session(sched);
|
|
|
|
return;
|
|
}
|
|
|
|
void sched_end_session(xce_scheduler_t* sched)
|
|
{
|
|
plugin_end_session(sched);
|
|
|
|
return;
|
|
}
|
|
|
|
int sched_add_job(xce_scheduler_t* sched, xce_sched_job_t* j)
|
|
{
|
|
return kvmgr_add(&(sched->jobs), (void*)j->jobid, j);
|
|
}
|
|
|
|
xce_sched_job_t* sched_get_job(xce_scheduler_t* sched, uint64_t jobid)
|
|
{
|
|
return kvmgr_lock(&(sched->jobs), (void*)jobid, OBJREF_NOP);
|
|
}
|
|
|
|
int sched_add_queue(xce_scheduler_t* sched, xce_sched_queue_t* q)
|
|
{
|
|
return kvmgr_add(&(sched->queues), q->name, q);
|
|
}
|
|
|
|
xce_sched_queue_t* sched_get_queue(xce_scheduler_t* sched, char* name)
|
|
{
|
|
return kvmgr_lock(&(sched->queues), (void*)name, OBJREF_NOP);
|
|
}
|
|
|
|
int sched_add_worker(xce_scheduler_t* sched, xce_sched_worker_t* w)
|
|
{
|
|
return kvmgr_add(&(sched->workers), w->name, w);
|
|
}
|
|
|
|
xce_sched_worker_t* sched_get_worker(xce_scheduler_t* sched, char* name)
|
|
{
|
|
return kvmgr_lock(&(sched->workers), (void*)name, OBJREF_NOP);
|
|
}
|
|
|
|
int sched_add_user(xce_scheduler_t* sched, xce_sched_user_t* u)
|
|
{
|
|
return kvmgr_add(&(sched->users), u->name, u);
|
|
}
|
|
|
|
xce_sched_user_t* sched_get_user(xce_scheduler_t* sched, char* name)
|
|
{
|
|
return kvmgr_lock(&(sched->users), (void*)name, OBJREF_NOP);
|
|
}
|
|
|
|
int sched_add_project(xce_scheduler_t* sched, xce_sched_project_t* p)
|
|
{
|
|
return kvmgr_add(&(sched->projects), p->name, p);
|
|
}
|
|
|
|
xce_sched_project_t* sched_get_project(xce_scheduler_t* sched, char* name)
|
|
{
|
|
return kvmgr_lock(&(sched->projects), (void*)name, OBJREF_NOP);
|
|
}
|
|
|
|
int sched_job_is_ready(xce_sched_job_t* j) { return 1; }
|
|
|
|
int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
|
|
|
|
int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
|
|
{
|
|
return plugin_commit_decision(sched, decision);
|
|
}
|
|
|
|
int sched_publish_pend_reason(xce_scheduler_t* sched,
|
|
xce_preason_set_t* reasons)
|
|
{
|
|
return xcecli_set_pend_reason(sched->apictx, reasons);
|
|
}
|
|
|
|
int sched_publish_dispatch(xce_scheduler_t* sched, xce_job_dispatch_t* dispatch)
|
|
{
|
|
return xcecli_jobdispatch(sched->apictx, dispatch);
|
|
}
|