update controller code
This commit is contained in:
Rongsong Shen
parent
2836d196da
commit
41f56fa6ed
54 changed files with 9688 additions and 214 deletions
620
src/controller/sched.c
Normal file
620
src/controller/sched.c
Normal file
|
|
@ -0,0 +1,620 @@
|
|||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "cluster.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "objmgr.h"
|
||||
#include "request.h"
|
||||
#include "worker.h"
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
extern void collect_shared_resvals(cluster_t*, collect_check_fn_t,
|
||||
int*, xce_val_with_idx_t***);
|
||||
|
||||
static int xcesched_push_joblist(cluster_t* c,
|
||||
xlist_t* l)
|
||||
{
|
||||
xce_data_t data;
|
||||
xce_jobs_data_t jobs = {
|
||||
.n_total = l->count,
|
||||
.pos = 0,
|
||||
.count = l->count,
|
||||
.jobs = (xce_job_t**)l->data
|
||||
};
|
||||
int rc = 0;
|
||||
|
||||
memset(&data, 0x0, sizeof(data));
|
||||
|
||||
data.table = Data_data_schedjobs;
|
||||
data.data.jobs = &jobs;
|
||||
|
||||
rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
|
||||
check_controller, c, 100);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int xcesched_push_workerlist(cluster_t* c,
|
||||
xlist_t* l)
|
||||
{
|
||||
xce_data_t data;
|
||||
xce_workers_data_t workers = {
|
||||
.n_total = l->count,
|
||||
.pos = 0,
|
||||
.count = l->count,
|
||||
.workers = (xce_worker_t**)l->data
|
||||
};
|
||||
int rc = 0;
|
||||
|
||||
memset(&data, 0x0, sizeof(data));
|
||||
|
||||
data.table = Data_data_schedworkers;
|
||||
data.data.workers = &workers;
|
||||
|
||||
rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
|
||||
check_controller, c, 100);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int xcesched_push_configs(cluster_t* c)
|
||||
{
|
||||
xce_configdata_t configdata;
|
||||
int rc = 0;
|
||||
|
||||
memset(&configdata, 0x0, sizeof(configdata));
|
||||
|
||||
configdata.data.conf4sched = (xce_conf4sched_t*)calloc(1, sizeof(xce_conf4sched_t));
|
||||
if (configdata.data.conf4sched == NULL) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
collect_confvars(c, NULL, &(configdata.data.conf4sched->n_confvars),
|
||||
&(configdata.data.conf4sched->confvars));
|
||||
collect_queues(c, NULL, &(configdata.data.conf4sched->n_queues),
|
||||
&(configdata.data.conf4sched->queues));
|
||||
collect_resources(c, NULL, &(configdata.data.conf4sched->n_resources),
|
||||
&(configdata.data.conf4sched->resources));
|
||||
collect_shared_resvals(c, NULL, &(configdata.data.conf4sched->nresvals),
|
||||
&(configdata.data.conf4sched->resvals));
|
||||
|
||||
configdata.data.conf4sched->n_ugroups = c->myconf->cluster_config->nugroups;
|
||||
configdata.data.conf4sched->ugroups = c->myconf->cluster_config->ugroups;
|
||||
configdata.data.conf4sched->n_hgroups = c->myconf->cluster_config->nhgroups;
|
||||
configdata.data.conf4sched->hgroups = c->myconf->cluster_config->hgroups;
|
||||
|
||||
/* add resources */
|
||||
configdata.data_which = ConfigData_data_schedconfig;
|
||||
configdata.data.conf4sched->clustername = c->myconf->cluster_config->clustername;
|
||||
|
||||
rc = xce_pushconfig_req(c->sched.ep, c->myhost,
|
||||
&configdata);
|
||||
|
||||
free_resval_with_idx_list(&(configdata.data.conf4sched->resvals),
|
||||
configdata.data.conf4sched->nresvals);
|
||||
|
||||
deref_objs(configdata.data.conf4sched->n_confvars,
|
||||
(void**)configdata.data.conf4sched->confvars);
|
||||
deref_objs(configdata.data.conf4sched->n_queues,
|
||||
(void**)configdata.data.conf4sched->queues);
|
||||
deref_objs(configdata.data.conf4sched->n_resources,
|
||||
(void**)configdata.data.conf4sched->resources);
|
||||
|
||||
FREEUP(configdata.data.conf4sched->confvars);
|
||||
FREEUP(configdata.data.conf4sched->queues);
|
||||
FREEUP(configdata.data.conf4sched->resources);
|
||||
|
||||
FREEUP(configdata.data.conf4sched);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int check_job_(xce_job_t* j)
|
||||
{
|
||||
if (j == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int xcesched_push_jobs(cluster_t* c)
|
||||
{
|
||||
xce_data_t data;
|
||||
xce_jobs_data_t jobs = {
|
||||
.n_total = 0,
|
||||
.pos = 0,
|
||||
.count = 0,
|
||||
.jobs = NULL
|
||||
};
|
||||
int rc = 0;
|
||||
|
||||
memset(&data, 0x0, sizeof(data));
|
||||
|
||||
data.table = Data_data_schedjobs;
|
||||
data.data.jobs = &jobs;
|
||||
|
||||
collect_jobs(c, (collect_check_fn_t)check_job_,
|
||||
&(jobs.n_total), &(jobs.jobs));
|
||||
jobs.count = jobs.n_total;
|
||||
|
||||
rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
|
||||
check_controller, c, 100);
|
||||
deref_objs(jobs.n_total, (void**)jobs.jobs);
|
||||
FREEUP(jobs.jobs);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int xcesched_push_workers(cluster_t* c)
|
||||
{
|
||||
xce_data_t data;
|
||||
xce_workers_data_t workers = {
|
||||
.n_total = 0,
|
||||
.pos = 0,
|
||||
.count = 0,
|
||||
.workers = NULL
|
||||
};
|
||||
int rc = 0;
|
||||
int i;
|
||||
|
||||
memset(&data, 0x0, sizeof(data));
|
||||
|
||||
data.table = Data_data_schedworkers;
|
||||
data.data.workers = &workers;
|
||||
|
||||
collect_workers(c, (collect_check_fn_t)check_worker_,
|
||||
&(workers.n_total), &(workers.workers));
|
||||
workers.count = workers.n_total;
|
||||
|
||||
rc = xce_push_data_req(c->sched.ep, c->myhost, &data,
|
||||
check_controller, c, 100);
|
||||
|
||||
/* free up collected resource value which used
|
||||
passed to scheduler
|
||||
*/
|
||||
for (i = 0; i < workers.n_total; i++) {
|
||||
xce_worker_t* w = workers.workers[i];
|
||||
|
||||
free_resval_with_idx_list(&(w->resvals_to_sched),
|
||||
w->nresvals_to_sched);
|
||||
}
|
||||
|
||||
deref_objs(workers.n_total, (void**)workers.workers);
|
||||
FREEUP(workers.workers);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_new_fun(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
xlist_t* l = va_arg(ap, xlist_t*);
|
||||
int rc = 0;
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = -EINVAL;
|
||||
if (l != NULL) {
|
||||
rc = xcesched_push_joblist(c, l);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_start_fun(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
job_pidinfo_t* info = va_arg(ap, job_pidinfo_t*);
|
||||
int rc = 0;
|
||||
/* clang-format off */
|
||||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedStartJob,
|
||||
.data = {
|
||||
.sched_start_job = info->jobid
|
||||
}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = notify(c, c->sched.ep, &xdata);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_exit_fun(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
xce_jobfinish_t* info = va_arg(ap, xce_jobfinish_t*);
|
||||
int rc = 0;
|
||||
/* clang-format off */
|
||||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedExitJob,
|
||||
.data = {
|
||||
.sched_exit_job = info->jobid
|
||||
}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = notify(c, c->sched.ep, &xdata);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_fail_fun(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
job_failinfo_t* info = va_arg(ap, job_failinfo_t*);
|
||||
int rc = 0;
|
||||
/* clang-format off */
|
||||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedFailJob,
|
||||
.data = {
|
||||
.sched_fail_job = info->jobid
|
||||
}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = notify(c, c->sched.ep, &xdata);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_update_fun(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
job_t* job = va_arg(ap, job_t*);
|
||||
int rc = 0;
|
||||
/* clang-format off */
|
||||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedUpdateJob,
|
||||
.data = {
|
||||
.sched_update_job = &(job->job)
|
||||
}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = notify(c, c->sched.ep, &xdata);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_job_hook_fun(int type,
|
||||
int situation,
|
||||
int detail,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
switch (detail) {
|
||||
case HOOK_DETAIL_JOB_NEW:
|
||||
return sched_job_new_fun(type, situation,
|
||||
data, ap);
|
||||
case HOOK_DETAIL_JOB_START:
|
||||
return sched_job_start_fun(type, situation,
|
||||
data, ap);
|
||||
case HOOK_DETAIL_JOB_EXIT:
|
||||
return sched_job_exit_fun(type, situation,
|
||||
data, ap);
|
||||
case HOOK_DETAIL_JOB_UPDATE:
|
||||
return sched_job_update_fun(type, situation,
|
||||
data, ap);
|
||||
case HOOK_DETAIL_JOB_FAIL:
|
||||
return sched_job_fail_fun(type, situation,
|
||||
data, ap);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
static int sched_worker_change(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
cluster_t* c,
|
||||
worker_t* w)
|
||||
{
|
||||
int rc = 0;
|
||||
xlist_t wlist;
|
||||
|
||||
TRACE("%s worker %s idx %d situation %d",
|
||||
__func__, w->host.name,
|
||||
w->idx, situation);
|
||||
|
||||
collect_resval_with_idx(&(w->host.resvals),
|
||||
&(w->host.nresvals_to_sched),
|
||||
&(w->host.resvals_to_sched));
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
xlist_init(&wlist, 2);
|
||||
xlist_store(&wlist, w);
|
||||
rc = xcesched_push_workerlist(c, &wlist);
|
||||
xlist_fini(&wlist);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
free_resval_with_idx_list(&(w->host.resvals_to_sched),
|
||||
w->host.nresvals_to_sched);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_worker_join(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
char* name = va_arg(ap, char*);
|
||||
worker_t* w = NULL;
|
||||
int rc;
|
||||
|
||||
if (situation == HOOK_BEFORE_EVENT) {
|
||||
/* nothing to do */
|
||||
return 0;
|
||||
}
|
||||
|
||||
w = workermgr_lock(&(c->workermgr), name,
|
||||
OBJREF_READ);
|
||||
|
||||
rc = sched_worker_change(type, situation, data,
|
||||
c, w);
|
||||
|
||||
/* FIXME: need to mark the jobs in this worker */
|
||||
obj_deref(w);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_worker_dead(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
worker_t* w = va_arg(ap, worker_t*);
|
||||
|
||||
/* FIXME: need to handle job status in this worker */
|
||||
if (situation == HOOK_BEFORE_EVENT) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return sched_worker_change(type, situation, data,
|
||||
c, w);
|
||||
}
|
||||
|
||||
static int sched_worker_update(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
worker_t* w = va_arg(ap, worker_t*);
|
||||
|
||||
if (situation == HOOK_BEFORE_EVENT) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return sched_worker_change(type, situation, data,
|
||||
c, w);
|
||||
}
|
||||
static int sched_worker_hook_fun(int type,
|
||||
int situation,
|
||||
int detail,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
switch (detail) {
|
||||
case HOOK_DETAIL_WORKER_JOIN:
|
||||
return sched_worker_join(type, situation,
|
||||
data, ap);
|
||||
case HOOK_DETAIL_WORKER_DEAD:
|
||||
return sched_worker_dead(type, situation,
|
||||
data, ap);
|
||||
case HOOK_DETAIL_WORKER_UPDATE:
|
||||
return sched_worker_update(type, situation,
|
||||
data, ap);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sched_resource_hook_fun(int type,
|
||||
int situation,
|
||||
int detail,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
xce_resval_list_t* resvals = va_arg(ap, xce_resval_list_t*);
|
||||
/* clang-format on */
|
||||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedUpdateResvals,
|
||||
.data = {
|
||||
.sched_update_resvals = resvals }
|
||||
};
|
||||
/* clang-format off */
|
||||
int rc = 0;
|
||||
|
||||
if (detail != HOOK_DETAIL_RESOURCE_UPDATE) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
xce_use(resvals);
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = notify(c, c->sched.ep, &xdata);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int sched_queue_update(int type,
|
||||
int situation,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
queue_t* q = va_arg(ap, queue_t*);
|
||||
int rc = 0;
|
||||
/* clang-format off */
|
||||
xce_notify_data_t xdata = {
|
||||
.event = NotifyData_data_schedUpdateQueue,
|
||||
.data = {
|
||||
.sched_update_queue = &(q->queue)
|
||||
}
|
||||
};
|
||||
/* clang-format on */
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
switch (situation) {
|
||||
case HOOK_AFTER_EVENT:
|
||||
rc = notify(c, c->sched.ep, &xdata);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
return rc;
|
||||
}
|
||||
static int sched_queue_hook_fun(int type,
|
||||
int situation,
|
||||
int detail,
|
||||
void* data,
|
||||
va_list ap)
|
||||
{
|
||||
switch (detail) {
|
||||
case HOOK_DETAIL_QUEUE_UPDATE:
|
||||
return sched_queue_update(type, situation, data, ap);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int on_scheduler_join(cluster_t* c)
|
||||
{
|
||||
static int hook_installed = 0;
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
xcesched_push_configs(c);
|
||||
xcesched_push_workers(c);
|
||||
xcesched_push_jobs(c);
|
||||
}
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
|
||||
if (!hook_installed) {
|
||||
hook_installed = 1;
|
||||
|
||||
xce_add_hook(&(c->hooksys),
|
||||
XCE_CONTROLLER_HOOK_JOB,
|
||||
(hook_fn_t)sched_job_hook_fun,
|
||||
NULL);
|
||||
xce_add_hook(&(c->hooksys),
|
||||
XCE_CONTROLLER_HOOK_WORKER,
|
||||
(hook_fn_t)sched_worker_hook_fun,
|
||||
NULL);
|
||||
xce_add_hook(&(c->hooksys),
|
||||
XCE_CONTROLLER_HOOK_RESOURCE,
|
||||
(hook_fn_t)sched_resource_hook_fun,
|
||||
NULL);
|
||||
xce_add_hook(&(c->hooksys),
|
||||
XCE_CONTROLLER_HOOK_QUEUE,
|
||||
(hook_fn_t)sched_queue_hook_fun,
|
||||
NULL);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int handle_scheduler_leave(cluster_t* c)
|
||||
{
|
||||
/* we need to mark the cluster status */
|
||||
TRACE("%s scheduler leave",
|
||||
__func__);
|
||||
mylock_lock(&(c->sched.lock));
|
||||
c->sched.ep = NULL;
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
return 0;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue