#include #include #include "cluster.h" #include "controller.capnp.h" #include "objmgr.h" #include "request.h" #include "worker.h" #include "xce/cluster_def.h" #include "xce/types.h" extern void collect_shared_resvals(cluster_t*, collect_check_fn_t, int*, xce_val_with_idx_t***); static int xcesched_push_joblist(cluster_t* c, xlist_t* l) { xce_data_t data; xce_jobs_data_t jobs = { .n_total = l->count, .pos = 0, .count = l->count, .jobs = (xce_job_t**)l->data }; int rc = 0; memset(&data, 0x0, sizeof(data)); data.table = Data_data_schedjobs; data.data.jobs = &jobs; rc = xce_push_data_req(c->sched.ep, c->myhost, &data, check_controller, c, 100); return rc; } static int xcesched_push_workerlist(cluster_t* c, xlist_t* l) { xce_data_t data; xce_workers_data_t workers = { .n_total = l->count, .pos = 0, .count = l->count, .workers = (xce_worker_t**)l->data }; int rc = 0; memset(&data, 0x0, sizeof(data)); data.table = Data_data_schedworkers; data.data.workers = &workers; rc = xce_push_data_req(c->sched.ep, c->myhost, &data, check_controller, c, 100); return rc; } static int xcesched_push_configs(cluster_t* c) { xce_configdata_t configdata; int rc = 0; memset(&configdata, 0x0, sizeof(configdata)); configdata.data.conf4sched = (xce_conf4sched_t*)calloc(1, sizeof(xce_conf4sched_t)); if (configdata.data.conf4sched == NULL) { return -ENOMEM; } collect_confvars(c, NULL, &(configdata.data.conf4sched->n_confvars), &(configdata.data.conf4sched->confvars)); collect_queues(c, NULL, &(configdata.data.conf4sched->n_queues), &(configdata.data.conf4sched->queues)); collect_resources(c, NULL, &(configdata.data.conf4sched->n_resources), &(configdata.data.conf4sched->resources)); collect_shared_resvals(c, NULL, &(configdata.data.conf4sched->nresvals), &(configdata.data.conf4sched->resvals)); configdata.data.conf4sched->n_ugroups = c->myconf->cluster_config->nugroups; configdata.data.conf4sched->ugroups = c->myconf->cluster_config->ugroups; configdata.data.conf4sched->n_hgroups = c->myconf->cluster_config->nhgroups; configdata.data.conf4sched->hgroups = c->myconf->cluster_config->hgroups; /* add resources */ configdata.data_which = ConfigData_data_schedconfig; configdata.data.conf4sched->clustername = c->myconf->cluster_config->clustername; rc = xce_pushconfig_req(c->sched.ep, c->myhost, &configdata); free_resval_with_idx_list(&(configdata.data.conf4sched->resvals), configdata.data.conf4sched->nresvals); deref_objs(configdata.data.conf4sched->n_confvars, (void**)configdata.data.conf4sched->confvars); deref_objs(configdata.data.conf4sched->n_queues, (void**)configdata.data.conf4sched->queues); deref_objs(configdata.data.conf4sched->n_resources, (void**)configdata.data.conf4sched->resources); FREEUP(configdata.data.conf4sched->confvars); FREEUP(configdata.data.conf4sched->queues); FREEUP(configdata.data.conf4sched->resources); FREEUP(configdata.data.conf4sched); return rc; } static int check_job_(xce_job_t* j) { if (j == NULL) { return 0; } return 1; } static int xcesched_push_jobs(cluster_t* c) { xce_data_t data; xce_jobs_data_t jobs = { .n_total = 0, .pos = 0, .count = 0, .jobs = NULL }; int rc = 0; memset(&data, 0x0, sizeof(data)); data.table = Data_data_schedjobs; data.data.jobs = &jobs; collect_jobs(c, (collect_check_fn_t)check_job_, &(jobs.n_total), &(jobs.jobs)); jobs.count = jobs.n_total; rc = xce_push_data_req(c->sched.ep, c->myhost, &data, check_controller, c, 100); deref_objs(jobs.n_total, (void**)jobs.jobs); FREEUP(jobs.jobs); return rc; } static int xcesched_push_workers(cluster_t* c) { xce_data_t data; xce_workers_data_t workers = { .n_total = 0, .pos = 0, .count = 0, .workers = NULL }; int rc = 0; int i; memset(&data, 0x0, sizeof(data)); data.table = Data_data_schedworkers; data.data.workers = &workers; collect_workers(c, (collect_check_fn_t)check_worker_, &(workers.n_total), &(workers.workers)); workers.count = workers.n_total; rc = xce_push_data_req(c->sched.ep, c->myhost, &data, check_controller, c, 100); /* free up collected resource value which used passed to scheduler */ for (i = 0; i < workers.n_total; i++) { xce_worker_t* w = workers.workers[i]; free_resval_with_idx_list(&(w->resvals_to_sched), w->nresvals_to_sched); } deref_objs(workers.n_total, (void**)workers.workers); FREEUP(workers.workers); return rc; } static int sched_job_new_fun(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); xlist_t* l = va_arg(ap, xlist_t*); int rc = 0; mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = -EINVAL; if (l != NULL) { rc = xcesched_push_joblist(c, l); } break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_job_start_fun(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); job_pidinfo_t* info = va_arg(ap, job_pidinfo_t*); int rc = 0; /* clang-format off */ xce_notify_data_t xdata = { .event = NotifyData_data_schedStartJob, .data = { .sched_start_job = info->jobid } }; /* clang-format on */ mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = notify(c, c->sched.ep, &xdata); break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_job_exit_fun(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); xce_jobfinish_t* info = va_arg(ap, xce_jobfinish_t*); int rc = 0; /* clang-format off */ xce_notify_data_t xdata = { .event = NotifyData_data_schedExitJob, .data = { .sched_exit_job = info->jobid } }; /* clang-format on */ mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = notify(c, c->sched.ep, &xdata); break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_job_fail_fun(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); job_failinfo_t* info = va_arg(ap, job_failinfo_t*); int rc = 0; /* clang-format off */ xce_notify_data_t xdata = { .event = NotifyData_data_schedFailJob, .data = { .sched_fail_job = info->jobid } }; /* clang-format on */ mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = notify(c, c->sched.ep, &xdata); break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_job_update_fun(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); job_t* job = va_arg(ap, job_t*); int rc = 0; /* clang-format off */ xce_notify_data_t xdata = { .event = NotifyData_data_schedUpdateJob, .data = { .sched_update_job = &(job->job) } }; /* clang-format on */ mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = notify(c, c->sched.ep, &xdata); break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_job_hook_fun(int type, int situation, int detail, void* data, va_list ap) { switch (detail) { case HOOK_DETAIL_JOB_NEW: return sched_job_new_fun(type, situation, data, ap); case HOOK_DETAIL_JOB_START: return sched_job_start_fun(type, situation, data, ap); case HOOK_DETAIL_JOB_EXIT: return sched_job_exit_fun(type, situation, data, ap); case HOOK_DETAIL_JOB_UPDATE: return sched_job_update_fun(type, situation, data, ap); case HOOK_DETAIL_JOB_FAIL: return sched_job_fail_fun(type, situation, data, ap); } return 0; } static int sched_worker_change(int type, int situation, void* data, cluster_t* c, worker_t* w) { int rc = 0; xlist_t wlist; TRACE("%s worker %s idx %d situation %d", __func__, w->host.name, w->idx, situation); collect_resval_with_idx(&(w->host.resvals), &(w->host.nresvals_to_sched), &(w->host.resvals_to_sched)); mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: xlist_init(&wlist, 2); xlist_store(&wlist, w); rc = xcesched_push_workerlist(c, &wlist); xlist_fini(&wlist); break; default: break; } } mylock_unlock(&(c->sched.lock)); free_resval_with_idx_list(&(w->host.resvals_to_sched), w->host.nresvals_to_sched); return rc; } static int sched_worker_join(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); char* name = va_arg(ap, char*); worker_t* w = NULL; int rc; if (situation == HOOK_BEFORE_EVENT) { /* nothing to do */ return 0; } w = workermgr_lock(&(c->workermgr), name, OBJREF_READ); rc = sched_worker_change(type, situation, data, c, w); /* FIXME: need to mark the jobs in this worker */ obj_deref(w); return rc; } static int sched_worker_dead(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); worker_t* w = va_arg(ap, worker_t*); /* FIXME: need to handle job status in this worker */ if (situation == HOOK_BEFORE_EVENT) { return 0; } return sched_worker_change(type, situation, data, c, w); } static int sched_worker_update(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); worker_t* w = va_arg(ap, worker_t*); if (situation == HOOK_BEFORE_EVENT) { return 0; } return sched_worker_change(type, situation, data, c, w); } static int sched_worker_hook_fun(int type, int situation, int detail, void* data, va_list ap) { switch (detail) { case HOOK_DETAIL_WORKER_JOIN: return sched_worker_join(type, situation, data, ap); case HOOK_DETAIL_WORKER_DEAD: return sched_worker_dead(type, situation, data, ap); case HOOK_DETAIL_WORKER_UPDATE: return sched_worker_update(type, situation, data, ap); } return 0; } static int sched_resource_hook_fun(int type, int situation, int detail, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); xce_resval_list_t* resvals = va_arg(ap, xce_resval_list_t*); /* clang-format on */ xce_notify_data_t xdata = { .event = NotifyData_data_schedUpdateResvals, .data = { .sched_update_resvals = resvals } }; /* clang-format off */ int rc = 0; if (detail != HOOK_DETAIL_RESOURCE_UPDATE) { return 0; } xce_use(resvals); mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = notify(c, c->sched.ep, &xdata); break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_queue_update(int type, int situation, void* data, va_list ap) { cluster_t* c = va_arg(ap, cluster_t*); queue_t* q = va_arg(ap, queue_t*); int rc = 0; /* clang-format off */ xce_notify_data_t xdata = { .event = NotifyData_data_schedUpdateQueue, .data = { .sched_update_queue = &(q->queue) } }; /* clang-format on */ mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { switch (situation) { case HOOK_AFTER_EVENT: rc = notify(c, c->sched.ep, &xdata); break; default: break; } } mylock_unlock(&(c->sched.lock)); return rc; } static int sched_queue_hook_fun(int type, int situation, int detail, void* data, va_list ap) { switch (detail) { case HOOK_DETAIL_QUEUE_UPDATE: return sched_queue_update(type, situation, data, ap); default: break; } return 0; } int on_scheduler_join(cluster_t* c) { static int hook_installed = 0; mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { xcesched_push_configs(c); xcesched_push_workers(c); xcesched_push_jobs(c); } mylock_unlock(&(c->sched.lock)); if (!hook_installed) { hook_installed = 1; xce_add_hook(&(c->hooksys), XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)sched_job_hook_fun, NULL); xce_add_hook(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)sched_worker_hook_fun, NULL); xce_add_hook(&(c->hooksys), XCE_CONTROLLER_HOOK_RESOURCE, (hook_fn_t)sched_resource_hook_fun, NULL); xce_add_hook(&(c->hooksys), XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)sched_queue_hook_fun, NULL); } return 0; } int handle_scheduler_leave(cluster_t* c) { /* we need to mark the cluster status */ TRACE("%s scheduler leave", __func__); mylock_lock(&(c->sched.lock)); c->sched.ep = NULL; mylock_unlock(&(c->sched.lock)); return 0; }