scheduling using real slots number
This commit is contained in:
parent
8db4458375
commit
780e9fb313
22 changed files with 980 additions and 105 deletions
|
|
@ -35,7 +35,7 @@ typedef int (*plugin_begin_session_fn_t)(void*);
|
|||
|
||||
typedef int (*plugin_end_session_fn_t)(void*);
|
||||
/* allocation function of job */
|
||||
typedef xce_job_alloc_t* (*plugin_job_allocate_fn_t)(void*, int64_t,
|
||||
typedef xce_job_alloc_t* (*plugin_job_allocate_fn_t)(void*, int64_t, resreq_t*,
|
||||
xce_pendreason_t**);
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -10,14 +10,36 @@
|
|||
#include "xce/types.h"
|
||||
#include "zarray.h"
|
||||
|
||||
/* FIXME: improve later */
|
||||
typedef void* host_selection_t;
|
||||
|
||||
typedef enum {
|
||||
RESREQ_OP_SIMPLE
|
||||
= 0, /* simple just a list of resources and require values */
|
||||
RESREQ_OP_AFFINITY, /* with affinity allocation */
|
||||
RESREQ_OP_GPU, /* with GPU allocation */
|
||||
RESREQ_OP_PARALL, /* parall job nhosts, ptile and other constraints */
|
||||
RESREQ_OP_AND, /* success for all */
|
||||
RESREQ_OP_OR /* success for any */
|
||||
} resreq_op_t;
|
||||
|
||||
/* simple resource allocation just have numeric resource values */
|
||||
typedef struct {
|
||||
host_selection_t* select;
|
||||
int nres;
|
||||
uint32_t* resources;
|
||||
float* amount;
|
||||
} rank_req_t;
|
||||
host_selection_t* selection;
|
||||
int nres_shared;
|
||||
int* idxs_shared;
|
||||
float* vals_shared;
|
||||
int nres_host;
|
||||
int* idxs_host;
|
||||
float* vals_host;
|
||||
} resreq_simple_t;
|
||||
|
||||
typedef struct {
|
||||
resreq_op_t op;
|
||||
union {
|
||||
resreq_simple_t simple;
|
||||
} u;
|
||||
} resreq_t;
|
||||
|
||||
typedef struct {
|
||||
char* name;
|
||||
|
|
@ -48,10 +70,11 @@ typedef struct {
|
|||
xce_sched_queue_t* queue;
|
||||
xce_sched_user_t* user;
|
||||
xce_sched_project_t* project;
|
||||
void* allocator;
|
||||
resreq_t* resreq;
|
||||
uint32_t mintasks;
|
||||
uint32_t maxtasks;
|
||||
xce_job_alloc_t* allocs;
|
||||
xce_job_account_t* accounts;
|
||||
} xce_sched_job_t;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -52,11 +52,17 @@ typedef struct {
|
|||
mylock_t lock;
|
||||
int stop;
|
||||
xcecli_ctx_t apictx;
|
||||
uint64_t nextid;
|
||||
xce_conf4sched_t* config;
|
||||
kvmgr_t plugins;
|
||||
void* simple_allocator;
|
||||
void* affinity_allocator;
|
||||
void* gpu_allocator;
|
||||
void* parallel_allocator;
|
||||
dataq_t* pend_updates[2];
|
||||
resmgr_t resmgr;
|
||||
zarray_t sharedInst;
|
||||
zarray_t sharedInUse;
|
||||
kvmgr_t jobs;
|
||||
kvmgr_t queues;
|
||||
kvmgr_t workers;
|
||||
|
|
@ -92,5 +98,17 @@ extern int sched_queue_is_ready(xce_sched_queue_t*);
|
|||
extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
|
||||
extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
|
||||
extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);
|
||||
extern int sched_meet_host_selection(xce_scheduler_t*, xce_sched_worker_t*,
|
||||
host_selection_t*);
|
||||
extern int sched_check_shared_satisfy(xce_scheduler_t*, int, int*, float*, int*,
|
||||
int*, float*);
|
||||
extern int sched_check_host_satisfy(xce_scheduler_t*, xce_sched_worker_t*, int,
|
||||
int*, float*, int*, int*, float*);
|
||||
extern int sched_collect_workers(xce_scheduler_t*, xlist_t*,
|
||||
zarray_iter_skip_fn_t);
|
||||
extern int sched_commit_job_alloc(xce_scheduler_t*, xce_sched_job_t*);
|
||||
extern uint64_t sched_get_id(xce_scheduler_t*);
|
||||
extern xce_alloc_t** sched_make_allocs(xce_scheduler_t*, char*, int, int*,
|
||||
float*, int*);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ scheduler_srcs = [
|
|||
'src/decision.c',
|
||||
'src/schedobj.c',
|
||||
'src/plugin.c',
|
||||
'src/resource.c',
|
||||
'src/plugins/fcfs.c',
|
||||
'src/plugins/simple_allocator.c',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -5,17 +5,23 @@
|
|||
#include <string.h>
|
||||
|
||||
#include "decision.h"
|
||||
#include "kvmgr.h"
|
||||
#include "objmgr.h"
|
||||
#include "plugin.h"
|
||||
#include "preason.h"
|
||||
#include "scheduler.h"
|
||||
#include "utils.h"
|
||||
#include "xce/types.h"
|
||||
#include "xcealloc.h"
|
||||
#include "xlist.h"
|
||||
|
||||
static uint32_t maxslots = 8;
|
||||
static uint32_t slotsInUse = 0;
|
||||
static int simple_allocator_init(xce_sched_plugin_t* plugin)
|
||||
{
|
||||
xce_scheduler_t* sched = (xce_scheduler_t*)plugin->sched;
|
||||
|
||||
static int simple_allocator_init(xce_sched_plugin_t* plugin) { return 1; }
|
||||
sched->simple_allocator = plugin;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void simple_allocator_fini(xce_sched_plugin_t* plugin)
|
||||
{
|
||||
|
|
@ -28,10 +34,7 @@ static void simple_allocator_fini(xce_sched_plugin_t* plugin)
|
|||
static int my_update_job(xce_sched_plugin_t* plugin, xce_sched_job_t* jold,
|
||||
xce_sched_job_t* j)
|
||||
{
|
||||
/* FIXME: we need to add logic to check whether the job should be handled by
|
||||
* me */
|
||||
j->allocator = plugin;
|
||||
|
||||
#if 0
|
||||
if (jold == NULL) {
|
||||
if (JOB_IS_RUN(j->state)) {
|
||||
slotsInUse++;
|
||||
|
|
@ -41,36 +44,134 @@ static int my_update_job(xce_sched_plugin_t* plugin, xce_sched_job_t* jold,
|
|||
slotsInUse--;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static xce_job_alloc_t* my_alloc(xce_sched_plugin_t* plugin, int64_t jobid,
|
||||
xce_pendreason_t** preason)
|
||||
static int skip_bad_worker_(xce_sched_worker_t* w)
|
||||
{
|
||||
xce_job_alloc_t* jalloc = NULL;
|
||||
if (w == NULL) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
*preason = NULL;
|
||||
/* add more check here */
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (slotsInUse < maxslots) {
|
||||
xce_alloc_t* alloc = NULL;
|
||||
xce_alloc_t** allocs = NULL;
|
||||
xce_host_alloc_t* halloc = NULL;
|
||||
xce_host_alloc_t** hallocs = NULL;
|
||||
static int worker_is_satisfy_(xce_sched_worker_t* w, va_list ap)
|
||||
{
|
||||
xce_scheduler_t* sched = va_arg(ap, xce_scheduler_t*);
|
||||
uint64_t jobid = va_arg(ap, uint64_t);
|
||||
resreq_t* resreq = va_arg(ap, resreq_t*);
|
||||
xce_job_alloc_t** jalloc = va_arg(ap, xce_job_alloc_t**);
|
||||
xce_pendreason_t** preason = va_arg(ap, xce_pendreason_t**);
|
||||
int rc;
|
||||
int unmeet_idxs[resreq->u.simple.nres_host + 1];
|
||||
float unmeet_vals[resreq->u.simple.nres_host + 1];
|
||||
int n_unmeet;
|
||||
xce_host_alloc_t** hallocs = NULL;
|
||||
xce_host_alloc_t* halloc = NULL;
|
||||
xce_alloc_t** allocs = NULL;
|
||||
|
||||
alloc = make_alloc("slots", "", -1, 1.0, -1);
|
||||
allocs = make_allocs(1, alloc);
|
||||
halloc = make_host_alloc("pc13v", 1, allocs);
|
||||
hallocs = make_host_allocs(1, halloc);
|
||||
rc = sched_meet_host_selection(sched, w, resreq->u.simple.selection);
|
||||
if (rc == 0) {
|
||||
add_job_preason((*preason)->reasons, PREASON_MATCH_FAIL, w->name, NULL);
|
||||
|
||||
jalloc = make_job_alloc(jobid, 1, hallocs);
|
||||
} else {
|
||||
xce_pendreason_t* reason = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
reason = make_pendreason(jobid);
|
||||
add_job_preason(reason->reasons, PREASON_NO_RESOURCE, "pc13v", "slots");
|
||||
rc = sched_check_host_satisfy(
|
||||
sched, w, resreq->u.simple.nres_host, resreq->u.simple.idxs_host,
|
||||
resreq->u.simple.vals_host, &n_unmeet, unmeet_idxs, unmeet_vals);
|
||||
if (rc <= 0) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < n_unmeet; i++) {
|
||||
xce_resource_t* r = resmgr_lock_byid(&(sched->resmgr),
|
||||
unmeet_idxs[i], OBJREF_NOP);
|
||||
|
||||
add_job_preason((*preason)->reasons, PREASON_NO_RESOURCE, w->name,
|
||||
r->name);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
allocs = sched_make_allocs(
|
||||
sched, "", resreq->u.simple.nres_host, resreq->u.simple.idxs_host,
|
||||
resreq->u.simple.vals_host, /* FIXME: no duration now */ NULL);
|
||||
|
||||
halloc = make_host_alloc(w->name, resreq->u.simple.nres_host, allocs);
|
||||
hallocs = make_host_allocs(1, halloc);
|
||||
|
||||
*jalloc = make_job_alloc(jobid, 1, hallocs);
|
||||
|
||||
/* host is OK */
|
||||
return 1;
|
||||
}
|
||||
|
||||
static xce_job_alloc_t* my_alloc(xce_sched_plugin_t* plugin, int64_t jobid,
|
||||
resreq_t* resreq, xce_pendreason_t** preason)
|
||||
{
|
||||
xce_scheduler_t* sched = (xce_scheduler_t*)plugin->sched;
|
||||
xce_job_alloc_t* jalloc = NULL;
|
||||
int n_unmeet;
|
||||
int unmeet_idxs[resreq->u.simple.nres_shared + 1];
|
||||
float unmeet_vals[resreq->u.simple.nres_shared + 1];
|
||||
int rc;
|
||||
xlist_t workers;
|
||||
xce_pendreason_t* reason = NULL;
|
||||
|
||||
reason = make_pendreason(jobid);
|
||||
if (reason == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
*preason = reason;
|
||||
|
||||
rc = sched_check_shared_satisfy(
|
||||
sched, resreq->u.simple.nres_shared, resreq->u.simple.idxs_shared,
|
||||
resreq->u.simple.vals_shared, &n_unmeet, unmeet_idxs, unmeet_vals);
|
||||
if (rc < 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (rc == 0) {
|
||||
int i;
|
||||
|
||||
for (i = 0; i < n_unmeet; i++) {
|
||||
xce_resource_t* r = resmgr_lock_byid(&(sched->resmgr),
|
||||
unmeet_idxs[i], OBJREF_NOP);
|
||||
add_job_preason(reason->reasons, PREASON_NO_RESOURCE, NULL,
|
||||
r->name);
|
||||
}
|
||||
|
||||
*preason = reason;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
xlist_init(&workers, 128);
|
||||
|
||||
rc = sched_collect_workers(sched, &workers,
|
||||
(zarray_iter_skip_fn_t)skip_bad_worker_);
|
||||
if (rc < 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
xlist_find(&workers, (xlist_find_fn_t)worker_is_satisfy_, sched, jobid,
|
||||
resreq, &jalloc, preason);
|
||||
|
||||
if (jalloc != NULL) {
|
||||
if (resreq->u.simple.nres_shared > 0) {
|
||||
jalloc->allocs_shared = sched_make_allocs(
|
||||
sched, "", resreq->u.simple.nres_shared,
|
||||
resreq->u.simple.idxs_shared, resreq->u.simple.vals_shared,
|
||||
/*FIXME: no duration now*/ NULL);
|
||||
jalloc->nres_shared = resreq->u.simple.nres_shared;
|
||||
} else {
|
||||
jalloc->allocs_shared = NULL;
|
||||
jalloc->nres_shared = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return jalloc;
|
||||
|
|
@ -78,34 +179,17 @@ static xce_job_alloc_t* my_alloc(xce_sched_plugin_t* plugin, int64_t jobid,
|
|||
|
||||
static int my_commit(xce_sched_plugin_t* plugin, xce_decision_t* decision)
|
||||
{
|
||||
int rc = 0;
|
||||
uint64_t jobid = 0;
|
||||
xce_sched_job_t* job = NULL;
|
||||
|
||||
switch (decision->op) {
|
||||
case XCE_DECISION_DISPATCH:
|
||||
/* FIXME: */
|
||||
jobid = decision->u.alloc->jobid;
|
||||
job = sched_get_job(plugin->sched, jobid);
|
||||
if (job != NULL) {
|
||||
slotsInUse++;
|
||||
job->starting = 1;
|
||||
}
|
||||
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return rc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int my_jstate_changed(xce_sched_plugin_t* plugin, xce_sched_job_t* job,
|
||||
uint32_t state)
|
||||
{
|
||||
#if 0
|
||||
if (JOB_IS_RUN(job->state) && JOB_IS_FINISH(state)) {
|
||||
slotsInUse--;
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
266
src/scheduler/src/resource.c
Normal file
266
src/scheduler/src/resource.c
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "kvmgr.h"
|
||||
#include "sched/types.h"
|
||||
#include "scheduler.h"
|
||||
#include "utils.h"
|
||||
#include "xce/types.h"
|
||||
#include "zarray.h"
|
||||
#include "objmgr.h"
|
||||
|
||||
static int get_resvals(zarray_t* array, int nres, int* idxs, float* vals)
|
||||
{
|
||||
xce_val_t* xvals[nres + 1];
|
||||
int rc = 0;
|
||||
int i;
|
||||
|
||||
if ((array == NULL) || (nres < 0) || (idxs == NULL) || (vals == NULL)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (nres == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
rc = zarray_get_ex(array, nres, idxs, (void**)xvals);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
for (i = 0; i < nres; i++) {
|
||||
if (xvals[i] == NULL) {
|
||||
vals[i] = 0.0;
|
||||
} else {
|
||||
vals[i] = xvals[i]->u.fvalue;
|
||||
}
|
||||
}
|
||||
|
||||
return nres;
|
||||
}
|
||||
|
||||
static int put_resvals(zarray_t* array, int nres, int* idxs, float* vals)
|
||||
{
|
||||
xce_val_t* xvals[nres + 1];
|
||||
int rc, i;
|
||||
|
||||
if ((array == NULL) || (nres < 0) || (idxs == NULL) || (vals == NULL)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (nres == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
rc = zarray_get_ex(array, nres, idxs, (void**)xvals);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
for (i = 0; i < nres; i++) {
|
||||
if (idxs[i] <= 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (xvals[i] == NULL) {
|
||||
xce_val_t* val = (xce_val_t*)calloc(1, sizeof(*val));
|
||||
if (val == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
val->which = Value_u_fvalue;
|
||||
val->u.fvalue = 0.0;
|
||||
zarray_put(array, idxs[i], val);
|
||||
xvals[i] = val;
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < nres; i++) {
|
||||
xvals[i]->u.fvalue = vals[i];
|
||||
}
|
||||
|
||||
return nres;
|
||||
}
|
||||
|
||||
static int unsatisfy(int nres, int* idxs, float* total, float* inuse,
|
||||
float* wanted, int* unmeet_idxs, float* unmeet_vals)
|
||||
{
|
||||
int i, j;
|
||||
int count;
|
||||
|
||||
count = 0;
|
||||
for (i = 0; i < nres; i++) {
|
||||
if (total[i] < inuse[i] + wanted[i]) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
if (count == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
j = 0;
|
||||
for (i = 0; i < nres; i++) {
|
||||
if (total[i] < inuse[i] + wanted[i]) {
|
||||
unmeet_idxs[j] = idxs[i];
|
||||
unmeet_vals[j] = inuse[i] + wanted[i] - total[i];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
|
||||
return count;
|
||||
}
|
||||
|
||||
int sched_meet_host_selection(xce_scheduler_t* sched, xce_sched_worker_t* w,
|
||||
host_selection_t* s)
|
||||
{
|
||||
/* FIXME: add real work later */
|
||||
return 1;
|
||||
}
|
||||
|
||||
int sched_check_shared_satisfy(xce_scheduler_t* sched, int nres, int* idxs,
|
||||
float* wanted, int* n_unmeet, int* unmeet_idxs,
|
||||
float* unmeet_vals)
|
||||
{
|
||||
float total[nres];
|
||||
float inuse[nres];
|
||||
int rc;
|
||||
|
||||
*n_unmeet = 0;
|
||||
|
||||
if (nres == 0) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
rc = get_resvals(&(sched->sharedInst), nres, idxs, total);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
rc = get_resvals(&(sched->sharedInUse), nres, idxs, inuse);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
rc = unsatisfy(nres, idxs, total, inuse, wanted, unmeet_idxs, unmeet_vals);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (rc > 0) {
|
||||
*n_unmeet = rc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int sched_check_host_satisfy(xce_scheduler_t* sched, xce_sched_worker_t* w,
|
||||
int nres, int* idxs, float* wanted, int* n_unmeet,
|
||||
int* unmeet_idxs, float* unmeet_vals)
|
||||
{
|
||||
float total[nres];
|
||||
float inuse[nres];
|
||||
int rc;
|
||||
|
||||
rc = get_resvals(&(w->resvals), nres, idxs, total);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
rc = get_resvals(&(w->resInUse), nres, idxs, inuse);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
rc = unsatisfy(nres, idxs, total, inuse, wanted, unmeet_idxs, unmeet_vals);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (rc > 0) {
|
||||
*n_unmeet = rc;
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int sched_charge_res_accounts(xce_scheduler_t* sched, zarray_t* array,
|
||||
int naccounts,
|
||||
xce_res_account_t** accounts)
|
||||
{
|
||||
int i;
|
||||
int idxs[naccounts + 1];
|
||||
float vals[naccounts + 1];
|
||||
|
||||
if (naccounts < 0) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (naccounts == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (i = 0; i < naccounts; i++) {
|
||||
xce_res_account_t* acct = accounts[i];
|
||||
|
||||
if (acct->residx < 0) {
|
||||
xce_resource_t* r
|
||||
= resmgr_lockres(&(sched->resmgr), acct->resname, OBJREF_NOP);
|
||||
|
||||
if (r != NULL) {
|
||||
acct->residx = r->idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < naccounts; i++) {
|
||||
idxs[i] = accounts[i]->residx;
|
||||
}
|
||||
|
||||
get_resvals(array, naccounts, idxs, vals);
|
||||
|
||||
for (i = 0; i < naccounts; i++) {
|
||||
vals[i] += (accounts[i]->used + accounts[i]->reserved);
|
||||
}
|
||||
|
||||
put_resvals(array, naccounts, idxs, vals);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sched_charge_account(xce_scheduler_t* sched,
|
||||
xce_job_account_t* account)
|
||||
{
|
||||
int i;
|
||||
|
||||
sched_charge_res_accounts(sched, &(sched->sharedInUse),
|
||||
account->nres_shared, account->accounts_shared);
|
||||
for (i = 0; i < account->nhosts; i++) {
|
||||
xce_host_account_t* haccount = account->accounts[i];
|
||||
xce_sched_worker_t* w = NULL;
|
||||
|
||||
if (haccount == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
w = sched_get_worker(sched, haccount->host);
|
||||
if (w == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
sched_charge_res_accounts(sched, &(w->resInUse), haccount->naccount,
|
||||
haccount->accounts);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
int sched_commit_job_alloc(xce_scheduler_t* sched, xce_sched_job_t* job)
|
||||
{
|
||||
if (job->accounts == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return sched_charge_account(sched, job->accounts);
|
||||
}
|
||||
|
|
@ -3,9 +3,12 @@
|
|||
#include <string.h>
|
||||
|
||||
#include "controller.capnp.h"
|
||||
#include "kvmgr.h"
|
||||
#include "objmgr.h"
|
||||
#include "sched/types.h"
|
||||
#include "scheduler.h"
|
||||
#include "xce/types.h"
|
||||
#include "xcealloc.h"
|
||||
|
||||
void kill_sched_job(void* arg)
|
||||
{
|
||||
|
|
@ -14,6 +17,32 @@ void kill_sched_job(void* arg)
|
|||
}
|
||||
}
|
||||
|
||||
static resreq_t* make_sched_resreq(xce_scheduler_t* sched, xce_job_t* j)
|
||||
{
|
||||
resreq_t* resreq = NULL;
|
||||
xce_resource_t* slots
|
||||
= resmgr_lockres(&(sched->resmgr), (void*)"slots", OBJREF_NOP);
|
||||
|
||||
resreq = (resreq_t*)calloc(1, sizeof(*resreq));
|
||||
if (resreq == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* FIXME: need parse real resreq from job */
|
||||
resreq->op = RESREQ_OP_SIMPLE;
|
||||
resreq->u.simple.selection = NULL;
|
||||
resreq->u.simple.nres_shared = 0;
|
||||
resreq->u.simple.idxs_shared = NULL;
|
||||
resreq->u.simple.vals_shared = NULL;
|
||||
resreq->u.simple.nres_host = 1;
|
||||
resreq->u.simple.idxs_host = (int*)calloc(1, sizeof(int));
|
||||
resreq->u.simple.vals_host = (float*)calloc(1, sizeof(float));
|
||||
resreq->u.simple.idxs_host[0] = slots->idx;
|
||||
resreq->u.simple.vals_host[0] = 1.0;
|
||||
|
||||
return resreq;
|
||||
}
|
||||
|
||||
xce_sched_job_t* make_sched_job(void* arg, xce_job_t* j)
|
||||
{
|
||||
xce_sched_job_t* job = (xce_sched_job_t*)calloc(1, sizeof(*job));
|
||||
|
|
@ -34,6 +63,13 @@ xce_sched_job_t* make_sched_job(void* arg, xce_job_t* j)
|
|||
job->mintasks = j->submit->mintasks;
|
||||
job->maxtasks = j->submit->maxtasks;
|
||||
job->priority = 0; /* FIXME: support job level priority */
|
||||
job->resreq = make_sched_resreq(sched, j);
|
||||
job->allocs = NULL;
|
||||
job->accounts = NULL;
|
||||
if (j->allocs != NULL) {
|
||||
MOVE_PTR(job->allocs, j->allocs);
|
||||
job->accounts = make_job_account(job->allocs);
|
||||
}
|
||||
|
||||
return job;
|
||||
}
|
||||
|
|
@ -89,6 +125,7 @@ xce_sched_worker_t* make_sched_worker(void* arg, xce_worker_t* w)
|
|||
{
|
||||
xce_scheduler_t* sched = (xce_scheduler_t*)arg;
|
||||
xce_sched_worker_t* worker = NULL;
|
||||
int i;
|
||||
|
||||
if ((sched == NULL) || (w == NULL)) {
|
||||
return NULL;
|
||||
|
|
@ -102,9 +139,19 @@ xce_sched_worker_t* make_sched_worker(void* arg, xce_worker_t* w)
|
|||
worker->name = strdup(w->name);
|
||||
worker->status = w->status;
|
||||
|
||||
zarray_init(&(worker->resvals), 128, (zarray_item_killer_t)kill_sched_resval);
|
||||
zarray_init(&(worker->resInUse), 128, (zarray_item_killer_t)kill_sched_resval);
|
||||
/* FIXME: */
|
||||
zarray_init(&(worker->resvals), 128,
|
||||
(zarray_item_killer_t)kill_sched_resval);
|
||||
zarray_init(&(worker->resInUse), 128,
|
||||
(zarray_item_killer_t)kill_sched_resval);
|
||||
|
||||
/* fill resource values */
|
||||
for (i = 0; i < w->nresvals_to_sched; i++) {
|
||||
int idx = w->resvals_to_sched[i]->idx;
|
||||
xce_val_t* val = NULL;
|
||||
|
||||
MOVE_PTR(val, w->resvals_to_sched[i]->val);
|
||||
zarray_put(&(worker->resvals), idx, val);
|
||||
}
|
||||
|
||||
return worker;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,8 +11,10 @@
|
|||
#include "kvmgr.h"
|
||||
#include "objmgr.h"
|
||||
#include "plugin.h"
|
||||
#include "preason.h"
|
||||
#include "scheduler.h"
|
||||
#include "utils.h"
|
||||
#include "xcealloc.h"
|
||||
|
||||
static void* sched_job_picker(void* arg)
|
||||
{
|
||||
|
|
@ -40,19 +42,32 @@ static xce_decision_t* calc_decision(xce_scheduler_t* sched, uint64_t jobid)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
if (job->allocator == NULL) {
|
||||
ERROR("%s job %lu has invalid allocator", __func__, jobid);
|
||||
return NULL;
|
||||
if (job->resreq == NULL) {
|
||||
xce_pendreason_t* reason = NULL;
|
||||
|
||||
ERROR("%s job %lu has invalid resource requirement", __func__, jobid);
|
||||
reason = make_pendreason(jobid);
|
||||
|
||||
add_job_preason(reason->reasons, PREASON_BAD_RESREQ, NULL, NULL);
|
||||
|
||||
return make_pend_decision(reason);
|
||||
}
|
||||
|
||||
plugin = (xce_sched_plugin_t*)job->allocator;
|
||||
if (plugin->job_alloc_fn == NULL) {
|
||||
ERROR("%s plugin %s has no allocation function", __func__,
|
||||
plugin->name);
|
||||
return NULL;
|
||||
switch (job->resreq->op) {
|
||||
case RESREQ_OP_SIMPLE:
|
||||
plugin = sched->simple_allocator;
|
||||
if (plugin->job_alloc_fn == NULL) {
|
||||
ERROR("%s plugin %s has no allocation function", __func__,
|
||||
plugin->name);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
alloc = plugin->job_alloc_fn(plugin, jobid, job->resreq, &reason);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
alloc = plugin->job_alloc_fn(job->allocator, jobid, &reason);
|
||||
if (alloc == NULL) {
|
||||
return make_pend_decision(reason);
|
||||
}
|
||||
|
|
@ -86,10 +101,10 @@ static void* sched_allocator(void* arg)
|
|||
|
||||
/* calculate decision for jobid */
|
||||
decision = calc_decision(sched, jobid);
|
||||
|
||||
commit_decision(sched, decision);
|
||||
|
||||
cbuf_put(&(sched->decisions), &decision);
|
||||
if (decision != NULL) {
|
||||
commit_decision(sched, decision);
|
||||
cbuf_put(&(sched->decisions), &decision);
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
|
|
@ -153,6 +168,8 @@ int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|||
|
||||
memset(sched, 0x0, sizeof(*sched));
|
||||
|
||||
sched->nextid = 1;
|
||||
|
||||
mylock_init(&(sched->lock));
|
||||
cbuf_init(&(sched->wbuf), 1, sizeof(int));
|
||||
cbuf_init(&(sched->jbuf), njobs, sizeof(uint64_t));
|
||||
|
|
@ -185,6 +202,7 @@ int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|||
|
||||
zarray_init(&(sched->sharedInst), 128,
|
||||
(zarray_item_killer_t)kill_sched_resval);
|
||||
zarray_init(&(sched->sharedInUse), 128, NULL);
|
||||
|
||||
sched_register_plugins(sched);
|
||||
|
||||
|
|
@ -261,6 +279,21 @@ static int sched_init_resources(xce_scheduler_t* sched)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int sched_init_sharedInst(xce_scheduler_t* sched)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < sched->config->nresvals; i++) {
|
||||
int idx = sched->config->resvals[i]->idx;
|
||||
xce_val_t* val = NULL;
|
||||
|
||||
MOVE_PTR(val, sched->config->resvals[i]->val);
|
||||
zarray_put(&(sched->sharedInst), idx, val);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sched_init_queues(xce_scheduler_t* sched)
|
||||
{
|
||||
int i;
|
||||
|
|
@ -281,6 +314,7 @@ int sched_init_config(xce_scheduler_t* sched, xce_conf4sched_t* config)
|
|||
sched->config = config;
|
||||
|
||||
sched_init_resources(sched);
|
||||
sched_init_sharedInst(sched);
|
||||
sched_init_queues(sched);
|
||||
|
||||
return 0;
|
||||
|
|
@ -477,7 +511,23 @@ int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
|
|||
|
||||
int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
|
||||
{
|
||||
return plugin_commit_decision(sched, decision);
|
||||
int rc = 0;
|
||||
xce_sched_job_t* job = NULL;
|
||||
|
||||
rc = plugin_commit_decision(sched, decision);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
if (decision->op == XCE_DECISION_DISPATCH) {
|
||||
job = sched_get_job(sched, decision->u.alloc->jobid);
|
||||
job->allocs = decision->u.alloc;
|
||||
job->accounts = make_job_account(job->allocs);
|
||||
|
||||
return sched_commit_job_alloc(sched, job);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int sched_publish_pend_reason(xce_scheduler_t* sched,
|
||||
|
|
@ -490,3 +540,76 @@ int sched_publish_dispatch(xce_scheduler_t* sched, xce_job_dispatch_t* dispatch)
|
|||
{
|
||||
return xcecli_jobdispatch(sched->apictx, dispatch);
|
||||
}
|
||||
|
||||
static int _add_worker(void* arg1, void* arg2)
|
||||
{
|
||||
xlist_t* l = (xlist_t*)arg1;
|
||||
xce_sched_worker_t* w = (xce_sched_worker_t*)arg2;
|
||||
|
||||
if ((l == NULL) || (w == NULL)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
xlist_store(l, w);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int sched_collect_workers(xce_scheduler_t* sched, xlist_t* lst,
|
||||
zarray_iter_skip_fn_t skipfn)
|
||||
{
|
||||
xlist_t l2;
|
||||
int rc = 0;
|
||||
|
||||
xlist_init(&l2, 2);
|
||||
xlist_store(&l2, (void*)_add_worker);
|
||||
|
||||
rc = kvmgr_iter(&(sched->workers), skipfn, &l2, lst);
|
||||
xlist_fini(&l2);
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
uint64_t sched_get_id(xce_scheduler_t* sched)
|
||||
{
|
||||
uint64_t id;
|
||||
|
||||
mylock_lock(&(sched->lock));
|
||||
id = sched->nextid++;
|
||||
mylock_unlock(&(sched->lock));
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
xce_alloc_t** sched_make_allocs(xce_scheduler_t* sched, char* reserve, int nres,
|
||||
int* idxs, float* vals, int* duration)
|
||||
{
|
||||
xce_alloc_t** allocs = NULL;
|
||||
int i;
|
||||
|
||||
if ((sched == NULL) || (nres <= 0) || (idxs == NULL) || (vals == NULL)) {
|
||||
errno = EINVAL;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
allocs = (xce_alloc_t**)calloc(nres, sizeof(*allocs));
|
||||
if (allocs == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (i = 0; i < nres; i++) {
|
||||
xce_resource_t* r = (xce_resource_t*)resmgr_lock_byid(
|
||||
&(sched->resmgr), idxs[i], OBJREF_NOP);
|
||||
int d = (duration != NULL) ? duration[i] : -1;
|
||||
|
||||
allocs[i] = NULL;
|
||||
if (r == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
allocs[i]
|
||||
= make_alloc(r->name, reserve, sched_get_id(sched), vals[i], d);
|
||||
}
|
||||
|
||||
return allocs;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue