merge job rusage and pending reason support
This commit is contained in:
parent
30824ae5ea
commit
cb0c7fe9e7
82 changed files with 4280 additions and 2561 deletions
|
|
@ -17,7 +17,9 @@
|
|||
|
||||
#include "client.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "delay.h"
|
||||
#include "idgen.h"
|
||||
#include "preason.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
typedef int (*cmd_func_t)(int, char*[]);
|
||||
|
|
@ -73,6 +75,7 @@ typedef struct {
|
|||
#define MAX_TASK_HOSTS (5)
|
||||
#define MAX_RUN_TASKS (50)
|
||||
#define RUSAGE_UPDATE_INTERVAL (60)
|
||||
#define MAX_LINE_LEN (40960)
|
||||
|
||||
#define S2J_JSON_SET_var_string_ELEMENT(to_json, from_struct, _element) \
|
||||
if (((from_struct) != NULL) && (((from_struct)->_element) != NULL)) { \
|
||||
|
|
@ -109,14 +112,12 @@ extern int cmd_jobkill(int, char*[]);
|
|||
extern int cmd_jobckpt(int, char*[]);
|
||||
extern int cmd_qctrl(int, char*[]);
|
||||
extern int cmd_runtasks(int, char*[]);
|
||||
extern int cmd_setpend(int, char*[]);
|
||||
extern int cmd_jobinfo(int, char*[]);
|
||||
static int worker_house_keeping(void*);
|
||||
|
||||
static void handler(int, siginfo_t*, void*);
|
||||
|
||||
#define CLOCKID CLOCK_REALTIME
|
||||
#define SIG SIGRTMIN
|
||||
|
||||
static timer_t timerid;
|
||||
static idgen_ctx_t idctx;
|
||||
delaymgr_t delaymgr;
|
||||
|
||||
static setting_t settings[] = {
|
||||
{ (char*)"host", (char*)NULL, (char*)"localhost" },
|
||||
|
|
@ -168,6 +169,7 @@ static command_t client_commands[] = {
|
|||
{ "jobsub", cmd_jobsub, "submit a job" },
|
||||
{ "jobkill", cmd_jobkill, "signal a job" },
|
||||
{ "jobckpt", cmd_jobckpt, "checkpoint a job" },
|
||||
{ "jobinfo", cmd_jobinfo, "job information" },
|
||||
{ "resval", cmd_resval, "get value of res" },
|
||||
{ "taskrun", cmd_runtasks, "run tasks" },
|
||||
{ "help", cmd_help, "Display this text" },
|
||||
|
|
@ -186,6 +188,7 @@ static command_t scheduler_commands[] = {
|
|||
{ "quit", cmd_quit, "Quit xce test" },
|
||||
{ "join", cmd_join_sched, "Join the cluster" },
|
||||
{ "getconf", cmd_getconf, "Get conf of cluster" },
|
||||
{ "setpend", cmd_setpend, "set job pend reason" },
|
||||
{ (char*)NULL, (cmd_func_t)NULL, (char*)NULL }
|
||||
};
|
||||
|
||||
|
|
@ -278,7 +281,7 @@ int leave_ctx(command_ctx_t* ctx)
|
|||
}
|
||||
|
||||
if (ctx == &workerctx) {
|
||||
timer_delete(timerid);
|
||||
delaymgr_fini(&delaymgr);
|
||||
}
|
||||
|
||||
xcecli_wakeup(active_ctx->ctx, (int)-EINTR);
|
||||
|
|
@ -392,10 +395,7 @@ int cmd_up(int argc, char* argv[])
|
|||
|
||||
int cmd_worker(int argc, char* argv[])
|
||||
{
|
||||
int rc;
|
||||
struct sigevent sev;
|
||||
struct sigaction sa;
|
||||
struct itimerspec its;
|
||||
int rc;
|
||||
|
||||
if ((rc = enter_ctx(&workerctx)) < 0) {
|
||||
return rc;
|
||||
|
|
@ -403,22 +403,10 @@ int cmd_worker(int argc, char* argv[])
|
|||
|
||||
active_ctx = &workerctx;
|
||||
|
||||
sa.sa_flags = SA_SIGINFO;
|
||||
sa.sa_sigaction = handler;
|
||||
sigemptyset(&(sa.sa_mask));
|
||||
sigaction(SIG, &sa, NULL);
|
||||
|
||||
sev.sigev_notify = SIGEV_SIGNAL;
|
||||
sev.sigev_signo = SIG;
|
||||
sev.sigev_value.sival_ptr = &timerid;
|
||||
|
||||
timer_create(CLOCKID, &sev, &timerid);
|
||||
|
||||
memset(&its, 0x0, sizeof(its));
|
||||
its.it_value.tv_sec = 1;
|
||||
|
||||
timer_settime(timerid, 0, &its, NULL);
|
||||
|
||||
delaymgr_init(&delaymgr, 2, NULL);
|
||||
delaymgr_start(&delaymgr);
|
||||
delay_task(&delaymgr, 5, -1, (delay_task_entry_fn_t)worker_house_keeping,
|
||||
&workerctx);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -631,11 +619,94 @@ int cmd_joblist(int argc, char* argv[])
|
|||
jobs[i]->submit->start_info->cmdline);
|
||||
}
|
||||
|
||||
for (i = 0; i < rc; i++) {
|
||||
free_Job_ptr(&(jobs[i]));
|
||||
}
|
||||
|
||||
free(jobs);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void print_jobinfo(xce_job_t* job)
|
||||
{
|
||||
char idstring[128];
|
||||
|
||||
if (job == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
fmt_jobid(idstring, sizeof(idstring) - 1, job->jobid);
|
||||
printf("JobID: %s\n", idstring);
|
||||
printf("state: %s\n", job_state_str(job->state));
|
||||
printf("Queue: %s\n", job->submit->queue);
|
||||
printf("User: %s\n", job->submit->user);
|
||||
printf("Slots: %d,%d\n", job->submit->mintasks, job->submit->maxtasks);
|
||||
printf("Command: %s\n", job->submit->start_info->cmdline);
|
||||
|
||||
if (job->pend_reasons != NULL) {
|
||||
printf("Pend Reason:\n");
|
||||
if (job->pend_reasons->main_reason != 0) {
|
||||
printf("\tmain Reason: %d\n", job->pend_reasons->main_reason);
|
||||
} else {
|
||||
if (job->pend_reasons->nhost_reasons > 0) {
|
||||
int i;
|
||||
|
||||
printf("\tHost Reasons:\n");
|
||||
for (i = 0; i < job->pend_reasons->nhost_reasons; i++) {
|
||||
printf("\t\t(%s, %d)\n",
|
||||
job->pend_reasons->host_based_reasons[i]->host,
|
||||
job->pend_reasons->host_based_reasons[i]->reason);
|
||||
}
|
||||
}
|
||||
|
||||
if (job->pend_reasons->nres_reasons > 0) {
|
||||
int i;
|
||||
|
||||
printf("\tResource Reasons:\n");
|
||||
for (i = 0; i < job->pend_reasons->nres_reasons; i++) {
|
||||
printf("\t\t(%s, %s, %d)\n",
|
||||
job->pend_reasons->res_based_reasons[i]->resource,
|
||||
job->pend_reasons->res_based_reasons[i]->host,
|
||||
job->pend_reasons->res_based_reasons[i]->reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int cmd_jobinfo(int argc, char* argv[])
|
||||
{
|
||||
int opt;
|
||||
|
||||
optind = 1;
|
||||
|
||||
while ((opt = getopt(argc, argv, "j:")) != -1) {
|
||||
uint64_t jobid;
|
||||
xce_job_t* job;
|
||||
int rc = 0;
|
||||
|
||||
switch (opt) {
|
||||
case 'j':
|
||||
parse_jobid(&jobid, optarg);
|
||||
|
||||
if (jobid > 0) {
|
||||
rc = xcecli_jobinfo(active_ctx->ctx, jobid, &job);
|
||||
if (rc < 0) {
|
||||
perror("xcecli_jobinfo");
|
||||
continue;
|
||||
}
|
||||
|
||||
print_jobinfo(job);
|
||||
free_Job_ptr(&job);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int fill_job_alloc(xce_job_alloc_t* jalloc, int nhosts,
|
||||
host_task_t* hosts)
|
||||
{
|
||||
|
|
@ -1256,6 +1327,91 @@ int cmd_runtasks(int argc, char* argv[])
|
|||
return rc;
|
||||
}
|
||||
|
||||
static int add_pendreason(xce_job_preason_t* preason, char* s)
|
||||
{
|
||||
char s1[MAX_LINE_LEN];
|
||||
char* pcode = NULL;
|
||||
char* phost = NULL;
|
||||
char* pres = NULL;
|
||||
char* p;
|
||||
int code;
|
||||
|
||||
if ((preason == NULL) || (s == NULL)) {
|
||||
fprintf(stderr, "bad parameter\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if ((p = strrchr(s, (int)':')) == NULL) {
|
||||
pcode = s;
|
||||
} else {
|
||||
pcode = p + 1;
|
||||
memset(s1, 0x0, sizeof(s1));
|
||||
strncpy(s1, s, p - s);
|
||||
|
||||
if ((p = strrchr(s1, (int)':')) == NULL) {
|
||||
phost = s1;
|
||||
} else {
|
||||
*p = 0x0;
|
||||
phost = p + 1;
|
||||
pres = s1;
|
||||
}
|
||||
}
|
||||
|
||||
code = atoi(pcode);
|
||||
|
||||
return add_job_preason(preason, code, phost, pres);
|
||||
}
|
||||
|
||||
int cmd_setpend(int argc, char* argv[])
|
||||
{
|
||||
int opt;
|
||||
xce_job_preason_t jreasons;
|
||||
xce_pendreason_t preason = { .reasons = &jreasons };
|
||||
xce_preason_set_t reasons
|
||||
= { .nreasons = 1, .reasons = (xce_pendreason_t*[]) { &preason } };
|
||||
int rc = 0;
|
||||
|
||||
optind = 1;
|
||||
|
||||
while ((opt = getopt(argc, argv, "j:p:")) != -1) {
|
||||
int rc = 0;
|
||||
switch (opt) {
|
||||
case 'j':
|
||||
parse_jobid(&(preason.jobid), optarg);
|
||||
break;
|
||||
case 'p':
|
||||
rc = add_pendreason(&jreasons, optarg);
|
||||
if (rc < 0) {
|
||||
perror("add_pendreason");
|
||||
return rc;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (preason.jobid == 0) {
|
||||
fprintf(stderr, "no or incorrect jobid specified\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if ((jreasons.main_reason == 0) && (jreasons.nhost_reasons == 0)
|
||||
&& (jreasons.nres_reasons == 0)) {
|
||||
fprintf(stderr, "no or incorrect pend reason set\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
rc = xcecli_set_pend_reason(active_ctx->ctx, &reasons);
|
||||
if (rc < 0) {
|
||||
perror("xcecli_set_pend_reason");
|
||||
erase_job_preason(&jreasons);
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("ok\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmd_join_worker(int argc, char* argv[])
|
||||
{
|
||||
int rc;
|
||||
|
|
@ -2263,18 +2419,24 @@ static void check_run_tasks()
|
|||
}
|
||||
}
|
||||
|
||||
static void handler(int sig, siginfo_t* si, void* uc)
|
||||
#define HEARTBEAT_INTERVAL (60)
|
||||
|
||||
static int worker_house_keeping(void* arg)
|
||||
{
|
||||
struct itimerspec its;
|
||||
command_ctx_t* ctx = (command_ctx_t*)arg;
|
||||
int rc = 0;
|
||||
|
||||
xce_use(ctx);
|
||||
|
||||
check_run_jobs();
|
||||
check_run_tasks();
|
||||
|
||||
memset(&its, 0x0, sizeof(its));
|
||||
rc = xcecli_heartbeat(ctx->ctx, HEARTBEAT_INTERVAL);
|
||||
if (rc < 0) {
|
||||
perror("xcecli_heartbeat");
|
||||
}
|
||||
|
||||
its.it_value.tv_sec = 1;
|
||||
|
||||
timer_settime(timerid, 0, &its, NULL);
|
||||
return rc;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue