merge job rusage and pending reason support

src/include/client.h

@@ -30,6 +30,8 @@ extern int  xcecli_start(xcecli_ctx_t);
 extern int  xcecli_stop(xcecli_ctx_t);
 extern int  xcecli_wakeup(xcecli_ctx_t, int);
 extern int  xcecli_hello(xcecli_ctx_t, char*);
+extern int  xcecli_nop(xcecli_ctx_t);
+extern int  xcecli_heartbeat(xcecli_ctx_t, int);
 extern int  xcecli_shutdown(xcecli_ctx_t);
 extern int  xcecli_reslist(xcecli_ctx_t, xce_resource_t***);
 extern int  xcecli_join(xcecli_ctx_t, int, hostattrs_t*);
@@ -60,6 +62,9 @@ extern endpoint_t*     xcecli_get_my_endpoint(xcecli_ctx_t);
 extern xce_peerinfo_t* xcecli_get_myinfo(xcecli_ctx_t);
 extern void            xcecli_block_tasks(xcecli_ctx_t);
 extern void            xcecli_unblock_tasks(xcecli_ctx_t);
+extern int             xcecli_set_pend_reason(xcecli_ctx_t, xce_preason_set_t*);
+extern int             xcecli_clear_reason(xcecli_ctx_t, int, uint64_t*);
+extern int             xcecli_jobinfo(xcecli_ctx_t, uint64_t, xce_job_t**);
 
 #define xcecli_free_ress(nres, ress) xce_free_ress(nres, ress)
 #define xcecli_free_workers(nworkers, workers)                                 \

src/include/clusterconfig.h

@@ -28,6 +28,14 @@ typedef struct {
     int   port;
 } master_candidate_t;
 
+typedef struct {
+    char* name;
+    char* prov_type;
+    char* conf_path;
+    char* script_path;
+    char* script_option;
+} scaler_config_t;
+
 typedef struct {
     char*                clustername;
     int                  port;
@@ -53,6 +61,8 @@ typedef struct {
     qmgr_config_t*       qmgr;
     resmgr_config_t*     resmgr;
     xstrings_config_t*   xstrings;
+    int                  nhproviders;
+    scaler_config_t**    hproviders;
 } cluster_config_t;
 
 #endif

src/include/delay.h

@@ -0,0 +1,37 @@
+#if !defined(_DELAY_H_)
+
+#define _DELAY_H_ 1
+
+#include <pthread.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <time.h>
+
+#include "ordq.h"
+
+typedef int (*delay_task_entry_fn_t)(void*);
+typedef int (*task_run_fn_t)(delay_task_entry_fn_t, void*);
+
+typedef struct {
+    int                   run_period;
+    int                   runcount;
+    delay_task_entry_fn_t entry;
+    void*                 args;
+} delay_task_t;
+
+typedef struct {
+    pthread_t     worker;
+    int           running;
+    int           precision;
+    task_run_fn_t runner_fn;
+    timer_t       timerid;
+    ordq_t        tasks;
+} delaymgr_t;
+
+extern int  delaymgr_init(delaymgr_t*, int, task_run_fn_t);
+extern void delaymgr_stop(delaymgr_t*);
+extern void delaymgr_fini(delaymgr_t*);
+extern int  delaymgr_start(delaymgr_t*);
+extern int  delay_task(delaymgr_t*, int, int, delay_task_entry_fn_t, void*);
+
+#endif

src/include/netio.h

@@ -79,6 +79,8 @@ typedef struct {
     xce_peerinfo_t* peer;
     msg_handler_t   msg_handler;
     err_handler_t   err_handler;
+    time_t          ts_last_in;
+    time_t          ts_last_out;
 } endpoint_t;
 
 #define INVOKE_MSG_HANDLER(ep)                                                 \

src/include/ordq.h

@@ -0,0 +1,35 @@
+#if !defined(_ORDQ_H_)
+
+#define _ORDQ_H_ 1
+
+#include <stdint.h>
+
+#include "mylock.h"
+
+typedef void* ordkey_t;
+
+typedef struct _ordq_item {
+    struct _ordq_item* next;
+    ordkey_t           key;
+    void*              data;
+} ordq_item_t;
+
+typedef int (*ordq_cmp_fn_t)(ordkey_t, ordkey_t);
+typedef void (*ordq_data_killer_t)(void*);
+
+typedef struct {
+    mylock_t           lock;
+    ordq_item_t*       head;
+    ordq_data_killer_t killer;
+    ordq_cmp_fn_t      cmp;
+} ordq_t;
+
+#define mk_ordkey(k) (ordkey_t)((long)(k))
+
+extern int          ordq_init(ordq_t*, ordq_data_killer_t, ordq_cmp_fn_t);
+extern void         ordq_fini(ordq_t* q);
+extern int          ordq_put(ordq_t*, ordkey_t, void*);
+extern ordq_item_t* ordq_get(ordq_t*);
+extern ordq_item_t* ordq_peek(ordq_t*);
+
+#endif

src/include/preason.h

@@ -0,0 +1,10 @@
+#if !defined(_PREASON_H_)
+
+#define _PREASON_H_ 1
+
+#include "xce/types.h"
+
+extern void erase_job_preason(xce_job_preason_t*);
+extern int  add_job_preason(xce_job_preason_t*, int, char*, char*);
+
+#endif

src/include/request.h

@@ -55,5 +55,7 @@ extern int xce_update_req(endpoint_t*, xce_update_data_t*);
 extern int xce_jobctrl_req(endpoint_t*, xce_jobctrl_t*);
 extern int xce_queuectrl_req(endpoint_t*, xce_queuectrl_t*);
 extern int xce_runtask_req(endpoint_t*, xce_taskrun_t*);
+extern int xce_set_pend_reason_req(endpoint_t*, xce_preason_set_t*);
+extern int xce_clear_reason_req(endpoint_t*, xce_clear_reason_t*);
 
 #endif

src/include/typemap.h

@@ -141,6 +141,7 @@ typedef struct {
         xce_task_failinfo_t* taskfailinfo;
         xce_task_finish_t*   taskfinish;
         uint64_t             pdone_job;
+        xce_job_rusages_t*   sched_job_rusages;
     } data;
 } xce_notify_data_t;
 
@@ -186,6 +187,8 @@ typedef struct {
         xce_jobctrl_t*      jobctrlmsg;
         xce_queuectrl_t*    queuectrlmsg;
         xce_taskrun_t*      runtaskmsg;
+        xce_preason_set_t*  preasonmsg;
+        xce_clear_reason_t* clrpreasonmsg;
     } body;
 } xce_request_t;
 

src/include/utils.h

@@ -130,5 +130,7 @@ extern int      fmt_jobid(char*, size_t, uint64_t);
 extern int      parse_jobid(uint64_t*, char*);
 extern uint64_t array_jobid(uint64_t);
 extern uint64_t array_idx(uint64_t);
+extern void     unblock_sig(int);
+extern void     block_sig(int);
 
 #endif

src/include/xce/cluster_def.h

@@ -6,6 +6,7 @@
 #include <pthread.h>
 #include <stdint.h>
 
+#include "delay.h"
 #include "hook.h"
 #include "htab.h"
 #include "kvmgr.h"
@@ -39,6 +40,8 @@ enum {
     HOOK_DETAIL_JOB_FAIL,
     HOOK_DETAIL_JOB_UPDATE,
     HOOK_DETAIL_JOB_CKPT,
+    HOOK_DETAIL_SETPREASON,
+    HOOK_DETAIL_CLRPREASON,
     HOOK_DETAIL_WORKER_JOIN,
     HOOK_DETAIL_WORKER_DEAD,
     HOOK_DETAIL_WORKER_UPDATE,
@@ -50,7 +53,8 @@ enum {
     HOOK_DETAIL_RESOURCE_LOC_UPDATE,
     HOOK_DETAIL_QUEUE_UPDATE,
     HOOK_DETAIL_USERGROUP_UPDATE,
-    HOOK_DETAIL_HOSTGROUP_UPDATE
+    HOOK_DETAIL_HOSTGROUP_UPDATE,
+    HOOK_DETAIL_JOB_RUSAGE_UPDATE
 };
 
 #define XCE_NS_CONFIG (0)
@@ -125,6 +129,7 @@ typedef struct {
     int             myidx;
     int             master;
     int             ncontrollers;
+    delaymgr_t      delaymgr;
     scheduler_t     sched;
     jobmgr_t        jobmgr;
     resmgr_t        resmgr;

src/include/xce/types.h

@@ -230,16 +230,16 @@ typedef struct {
 } xce_launch_t;
 
 typedef struct {
-    char*       exechome;
-    char*       execcwd;
-    char*       execuser;
-    char*       cmdline;
-    char*       exdata;
-    char*       launch_type;
-    char*       q_pre_exec;
-    char*       q_post_exec;
-    char*       job_pre_exec;
-    char*       job_post_exec;
+    char* exechome;
+    char* execcwd;
+    char* execuser;
+    char* cmdline;
+    char* exdata;
+    char* launch_type;
+    char* q_pre_exec;
+    char* q_post_exec;
+    char* job_pre_exec;
+    char* job_post_exec;
 } job_startinfo_t;
 
 typedef struct {
@@ -389,6 +389,30 @@ typedef struct {
     char*    ckptinfo;
 } xce_ckptinfo_t;
 
+typedef struct {
+    char* host;
+    int   reason;
+} xce_host_reason_t;
+
+typedef struct {
+    char* resource;
+    char* host;
+    int   reason;
+} xce_res_reason_t;
+
+typedef struct {
+    int       njobs;
+    uint64_t* jobs;
+} xce_clear_reason_t;
+
+typedef struct {
+    int                 main_reason;
+    int                 nhost_reasons;
+    xce_host_reason_t** host_based_reasons;
+    int                 nres_reasons;
+    xce_res_reason_t**  res_based_reasons;
+} xce_job_preason_t;
+
 typedef struct {
     uint64_t            jobid;
     job_submit_data_t*  submit;
@@ -405,6 +429,7 @@ typedef struct {
     job_exitinfo_t*     exit_info;
     char*               ckptinfo;
     int64_t             pending_ctrl;
+    xce_job_preason_t*  pend_reasons;
 } xce_job_t;
 
 typedef struct {
@@ -447,6 +472,7 @@ typedef struct {
     int64_t                  pending_ctrl;
     char*                    pre_exec;
     char*                    post_exec;
+    char*                    autopscaler;
 } xce_queue_t;
 
 typedef struct {
@@ -615,4 +641,14 @@ typedef struct {
     float value;
 } xce_rusage_t;
 
+typedef struct {
+    uint64_t           jobid;
+    xce_job_preason_t* reasons;
+} xce_pendreason_t;
+
+typedef struct {
+    int                nreasons;
+    xce_pendreason_t** reasons;
+} xce_preason_set_t;
+
 #endif

src/include/xceapi.h

@@ -8,6 +8,8 @@
 #include "xcectx.h"
 
 extern int xce_hello(xce_ctx_t*, char*);
+extern int xce_nop(xce_ctx_t*);
+extern int xce_heartbeat(xce_ctx_t*, int);
 extern int xce_shutdown(xce_ctx_t*);
 extern int xce_reslist(xce_ctx_t*, xce_resource_t***);
 extern int xce_join(xce_ctx_t*, int, hostattrs_t*);
@@ -27,5 +29,8 @@ extern int xce_queuectrl(xce_ctx_t*, xce_queuectrl_t*);
 extern int xce_runtask(xce_ctx_t*, xce_taskrun_t*);
 extern int xce_update_jusage(xce_ctx_t*, uint64_t, int, int*, int, int*, int,
                              xce_rusage_t**);
+extern int xce_set_pend_reason(xce_ctx_t*, xce_preason_set_t*);
+extern int xce_clear_reason(xce_ctx_t*, xce_clear_reason_t*);
+extern int xce_jobinfo(xce_ctx_t*, uint64_t, xce_job_t**);
 
 #endif