use nix build & meson & zlog

src/include/xce/cluster_def.h

@@ -0,0 +1,168 @@
+#ifndef XCE_CLUSTER_DEF_H
+#define XCE_CLUSTER_DEF_H
+
+#include "config.h"
+
+#include <pthread.h>
+#include <stdint.h>
+
+#include "hook.h"
+#include "htab.h"
+#include "kvmgr.h"
+#include "myconf.h"
+#include "mylock.h"
+#include "netio.h"
+#include "objmgr.h"
+#include "vtaskmgr.h"
+#include "xce/types.h"
+#include "xlist.h"
+#include "zarray.h"
+
+#define N_CONTROLLER_MAX (16)
+
+enum {
+    XCE_CONTROLLER_HOOK_JOB = 0,
+    XCE_CONTROLLER_HOOK_WORKER,
+    XCE_CONTROLLER_HOOK_SCHEDULER,
+    XCE_CONTROLLER_HOOK_CLUSTER,
+    XCE_CONTROLLER_HOOK_RESOURCE,
+    XCE_CONTROLLER_HOOK_CTRL_REQ,
+    XCE_CONTROLLER_HOOK_QUEUE,
+    XCE_CONTROLLER_HOOK_USERGROUP,
+    XCE_CONTROLLER_HOOK_HOSTGROUP,
+    XCE_CONTROLLER_HOOK_MAX /* keep this as the max */
+};
+
+enum {
+    HOOK_DETAIL_JOB_NEW = 0,
+    HOOK_DETAIL_JOB_START,
+    HOOK_DETAIL_JOB_EXIT,
+    HOOK_DETAIL_JOB_FAIL,
+    HOOK_DETAIL_JOB_UPDATE,
+    HOOK_DETAIL_JOB_CKPT,
+    HOOK_DETAIL_WORKER_JOIN,
+    HOOK_DETAIL_WORKER_DEAD,
+    HOOK_DETAIL_WORKER_UPDATE,
+    HOOK_DETAIL_SCHEDULER_JOIN,
+    HOOK_DETAIL_SCHEDULER_DEAD,
+    HOOK_DETAIL_CLUSTER_INIT,
+    HOOK_DETAIL_DB_INIT,
+    HOOK_DETAIL_CLUSTER_SHUTDOWN,
+    HOOK_DETAIL_DB_SHUTDOWN,
+    HOOK_DETAIL_RESOURCE_UPDATE,
+    HOOK_DETAIL_RESOURCE_LOC_UPDATE,
+    HOOK_DETAIL_CTRL_REQ_FINISH,
+    HOOK_DETAIL_QUEUE_UPDATE,
+    HOOK_DETAIL_USERGROUP_UPDATE,
+    HOOK_DETAIL_HOSTGROUP_UPDATE
+};
+
+#define XCE_NS_CONFIG (0)
+#define XCE_NS_UGROUP (1)
+#define XCE_NS_HGROUP (2)
+#define XCE_NS_MAX    (2)
+
+typedef struct {
+    int role;
+    union {
+        int idx;
+    } u;
+} ep_private_t;
+
+typedef struct {
+    endpoint_t* ep;
+} controller_t;
+
+typedef struct {
+} metrics_t;
+
+typedef struct {
+    xce_queue_t queue;
+} queue_t;
+
+typedef struct {
+    xce_job_t job;
+} job_t;
+
+typedef struct {
+    mylock_t lock;
+    htab_t   htab;
+    zarray_t jobs;
+} jobmgr_t;
+
+typedef struct {
+    mylock_t lock;
+    htab_t   htab;
+    int      nshared;
+    int      nunshared;
+    zarray_t defs;
+    zarray_t sharedinst;
+} resmgr_t;
+
+typedef struct {
+    mylock_t lock;
+    htab_t   htab;
+    zarray_t workers;
+} workermgr_t;
+
+typedef struct {
+    mylock_t lock;
+    htab_t   htab;
+    zarray_t queues;
+} qmgr_t;
+
+typedef struct {
+    mylock_t    lock;
+    endpoint_t* ep;
+} scheduler_t;
+
+typedef struct {
+    DBPool*   db_pool;
+    DBHandle* db_handle;
+    pthread_t fetch_thread;
+    int64_t   fetch_job_pos;
+    int       fetch_job_interval;
+    int       running;
+    mylock_t  db_lock;
+} db_ctx_t;
+
+typedef struct {
+    pthread_t thread;
+    int       runing;
+    int       interval;
+} ctrlreq_ctx_t;
+
+typedef struct {
+    mylock_t      lock;
+    ctrlreq_ctx_t ctrlreq_ctx;
+} ctrl_req_mgr_t;
+
+typedef struct {
+    myconf_t*       myconf;
+    xce_peerinfo_t* myhost;
+    hooksys_t       hooksys;
+    iomgr_t         iomgr;
+    objmgr_t        objmgr;
+    zarray_t        endpoints;
+    endpoint_t      ep;
+    zarray_t        configs;
+    controller_t    controllers[N_CONTROLLER_MAX];
+    int             myidx;
+    int             master;
+    int             ncontrollers;
+    scheduler_t     sched;
+    jobmgr_t        jobmgr;
+    resmgr_t        resmgr;
+    workermgr_t     workermgr;
+    qmgr_t          qmgr;
+    kvmgr_t         configmgr;
+    kvmgr_t         ugmgr;
+    kvmgr_t         hgmgr;
+    vtaskmgr_t      vtaskmgr;
+    cbuf_t          wbuf;
+    int             running;
+    db_ctx_t        dbctx;
+    ctrl_req_mgr_t  ctrl_req_mgr;
+} cluster_t;
+
+#endif /* XCE_CLUSTER_DEF_H */

src/include/xce/errnox.h

@@ -0,0 +1,26 @@
+#if !defined(__XCE_ERRNOX_H__)
+
+#define __XCE_ERRNOX_H__ 1
+
+#include <errno.h>
+#include <string.h>
+#include <stddef.h>
+
+#define XCE_ERRNO_BASE  (10001)
+
+enum {
+    E_XCE_BAD_JOB = XCE_ERRNO_BASE,
+    E_XCE_BAD_WORKER,
+    E_XCE_DUP_JOB,
+    E_XCE_DUP_WORKER,
+    E_XCE_BAD_OPCODE,
+    E_XCE_BAD_ROLE,
+    E_XCE_BAD_RES,
+    E_XCE_BAD_QUEUE,
+    E_XCE_BAD_GROUP,
+    E_XCE_INTERNAL,
+    E_XCE_MAX
+};
+    
+
+#endif

src/include/xce/types.h

@@ -0,0 +1,619 @@
+#if !defined(_XCE_TYPES_H_)
+
+#define _XCE_TYPES_H_
+
+#include "config.h"
+
+#include <stdint.h>
+#include <time.h>
+
+#include "xlist.h"
+#include "zarray.h"
+
+#define XCE_VAL_STRING  (0)
+#define XCE_VAL_NUMERIC (1)
+#define XCE_VAL_BOOLEAN (2)
+
+typedef enum {
+    XCE_ORD_INCR = 0,
+    XCE_ORD_DECR,
+    XCE_ORD_NA
+} xce_order_t;
+
+typedef struct {
+    int which;
+    union {
+        int32_t  ivalue32;
+        uint32_t uvalue32;
+        int64_t  ivalue64;
+        uint64_t uvalue64;
+        char*    svalue;
+        double   fvalue;
+        int      bvalue;
+    } u;
+} xce_val_t;
+
+typedef struct {
+    int        idx;
+    xce_val_t* val;
+} xce_val_with_idx_t;
+
+typedef struct {
+    char*      name;
+    char*      host;
+    xce_val_t* val;
+} xce_resval_t;
+
+typedef struct {
+    int       field;
+    xce_val_t value;
+} xce_kv_t;
+
+typedef struct {
+    int       count;
+    xce_kv_t* vals;
+} xce_kv_list_t;
+
+typedef struct {
+    int day;  /* 1 - 7 (Mon to Sun, 0 unspecified) */
+    int hour; /* 0 - 23 */
+    int min;  /* 0 - 59 */
+} xce_timewindow_time_t;
+
+typedef struct {
+    char*                  tz;
+    xce_timewindow_time_t* open;
+    xce_timewindow_time_t* close;
+} xce_timewindow_seg_config_t;
+
+typedef struct {
+    int                           nsegs;
+    xce_timewindow_seg_config_t** segs;
+} xce_timewindow_config_t;
+
+#define XCE_RESF_BUILTIN      (0x01)
+#define XCE_RESF_DYNAMIC      (0x02)
+#define XCE_RESF_GLOBAL       (0x04)
+#define XCE_RESF_SHARED       (0x08)
+#define XCE_RESF_EXTERNAL     (0x10)
+#define XCE_RESF_RELEASE      (0x20)
+#define XCE_RESF_DEFAULT_HOST (0x40)
+#define XCE_RESF_CONSUMABLE   (0x80)
+
+typedef struct {
+    int   count;
+    void* data;
+} xce_list_t;
+
+typedef struct {
+    int  count;
+    int* data;
+} xce_intlist_t;
+
+typedef struct {
+    int    count;
+    char** data;
+} xce_strlist_t;
+
+typedef struct {
+    char* name;
+    char* key;
+} xce_peerinfo_t;
+
+typedef struct {
+    char* name;
+    char* value;
+} xce_config_t;
+
+typedef struct {
+    int            count;
+    xce_config_t** configs;
+} xce_configlist_t;
+
+typedef struct {
+    char*         name;
+    xce_strlist_t members;
+    xce_strlist_t ex_members;
+} xce_group_t;
+
+typedef struct {
+    int           count;
+    xce_group_t** groups;
+} xce_grouplist_t;
+
+typedef struct {
+    xce_val_t* value;
+    int        nhosts;
+    char**     hosts;
+} res_distribution_t;
+
+typedef struct {
+    char*                name;
+    char*                desc;
+    int                  idx;
+    int                  valtype;
+    xce_order_t          ordtype;
+    int                  flags;
+    int                  interval;
+    time_t               last_report;
+    int                  ndistributions;
+    res_distribution_t** distributions;
+    xce_val_t*           hbased_defined;
+} xce_resource_t;
+
+typedef struct {
+    int              count;
+    xce_resource_t** res;
+} xce_reslist_t;
+
+typedef struct {
+    char*  name;
+    double sched;
+    double stop;
+} xce_res_threshold_t;
+
+typedef struct {
+    char*                    name;
+    char*                    hostmodel;
+    char*                    hosttype;
+    int                      mxj;
+    int                      nresources;
+    xce_config_t**           resources;
+    int                      nthresholds;
+    xce_res_threshold_t**    thresholds;
+    xce_timewindow_config_t* dispatchwindow;
+} xce_hostconfig_t;
+
+#define XCE_WORKER_UNAVAIL (0x00010000)
+#define XCE_WORKER_LOCKEDU (0x00020000)
+#define XCE_WORKER_LOCKEDW (0x00040000)
+#define XCE_WORKER_BUSY    (0x00080000)
+#define XCE_WORKER_CLOSED  (0x00100000)
+
+#define IS_WORKER_UNAVAIL(status) \
+    (((status) & XCE_WORKER_UNAVAIL) != 0)
+#define IS_WORKER_LOCKEDU(status) \
+    (((status) & XCE_WORKER_LOCKEDU) != 0)
+#define IS_WORKER_LOCKEDW(status) \
+    (((status) & XCE_WORKER_LOCKEDW) != 0)
+#define IS_WORKER_LOCKED(status) \
+    (IS_WORKER_LOCKEDU(status) || IS_WORKER_LOCKEDW(status))
+#define IS_WORKER_BUSY(status) \
+    (((status) & XCE_WORKER_BUSY) != 0)
+
+typedef struct {
+    char*  hosttype;
+    char*  hostmodel;
+    double cpufactor;
+    int    maxcpus;
+    double maxmem;
+    double maxswap;
+    double maxtmp;
+    int    ndisks;
+} hostattrs_t;
+
+typedef struct {
+    uint64_t start;
+    uint64_t end;
+} xce_timerange_t;
+
+typedef struct {
+    int              nsegs;
+    xce_timerange_t* ranges;
+} xce_timewindow_t;
+
+typedef struct {
+    char*                    name;
+    hostattrs_t*             attrs;
+    uint32_t                 status;
+    zarray_t                 resvals;
+    time_t                   dead_since;
+    int64_t                  pending_ctrl;
+    xce_timewindow_config_t* dispatch_window_config;
+    int                      nthresholds;
+    xce_res_threshold_t**    thresholds;
+    /* following two vars used to push resource values to sched */
+    int                  nresvals_to_sched;
+    xce_val_with_idx_t** resvals_to_sched;
+} xce_worker_t;
+
+typedef struct {
+    int            count;
+    xce_worker_t** workers;
+} xce_workerlist_t;
+
+typedef struct {
+    char* host;
+    int   nranks;
+} xce_host_launch_t;
+
+typedef struct {
+    int                 nhosts;
+    xce_host_launch_t** hosts;
+} xce_launch_t;
+
+typedef struct {
+    char* exechome;
+    char* execcwd;
+    char* execuser;
+    char* cmdline;
+    char* exdata;
+    char* launch_type;
+    char* q_pre_exec;
+    char* q_post_exec;
+    char* job_pre_exec;
+    char* job_post_exec;
+} job_startinfo_t;
+
+typedef struct {
+    uint64_t      jobid;
+    int           npids;
+    unsigned int* pids;
+    int           npgids;
+    unsigned int* pgids;
+    char*         exdata;
+} job_pidinfo_t;
+
+typedef struct {
+    uint64_t jobid;
+    uint32_t stage;
+    int      code;
+    char*    detail;
+} job_failinfo_t;
+
+typedef struct {
+    char*    resname;
+    char*    reserve;
+    uint32_t instid;
+    double   value;
+    uint64_t from;
+    int64_t  duration;
+} xce_alloc_t;
+
+typedef struct {
+    char*         host;
+    int           nallocs;
+    xce_alloc_t** allocs;
+} xce_host_alloc_t;
+
+typedef struct {
+    uint64_t           jobid;
+    int                nhosts;
+    xce_host_alloc_t** allocs;
+} xce_job_alloc_t;
+
+typedef struct {
+    int               njobs;
+    xce_job_alloc_t** allocs;
+} xce_job_dispatch_t;
+
+typedef struct {
+    char*    queue;
+    char*    user;
+    char*    cmdline;
+    char*    resreq;
+    char*    exdata;
+    char*    pre_exec;
+    char*    post_exec;
+    uint32_t mintasks;
+    uint32_t maxtasks;
+} xce_job_submit_t;
+
+typedef struct {
+    int exitcode;
+} job_exitinfo_t;
+
+#define JOB_STAT_NULL  0x00     /**< NULL. */
+#define JOB_STAT_PEND  0x01     /**< Job is submitted. */
+#define JOB_STAT_RUN   0x02     /**< Job is running. */
+#define JOB_STAT_EXIT  0x04     /**< Job is exit. */
+#define JOB_STAT_DONE  0x08     /**< Job is done. */
+#define JOB_STAT_PSUSP 0x10     /**< Job is pending and suspend*/
+#define JOB_STAT_USUSP 0x20     /**< Job is user suspend*/
+#define JOB_STAT_SSUSP 0x40     /**< Job is system suspend*/
+
+#define JOB_STAT_PDONE  (0x80)  /**< Post job process done successfully */
+#define JOB_STAT_PERR   (0x100) /**< Post job process has error */
+#define JOB_STAT_WAIT   (0x200) /**< Chunk job waiting its turn to exec */
+#define JOB_STAT_RUNKWN 0x8000  /* Flag : Job status is UNKWN caused by losting contact with remote cluster */
+#define JOB_STAT_UNKWN  0x10000 /**< The worker on the host on which the \
+                                 * job is processed has lost contact with controller */
+#define JOB_STAT_PROV (0x20000) /**< this state shows that the job is dispatched to a standby \
+                                 * power saved host, and this host is                         \
+                                 * being waken up or started up */
+#define JOB_STAT_ERR (0x40000)  /**ERROR */
+
+#define JOB_EXEC_STAGE_Q_PRE_EXEC  (0)
+#define JOB_EXEC_STAGE_PRE_EXEC    (1)
+#define JOB_EXEC_STAGE_MAIN        (2)
+#define JOB_EXEC_STAGE_POST_EXEC   (3)
+#define JOB_EXEC_STAGE_Q_POST_EXEC (4)
+
+#define QUEUE_STAT_OPEN              0x00 /**< OPEN. */
+#define QUEUE_STAT_CLOSED            0x01 /**< The queue is closed */
+#define QUEUE_STAT_INACT_WINDOW      0x02 /**< The queue inactivated due to dispatch or run windows closed */
+#define QUEUE_STAT_INACT_ADMIN       0x04 /**< The queue inactivated by admin */
+#define QUEUE_STAT_NOPERM            0x08 /**< Remote queue rejecting jobs. */
+#define QUEUE_STAT_DISC              0x10 /**< Remote queue status is disconnected. */
+#define QUEUE_STAT_FAIRSHARE_DEFAULT 0x20 /**< Fairshare queue with null user share defined*/
+
+typedef enum {
+    NORMAL = 0,
+    TTY,
+    XJOB,
+    DISPLAY
+} xce_job_type;
+
+typedef struct {
+    char* host_name;
+    int   num_slots;
+} xce_job_host_slots_t;
+
+typedef struct {
+    int                   num_hosts;
+    xce_job_host_slots_t* host_slots;
+} xce_job_slots_alloc_t;
+
+typedef struct {
+    char*  host_name;
+    int    cpu_time;
+    double mem_usage;
+} xce_host_runusage_t;
+
+typedef struct {
+    double                max_mem;
+    double                avg_mem;
+    double                used_mem;
+    int                   cpu_time;
+    int                   peak_cpu_time;
+    int                   peak_cpu_duration;
+    float                 mem_efficiency;
+    float                 avg_cpu_efficiency;
+    float                 peak_cpu_efficiency;
+    int                   num_hosts;
+    xce_host_runusage_t** host_runusage;
+} xce_job_runusage_t;
+
+typedef struct {
+    uint64_t jobid;
+    char*    ckptinfo;
+} xce_ckptinfo_t;
+
+typedef struct {
+    uint64_t            jobid;
+    uint64_t            array_base_id;
+    int                 array_idx;
+    char*               queue;
+    char*               user;
+    int64_t             idx;
+    time_t              sub_time;
+    time_t              queue_time;
+    time_t              start_time;
+    time_t              finish_time;
+    uint32_t            mintasks;
+    uint32_t            maxtasks;
+    job_pidinfo_t*      pidinfo;
+    job_startinfo_t*    start_info;
+    unsigned int        state;
+    char*               job_name;
+    char*               project_name;
+    char*               user_group;
+    char*               job_description;
+    char*               resreq;
+    xce_job_alloc_t*    allocs;
+    int                 run_time;
+    xce_job_runusage_t* run_usage;
+    job_exitinfo_t*     exit_info;
+    char*               ckptinfo;
+    int64_t             pending_ctrl;
+} xce_job_t;
+
+typedef struct {
+    int         count;
+    xce_job_t** data;
+} xce_job_list_t;
+
+typedef struct {
+    uint64_t         jobid;
+    job_startinfo_t* start_info;
+    xce_launch_t*    launchinfo;
+} xce_job_exec_t;
+
+typedef struct {
+    uint64_t      jobid;
+    job_pidinfo_t pidinfo;
+} xce_job_runinfo_t;
+
+typedef struct {
+    uint64_t jobid;
+    int      exit_code;
+} xce_jobfinish_t;
+
+typedef struct {
+    char*                    queue;
+    uint32_t                 attrs;
+    uint32_t                 status;
+    int32_t                  priority;
+    int32_t                  accept_interval;
+    int32_t                  sched_delay;
+    char*                    resreq;
+    xce_group_t*             users;
+    xce_group_t*             hosts;
+    int32_t                  nthresholds;
+    xce_res_threshold_t**    thresholds;
+    xce_timewindow_t*        runwindow;
+    xce_timewindow_t*        dispatchwindow;
+    xce_timewindow_config_t* runwindow_config;
+    xce_timewindow_config_t* dispatchwindow_config;
+    int64_t                  pending_ctrl;
+    char*                    pre_exec;
+    char*                    post_exec;
+} xce_queue_t;
+
+typedef struct {
+    int           count;
+    xce_queue_t** queues;
+} xce_queuelist_t;
+
+typedef struct {
+    char*            name;
+    xce_reslist_t    reslist;
+    xce_queuelist_t  qlist;
+    xce_workerlist_t wlist;
+    xce_grouplist_t  uglist;
+    xce_grouplist_t  hglist;
+    xce_configlist_t cfglist;
+} xce_clusterconf_t;
+
+typedef enum {
+    CONTROL_NONE = 0,
+    CONTROL_JOB,
+    CONTROL_HOST,
+    CONTROL_QUEUE,
+    CONTROL_CLUSTER,
+    CONTROL_DEBUG
+} ctrl_target_type_t;
+
+typedef enum {
+    JOB_DO_NONE = 0,
+    JOB_STOP,
+    JOB_RESUME,
+    JOB_KILL,
+    JOB_MIG,
+    JOB_REQUEUE,
+    JOB_MODIFY,
+    JOB_SWITCH,
+    JOB_CHKPNT,
+    JOB_POST,
+    JOB_READ,
+    JOB_TOP,
+    JOB_BOT
+} job_ctrl_type_t;
+
+typedef enum {
+    HOST_DO_NONE = 0,
+    HOST_OPEN,
+    HOST_CLOSE,
+    HOST_TAG,
+    HOST_UNTAG
+} host_ctrl_type_t;
+
+typedef enum {
+    QUEUE_DO_NONE = 0,
+    QUEUE_OPEN,
+    QUEUE_CLOSE,
+    QUEUE_INACTIVE,
+    QUEUE_ACTIVE
+} queue_ctrl_type_t;
+
+typedef struct {
+    uint64_t jobid;
+    int      signal;
+    int      forcekill;
+    char*    method;
+    char*    comment;
+} job_ctrl_content_t;
+
+typedef struct {
+    char* hostname;
+    char* comment;
+} host_ctrl_content_t;
+
+typedef struct {
+    char* queuename;
+    char* comment;
+} queue_ctrl_content_t;
+
+typedef struct {
+    int64_t id;
+    int     target_type;
+    int     action_type;
+    time_t  request_time;
+    char*   user;
+    void*   content;
+} xce_ctrl_req_t;
+
+typedef struct {
+    int signo;
+    int force;
+} xce_signal_t;
+
+typedef struct {
+    int   kill;
+    char* method;
+} xce_ckpt_options_t;
+
+typedef struct {
+    int      action;
+    uint64_t jobid;
+    union {
+        xce_signal_t*       siginfo;
+        xce_ckpt_options_t* ckptoption;
+    } u;
+} xce_jobctrl_t;
+
+typedef struct {
+    int   action;
+    char* queue;
+} xce_queuectrl_t;
+
+typedef struct {
+    int   action;
+    char* hostname;
+} xce_hostctrl_t;
+
+typedef struct {
+    char* host;
+    char* launch_type; /* MPI ranks or something others */
+    int   ntasks;
+    char* exechome;
+    char* execcwd;
+    char* execuser;
+    char* cmdline;
+    char* exdata;
+} xce_taskhost_t;
+
+typedef struct {
+    uint64_t         jobid;
+    int              nhosts;
+    int              clientidx;
+    xce_taskhost_t** taskhosts;
+} xce_taskrun_t;
+
+typedef struct {
+    uint64_t      jobid;
+    char*         host;
+    int           clientidx;
+    int           taskidx;
+    int           npids;
+    unsigned int* pids;
+    int           npgids;
+    unsigned int* pgids;
+    char*         exdata;
+} xce_task_pidinfo_t;
+
+typedef struct {
+    uint64_t jobid;
+    char*    host;
+    int      clientidx;
+    int      taskidx; /* taskidx == -1 for fail of all tasks
+                       *  in the same host of xcecli_runtask()
+                       */
+    int   code;
+    char* detail;
+} xce_task_failinfo_t;
+
+typedef struct {
+    uint64_t jobid;
+    char*    host;
+    int      taskidx;
+    int      clientidx;
+    int      exit_code;
+} xce_task_finish_t;
+
+typedef struct {
+    char* resname;
+    float value;
+} xce_rusage_t;
+
+#endif