merge job rusage and pending reason support

src/controller/cluster.c

@@ -5,6 +5,7 @@
 #include <unistd.h>
 
 #include "cluster.h"
+#include "dataq.h"
 #include "htab.h"
 #include "mylock.h"
 #include "request.h"
@@ -188,6 +189,12 @@ static int myreq_handler(vtask_t* task)
     case OPCode_runtasks:
         rc = do_runtasks(c, ep, &request);
         break;
+    case OPCode_setpreason:
+        rc = do_set_preason(c, ep, &request);
+        break;
+    case OPCode_clrpreason:
+        rc = do_clr_preason(c, ep, &request);
+        break;
     default:
         rc = -E_XCE_BAD_OPCODE;
         ERROR("Unknown opcode: %d", request.opcode);
@@ -447,6 +454,11 @@ static int xce_install_hooks(cluster_t* c)
     return 0;
 }
 
+static int run_delay_tasks(delay_task_entry_fn_t entry, void* arg)
+{
+    return entry(arg);
+}
+
 int cluster_init(cluster_t* c)
 {
     int      i, rc;
@@ -524,12 +536,90 @@ int cluster_init(cluster_t* c)
 
     vtaskmgr_init(&(c->vtaskmgr), c->myconf->cluster_config->vcpus);
 
+    delaymgr_init(&(c->delaymgr), 2, run_delay_tasks);
+
     xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_AFTER_EVENT,
                   HOOK_DETAIL_CLUSTER_INIT, c);
 
     return 0;
 }
 
+static int make_heartbeat(cluster_t* c, endpoint_t* ep)
+{
+    return xce_nop_req(ep);
+}
+
+static int check_worker_timeout_(void* arg1, void* arg2)
+{
+    cluster_t*        c = (cluster_t*)arg1;
+    worker_t*         w = (worker_t*)arg2;
+    endpoint_t*       ep;
+    time_t            now = time(NULL);
+    int               timeout;
+    cluster_config_t* config;
+
+    if (w == NULL) {
+        return 0;
+    }
+
+    if (c == NULL) {
+        return -EINVAL;
+    }
+
+    ep = w->ep;
+    if (ep == NULL) {
+        return 0;
+    }
+
+    config  = c->myconf->cluster_config;
+    timeout = config->heartbeat_interval * config->inactive_heartbeat_count;
+
+    if (now - ep->ts_last_in > timeout) {
+        TRACE("%s worker %s becomes UNAVAIL as heartbeat timeout", __func__,
+              w->host.name);
+
+        mark_worker_unavail(c, w);
+
+        return 0;
+    } else {
+        if (IS_WORKER_UNAVAIL(w->host.status)) {
+            mark_worker_back(c, w);
+        }
+    }
+
+    if (now - ep->ts_last_out >= config->heartbeat_interval) {
+        if (DATAQ_EMPTY(&(ep->outq))) {
+            int rc = 0;
+
+            TRACE("%s send heartbeat to %s", __func__, w->host.name);
+
+            rc = make_heartbeat(c, ep);
+
+            if (rc < 0) {
+                ERROR("%s make_heartbeat to %s failed", __func__, w->host.name);
+                return rc;
+            }
+        }
+    }
+
+    return 0;
+}
+
+static int check_workers_timeout(void* args)
+{
+    xlist_t    lst;
+    int        rc;
+    cluster_t* c = (cluster_t*)args;
+
+    xlist_init(&lst, 3);
+    xlist_store(&lst, check_worker_timeout_);
+    rc = workermgr_iter(&(c->workermgr), &lst, c);
+
+    xlist_fini(&lst);
+
+    return rc;
+}
+
 int cluster_start(cluster_t* c)
 {
     int rc;
@@ -546,6 +636,10 @@ int cluster_start(cluster_t* c)
 
     iomgr_start(&(c->iomgr));
     vtaskmgr_start(&(c->vtaskmgr));
+    delaymgr_start(&(c->delaymgr));
+
+    delay_task(&(c->delaymgr), c->myconf->cluster_config->heartbeat_interval,
+               -1, (delay_task_entry_fn_t)check_workers_timeout, (void*)c);
 
     c->running = 1;
     return 0;
@@ -565,6 +659,9 @@ int cluster_run(cluster_t* c)
     xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_CLUSTER, HOOK_BEFORE_EVENT,
                   HOOK_DETAIL_CLUSTER_SHUTDOWN, c);
     c->running = 0;
+
+    delaymgr_stop(&(c->delaymgr));
+
     iomgr_unmonitor(&(c->iomgr), &(c->ep));
     iomgr_stop(&(c->iomgr));
 
@@ -573,6 +670,7 @@ int cluster_run(cluster_t* c)
 
 void cluster_fini(cluster_t* c)
 {
+    delaymgr_fini(&(c->delaymgr));
     vtaskmgr_fini(&(c->vtaskmgr));
     endpoint_fini(&(c->ep));