Init commit of jobs output/error message peek project.

src/controller/job.c

@@ -1,10 +1,12 @@
 #include <errno.h>
+#include <pthread.h>
 #include <signal.h>
 #include <time.h>
 
 #include "cluster.h"
 #include "jobj.h"
 #include "preason.h"
+#include "request.h"
 #include "xce/cluster_def.h"
 #include "xce/errnox.h"
 #include "xce/types.h"
@@ -1047,3 +1049,188 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
 
     return 0;
 }
+
+/* ---- fetchfile pending-request infrastructure ---- */
+
+typedef struct fetchfile_pending {
+    uint64_t               jobid;
+    pthread_mutex_t        mu;
+    pthread_cond_t         cv;
+    int                    done;
+    int                    status;
+    xce_fetchfile_reply_t* reply;
+    struct fetchfile_pending* next;
+} fetchfile_pending_t;
+
+static pthread_mutex_t    g_fetchfile_mu   = PTHREAD_MUTEX_INITIALIZER;
+static fetchfile_pending_t* g_fetchfile_list = NULL;
+
+static void fetchfile_register(fetchfile_pending_t* p)
+{
+    pthread_mutex_lock(&g_fetchfile_mu);
+    p->next        = g_fetchfile_list;
+    g_fetchfile_list = p;
+    pthread_mutex_unlock(&g_fetchfile_mu);
+}
+
+static void fetchfile_unregister(fetchfile_pending_t* p)
+{
+    fetchfile_pending_t** pp;
+    pthread_mutex_lock(&g_fetchfile_mu);
+    for (pp = &g_fetchfile_list; *pp != NULL; pp = &(*pp)->next) {
+        if (*pp == p) {
+            *pp = p->next;
+            break;
+        }
+    }
+    pthread_mutex_unlock(&g_fetchfile_mu);
+}
+
+fetchfile_pending_t* fetchfile_find(uint64_t jobid)
+{
+    fetchfile_pending_t* p;
+    pthread_mutex_lock(&g_fetchfile_mu);
+    for (p = g_fetchfile_list; p != NULL; p = p->next) {
+        if (p->jobid == jobid) break;
+    }
+    pthread_mutex_unlock(&g_fetchfile_mu);
+    return p;
+}
+
+/* Signal a pending fetchfile request with the worker's reply. */
+void fetchfile_signal(uint64_t jobid, int status, xce_fetchfile_reply_t* reply)
+{
+    fetchfile_pending_t* p = fetchfile_find(jobid);
+    if (p == NULL) return;
+
+    pthread_mutex_lock(&p->mu);
+    p->status = status;
+    p->reply  = reply;
+    p->done   = 1;
+    pthread_cond_signal(&p->cv);
+    pthread_mutex_unlock(&p->mu);
+}
+
+/* ---- on_fetchfile: proxy fetchfile from apiserver to worker ---- */
+
+int on_fetchfile(cluster_t* c, endpoint_t* caller_ep, xce_request_t* request)
+{
+    xce_fetchfile_t* msg = request->body.fetchfilemsg;
+    xce_reply_t      reply;
+    job_t*           j        = NULL;
+    worker_t*        w        = NULL;
+    int              rc       = 0;
+    char*            exechost = NULL;
+    struct timespec  deadline;
+
+    fetchfile_pending_t pending = {
+        .mu   = PTHREAD_MUTEX_INITIALIZER,
+        .cv   = PTHREAD_COND_INITIALIZER,
+        .done = 0, .status = -ETIMEDOUT, .reply = NULL, .next = NULL
+    };
+
+    init_reply_by_req(&reply, request);
+
+    if (msg == NULL || msg->filepath == NULL || msg->filepath[0] == '\0') {
+        reply.status     = -EINVAL;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(caller_ep, c->myhost, &reply);
+        return -EINVAL;
+    }
+
+    pending.jobid = msg->jobid;
+
+    /* Look up the job */
+    lock(c->jobmgr, RTTI_LOCK_READ);
+    key_find(c->jobmgr, (void*)msg->jobid, (void**)&j);
+    if (j != NULL) reference(j);
+    unlock(c->jobmgr);
+
+    if (j == NULL) {
+        reply.status     = -ENOENT;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(caller_ep, c->myhost, &reply);
+        return -ENOENT;
+    }
+
+    /* Get the first exec-host from job allocs */
+    lock(j, RTTI_LOCK_READ);
+    if (j->job.allocs != NULL && j->job.allocs->nhosts > 0
+        && j->job.allocs->allocs != NULL && j->job.allocs->allocs[0] != NULL
+        && j->job.allocs->allocs[0]->host != NULL) {
+        exechost = strdup(j->job.allocs->allocs[0]->host);
+    }
+    unlock(j);
+    dereference(j);
+
+    if (exechost == NULL) {
+        reply.status     = -ENOENT;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(caller_ep, c->myhost, &reply);
+        return -ENOENT;
+    }
+
+    /* Find the worker */
+    lock(c->workermgr, RTTI_LOCK_READ);
+    rc = key_find(c->workermgr, (void*)exechost, (void**)&w);
+    if (w != NULL) reference(w);
+    unlock(c->workermgr);
+    free(exechost);
+
+    if (w == NULL) {
+        reply.status     = -ENOENT;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(caller_ep, c->myhost, &reply);
+        return -ENOENT;
+    }
+
+    lock(w, RTTI_LOCK_READ);
+    if (w->ep == NULL) {
+        unlock(w);
+        dereference(w);
+        reply.status     = -ENOENT;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(caller_ep, c->myhost, &reply);
+        return -ENOENT;
+    }
+
+    /* Register pending before sending to avoid a race with the reply */
+    fetchfile_register(&pending);
+    rc = xce_fetchfile_req(w->ep, msg);
+    unlock(w);
+    dereference(w);
+
+    if (rc < 0) {
+        fetchfile_unregister(&pending);
+        reply.status     = rc;
+        reply.data_which = RequestReply_data_nodata;
+        xce_reply(caller_ep, c->myhost, &reply);
+        return rc;
+    }
+
+    /* Wait up to 10 seconds for the worker to respond */
+    clock_gettime(CLOCK_REALTIME, &deadline);
+    deadline.tv_sec += 10;
+
+    pthread_mutex_lock(&pending.mu);
+    while (!pending.done) {
+        rc = pthread_cond_timedwait(&pending.cv, &pending.mu, &deadline);
+        if (rc == ETIMEDOUT) break;
+    }
+    pthread_mutex_unlock(&pending.mu);
+
+    fetchfile_unregister(&pending);
+
+    /* Relay the reply back to the apiserver */
+    if (pending.done && pending.status >= 0 && pending.reply != NULL) {
+        reply.status              = 0;
+        reply.data_which          = RequestReply_data_fetchfilereply;
+        reply.data.fetchfilereply = pending.reply;
+    } else {
+        reply.status     = pending.done ? pending.status : -ETIMEDOUT;
+        reply.data_which = RequestReply_data_nodata;
+        FREEUP(pending.reply);
+    }
+    xce_reply(caller_ep, c->myhost, &reply);
+    return 0;
+}