merge
This commit is contained in:
commit
ff1be48001
38 changed files with 2292 additions and 35 deletions
|
|
@ -404,6 +404,9 @@ static int myreq_handler(vtask_t* task)
|
|||
case OPCode_clrpreason:
|
||||
rc = do_clr_preason(c, ep, &request);
|
||||
break;
|
||||
case OPCode_fetchfile:
|
||||
rc = on_fetchfile(c, ep, &request);
|
||||
break;
|
||||
case OPCode_requestVote: {
|
||||
/* Task 6.3: delegate vote-request handling to raft library.
|
||||
*
|
||||
|
|
@ -691,22 +694,41 @@ static int myreq_handler(vtask_t* task)
|
|||
}
|
||||
break;
|
||||
#endif
|
||||
case OPCode_reply:
|
||||
/* Config-push ACK arriving on an inbound standby endpoint (the
|
||||
* join.c-registered standby_endpoints channel — these eps reply on
|
||||
* the same connection the primary pushed on, so the ACK lands here
|
||||
* instead of peer_ctrl_msg_handler). Feed it to the replicator so
|
||||
* the configAck barrier can observe it (ha-commit-barrier-rollout
|
||||
* task 5.2 remediation r1; same ingestion as
|
||||
* repl_ack_decode_and_dispatch in ha.c — completes the unlanded
|
||||
* half of ha-config-bootstrap task 8.1). */
|
||||
case OPCode_reply: {
|
||||
/* Reply arriving on an inbound endpoint. Two independent producers
|
||||
* feed this case:
|
||||
* 1. fetchfile proxy replies from a worker (fetchfile feature) —
|
||||
* routed to the pending waiter via fetchfile_signal.
|
||||
* 2. Config-push ACKs arriving on an inbound standby endpoint (the
|
||||
* join.c-registered standby_endpoints channel — these eps reply
|
||||
* on the same connection the primary pushed on, so the ACK
|
||||
* lands here instead of peer_ctrl_msg_handler). Fed to the
|
||||
* replicator so the configAck barrier can observe it
|
||||
* (ha-commit-barrier-rollout task 5.2 remediation r1; same
|
||||
* ingestion as repl_ack_decode_and_dispatch in ha.c — completes
|
||||
* the unlanded half of ha-config-bootstrap task 8.1). */
|
||||
xce_reply_t* workreply = request.body.replymsg;
|
||||
|
||||
if (workreply != NULL && workreply->opcode == OPCode_fetchfile) {
|
||||
xce_fetchfile_reply_t* fr =
|
||||
(workreply->data_which == RequestReply_data_fetchfilereply)
|
||||
? workreply->data.fetchfilereply
|
||||
: NULL;
|
||||
uint64_t jid = fr ? fr->jobid : 0;
|
||||
fetchfile_signal(jid, workreply->status, fr);
|
||||
/* Ownership of fr transfers to the pending waiter. */
|
||||
workreply->data.fetchfilereply = NULL;
|
||||
rc = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
if (request.body_which == Request_body_replymsg
|
||||
&& request.body.replymsg
|
||||
&& request.body.replymsg->data_which
|
||||
&& workreply != NULL
|
||||
&& workreply->data_which
|
||||
== RequestReply_data_replconfigpushreplymsg
|
||||
&& request.body.replymsg->data.replconfigpushreplymsg) {
|
||||
&& workreply->data.replconfigpushreplymsg) {
|
||||
repl_config_push_reply_t *cack =
|
||||
request.body.replymsg->data.replconfigpushreplymsg;
|
||||
workreply->data.replconfigpushreplymsg;
|
||||
if (cack->accepted) {
|
||||
repl_on_config_ack(&c->persist.repl, ep, cack->version_tag);
|
||||
TRACE("myreq_handler: config ACK from inbound standby"
|
||||
|
|
@ -718,8 +740,10 @@ static int myreq_handler(vtask_t* task)
|
|||
rc = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
/* Any other reply body keeps the legacy unknown-opcode handling. */
|
||||
/* fall through */
|
||||
}
|
||||
default:
|
||||
rc = -E_XCE_BAD_OPCODE;
|
||||
ERROR("Unknown opcode: %d", request.opcode);
|
||||
|
|
|
|||
|
|
@ -220,6 +220,8 @@ extern int release_resource_from_worker(char*, cluster_t*);
|
|||
extern int collect_resval(var, zarray_t*, xlist_t*);
|
||||
extern int do_set_preason(cluster_t*, endpoint_t*, xce_request_t*);
|
||||
extern int do_clr_preason(cluster_t*, endpoint_t*, xce_request_t*);
|
||||
extern int on_fetchfile(cluster_t*, endpoint_t*, xce_request_t*);
|
||||
extern void fetchfile_signal(uint64_t jobid, int status, xce_fetchfile_reply_t*);
|
||||
extern int handle_apiserver_leave(cluster_t*, endpoint_t*);
|
||||
extern int add_apiserver(cluster_t*, char*, endpoint_t*);
|
||||
extern int apiserver_init(cluster_t*);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <signal.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
|
|
@ -10,6 +11,7 @@
|
|||
#include "jobj.h"
|
||||
#include "persist.h"
|
||||
#include "preason.h"
|
||||
#include "request.h"
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/errnox.h"
|
||||
#include "xce/types.h"
|
||||
|
|
@ -2402,6 +2404,191 @@ int do_clr_preason(cluster_t* c, endpoint_t* ep, xce_request_t* request)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* ---- fetchfile pending-request infrastructure ---- */
|
||||
|
||||
typedef struct fetchfile_pending {
|
||||
uint64_t jobid;
|
||||
pthread_mutex_t mu;
|
||||
pthread_cond_t cv;
|
||||
int done;
|
||||
int status;
|
||||
xce_fetchfile_reply_t* reply;
|
||||
struct fetchfile_pending* next;
|
||||
} fetchfile_pending_t;
|
||||
|
||||
static pthread_mutex_t g_fetchfile_mu = PTHREAD_MUTEX_INITIALIZER;
|
||||
static fetchfile_pending_t* g_fetchfile_list = NULL;
|
||||
|
||||
static void fetchfile_register(fetchfile_pending_t* p)
|
||||
{
|
||||
pthread_mutex_lock(&g_fetchfile_mu);
|
||||
p->next = g_fetchfile_list;
|
||||
g_fetchfile_list = p;
|
||||
pthread_mutex_unlock(&g_fetchfile_mu);
|
||||
}
|
||||
|
||||
static void fetchfile_unregister(fetchfile_pending_t* p)
|
||||
{
|
||||
fetchfile_pending_t** pp;
|
||||
pthread_mutex_lock(&g_fetchfile_mu);
|
||||
for (pp = &g_fetchfile_list; *pp != NULL; pp = &(*pp)->next) {
|
||||
if (*pp == p) {
|
||||
*pp = p->next;
|
||||
break;
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&g_fetchfile_mu);
|
||||
}
|
||||
|
||||
fetchfile_pending_t* fetchfile_find(uint64_t jobid)
|
||||
{
|
||||
fetchfile_pending_t* p;
|
||||
pthread_mutex_lock(&g_fetchfile_mu);
|
||||
for (p = g_fetchfile_list; p != NULL; p = p->next) {
|
||||
if (p->jobid == jobid) break;
|
||||
}
|
||||
pthread_mutex_unlock(&g_fetchfile_mu);
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Signal a pending fetchfile request with the worker's reply. */
|
||||
void fetchfile_signal(uint64_t jobid, int status, xce_fetchfile_reply_t* reply)
|
||||
{
|
||||
fetchfile_pending_t* p = fetchfile_find(jobid);
|
||||
if (p == NULL) return;
|
||||
|
||||
pthread_mutex_lock(&p->mu);
|
||||
p->status = status;
|
||||
p->reply = reply;
|
||||
p->done = 1;
|
||||
pthread_cond_signal(&p->cv);
|
||||
pthread_mutex_unlock(&p->mu);
|
||||
}
|
||||
|
||||
/* ---- on_fetchfile: proxy fetchfile from apiserver to worker ---- */
|
||||
|
||||
int on_fetchfile(cluster_t* c, endpoint_t* caller_ep, xce_request_t* request)
|
||||
{
|
||||
xce_fetchfile_t* msg = request->body.fetchfilemsg;
|
||||
xce_reply_t reply;
|
||||
job_t* j = NULL;
|
||||
worker_t* w = NULL;
|
||||
int rc = 0;
|
||||
char* exechost = NULL;
|
||||
struct timespec deadline;
|
||||
|
||||
fetchfile_pending_t pending = {
|
||||
.mu = PTHREAD_MUTEX_INITIALIZER,
|
||||
.cv = PTHREAD_COND_INITIALIZER,
|
||||
.done = 0, .status = -ETIMEDOUT, .reply = NULL, .next = NULL
|
||||
};
|
||||
|
||||
init_reply_by_req(&reply, request);
|
||||
|
||||
if (msg == NULL || msg->filepath == NULL || msg->filepath[0] == '\0') {
|
||||
reply.status = -EINVAL;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
pending.jobid = msg->jobid;
|
||||
|
||||
/* Look up the job */
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)msg->jobid, (void**)&j);
|
||||
if (j != NULL) reference(j);
|
||||
unlock(c->jobmgr);
|
||||
|
||||
if (j == NULL) {
|
||||
reply.status = -ENOENT;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* Get the first exec-host from job allocs */
|
||||
lock(j, RTTI_LOCK_READ);
|
||||
if (j->job.allocs != NULL && j->job.allocs->nhosts > 0
|
||||
&& j->job.allocs->allocs != NULL && j->job.allocs->allocs[0] != NULL
|
||||
&& j->job.allocs->allocs[0]->host != NULL) {
|
||||
exechost = strdup(j->job.allocs->allocs[0]->host);
|
||||
}
|
||||
unlock(j);
|
||||
dereference(j);
|
||||
|
||||
if (exechost == NULL) {
|
||||
reply.status = -ENOENT;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* Find the worker */
|
||||
lock(c->workermgr, RTTI_LOCK_READ);
|
||||
rc = key_find(c->workermgr, (void*)exechost, (void**)&w);
|
||||
if (w != NULL) reference(w);
|
||||
unlock(c->workermgr);
|
||||
free(exechost);
|
||||
|
||||
if (w == NULL) {
|
||||
reply.status = -ENOENT;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
lock(w, RTTI_LOCK_READ);
|
||||
if (w->ep == NULL) {
|
||||
unlock(w);
|
||||
dereference(w);
|
||||
reply.status = -ENOENT;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return -ENOENT;
|
||||
}
|
||||
|
||||
/* Register pending before sending to avoid a race with the reply */
|
||||
fetchfile_register(&pending);
|
||||
rc = xce_fetchfile_req(w->ep, msg);
|
||||
unlock(w);
|
||||
dereference(w);
|
||||
|
||||
if (rc < 0) {
|
||||
fetchfile_unregister(&pending);
|
||||
reply.status = rc;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Wait up to 10 seconds for the worker to respond */
|
||||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||||
deadline.tv_sec += 10;
|
||||
|
||||
pthread_mutex_lock(&pending.mu);
|
||||
while (!pending.done) {
|
||||
rc = pthread_cond_timedwait(&pending.cv, &pending.mu, &deadline);
|
||||
if (rc == ETIMEDOUT) break;
|
||||
}
|
||||
pthread_mutex_unlock(&pending.mu);
|
||||
|
||||
fetchfile_unregister(&pending);
|
||||
|
||||
/* Relay the reply back to the apiserver */
|
||||
if (pending.done && pending.status >= 0 && pending.reply != NULL) {
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_fetchfilereply;
|
||||
reply.data.fetchfilereply = pending.reply;
|
||||
} else {
|
||||
reply.status = pending.done ? pending.status : -ETIMEDOUT;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
FREEUP(pending.reply);
|
||||
}
|
||||
xce_reply(caller_ep, c->myhost, &reply);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* job_is_terminal and job_should_retain live in persist.c —
|
||||
* they only depend on xce_job_t fields and persist_config_t,
|
||||
* and must be accessible from the unit test binary which links persist.c. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue