Merge branch 'persistent_ha' into scheduler
This commit is contained in:
commit
a07a74a943
106 changed files with 18528 additions and 637 deletions
|
|
@ -15,6 +15,7 @@
|
|||
#include "kvmgr.h"
|
||||
#include "mylock.h"
|
||||
#include "plugin.h"
|
||||
#include "xce/cluster.h"
|
||||
#include "xce/types.h"
|
||||
#include "xlist.h"
|
||||
#include "xvec.h"
|
||||
|
|
@ -110,10 +111,12 @@ typedef struct {
|
|||
_Atomic int eval_in_progress; /* 1 while eval_select loop runs */
|
||||
_Atomic int staging_merge_complete; /* 1 after staging merge completes */
|
||||
select_eval_ctx_t* sel_eval_ctx; /* session-level memoisation context; NULL between sessions */
|
||||
xce_cluster_config_t cluster_cfg; /* HA candidate list — empty means single-host mode */
|
||||
} xce_scheduler_t;
|
||||
|
||||
extern int sched_init(xce_scheduler_t*, xce_client_config_t*,
|
||||
xce_request_handler_t);
|
||||
extern void sched_set_cluster_cfg(xce_scheduler_t*, const xce_cluster_config_t*);
|
||||
extern void sched_fini(xce_scheduler_t*);
|
||||
extern int sched_init_config(xce_scheduler_t* sched, xce_conf4sched_t*);
|
||||
extern int sched_update(xce_scheduler_t*);
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@
|
|||
#include <signal.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "client.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "xce/cluster.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
#include "scheduler.h"
|
||||
|
|
@ -13,6 +15,7 @@
|
|||
#include "resreq.h"
|
||||
#include "utils.h"
|
||||
|
||||
|
||||
extern int scheduler_req_handler(xce_request_t*, void*);
|
||||
|
||||
static char doc[] = "xce_scheduler";
|
||||
|
|
@ -24,11 +27,15 @@ const char* argp_program_bug_address = BUG_EMAIL;
|
|||
struct arguments {
|
||||
char* logfile;
|
||||
int verbose;
|
||||
char* peers; /* --peers value (optional) */
|
||||
char* ca_cert; /* --ca-cert value (optional) */
|
||||
};
|
||||
|
||||
static struct argp_option options[] = {
|
||||
{ "logfile", 'l', "FILE", 0, "The full path of log file" },
|
||||
{ "verbose", 'v', 0, 0, "Produce verbose log" },
|
||||
{ "logfile", 'l', "FILE", 0, "The full path of log file" },
|
||||
{ "verbose", 'v', 0, 0, "Produce verbose log" },
|
||||
{ "peers", 'P', "LIST", 0, "HA candidate list: [name=]host:port,..." },
|
||||
{ "ca-cert", 'C', "FILE", 0, "CA certificate path for TLS verification" },
|
||||
{ 0 }
|
||||
};
|
||||
|
||||
|
|
@ -43,6 +50,12 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
|||
case 'v':
|
||||
arguments->verbose++;
|
||||
break;
|
||||
case 'P':
|
||||
arguments->peers = arg;
|
||||
break;
|
||||
case 'C':
|
||||
arguments->ca_cert = arg;
|
||||
break;
|
||||
default:
|
||||
return ARGP_ERR_UNKNOWN;
|
||||
}
|
||||
|
|
@ -109,6 +122,42 @@ int main(int argc, char* argv[])
|
|||
|
||||
mylog_init(logfile, loglevel, logsize, logcount);
|
||||
|
||||
/* P2.5-6: populate and validate xce_cluster_config_t when --peers given */
|
||||
if (arguments.peers) {
|
||||
xce_cluster_config_t cluster_cfg = {0};
|
||||
|
||||
if (xce_parse_peers(arguments.peers, &cluster_cfg) != 0) {
|
||||
fprintf(stderr,
|
||||
"xce_scheduler: --peers parse error: expected "
|
||||
"\"[name=]host:port,...\" got \"%s\"\n",
|
||||
arguments.peers);
|
||||
return 1;
|
||||
}
|
||||
if (arguments.ca_cert) {
|
||||
strncpy(cluster_cfg.ca_cert_path, arguments.ca_cert,
|
||||
sizeof(cluster_cfg.ca_cert_path) - 1);
|
||||
}
|
||||
strncpy(cluster_cfg.identity, "xce_scheduler",
|
||||
sizeof(cluster_cfg.identity) - 1);
|
||||
if (xce_cluster_config_validate(&cluster_cfg) != 0) {
|
||||
fprintf(stderr,
|
||||
"xce_scheduler: invalid cluster config (missing "
|
||||
"--ca-cert?).\n");
|
||||
return 1;
|
||||
}
|
||||
if (cluster_cfg.candidates_count > 3) {
|
||||
WARN("xce_scheduler: --peers has %d candidates; "
|
||||
"only <=3 validated for HA",
|
||||
cluster_cfg.candidates_count);
|
||||
}
|
||||
INFO("HA cluster config: %d candidate(s), identity=%s, ca_cert=%s",
|
||||
cluster_cfg.candidates_count, cluster_cfg.identity,
|
||||
cluster_cfg.ca_cert_path);
|
||||
sched_set_cluster_cfg(&sched, &cluster_cfg);
|
||||
} else {
|
||||
INFO("No --peers flag; using single-controller path");
|
||||
}
|
||||
|
||||
init_config(&config);
|
||||
|
||||
sched_init(&sched, &config, scheduler_req_handler);
|
||||
|
|
|
|||
|
|
@ -207,6 +207,7 @@ static void* fcfs_elect_job(xce_sched_plugin_t* plugin)
|
|||
ordq_item_t* it = NULL;
|
||||
int go = 0;
|
||||
uint64_t jobid = 0;
|
||||
char jobidString[64];
|
||||
|
||||
pthread_testcancel();
|
||||
|
||||
|
|
@ -233,7 +234,8 @@ static void* fcfs_elect_job(xce_sched_plugin_t* plugin)
|
|||
if (sched_job_is_ready(j->job)) {
|
||||
jobid = j->job->jobid;
|
||||
|
||||
TRACE("%s jobid %ld", __func__, jobid);
|
||||
fmt_jobid(jobidString, sizeof(jobidString), jobid);
|
||||
TRACE("%s jobid %s", __func__, jobidString);
|
||||
cbuf_put(&(sched->jbuf), &jobid);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -226,9 +226,11 @@ static int limit_job_election(void* arg, xce_sched_job_t* job)
|
|||
xce_sched_obj_t* obj = job->schedobj;
|
||||
int rc = 0;
|
||||
iterator_t it;
|
||||
xce_sched_ctx_t ctx = { 0 };
|
||||
xce_sched_ctx_t ctx = { 0 };
|
||||
char jobidString[64];
|
||||
|
||||
TRACE("%s job %lu plugin %s", __func__, job->jobid, plugin->name);
|
||||
fmt_jobid(jobidString, sizeof(jobidString), job->jobid);
|
||||
TRACE("%s job %s plugin %s", __func__, jobidString, plugin->name);
|
||||
|
||||
if (obj->configver >= sched->configver) {
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -22,14 +22,12 @@
|
|||
|
||||
static int sched_session_active(xce_scheduler_t* sched)
|
||||
{
|
||||
return atomic_load_explicit(&sched->session_active,
|
||||
memory_order_acquire);
|
||||
return atomic_load_explicit(&sched->session_active, memory_order_acquire);
|
||||
}
|
||||
|
||||
static int sched_eval_in_progress(xce_scheduler_t* sched)
|
||||
{
|
||||
return atomic_load_explicit(&sched->eval_in_progress,
|
||||
memory_order_acquire);
|
||||
return atomic_load_explicit(&sched->eval_in_progress, memory_order_acquire);
|
||||
}
|
||||
|
||||
static int sched_staging_merge_complete(xce_scheduler_t* sched)
|
||||
|
|
@ -224,6 +222,7 @@ static void* sched_allocator(void* arg)
|
|||
xce_use(sched);
|
||||
while (1) {
|
||||
uint64_t jobid;
|
||||
char jobidString[64];
|
||||
xce_decision_t* decision = NULL;
|
||||
xce_sched_job_t* job = NULL;
|
||||
int rc = 0;
|
||||
|
|
@ -240,7 +239,8 @@ static void* sched_allocator(void* arg)
|
|||
|
||||
job = sched_get_job(sched, jobid);
|
||||
if (job == NULL) {
|
||||
TRACE("%s job %lu can not be found", __func__, jobid);
|
||||
fmt_jobid(jobidString, sizeof(jobidString), jobid);
|
||||
TRACE("%s job %s can not be found", __func__, jobidString);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -255,9 +255,10 @@ static void* sched_allocator(void* arg)
|
|||
/* job is pend */
|
||||
xce_pendreason_t* reason = job->schedobj->reason;
|
||||
|
||||
WARN("job %ld is pending but pend reason is NULL", jobid);
|
||||
|
||||
if (reason != NULL) {
|
||||
if (reason == NULL) {
|
||||
fmt_jobid(jobidString, sizeof(jobidString), jobid);
|
||||
WARN("job %s is pending but pend reason is NULL", jobidString);
|
||||
} else {
|
||||
reason->jobid = job->jobid;
|
||||
decision = make_pend_decision(reason);
|
||||
}
|
||||
|
|
@ -395,8 +396,8 @@ int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|||
sched->nextid = 1;
|
||||
sched->configver = 0;
|
||||
|
||||
xstring_table_init(1024*1024);
|
||||
|
||||
xstring_table_init(1024 * 1024);
|
||||
|
||||
mylock_init(&(sched->lock));
|
||||
cbuf_init(&(sched->wbuf), 1, sizeof(int));
|
||||
cbuf_init(&(sched->jbuf), njobs, sizeof(uint64_t));
|
||||
|
|
@ -436,14 +437,54 @@ int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|||
|
||||
xcecli_set_handler(sched->apictx, handler, sched);
|
||||
|
||||
rc = xcecli_start(sched->apictx);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
/* HA mode: iterate candidates until one accepts as primary */
|
||||
if (sched->cluster_cfg.candidates_count > 0) {
|
||||
int joined = 0;
|
||||
for (int i = 0; i < sched->cluster_cfg.candidates_count; i++) {
|
||||
const char* host = sched->cluster_cfg.candidates[i].hostname;
|
||||
int port = sched->cluster_cfg.candidates[i].port;
|
||||
|
||||
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
config->nmasters = 1;
|
||||
config->masters[0] = (char*)host;
|
||||
config->master_ports[0] = port;
|
||||
|
||||
rc = xcecli_ctx_init(&(sched->apictx), config);
|
||||
if (rc < 0) {
|
||||
continue;
|
||||
}
|
||||
xcecli_set_handler(sched->apictx, handler, sched);
|
||||
|
||||
rc = xcecli_start(sched->apictx);
|
||||
if (rc < 0) {
|
||||
xcecli_ctx_fini(sched->apictx); /* free before next attempt */
|
||||
continue;
|
||||
}
|
||||
|
||||
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
||||
if (rc == 0) {
|
||||
joined = 1;
|
||||
break;
|
||||
}
|
||||
|
||||
/* Standby replied — tear down fully before next attempt */
|
||||
xcecli_stop(sched->apictx);
|
||||
xcecli_ctx_fini(sched->apictx);
|
||||
}
|
||||
|
||||
if (!joined) {
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
/* Single-host path (original) */
|
||||
rc = xcecli_start(sched->apictx);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
rc = xcecli_join(sched->apictx, HostRole_scheduler, NULL);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
|
||||
rc = pthread_create(&(sched->jobpicker), NULL, &sched_job_picker, sched);
|
||||
|
|
@ -456,6 +497,15 @@ int sched_init(xce_scheduler_t* sched, xce_client_config_t* config,
|
|||
return rc;
|
||||
}
|
||||
|
||||
void sched_set_cluster_cfg(xce_scheduler_t* sched,
|
||||
const xce_cluster_config_t* cfg)
|
||||
{
|
||||
if (sched == NULL || cfg == NULL) {
|
||||
return;
|
||||
}
|
||||
sched->cluster_cfg = *cfg;
|
||||
}
|
||||
|
||||
void sched_fini(xce_scheduler_t* sched)
|
||||
{
|
||||
void* res;
|
||||
|
|
@ -888,7 +938,8 @@ static int sched_account_jobs(xce_scheduler_t* sched)
|
|||
void sched_begin_session(xce_scheduler_t* sched)
|
||||
{
|
||||
atomic_store_explicit(&sched->session_active, 1, memory_order_release);
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 0, memory_order_release);
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 0,
|
||||
memory_order_release);
|
||||
|
||||
/* Allocate a fresh memoisation context sized to the current interned-node
|
||||
* population. sched_reset_workers (called below) frees each worker's
|
||||
|
|
@ -904,7 +955,8 @@ void sched_begin_session(xce_scheduler_t* sched)
|
|||
sched_account_jobs(sched);
|
||||
|
||||
/* Staging merge done — sched_end_session's GC guard reads this field. */
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 1, memory_order_release);
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 1,
|
||||
memory_order_release);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -920,8 +972,7 @@ void sched_end_session(xce_scheduler_t* sched)
|
|||
/* Guard: GC requires the evaluation pipeline to be idle */
|
||||
if (sched_eval_in_progress(sched) || !sched_staging_merge_complete(sched)) {
|
||||
TRACE("%s: GC skipped — eval_in_progress=%d staging_merge_complete=%d",
|
||||
__func__,
|
||||
sched_eval_in_progress(sched),
|
||||
__func__, sched_eval_in_progress(sched),
|
||||
sched_staging_merge_complete(sched));
|
||||
return;
|
||||
}
|
||||
|
|
@ -943,8 +994,8 @@ void sched_end_session(xce_scheduler_t* sched)
|
|||
}
|
||||
if (njobs >= cap) {
|
||||
int newcap = cap * 2;
|
||||
xce_sched_job_t** tmp = realloc(live_jobs,
|
||||
newcap * sizeof(*live_jobs));
|
||||
xce_sched_job_t** tmp
|
||||
= realloc(live_jobs, newcap * sizeof(*live_jobs));
|
||||
if (tmp == NULL) {
|
||||
free(live_jobs);
|
||||
live_jobs = NULL;
|
||||
|
|
@ -964,7 +1015,8 @@ void sched_end_session(xce_scheduler_t* sched)
|
|||
|
||||
if (live_jobs == NULL) {
|
||||
/* OOM: cannot safely GC — skip to avoid sweeping nodes still in use */
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 1, memory_order_release);
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 1,
|
||||
memory_order_release);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1016,7 +1068,8 @@ void sched_end_session(xce_scheduler_t* sched)
|
|||
recompact_node_ids();
|
||||
free(live_jobs);
|
||||
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 1, memory_order_release);
|
||||
atomic_store_explicit(&sched->staging_merge_complete, 1,
|
||||
memory_order_release);
|
||||
}
|
||||
|
||||
int sched_add_job(xce_scheduler_t* sched, xce_sched_job_t* j)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue