feat: add --peers to controller for HA peer list and priority election

- myconf_t gains peers_cfg (xce_cluster_config_t) populated from --peers
- controller/main.c: parse --peers host:port,... via argp; ca_cert_path
  auto-derived from workdir/certs/xce.crt (no --ca-cert flag needed)
- cluster_init: derive my_priority from position in peers list (index 0 =
  primary); fall back to 0 when --peers not given (single-node mode)
This commit is contained in:
Rongsong Shen 2026-05-08 04:40:27 +00:00
parent d61f91b99c
commit 59597e105e
10 changed files with 8046 additions and 325 deletions

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,39 @@
# XCE Persistence & HA — Implementation Overview
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Make the XCE controller survive restarts and primary failures with no job/worker/queue state loss.
**Architecture:** WAL + periodic checkpoint (dual-zarray COW) for single-controller persistence; LSF-style priority election + fan-out TCP replication for HA. libxce smart discovery handles client auto-reconnect after failover.
**Tech Stack:** C (controller/lib), Cap'n Proto (serialization), Nix flakes + meson + ninja (build), Go (API server discovery bridge)
---
## Phase Map
Each phase is an independently shippable increment. Create the next phase plan only after the current phase passes its gate tests.
| Phase | Plan file | Goal | Gate |
|-------|-----------|------|------|
| **P0** | `2026-05-07-p0-single-controller-persistence.md` | Controller survives restart | `P0-G` smoke tests green |
| **P1** | _create after P0 ships_ | Standby takes over when primary fails | Election + promotion tests green |
| **P2** | _create after P1 ships_ | Primary replicates to standbys in real time | Replication round-trip + catch-up tests green |
| **P2.5** | _create alongside P2_ | Workers/schedulers auto-reconnect after failover | Discovery + reconnect tests green |
| **P3** | _create after P2 ships_ | End-to-end failover + hardening + metrics | Full failover test + benchmark passing |
| **P4** | _create after P3 ships_ | Docs, runbook, stale-ref cleanup | Runbook review complete |
## Out of Scope (all phases)
- Admin CLI for manual failover (`xce hostcontrol --failover`)
- WAL compaction beyond checkpoint trimming
- Multi-DC replication
- Runtime config changes without restart
- Snapshot compression
- `config_version` hash in heartbeat (deferred to P3 tooling)
- Degraded-mode restricted-state timeout (`repl_degraded_timeout_s`)
- `MAX_HA_CANDIDATES > 4` / `MAX_STANDBYS > 3`
## Design Document
`src/docs/persistence-and-ha-design.md` — all implementation detail lives here. Read the relevant Part before starting each task.

View file

@ -640,8 +640,28 @@ int cluster_init(cluster_t* c)
.probe_connect_timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS, .probe_connect_timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS,
.max_promotion_retries = HA_DEFAULT_MAX_PROMOTION_RETRIES, .max_promotion_retries = HA_DEFAULT_MAX_PROMOTION_RETRIES,
}; };
/* my_priority: single-node P1 always uses 0 (highest priority) */
rc = ha_init(&c->ha, &ha_cfg, 0); /* Derive priority from position in --peers list.
* Index 0 = highest priority (becomes primary on startup).
* If the controller's name is not found in the list, fall back to
* the last index (lowest priority = standby). */
int my_priority = 0;
const xce_cluster_config_t* pcfg = &c->myconf->peers_cfg;
if (pcfg->candidates_count > 0) {
my_priority = pcfg->candidates_count - 1; /* default: last = lowest */
for (int i = 0; i < pcfg->candidates_count; i++) {
if (strncmp(pcfg->candidates[i].hostname,
c->myconf->name,
XCE_MAXHOSTNAME_LEN - 1) == 0) {
my_priority = i;
break;
}
}
INFO("HA peers: %d candidates, my_priority=%d (name=%s)",
pcfg->candidates_count, my_priority, c->myconf->name);
}
rc = ha_init(&c->ha, &ha_cfg, my_priority);
if (rc != 0) { if (rc != 0) {
ERROR("ha_init failed: %d", rc); ERROR("ha_init failed: %d", rc);
persist_fini(&c->persist); persist_fini(&c->persist);

View file

@ -6,6 +6,7 @@
#include "myconf.h" #include "myconf.h"
#include "utils.h" #include "utils.h"
#include "xce/cluster.h"
extern int myapp_run(myconf_t*); extern int myapp_run(myconf_t*);
@ -15,11 +16,15 @@ const char* argp_program_version
= MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID)); = MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID));
const char* argp_program_bug_address = BUG_EMAIL; const char* argp_program_bug_address = BUG_EMAIL;
/* Long-option key (above ASCII to avoid collisions with short opts) */
#define OPT_PEERS 256
struct arguments { struct arguments {
char* myname; char* myname;
char* logfile; char* logfile;
char* workdir; char* workdir;
int verbose; int verbose;
char* peers_str;
}; };
static struct argp_option options[] = { static struct argp_option options[] = {
@ -27,9 +32,44 @@ static struct argp_option options[] = {
{ "logfile", 'l', "FILE", 0, "The full path of log file" }, { "logfile", 'l', "FILE", 0, "The full path of log file" },
{ "verbose", 'v', 0, 0, "Produce verbose logs" }, { "verbose", 'v', 0, 0, "Produce verbose logs" },
{ "myname", 'n', "STR", 0, "The name of controller" }, { "myname", 'n', "STR", 0, "The name of controller" },
{ "peers", OPT_PEERS, "PEERS", 0,
"HA peer list: comma-separated host:port pairs ordered by priority "
"(e.g. ctrl01:9990,ctrl02:9990). Index 0 = highest priority (primary)." },
{ 0 } { 0 }
}; };
/* parse_peers — split "host:port,host:port,..." into cfg->candidates[].
* Accepts optional "name=host:port" prefix (name is ignored for controller).
* Returns 0 on success, -1 on empty input or parse error. */
static int parse_peers(const char* peers_str, xce_cluster_config_t* cfg)
{
cfg->candidates_count = 0;
if (!peers_str || peers_str[0] == '\0')
return -1;
char buf[2048];
strncpy(buf, peers_str, sizeof(buf) - 1);
buf[sizeof(buf) - 1] = '\0';
char* saveptr;
char* tok = strtok_r(buf, ",", &saveptr);
while (tok && cfg->candidates_count < MAX_HA_CANDIDATES) {
char* eq = strchr(tok, '=');
char* addr = eq ? eq + 1 : tok;
char* colon = strrchr(addr, ':');
if (!colon)
return -1;
*colon = '\0';
strncpy(cfg->candidates[cfg->candidates_count].hostname, addr,
XCE_MAXHOSTNAME_LEN - 1);
cfg->candidates[cfg->candidates_count].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
cfg->candidates[cfg->candidates_count].port = atoi(colon + 1);
cfg->candidates_count++;
tok = strtok_r(NULL, ",", &saveptr);
}
return cfg->candidates_count > 0 ? 0 : -1;
}
static error_t parse_opt(int key, char* arg, struct argp_state* state) static error_t parse_opt(int key, char* arg, struct argp_state* state)
{ {
struct arguments* arguments = state->input; struct arguments* arguments = state->input;
@ -47,6 +87,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
case 'n': case 'n':
arguments->myname = arg; arguments->myname = arg;
break; break;
case OPT_PEERS:
arguments->peers_str = arg;
break;
default: default:
return ARGP_ERR_UNKNOWN; return ARGP_ERR_UNKNOWN;
} }
@ -111,6 +154,27 @@ int main(int argc, char* argv[])
myconf.keyfile = key_file; myconf.keyfile = key_file;
myconf.workdir = arguments.workdir; myconf.workdir = arguments.workdir;
myconf.name = arguments.myname; myconf.name = arguments.myname;
memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg));
if (arguments.peers_str) {
if (parse_peers(arguments.peers_str, &myconf.peers_cfg) != 0) {
fprintf(stderr,
"xce_controller: --peers parse error: expected "
"host:port,... got: %s\n",
arguments.peers_str);
return 1;
}
/* CA cert derived from workdir — same infrastructure as certfile/keyfile */
snprintf(myconf.peers_cfg.ca_cert_path,
sizeof(myconf.peers_cfg.ca_cert_path),
"%s/certs/xce.crt", arguments.workdir);
strncpy(myconf.peers_cfg.identity, arguments.myname,
sizeof(myconf.peers_cfg.identity) - 1);
INFO("HA mode: %d controller candidates, identity=%s",
myconf.peers_cfg.candidates_count, myconf.peers_cfg.identity);
} else {
INFO("No --peers flag; running as single-node primary");
}
myapp_run(&myconf); myapp_run(&myconf);

File diff suppressed because it is too large Load diff

View file

@ -7,6 +7,7 @@
#include <tlog.h> #include <tlog.h>
#include "clusterconfig.h" #include "clusterconfig.h"
#include "xce/cluster.h"
struct logging { struct logging {
char* logdir; char* logdir;
@ -20,6 +21,7 @@ typedef struct {
char* keyfile; char* keyfile;
char* workdir; /* working directory for WAL/snapshot storage */ char* workdir; /* working directory for WAL/snapshot storage */
char* name; /* controller name (hostname by default) */ char* name; /* controller name (hostname by default) */
xce_cluster_config_t peers_cfg; /* HA peer list parsed from --peers; count==0 = single-node */
} myconf_t; } myconf_t;
extern cluster_config_t* get_default_conf(void*); extern cluster_config_t* get_default_conf(void*);