feat: add --peers to controller for HA peer list and priority election
- myconf_t gains peers_cfg (xce_cluster_config_t) populated from --peers - controller/main.c: parse --peers host:port,... via argp; ca_cert_path auto-derived from workdir/certs/xce.crt (no --ca-cert flag needed) - cluster_init: derive my_priority from position in peers list (index 0 = primary); fall back to 0 when --peers not given (single-node mode)
This commit is contained in:
parent
d61f91b99c
commit
59597e105e
10 changed files with 8046 additions and 325 deletions
File diff suppressed because it is too large
Load diff
1520
docs/superpowers/plans/2026-05-07-p1-ha-foundation.md
Normal file
1520
docs/superpowers/plans/2026-05-07-p1-ha-foundation.md
Normal file
File diff suppressed because it is too large
Load diff
1498
docs/superpowers/plans/2026-05-07-p2-replication.md
Normal file
1498
docs/superpowers/plans/2026-05-07-p2-replication.md
Normal file
File diff suppressed because it is too large
Load diff
1247
docs/superpowers/plans/2026-05-07-p25-discovery.md
Normal file
1247
docs/superpowers/plans/2026-05-07-p25-discovery.md
Normal file
File diff suppressed because it is too large
Load diff
1133
docs/superpowers/plans/2026-05-07-p3-hardening.md
Normal file
1133
docs/superpowers/plans/2026-05-07-p3-hardening.md
Normal file
File diff suppressed because it is too large
Load diff
39
docs/superpowers/plans/2026-05-07-persistence-ha-overview.md
Normal file
39
docs/superpowers/plans/2026-05-07-persistence-ha-overview.md
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
# XCE Persistence & HA — Implementation Overview
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Make the XCE controller survive restarts and primary failures with no job/worker/queue state loss.
|
||||||
|
|
||||||
|
**Architecture:** WAL + periodic checkpoint (dual-zarray COW) for single-controller persistence; LSF-style priority election + fan-out TCP replication for HA. libxce smart discovery handles client auto-reconnect after failover.
|
||||||
|
|
||||||
|
**Tech Stack:** C (controller/lib), Cap'n Proto (serialization), Nix flakes + meson + ninja (build), Go (API server discovery bridge)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase Map
|
||||||
|
|
||||||
|
Each phase is an independently shippable increment. Create the next phase plan only after the current phase passes its gate tests.
|
||||||
|
|
||||||
|
| Phase | Plan file | Goal | Gate |
|
||||||
|
|-------|-----------|------|------|
|
||||||
|
| **P0** | `2026-05-07-p0-single-controller-persistence.md` | Controller survives restart | `P0-G` smoke tests green |
|
||||||
|
| **P1** | _create after P0 ships_ | Standby takes over when primary fails | Election + promotion tests green |
|
||||||
|
| **P2** | _create after P1 ships_ | Primary replicates to standbys in real time | Replication round-trip + catch-up tests green |
|
||||||
|
| **P2.5** | _create alongside P2_ | Workers/schedulers auto-reconnect after failover | Discovery + reconnect tests green |
|
||||||
|
| **P3** | _create after P2 ships_ | End-to-end failover + hardening + metrics | Full failover test + benchmark passing |
|
||||||
|
| **P4** | _create after P3 ships_ | Docs, runbook, stale-ref cleanup | Runbook review complete |
|
||||||
|
|
||||||
|
## Out of Scope (all phases)
|
||||||
|
|
||||||
|
- Admin CLI for manual failover (`xce hostcontrol --failover`)
|
||||||
|
- WAL compaction beyond checkpoint trimming
|
||||||
|
- Multi-DC replication
|
||||||
|
- Runtime config changes without restart
|
||||||
|
- Snapshot compression
|
||||||
|
- `config_version` hash in heartbeat (deferred to P3 tooling)
|
||||||
|
- Degraded-mode restricted-state timeout (`repl_degraded_timeout_s`)
|
||||||
|
- `MAX_HA_CANDIDATES > 4` / `MAX_STANDBYS > 3`
|
||||||
|
|
||||||
|
## Design Document
|
||||||
|
|
||||||
|
`src/docs/persistence-and-ha-design.md` — all implementation detail lives here. Read the relevant Part before starting each task.
|
||||||
|
|
@ -640,8 +640,28 @@ int cluster_init(cluster_t* c)
|
||||||
.probe_connect_timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS,
|
.probe_connect_timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS,
|
||||||
.max_promotion_retries = HA_DEFAULT_MAX_PROMOTION_RETRIES,
|
.max_promotion_retries = HA_DEFAULT_MAX_PROMOTION_RETRIES,
|
||||||
};
|
};
|
||||||
/* my_priority: single-node P1 always uses 0 (highest priority) */
|
|
||||||
rc = ha_init(&c->ha, &ha_cfg, 0);
|
/* Derive priority from position in --peers list.
|
||||||
|
* Index 0 = highest priority (becomes primary on startup).
|
||||||
|
* If the controller's name is not found in the list, fall back to
|
||||||
|
* the last index (lowest priority = standby). */
|
||||||
|
int my_priority = 0;
|
||||||
|
const xce_cluster_config_t* pcfg = &c->myconf->peers_cfg;
|
||||||
|
if (pcfg->candidates_count > 0) {
|
||||||
|
my_priority = pcfg->candidates_count - 1; /* default: last = lowest */
|
||||||
|
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||||||
|
if (strncmp(pcfg->candidates[i].hostname,
|
||||||
|
c->myconf->name,
|
||||||
|
XCE_MAXHOSTNAME_LEN - 1) == 0) {
|
||||||
|
my_priority = i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
INFO("HA peers: %d candidates, my_priority=%d (name=%s)",
|
||||||
|
pcfg->candidates_count, my_priority, c->myconf->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
rc = ha_init(&c->ha, &ha_cfg, my_priority);
|
||||||
if (rc != 0) {
|
if (rc != 0) {
|
||||||
ERROR("ha_init failed: %d", rc);
|
ERROR("ha_init failed: %d", rc);
|
||||||
persist_fini(&c->persist);
|
persist_fini(&c->persist);
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
|
|
||||||
#include "myconf.h"
|
#include "myconf.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include "xce/cluster.h"
|
||||||
|
|
||||||
extern int myapp_run(myconf_t*);
|
extern int myapp_run(myconf_t*);
|
||||||
|
|
||||||
|
|
@ -15,11 +16,15 @@ const char* argp_program_version
|
||||||
= MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID));
|
= MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID));
|
||||||
const char* argp_program_bug_address = BUG_EMAIL;
|
const char* argp_program_bug_address = BUG_EMAIL;
|
||||||
|
|
||||||
|
/* Long-option key (above ASCII to avoid collisions with short opts) */
|
||||||
|
#define OPT_PEERS 256
|
||||||
|
|
||||||
struct arguments {
|
struct arguments {
|
||||||
char* myname;
|
char* myname;
|
||||||
char* logfile;
|
char* logfile;
|
||||||
char* workdir;
|
char* workdir;
|
||||||
int verbose;
|
int verbose;
|
||||||
|
char* peers_str;
|
||||||
};
|
};
|
||||||
|
|
||||||
static struct argp_option options[] = {
|
static struct argp_option options[] = {
|
||||||
|
|
@ -27,9 +32,44 @@ static struct argp_option options[] = {
|
||||||
{ "logfile", 'l', "FILE", 0, "The full path of log file" },
|
{ "logfile", 'l', "FILE", 0, "The full path of log file" },
|
||||||
{ "verbose", 'v', 0, 0, "Produce verbose logs" },
|
{ "verbose", 'v', 0, 0, "Produce verbose logs" },
|
||||||
{ "myname", 'n', "STR", 0, "The name of controller" },
|
{ "myname", 'n', "STR", 0, "The name of controller" },
|
||||||
|
{ "peers", OPT_PEERS, "PEERS", 0,
|
||||||
|
"HA peer list: comma-separated host:port pairs ordered by priority "
|
||||||
|
"(e.g. ctrl01:9990,ctrl02:9990). Index 0 = highest priority (primary)." },
|
||||||
{ 0 }
|
{ 0 }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* parse_peers — split "host:port,host:port,..." into cfg->candidates[].
|
||||||
|
* Accepts optional "name=host:port" prefix (name is ignored for controller).
|
||||||
|
* Returns 0 on success, -1 on empty input or parse error. */
|
||||||
|
static int parse_peers(const char* peers_str, xce_cluster_config_t* cfg)
|
||||||
|
{
|
||||||
|
cfg->candidates_count = 0;
|
||||||
|
if (!peers_str || peers_str[0] == '\0')
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
char buf[2048];
|
||||||
|
strncpy(buf, peers_str, sizeof(buf) - 1);
|
||||||
|
buf[sizeof(buf) - 1] = '\0';
|
||||||
|
|
||||||
|
char* saveptr;
|
||||||
|
char* tok = strtok_r(buf, ",", &saveptr);
|
||||||
|
while (tok && cfg->candidates_count < MAX_HA_CANDIDATES) {
|
||||||
|
char* eq = strchr(tok, '=');
|
||||||
|
char* addr = eq ? eq + 1 : tok;
|
||||||
|
char* colon = strrchr(addr, ':');
|
||||||
|
if (!colon)
|
||||||
|
return -1;
|
||||||
|
*colon = '\0';
|
||||||
|
strncpy(cfg->candidates[cfg->candidates_count].hostname, addr,
|
||||||
|
XCE_MAXHOSTNAME_LEN - 1);
|
||||||
|
cfg->candidates[cfg->candidates_count].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
|
||||||
|
cfg->candidates[cfg->candidates_count].port = atoi(colon + 1);
|
||||||
|
cfg->candidates_count++;
|
||||||
|
tok = strtok_r(NULL, ",", &saveptr);
|
||||||
|
}
|
||||||
|
return cfg->candidates_count > 0 ? 0 : -1;
|
||||||
|
}
|
||||||
|
|
||||||
static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
||||||
{
|
{
|
||||||
struct arguments* arguments = state->input;
|
struct arguments* arguments = state->input;
|
||||||
|
|
@ -47,6 +87,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
||||||
case 'n':
|
case 'n':
|
||||||
arguments->myname = arg;
|
arguments->myname = arg;
|
||||||
break;
|
break;
|
||||||
|
case OPT_PEERS:
|
||||||
|
arguments->peers_str = arg;
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
return ARGP_ERR_UNKNOWN;
|
return ARGP_ERR_UNKNOWN;
|
||||||
}
|
}
|
||||||
|
|
@ -111,6 +154,27 @@ int main(int argc, char* argv[])
|
||||||
myconf.keyfile = key_file;
|
myconf.keyfile = key_file;
|
||||||
myconf.workdir = arguments.workdir;
|
myconf.workdir = arguments.workdir;
|
||||||
myconf.name = arguments.myname;
|
myconf.name = arguments.myname;
|
||||||
|
memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg));
|
||||||
|
|
||||||
|
if (arguments.peers_str) {
|
||||||
|
if (parse_peers(arguments.peers_str, &myconf.peers_cfg) != 0) {
|
||||||
|
fprintf(stderr,
|
||||||
|
"xce_controller: --peers parse error: expected "
|
||||||
|
"host:port,... got: %s\n",
|
||||||
|
arguments.peers_str);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
/* CA cert derived from workdir — same infrastructure as certfile/keyfile */
|
||||||
|
snprintf(myconf.peers_cfg.ca_cert_path,
|
||||||
|
sizeof(myconf.peers_cfg.ca_cert_path),
|
||||||
|
"%s/certs/xce.crt", arguments.workdir);
|
||||||
|
strncpy(myconf.peers_cfg.identity, arguments.myname,
|
||||||
|
sizeof(myconf.peers_cfg.identity) - 1);
|
||||||
|
INFO("HA mode: %d controller candidates, identity=%s",
|
||||||
|
myconf.peers_cfg.candidates_count, myconf.peers_cfg.identity);
|
||||||
|
} else {
|
||||||
|
INFO("No --peers flag; running as single-node primary");
|
||||||
|
}
|
||||||
|
|
||||||
myapp_run(&myconf);
|
myapp_run(&myconf);
|
||||||
|
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -7,6 +7,7 @@
|
||||||
#include <tlog.h>
|
#include <tlog.h>
|
||||||
|
|
||||||
#include "clusterconfig.h"
|
#include "clusterconfig.h"
|
||||||
|
#include "xce/cluster.h"
|
||||||
|
|
||||||
struct logging {
|
struct logging {
|
||||||
char* logdir;
|
char* logdir;
|
||||||
|
|
@ -20,6 +21,7 @@ typedef struct {
|
||||||
char* keyfile;
|
char* keyfile;
|
||||||
char* workdir; /* working directory for WAL/snapshot storage */
|
char* workdir; /* working directory for WAL/snapshot storage */
|
||||||
char* name; /* controller name (hostname by default) */
|
char* name; /* controller name (hostname by default) */
|
||||||
|
xce_cluster_config_t peers_cfg; /* HA peer list parsed from --peers; count==0 = single-node */
|
||||||
} myconf_t;
|
} myconf_t;
|
||||||
|
|
||||||
extern cluster_config_t* get_default_conf(void*);
|
extern cluster_config_t* get_default_conf(void*);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue