feat: add --peers to controller for HA peer list and priority election
- myconf_t gains peers_cfg (xce_cluster_config_t) populated from --peers - controller/main.c: parse --peers host:port,... via argp; ca_cert_path auto-derived from workdir/certs/xce.crt (no --ca-cert flag needed) - cluster_init: derive my_priority from position in peers list (index 0 = primary); fall back to 0 when --peers not given (single-node mode)
This commit is contained in:
parent
d61f91b99c
commit
59597e105e
10 changed files with 8046 additions and 325 deletions
File diff suppressed because it is too large
Load diff
1520
docs/superpowers/plans/2026-05-07-p1-ha-foundation.md
Normal file
1520
docs/superpowers/plans/2026-05-07-p1-ha-foundation.md
Normal file
File diff suppressed because it is too large
Load diff
1498
docs/superpowers/plans/2026-05-07-p2-replication.md
Normal file
1498
docs/superpowers/plans/2026-05-07-p2-replication.md
Normal file
File diff suppressed because it is too large
Load diff
1247
docs/superpowers/plans/2026-05-07-p25-discovery.md
Normal file
1247
docs/superpowers/plans/2026-05-07-p25-discovery.md
Normal file
File diff suppressed because it is too large
Load diff
1133
docs/superpowers/plans/2026-05-07-p3-hardening.md
Normal file
1133
docs/superpowers/plans/2026-05-07-p3-hardening.md
Normal file
File diff suppressed because it is too large
Load diff
39
docs/superpowers/plans/2026-05-07-persistence-ha-overview.md
Normal file
39
docs/superpowers/plans/2026-05-07-persistence-ha-overview.md
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
# XCE Persistence & HA — Implementation Overview
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make the XCE controller survive restarts and primary failures with no job/worker/queue state loss.
|
||||
|
||||
**Architecture:** WAL + periodic checkpoint (dual-zarray COW) for single-controller persistence; LSF-style priority election + fan-out TCP replication for HA. libxce smart discovery handles client auto-reconnect after failover.
|
||||
|
||||
**Tech Stack:** C (controller/lib), Cap'n Proto (serialization), Nix flakes + meson + ninja (build), Go (API server discovery bridge)
|
||||
|
||||
---
|
||||
|
||||
## Phase Map
|
||||
|
||||
Each phase is an independently shippable increment. Create the next phase plan only after the current phase passes its gate tests.
|
||||
|
||||
| Phase | Plan file | Goal | Gate |
|
||||
|-------|-----------|------|------|
|
||||
| **P0** | `2026-05-07-p0-single-controller-persistence.md` | Controller survives restart | `P0-G` smoke tests green |
|
||||
| **P1** | _create after P0 ships_ | Standby takes over when primary fails | Election + promotion tests green |
|
||||
| **P2** | _create after P1 ships_ | Primary replicates to standbys in real time | Replication round-trip + catch-up tests green |
|
||||
| **P2.5** | _create alongside P2_ | Workers/schedulers auto-reconnect after failover | Discovery + reconnect tests green |
|
||||
| **P3** | _create after P2 ships_ | End-to-end failover + hardening + metrics | Full failover test + benchmark passing |
|
||||
| **P4** | _create after P3 ships_ | Docs, runbook, stale-ref cleanup | Runbook review complete |
|
||||
|
||||
## Out of Scope (all phases)
|
||||
|
||||
- Admin CLI for manual failover (`xce hostcontrol --failover`)
|
||||
- WAL compaction beyond checkpoint trimming
|
||||
- Multi-DC replication
|
||||
- Runtime config changes without restart
|
||||
- Snapshot compression
|
||||
- `config_version` hash in heartbeat (deferred to P3 tooling)
|
||||
- Degraded-mode restricted-state timeout (`repl_degraded_timeout_s`)
|
||||
- `MAX_HA_CANDIDATES > 4` / `MAX_STANDBYS > 3`
|
||||
|
||||
## Design Document
|
||||
|
||||
`src/docs/persistence-and-ha-design.md` — all implementation detail lives here. Read the relevant Part before starting each task.
|
||||
|
|
@ -640,8 +640,28 @@ int cluster_init(cluster_t* c)
|
|||
.probe_connect_timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS,
|
||||
.max_promotion_retries = HA_DEFAULT_MAX_PROMOTION_RETRIES,
|
||||
};
|
||||
/* my_priority: single-node P1 always uses 0 (highest priority) */
|
||||
rc = ha_init(&c->ha, &ha_cfg, 0);
|
||||
|
||||
/* Derive priority from position in --peers list.
|
||||
* Index 0 = highest priority (becomes primary on startup).
|
||||
* If the controller's name is not found in the list, fall back to
|
||||
* the last index (lowest priority = standby). */
|
||||
int my_priority = 0;
|
||||
const xce_cluster_config_t* pcfg = &c->myconf->peers_cfg;
|
||||
if (pcfg->candidates_count > 0) {
|
||||
my_priority = pcfg->candidates_count - 1; /* default: last = lowest */
|
||||
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||||
if (strncmp(pcfg->candidates[i].hostname,
|
||||
c->myconf->name,
|
||||
XCE_MAXHOSTNAME_LEN - 1) == 0) {
|
||||
my_priority = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
INFO("HA peers: %d candidates, my_priority=%d (name=%s)",
|
||||
pcfg->candidates_count, my_priority, c->myconf->name);
|
||||
}
|
||||
|
||||
rc = ha_init(&c->ha, &ha_cfg, my_priority);
|
||||
if (rc != 0) {
|
||||
ERROR("ha_init failed: %d", rc);
|
||||
persist_fini(&c->persist);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#include "myconf.h"
|
||||
#include "utils.h"
|
||||
#include "xce/cluster.h"
|
||||
|
||||
extern int myapp_run(myconf_t*);
|
||||
|
||||
|
|
@ -15,21 +16,60 @@ const char* argp_program_version
|
|||
= MK_VERSION("xce v0.0.1-", STR(VERSION_COMMIT_ID));
|
||||
const char* argp_program_bug_address = BUG_EMAIL;
|
||||
|
||||
/* Long-option key (above ASCII to avoid collisions with short opts) */
|
||||
#define OPT_PEERS 256
|
||||
|
||||
struct arguments {
|
||||
char* myname;
|
||||
char* logfile;
|
||||
char* workdir;
|
||||
int verbose;
|
||||
char* peers_str;
|
||||
};
|
||||
|
||||
static struct argp_option options[] = {
|
||||
{ "workdir", 'w', "DIR", 0, "The work directory of xce controller" },
|
||||
{ "logfile", 'l', "FILE", 0, "The full path of log file" },
|
||||
{ "verbose", 'v', 0, 0, "Produce verbose logs" },
|
||||
{ "myname", 'n', "STR", 0, "The name of controller" },
|
||||
{ "workdir", 'w', "DIR", 0, "The work directory of xce controller" },
|
||||
{ "logfile", 'l', "FILE", 0, "The full path of log file" },
|
||||
{ "verbose", 'v', 0, 0, "Produce verbose logs" },
|
||||
{ "myname", 'n', "STR", 0, "The name of controller" },
|
||||
{ "peers", OPT_PEERS, "PEERS", 0,
|
||||
"HA peer list: comma-separated host:port pairs ordered by priority "
|
||||
"(e.g. ctrl01:9990,ctrl02:9990). Index 0 = highest priority (primary)." },
|
||||
{ 0 }
|
||||
};
|
||||
|
||||
/* parse_peers — split "host:port,host:port,..." into cfg->candidates[].
|
||||
* Accepts optional "name=host:port" prefix (name is ignored for controller).
|
||||
* Returns 0 on success, -1 on empty input or parse error. */
|
||||
static int parse_peers(const char* peers_str, xce_cluster_config_t* cfg)
|
||||
{
|
||||
cfg->candidates_count = 0;
|
||||
if (!peers_str || peers_str[0] == '\0')
|
||||
return -1;
|
||||
|
||||
char buf[2048];
|
||||
strncpy(buf, peers_str, sizeof(buf) - 1);
|
||||
buf[sizeof(buf) - 1] = '\0';
|
||||
|
||||
char* saveptr;
|
||||
char* tok = strtok_r(buf, ",", &saveptr);
|
||||
while (tok && cfg->candidates_count < MAX_HA_CANDIDATES) {
|
||||
char* eq = strchr(tok, '=');
|
||||
char* addr = eq ? eq + 1 : tok;
|
||||
char* colon = strrchr(addr, ':');
|
||||
if (!colon)
|
||||
return -1;
|
||||
*colon = '\0';
|
||||
strncpy(cfg->candidates[cfg->candidates_count].hostname, addr,
|
||||
XCE_MAXHOSTNAME_LEN - 1);
|
||||
cfg->candidates[cfg->candidates_count].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
|
||||
cfg->candidates[cfg->candidates_count].port = atoi(colon + 1);
|
||||
cfg->candidates_count++;
|
||||
tok = strtok_r(NULL, ",", &saveptr);
|
||||
}
|
||||
return cfg->candidates_count > 0 ? 0 : -1;
|
||||
}
|
||||
|
||||
static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
||||
{
|
||||
struct arguments* arguments = state->input;
|
||||
|
|
@ -47,6 +87,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
|
|||
case 'n':
|
||||
arguments->myname = arg;
|
||||
break;
|
||||
case OPT_PEERS:
|
||||
arguments->peers_str = arg;
|
||||
break;
|
||||
default:
|
||||
return ARGP_ERR_UNKNOWN;
|
||||
}
|
||||
|
|
@ -111,6 +154,27 @@ int main(int argc, char* argv[])
|
|||
myconf.keyfile = key_file;
|
||||
myconf.workdir = arguments.workdir;
|
||||
myconf.name = arguments.myname;
|
||||
memset(&myconf.peers_cfg, 0, sizeof(myconf.peers_cfg));
|
||||
|
||||
if (arguments.peers_str) {
|
||||
if (parse_peers(arguments.peers_str, &myconf.peers_cfg) != 0) {
|
||||
fprintf(stderr,
|
||||
"xce_controller: --peers parse error: expected "
|
||||
"host:port,... got: %s\n",
|
||||
arguments.peers_str);
|
||||
return 1;
|
||||
}
|
||||
/* CA cert derived from workdir — same infrastructure as certfile/keyfile */
|
||||
snprintf(myconf.peers_cfg.ca_cert_path,
|
||||
sizeof(myconf.peers_cfg.ca_cert_path),
|
||||
"%s/certs/xce.crt", arguments.workdir);
|
||||
strncpy(myconf.peers_cfg.identity, arguments.myname,
|
||||
sizeof(myconf.peers_cfg.identity) - 1);
|
||||
INFO("HA mode: %d controller candidates, identity=%s",
|
||||
myconf.peers_cfg.candidates_count, myconf.peers_cfg.identity);
|
||||
} else {
|
||||
INFO("No --peers flag; running as single-node primary");
|
||||
}
|
||||
|
||||
myapp_run(&myconf);
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -7,6 +7,7 @@
|
|||
#include <tlog.h>
|
||||
|
||||
#include "clusterconfig.h"
|
||||
#include "xce/cluster.h"
|
||||
|
||||
struct logging {
|
||||
char* logdir;
|
||||
|
|
@ -14,12 +15,13 @@ struct logging {
|
|||
};
|
||||
|
||||
typedef struct {
|
||||
cluster_config_t* cluster_config;
|
||||
char* conffile;
|
||||
char* certfile;
|
||||
char* keyfile;
|
||||
char* workdir; /* working directory for WAL/snapshot storage */
|
||||
char* name; /* controller name (hostname by default) */
|
||||
cluster_config_t* cluster_config;
|
||||
char* conffile;
|
||||
char* certfile;
|
||||
char* keyfile;
|
||||
char* workdir; /* working directory for WAL/snapshot storage */
|
||||
char* name; /* controller name (hostname by default) */
|
||||
xce_cluster_config_t peers_cfg; /* HA peer list parsed from --peers; count==0 = single-node */
|
||||
} myconf_t;
|
||||
|
||||
extern cluster_config_t* get_default_conf(void*);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue