diff --git a/.kiro/specs/ha-config-bootstrap/design.md b/.kiro/specs/ha-config-bootstrap/design.md new file mode 100644 index 0000000..0fb7a5d --- /dev/null +++ b/.kiro/specs/ha-config-bootstrap/design.md @@ -0,0 +1,834 @@ +# Design: HA Config Bootstrap & Raft-Style Quorum Election + +## Overview + +This feature eliminates three critical safety gaps in the XCE controller's LSF-style priority-based election: unconfigured nodes can win elections and serve stale cluster state; concurrent-startup nodes falsely self-elect because the peer listener is not yet open; and a promoted standby with missing local config has no bootstrap path. The fix replaces priority election with a Raft-style quorum protocol that encodes config freshness in votes, moves the listener before election, and adds majority-commit durability for both WAL writes and config upgrades. + +**Purpose**: Delivers full CP high-availability to cluster operators — zero data loss on failover, no split-brain on concurrent startup, and no silent config loss on primary crash. The design chooses Consistency over Availability (CAP-CP): a minority partition cannot elect a primary; the cluster waits until majority is restored. + +**Impact**: Replaces `am_i_primary()` and `ha_check_heartbeat_nocluster()` with Raft quorum election and `EPOLLRDHUP`-based failover; extends the controller lifecycle from a 2-state (`CTRL_PRIMARY`/`CTRL_STANDBY`) to a 4-state FSM adding `CTRL_INIT` and `CTRL_ELECTING`; adds majority-commit gating to `persist_wal_sync_append()` and `push_config_to_standbys()`; establishes a persistent TLS control channel between each standby and the primary; adds transparent retry to libxce. + +### Goals + +- Raft-style quorum election with config-freshness tiebreaking (CP semantics, no minority-partition primaries) +- Zero data loss on failover via majority-commit WAL and config replication +- Sub-second failover detection via `EPOLLRDHUP` + heartbeat timeout vtask +- Transparent recovery for unconfigured nodes joining an existing cluster (deferred config fetch) +- Transparent client retry in libxce — worker/scheduler/apiserver code unchanged + +### Non-Goals + +- Raft snapshot protocol (existing dual-zarray COW is orthogonal and unchanged) +- Joint consensus for dynamic controller membership changes (`--peers` list is static at startup) +- mTLS between controllers (server-side TLS sufficient for trusted-LAN model) +- Config file format or storage layout changes +- Worker/scheduler daemon protocol changes + +--- + +## Boundary Commitments + +### This Spec Owns + +- Controller election protocol: full replacement of `am_i_primary()` with `am_i_primary_quorum()` +- Controller FSM: 4-state lifecycle (`CTRL_INIT`, `CTRL_ELECTING`, `CTRL_PRIMARY`, `CTRL_STANDBY`) and all state transitions +- Pre-election startup sequence: listener binding, iomgr/vtaskmgr init before election +- Config bootstrap: `bootstrap_config_if_needed()`, `xce_probe_config_tag()`, deferred fetch after election +- Startup validation: `--vcpus N` CLI, even-N peer count rejection, TLS cert pre-flight, ASAN leak fix +- Majority-commit WAL: `repl_broadcast()`, `repl_wait_majority_ack()`, `committed_lsn`, `persist_truncate_wal_above()` +- Majority-commit config push: `OP_upgradeConfigAck`, `committed_config_tag`, heartbeat piggybacking +- Persistent control channel: `ha_find_and_connect_to_primary()`, `EPOLLRDHUP` failover, WAL streaming over channel, three-tier primary discovery +- Election state durability: `ha-term.bin`, `ha_persist_term()`, `voted_for_term/name` across restarts +- libxce client retry: `xce_retry_config_t`, retry wrapper, `XCE_ERR_STANDBY` redirect fields in wire protocol +- Wire protocol extensions: `OP_requestVote`, `OP_probeConfigTag`, `OP_walAck`, `OP_upgradeConfigAck`, `OP_snapshotTransfer`, `OP_walCatchupEntry` +- Structured log observability for all new code paths (election, bootstrap, WAL commit, config push, FSM transitions) + +### Out of Boundary + +- `apply_wal_entry()` — must be extracted from the startup WAL replay path as a prerequisite to Change F; not authored in this spec +- `persist_checkpoint()` trigger policy — existing on-demand call in `repl_needs_snapshot()` path; unchanged +- Worker/scheduler daemon code — libxce retry is transparent; no caller code changes +- apiserver Go tier — no changes +- `scheduler-controller-msg-gap` spec and `worker-join-refactor` spec — orthogonal; no shared files +- Dynamic controller membership changes (joint consensus) +- mTLS between controllers + +### Allowed Dependencies + +- `iomgr` (epoll/`EPOLLIN`/`EPOLLRDHUP`): P0 — control channel and dispatch loop +- `vtaskmgr`: P0 — `vtaskmgr_init(int nvcpus)` confirmed to take vcpus directly, no cluster_config dependency +- `endpoint_connect()` / `connect_ssl_raw()`: P0 — SSL connection setup; TCP connect phase is blocking; vote request connections require per-peer pthreads +- `repl_add/remove/start_catchup/complete_catchup/needs_snapshot/on_ack/trim_lsn`: P0 — existing repl slot management reused unchanged +- `do_one_config_fetch()`: P1 — wired on branch (`2aae45f`); deferred fetch depends on it +- `xce_encode()` / `xce_decode()` / `xce_request()`: P0 — wire protocol encoding +- `ha_do_promote()` / `ha_do_demote()`: P0 — role transitions, reused +- `src/schema/controller.capnp`: P0 — new opcodes appended after existing @31 +- `xcelog.h` INFO/WARN/ERROR macros: P2 — observability + +### Revalidation Triggers + +- Changes to `ha_state_t` field layout require re-checking all ha.c callers +- Changes to `RequestReply` or `CtrlHeartbeat` capnp structs require checking all decode sites +- Moving `cluster_open_listener()` before election changes startup order; any code assuming phase2 runs before network is up must be re-verified +- `committed_lsn` cross-cuts `persist_t`, `repl.c`, and `ha_state_t`; changes to any require re-checking the others +- `ha-term.bin` format change requires migration or version field + +--- + +## Architecture + +### Existing Architecture Analysis + +The current controller has a 2-state election: `am_i_primary()` spawns per-peer pthreads that probe higher-priority peers and self-elect on timeout. `cluster_init_phase2()` contains `bind()+listen()`, `iomgr_init()`, and `vtaskmgr_init()` — all run after election, creating a 2–3 second window where the port is unreachable to election probes. + +Confirmed gaps from codebase inspection (see `research.md` §1 for full inventory): +- `ctrl_role_t` has only `CTRL_PRIMARY=0`, `CTRL_STANDBY=1` +- `ha_state_t` lacks: `voted_for_term`, `voted_for_name`, `primary_ctrl_fd`, `defer_config_fetch`, `elect_ctx`, `last_known_committed_lsn`, `heartbeat_seen`, `last_known_committed_config_tag` +- `repl_broadcast()` is a stub returning 0 without sending; `repl_wait_majority_ack()` does not exist +- `persist_t` lacks `committed_lsn`; `persist_truncate_wal_above()` does not exist +- `XCE_ERR_STANDBY` reply has no `redirect_host/port` fields — must be added to capnp schema +- `ha-term.bin` does not exist; `ha_init()` always starts from term=0 +- `ha_check_priority_stepdown()` is permanently unreachable after first quorum election (term always ≥ 1) +- `ha_check_heartbeat_nocluster()` uses 10s timestamp polling — replaced by dual-trigger failover + +### Architecture Pattern & FSM + +**Selected pattern**: 4-state FSM with Raft-style majority-quorum consensus (CP). Each startup step gates the next. + +```mermaid +stateDiagram-v2 + [*] --> CTRL_INIT : startup complete listener open + CTRL_INIT --> CTRL_STANDBY : heartbeat received from existing primary + CTRL_INIT --> CTRL_ELECTING : election timeout fires + CTRL_ELECTING --> CTRL_PRIMARY : quorum won and no fresher peer + CTRL_ELECTING --> CTRL_STANDBY : quorum lost or fresher peer yields + CTRL_STANDBY --> CTRL_ELECTING : EPOLLRDHUP or heartbeat timeout vtask + CTRL_PRIMARY --> CTRL_STANDBY : higher-term reply received +``` + +```mermaid +graph TB + subgraph Startup + V[validate_peer_count + validate_certs] --> P1[cluster_init_phase1] + P1 --> Net[iomgr_init + cluster_open_listener + vtaskmgr_init] + Net --> Boot[bootstrap_config_if_needed] + Boot --> Wait[ha_wait_for_heartbeat_or_timeout] + end + subgraph Election + Wait --> Quorum[am_i_primary_quorum] + Quorum --> Vote[OP_requestVote per peer thread] + Vote --> Eval[ha_elect_evaluate_and_signal] + end + subgraph Replication + WAL[persist_wal_sync_append] --> Bcast[repl_broadcast] + Bcast --> MAck[repl_wait_majority_ack] + MAck --> Commit[advance committed_lsn] + end + subgraph ControlChannel + SB[CTRL_STANDBY] --> Conn[ha_find_and_connect_to_primary] + Conn --> Epoll[EPOLLIN + EPOLLRDHUP] + Epoll --> Sync[state sync WAL streaming] + end +``` + +### Technology Stack + +| Layer | Choice | Role | Notes | +|-------|--------|------|-------| +| Runtime | C gnu17 | All new code | No C++ | +| Wire format | Cap'n Proto | 6 new opcodes in `controller.capnp` | Appended after existing @31 | +| SSL | OpenSSL (existing) | Vote request and probe connections | Reuses `connect_ssl_raw()` | +| Consensus | Raft subset (custom) | Leader election + majority-commit | No external library; bounded scope justified by static membership and no snapshot protocol need | +| IPC | iomgr epoll | `EPOLLIN` + `EPOLLRDHUP` on control channel | `EPOLLRDHUP` for sub-second failover | +| Persistence | Flat binary (`ha-term.bin`) | Term + voted_for across restarts | Atomic write+rename (existing pattern) | +| Build | Meson + Nix | capnp schema regeneration | `src/schema/controller.capnp` | + +--- + +## File Structure Plan + +### Directory Structure + +``` +src/ +├── schema/ +│ └── controller.capnp # Modified: 6 new opcodes + message structs +├── controller/ +│ ├── ha.h # Modified: CTRL_INIT/CTRL_ELECTING enum values; +│ │ # ha_vote_entry_t, ha_elect_ctx_t types; +│ │ # new ha_state_t fields; ha_persist_term() decl +│ ├── ha.c # Modified: replace am_i_primary() -> am_i_primary_quorum(); +│ │ # add handle_request_vote(), handle_request_vote_reply(), +│ │ # ha_wait_for_heartbeat_or_timeout(), ha_elect_evaluate_and_signal(), +│ │ # ha_find_and_connect_to_primary(), ha_persist_term(); +│ │ # retire ha_check_priority_stepdown(), ha_check_heartbeat_nocluster(); +│ │ # SSL_CTX_free ASAN fix in ha_fini() +│ ├── persist.h # Modified: add committed_lsn to persist_t +│ ├── persist.c # Modified: persist_truncate_wal_above() (new); +│ │ # upgrade persist_wal_sync_append() to repl_wait_majority_ack() +│ ├── repl.h # Modified: repl_wait_majority_ack() decl; +│ │ # pending_ack_count in standby_conn_t +│ ├── repl.c # Modified: implement repl_broadcast() (was stub); +│ │ # repl_wait_majority_ack() (new); backpressure; catch-up exclusion +│ ├── config_repl.h # Modified: bootstrap_config_if_needed() and xce_probe_config_tag() decls +│ ├── config_repl.c # Modified: bootstrap_config_if_needed() (new); +│ │ # xce_probe_config_tag() (new); +│ │ # upgrade push_config_to_standbys() to majority-commit +│ ├── cluster.h # Modified: remove vcpus from cluster_config_t +│ ├── cluster.c # Modified: extract cluster_open_listener() from phase2; +│ │ # extended handle_ctrlhello(); OP_snapshotTransfer and +│ │ # OP_walCatchupEntry dispatch handlers +│ ├── myapp.c # Modified: new startup order (listener before election); +│ │ # bootstrap_config_if_needed() call; deferred fetch integration +│ ├── main.c # Modified: --vcpus N CLI arg; even-N peer count validation; +│ │ # cert pre-flight access() check +│ └── testsuite/ +│ ├── test_ha_election.c # New: quorum election, vote protocol, FSM transitions +│ ├── test_ha_bootstrap.c # New: bootstrap_config_if_needed, probe RPC, deferred fetch +│ ├── test_ha_durability.c # New: ha-term.bin persist/restore, CRC corruption +│ ├── test_persist_truncate.c # New: persist_truncate_wal_above, boundary segment, idempotency +│ ├── test_repl_majority.c # New: repl_broadcast, repl_wait_majority_ack, backpressure +│ └── test_ha_startup.c # New: startup validation (vcpus, peer count, cert pre-flight) +└── lib/ + ├── request.h # Modified: xce_retry_config_t; xce_request_with_retry() decl + ├── request.c # Modified: retry wrapper; redirect handling; backoff logic + └── testsuite/ + └── test_libxce_retry.c # New: retry logic, backoff, redirect, non-retriable errors +``` + +### Modified Files Summary + +| File | Change Type | Requirements | +|------|-------------|--------------| +| `src/schema/controller.capnp` | Extend: 6 opcodes + structs | 2, 3, 5, 6, 7, 8 | +| `src/controller/ha.h` | Extend: enum + types + fields | 1, 2, 3, 5, 6, 9 | +| `src/controller/ha.c` | Major: replace election core; add control channel | 1, 2, 3, 6, 9 | +| `src/controller/persist.h` | Extend: committed_lsn | 5 | +| `src/controller/persist.c` | New function + upgrade sync path | 5 | +| `src/controller/repl.h` | Extend: standby_conn_t field + decl | 5, 8 | +| `src/controller/repl.c` | Implement stubs + add majority logic | 5, 6, 8 | +| `src/controller/config_repl.h` | Extend: two new function decls | 3, 8 | +| `src/controller/config_repl.c` | New functions + upgrade push | 3, 8 | +| `src/controller/cluster.h` | Remove vcpus field | 4 | +| `src/controller/cluster.c` | Extract listener; extend ctrlhello handler | 1, 6 | +| `src/controller/myapp.c` | New startup sequence | 1, 3 | +| `src/controller/main.c` | CLI validation + pre-flight | 4 | +| `src/lib/request.h` | New type + decl | 7 | +| `src/lib/request.c` | Retry wrapper | 7 | + +--- + +## System Flows + +### Flow 1: Controller Startup & Election + +```mermaid +sequenceDiagram + participant Main as main.c + participant MyApp as myapp.c + participant HA as ha.c + participant Cluster as cluster.c + participant Config as config_repl.c + Main->>Main: validate_peer_count odd-N check + Main->>Main: validate_certs access() + Main->>MyApp: myapp_run() + MyApp->>Cluster: cluster_init_phase1 reads ha-term.bin via ha_init + MyApp->>Cluster: iomgr_init + cluster_open_listener + vtaskmgr_init + Note over Cluster: listener open BEFORE election + MyApp->>Config: bootstrap_config_if_needed + Note over Config: tag=0 and peers have config? set defer_config_fetch and return + MyApp->>HA: ha_wait_for_heartbeat_or_timeout + alt heartbeat received in window + HA->>MyApp: set CTRL_STANDBY skip election + else election timeout fires + HA->>HA: am_i_primary_quorum send OP_requestVote per peer + HA->>MyApp: CTRL_PRIMARY or CTRL_STANDBY + end + alt defer_config_fetch and not primary + MyApp->>Config: do_one_config_fetch from proven winner + end + MyApp->>Cluster: cluster_init_phase2_rest kvmgr jobmgr apiserver etc. +``` + +### Flow 2: Quorum Election Vote Protocol + +```mermaid +sequenceDiagram + participant Cand as Candidate ha.c + participant PA as Peer A ha.c + participant PB as Peer B ha.c + Cand->>Cand: increment term persist ha-term.bin + Cand->>PA: OP_requestVote term config_tag last_lsn via per-peer thread + Cand->>PB: OP_requestVote term config_tag last_lsn via per-peer thread + PA->>PA: handle_request_vote check voted_for and log_ok + PA->>Cand: OP_requestVoteReply granted=true peer_tag peer_lsn + PB->>PB: handle_request_vote check voted_for and log_ok + PB->>Cand: OP_requestVoteReply granted=true peer_tag peer_lsn + Cand->>Cand: ha_elect_evaluate_and_signal + Note over Cand: votes_granted ge quorum AND no fresher peer? CTRL_PRIMARY +``` + +Key decisions: fresher-peer yield gate fires when any *responding* peer has higher `config_tag` or equal tag with higher LSN; non-responding peers never block election. Split-vote triggers one retry with incremented term and randomized jitter delay. + +### Flow 3: WAL Majority-Commit + +```mermaid +sequenceDiagram + participant Caller as Job submit caller + participant Persist as persist.c + participant Repl as repl.c + participant SA as Standby A + participant SB as Standby B + Caller->>Persist: persist_wal_sync_append WAL_JOB_SUBMIT + Persist->>Persist: write + fsync local advance applied_lsn + Persist->>Repl: repl_broadcast OP_walAck payload over persistent channel + Repl->>SA: WAL entry on control channel fd + Repl->>SB: WAL entry on control channel fd + SA->>Repl: OP_walAck lsn=N + Repl->>Persist: repl_wait_majority_ack quorum reached + Persist->>Persist: advance committed_lsn to N + Persist->>Caller: return 0 success + SB->>Repl: OP_walAck lsn=N arrives later counted but not blocking +``` + +Entry types `WAL_HOST_STATE` and `WAL_CHECKPOINT_REF` bypass this flow via `persist_wal_async()` (best-effort, unchanged). + +### Flow 4: Standby Failover Detection and Re-election + +```mermaid +sequenceDiagram + participant IOM as iomgr epoll + participant HA as ha.c standby + participant Peers as Peer controllers + IOM->>HA: EPOLLRDHUP on primary_ctrl_fd + HA->>HA: close primary_ctrl_fd set CTRL_ELECTING + HA->>HA: persist_truncate_wal_above last_known_committed_lsn + HA->>Peers: am_i_primary_quorum new term OP_requestVote + alt quorum won + HA->>HA: CTRL_PRIMARY accept inbound control channels + else quorum lost + HA->>Peers: ha_find_and_connect_to_primary three-tier discovery + HA->>HA: CTRL_STANDBY + end +``` + +Fallback trigger: heartbeat timeout vtask fires after `election_timeout_ms + rand_jitter` of silence on an open TCP channel (covers network partitions that keep TCP alive but suppress heartbeats). + +--- + +## Requirements Traceability + +| Requirement | Summary | Components | Key Interface | Flow | +|-------------|---------|------------|---------------|------| +| 1.1–1.4 | Listener + iomgr/vtaskmgr before election; dispatch gate for init/electing states | `cluster.c`, `ha.c`, `myapp.c` | `cluster_open_listener()`, vtaskmgr dispatch gate | Flow 1 | +| 2.1–2.12 | Quorum election: voted_for enforcement, freshness gate, split-vote retry, single-node shortcircuit | `ha.c` | `am_i_primary_quorum()`, `handle_request_vote()`, `OP_requestVote` | Flow 2 | +| 3.1–3.5 | Pre-election config probe, deferred fetch, retry backoff | `config_repl.c`, `myapp.c` | `bootstrap_config_if_needed()`, `xce_probe_config_tag()`, `OP_probeConfigTag` | Flow 1 | +| 4.1–4.4 | --vcpus CLI, even-N rejection, cert pre-flight, ASAN fix | `main.c`, `ha.c` | CLI argp, `access()`, `SSL_CTX_free` | Flow 1 | +| 5.1–5.8 | Majority-commit WAL: committed_lsn, broadcast, ack-wait, backpressure, truncation | `persist.c`, `repl.c`, `ha.c` | `repl_wait_majority_ack()`, `persist_truncate_wal_above()`, `OP_walAck` | Flow 3 | +| 6.1–6.8 | Persistent control channel: EPOLLRDHUP failover, state sync, catch-up, three-tier discovery | `ha.c`, `cluster.c`, `repl.c` | `ha_find_and_connect_to_primary()`, extended `OP_ctrlHello`, `OP_snapshotTransfer`, `OP_walCatchupEntry` | Flow 4 | +| 7.1–7.6 | Client retry: backoff, redirect, non-retriable passthrough | `request.c` (libxce) | `xce_request_with_retry()`, `xce_retry_config_t`, redirect fields in `OP_requestVoteReply` | — | +| 8.1–8.5 | Majority-commit config push, committed_config_tag, heartbeat piggybacking | `config_repl.c`, `repl.c` | `push_config_to_standbys()`, `OP_upgradeConfigAck` | Flow 3 analog | +| 9.1–9.3 | Term + voted_for durability across restarts via ha-term.bin | `ha.c` | `ha_persist_term()`, `ha_init()` read path | — | +| 10.1–10.5 | Structured logs for election outcome, bootstrap, WAL commit, config push, FSM transitions | All new code paths | `xcelog.h` INFO/WARN macros | All flows | + +--- + +## Components and Interfaces + +### Summary Table + +| Component | File(s) | Intent | Req Coverage | Key Dependencies | +|-----------|---------|--------|-------------|-----------------| +| Controller FSM | `ha.h`, `ha.c` | 4-state lifecycle + election logic | 1, 2, 9 | iomgr P0, vtaskmgr P0 | +| Config Bootstrap | `config_repl.c` | Pre-election probe + deferred fetch | 3 | `do_one_config_fetch` P1 | +| Startup Validation | `main.c`, `ha.c` | CLI, peer count, cert pre-flight, ASAN | 4 | argp, `access()` | +| WAL Majority-Commit | `persist.c`, `repl.c` | committed_lsn, broadcast, ack-wait | 5 | iomgr P0 | +| Persistent Control Channel | `ha.c`, `cluster.c` | TCP+SSL channel, EPOLLRDHUP, state sync | 6 | `repl.c` P0, iomgr P0 | +| Majority-Commit Config | `config_repl.c` | Config push with majority ACK | 8 | `repl_broadcast` P0 | +| Term Durability | `ha.c` | ha-term.bin persist/restore | 9 | atomic write+rename | +| Client Retry | `request.c` | Transparent retry + redirect | 7 | `XCE_ERR_*` codes P0 | +| Wire Protocol | `controller.capnp` | New opcodes and message types | 2, 3, 5, 6, 7, 8 | capnp toolchain P0 | + +--- + +### Controller Core (ha.h / ha.c) + +#### Controller FSM + +| Field | Detail | +|-------|--------| +| Intent | 4-state FSM governing election, standby, and primary lifecycle; owns all election logic and term durability | +| Requirements | 1.1, 1.2, 1.3, 1.4, 2.1–2.12, 9.1–9.3 | + +**Responsibilities & Constraints** +- Owns `ctrl_role_t` enum — add `CTRL_INIT=2`, `CTRL_ELECTING=3` (values above existing 0/1) +- Single allocation for `ha_elect_ctx_t` via C99 flexible array — one `calloc`, one `free` +- All role transitions via `atomic_store(&c->ha.role, ...)` — no lock held across transition +- `ha_check_priority_stepdown()` and `ha_check_heartbeat_nocluster()` are deleted in this spec + +**New `ha_state_t` fields**: +```c +uint64_t voted_for_term; // Req 9: term of last granted vote +char voted_for_name[XCE_MAXNAME_LEN]; // Req 9: candidate of last granted vote +int primary_ctrl_fd; // Req 6: persistent channel fd (-1 = none) +bool defer_config_fetch; // Req 3: tag=0 node defers fetch to post-election +_Atomic void *elect_ctx; // Req 2: ha_elect_ctx_t* during election; NULL otherwise +_Atomic uint64_t last_known_committed_lsn; // Req 5: from primary heartbeats +sem_t heartbeat_seen; // Req 1: posted by heartbeat handler during CTRL_INIT +uint64_t last_known_committed_config_tag; // Req 8: from primary heartbeats +``` + +**New types** (ha.h): +```c +typedef struct { + xce_candidate_t *peer; + uint64_t my_term; + uint64_t my_tag; + uint64_t my_lsn; + int vote_granted; + int replied; + uint64_t peer_tag; + uint64_t peer_lsn; +} ha_vote_entry_t; + +typedef struct { + int n_peers; + int votes_granted; // starts at 1 (self-vote) + int n_replied; + uint64_t max_resp_tag; + uint64_t max_resp_lsn; + int result; // 1=primary, 0=standby, <0=error + sem_t done; // startup thread blocks here; dispatch posts when done + ha_vote_entry_t votes[]; // C99 flexible array: n_peers-1 entries + // Allocate: calloc(1, sizeof(*ctx) + (n_peers-1) * sizeof(ctx->votes[0])) +} ha_elect_ctx_t; +``` + +**`ha_persist_term()` contract**: +- Precondition: called before any vote is granted or term incremented +- Action: writes `ha_term_file_t` atomically to `{workdir}/ha-term.bin` (write + rename) +- Postcondition: file reflects current state; durable before any network reply + +**Dispatch gate** (vtaskmgr switch, req 1.3): +```c +// CTRL_INIT or CTRL_ELECTING: only OP_requestVote and OP_probeConfigTag pass through +switch (atomic_load(&c->ha.role)) { +case CTRL_INIT: +case CTRL_ELECTING: + if (msg->op != OP_requestVote && msg->op != OP_probeConfigTag) { + xce_send_error(task, XCE_ERR_NOT_READY, "controller not ready"); + return; + } + break; +case CTRL_STANDBY: + // Primary-only ops return XCE_ERR_STANDBY + redirect_host/port when known + break; +case CTRL_PRIMARY: + break; +} +``` + +**Contracts**: Service [x] / State [x] + +**Implementation Notes** +- Risk: `endpoint_connect()` has a blocking TCP connect phase. Vote request sends require per-peer pthreads (not pure iomgr async). Election context `ha_elect_ctx_t` accumulates replies; threads post to `ctx->done` semaphore when quorum is reached or all peers replied. +- Integration: `ha_wait_for_heartbeat_or_timeout()` uses `sem_timedwait(&c->ha.heartbeat_seen, &deadline)` where `deadline = now + election_timeout_ms + rand_jitter`. Heartbeat handler posts `heartbeat_seen` only when `ha.role == CTRL_INIT`. +- `voted_for` enforcement: per-term, one vote per node. If `req->term > my_term`: update term, clear `voted_for_term`/`voted_for_name`. If `req->term < my_term`: deny. If already voted for a different candidate in same term: deny. + +--- + +### Persistence Layer (persist.h / persist.c) + +#### WAL Majority-Commit + +| Field | Detail | +|-------|--------| +| Intent | Gate job/queue WAL writes on majority-standby durability; new committed_lsn field and truncation function | +| Requirements | 5.1–5.8 | + +**New `persist_t` field**: +```c +_Atomic uint64_t committed_lsn; // highest majority-ACKed LSN; invariant: committed_lsn <= applied_lsn +``` + +**`persist_truncate_wal_above(persist_t *p, uint64_t cutoff_lsn)` contract**: +- Precondition: `cutoff_lsn <= p->applied_lsn`; may be called after promotion or standby re-join +- Action: find WAL segment containing `cutoff_lsn` by filename (filename encodes start LSN as hex); scan entries in that segment sequentially until LSN > cutoff_lsn; truncate at that byte offset; delete all segments starting above cutoff_lsn; atomically update `p->applied_lsn = cutoff_lsn` +- Postcondition: no WAL entry with LSN > cutoff_lsn exists on disk; idempotent (calling twice is safe) +- Note: WAL segments are 64 MB named `seg-{LSN_hex}.wal`; no built-in LSN-to-offset index; sequential scan required + +**Entry type dispatch** (5.1, 5.2): +- `WAL_JOB_SUBMIT`, `WAL_JOB_START`, `WAL_JOB_FINISH`, `WAL_QUEUE_CHANGE` → `persist_wal_sync_append()` → majority-commit path (blocking) +- `WAL_HOST_STATE`, `WAL_CHECKPOINT_REF` → `persist_wal_async()` → best-effort (unchanged) + +**WAL trim guard**: trim threshold changes from `applied_lsn` to `committed_lsn` to prevent trimming uncommitted entries. + +**Contracts**: Service [x] / State [x] + +--- + +### Replication Layer (repl.h / repl.c) + +#### Majority-Commit Broadcast + +| Field | Detail | +|-------|--------| +| Intent | Implement repl_broadcast (was stub); add repl_wait_majority_ack; manage per-standby backpressure | +| Requirements | 5.1–5.8, 8.1–8.5 | + +**New `standby_conn_t` field**: `int pending_ack_count` — backpressure counter; default max 64 + +**`repl_broadcast(cluster_t *c, uint8_t op, const void *payload, size_t len)` contract**: +- Sends payload to all connected standbys where `!in_catch_up && pending_ack_count <= max_pending_acks` +- Sends over persistent control channel fd (no new connection per entry) +- On send failure: remove standby from active set, trigger catch-up reconnect, log WARN +- Returns 0 when fanout initiated (not when ACKed) + +**`repl_wait_majority_ack(cluster_t *c, uint32_t timeout_ms, uint8_t ack_op)` contract**: +- Blocks (cond_timedwait) until `acked_count + 1 >= (n_active + 1) / 2` + - `+1` counts own write; `n_active` excludes `in_catch_up==1` standbys (5.4, Clarification 6) +- `ack_op` parameter selects `OP_walAck` vs `OP_upgradeConfigAck` to disambiguate concurrent commits +- Returns 0 on quorum, `-ETIMEDOUT` on timeout +- Single-node shortcircuit: `n_active == 0` → quorum immediately from own write (5.8, 8.5) + +**Contracts**: Service [x] / State [x] + +--- + +### Config Replication (config_repl.h / config_repl.c) + +#### Config Bootstrap + +| Field | Detail | +|-------|--------| +| Intent | Pre-election config tag discovery and deferred post-election fetch for unconfigured nodes | +| Requirements | 3.1–3.5 | + +**`bootstrap_config_if_needed(cluster_t *c)` contract**: +- Returns immediately if `c->myconf->config_version_tag != 0` +- Probes all peers via `xce_probe_config_tag()`; three outcomes: + 1. All peers `tag=0` → first-boot cluster, proceed to election (3.3) + 2. Any peer has `tag > 0` → set `c->ha.defer_config_fetch = true`, return (3.2) + 3. No peer reachable → retry with exponential backoff 500ms→10ms cap 10s; log WARN each attempt (3.4) +- Never returns an error; blocks until resolution or process killed + +**`xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out)` contract**: +- Single-RTT SSL request via `connect_ssl_raw()` + xce encode/decode +- Request: `OP_probeConfigTag` with `ProbeConfigTagRequest{name}` (requester name for peer logging) +- Reply: `ProbeConfigTagReply{config_tag}` (0 if peer has no config) +- Peer answers in any FSM state — dispatch gate exempts `OP_probeConfigTag` alongside `OP_requestVote` +- Returns 0 on success, `-ECONNREFUSED` / `-ETIMEDOUT` on unreachable peer + +#### Majority-Commit Config Push + +| Field | Detail | +|-------|--------| +| Intent | Upgrade push_config_to_standbys() from fire-and-forget to majority-commit using shared repl infrastructure | +| Requirements | 8.1–8.5 | + +**`push_config_to_standbys()` extended contract**: +1. Write config blob locally (existing atomic write+rename — unchanged) +2. `repl_broadcast(c, OP_upgradeConfig, blob, len)` +3. `repl_wait_majority_ack(c, commit_timeout_ms, OP_upgradeConfigAck)` +4. On quorum: `atomic_store(&c->myconf->committed_config_tag, tag)` (8.1) +5. On timeout: return `-ETIMEDOUT`; operator retries (8.3) + +**`committed_config_tag`**: new field in cluster config state; piggybacked on `CtrlHeartbeat` (`committed_config_tag @N :UInt64`). Standbys store in `ha.last_known_committed_config_tag`. On promotion: serve only configs up to `last_known_committed_config_tag` (8.4). + +**Contracts**: Service [x] / State [x] + +--- + +### Persistent Control Channel (ha.c / cluster.c) + +#### Primary Discovery and Control Channel + +| Field | Detail | +|-------|--------| +| Intent | Persistent TLS channel per standby; EPOLLRDHUP failover; full state sync on join | +| Requirements | 6.1–6.8 | + +**Three-tier primary discovery** (post-election, standby path): +1. **Tier 1** (O(1) for majority): if `ha.voted_for_name` set, try direct connect to voted-for peer; on `XCE_ERR_NOT_READY` peer still electing — backoff, retry Tier 1 +2. **Tier 2** (redirect): probe remaining peers; a `CTRL_STANDBY` peer that knows the primary replies with `XCE_ERR_STANDBY` + `redirect_host/port`; follow redirect immediately +3. **Tier 3** (blind scan): try remaining peers sequentially with 100ms→5s exponential backoff until primary found + +**Extended `OP_ctrlHello` schema** (controller.capnp): +``` +struct CtrlHelloRequest { + name @0 :Text; + key @1 :Data; # SHA-256(cluster_name + shared_secret); auth gate + applied_lsn @2 :UInt64; + config_tag @3 :UInt64; +} +struct CtrlHelloReply { + committed_lsn @0 :UInt64; + needs_snapshot @1 :Bool; + config_tag @2 :UInt64; +} +``` + +**Failover detection** — dual-trigger replacing timestamp polling (6.2, 6.3, 6.4): +- **Fast**: `EPOLLRDHUP` on `primary_ctrl_fd` → immediately close, null fd, set `CTRL_ELECTING` +- **Fallback**: heartbeat timeout vtask armed on each heartbeat receipt; fires after `election_timeout_ms + rand_jitter` of silence; same action as EPOLLRDHUP path + +**`XCE_ERR_STANDBY` redirect extension** (req 7.2, Tier 2): +Add to `RequestReply` (controller.capnp): +``` +redirect_host @N :Text; # empty string when primary unknown +redirect_port @M :UInt16; # 0 when primary unknown +``` +Standbys populate from `ha.primary_host/port` when known; clients without redirect fields fall back to normal retry. + +**State sync sequence on standby join** (6.5, 6.6, 6.7): + +Primary side (`handle_ctrlhello()`): +1. `repl_add_standby()` → `repl_start_catchup()` sets `in_catch_up=1` +2. Reply `CtrlHelloReply{committed_lsn, needs_snapshot, config_tag}` +3. If `needs_snapshot`: `persist_checkpoint()` → send `OP_snapshotTransfer` +4. Stream `OP_walCatchupEntry` [applied_lsn+1 → current committed_lsn] +5. On `OP_walAck` reaching committed_lsn → `repl_complete_catchup()` (`in_catch_up=0`) +6. If standby config_tag ≠ primary config_tag → `push_config_to_standbys()` + +Standby side: +1. Store `primary_committed_lsn = reply.committed_lsn`; update `ha.last_known_committed_lsn` +2. `persist_truncate_wal_above(primary_committed_lsn)` — discard orphaned uncommitted entries +3. `ha.in_catch_up = 1` — deny votes during sync (Clarification 6) +4. If `reply.needs_snapshot`: receive `OP_snapshotTransfer`, write, call `load_snapshot()` +5. Apply each `OP_walCatchupEntry` via `apply_wal_entry()`; send `OP_walAck` after each +6. On catch-up loop end from primary: `ha.in_catch_up = 0` + +**Role flip on promotion** (6.8): standby closes and nulls `primary_ctrl_fd` before returning from `am_i_primary_quorum()`; new primary accepts inbound `OP_ctrlHello` connections in a per-standby fd table. + +**Contracts**: Service [x] / State [x] + +**Implementation Notes** +- Risk: `apply_wal_entry()` must exist before Change F lands; extract from startup WAL replay path as a prerequisite task +- Risk: `persist_truncate_wal_above()` requires sequential scan of partial WAL segment; validate with unit tests before integrating into standby join path +- Integration: `repl_broadcast()` skips `in_catch_up==1` standbys during the streaming loop; `repl_complete_catchup()` fires only when standby's `acked_lsn` equals current `committed_lsn` + +--- + +### Client Retry (src/lib/request.c) + +#### libxce Retry Wrapper + +| Field | Detail | +|-------|--------| +| Intent | Transparent retry for transient controller errors; existing daemon code unchanged | +| Requirements | 7.1–7.6 | + +**`xce_retry_config_t`**: +```c +typedef struct { + int max_retries; // default 20 + int initial_retry_ms; // default 50 + int max_retry_ms; // default 5000 + double backoff_multiplier; // default 2.0 (exponential) +} xce_retry_config_t; +``` + +**`xce_request_with_retry(endpoint_t *ep, xce_request_t *req, const xce_retry_config_t *cfg)` contract**: +- NULL `cfg` → uses defaults +- `XCE_ERR_NOT_READY`: backoff, retry same endpoint (7.1) +- `XCE_ERR_STANDBY` with `redirect_host/port`: reconnect to redirect, retry (7.2) +- `XCE_ERR_STANDBY` without redirect: backoff, retry same endpoint (7.2 fallback) +- `ECONNREFUSED` / `ETIMEDOUT`: reconnect, retry (7.3) +- `XCE_ERR_AUTH`, `XCE_ERR_PERM`, `XCE_ERR_INVALID`, `XCE_ERR_NOENT`: surface immediately (7.4) +- Exhausted `max_retries`: surface last error to caller + +**Contracts**: Service [x] + +**Implementation Notes** +- Current `xce_request()` is async send-only; the retry wrapper handles the synchronous request-reply cycle around it +- Backoff jitter uses simple `rand_r()` to avoid thundering-herd (7.6) + +--- + +### Wire Protocol (src/schema/controller.capnp) + +#### New Opcodes and Message Types + +Append after existing `upgradeConfig @31`: + +| Opcode | @N | Purpose | Req | +|--------|----|---------|-----| +| `requestVote` | @32 | Raft vote request/reply | 2 | +| `probeConfigTag` | @33 | Lightweight config tag probe | 3 | +| `walAck` | @34 | WAL majority-commit ACK | 5 | +| `upgradeConfigAck` | @35 | Config push majority-commit ACK | 8 | +| `snapshotTransfer` | @36 | Snapshot blob from primary to standby | 6 | +| `walCatchupEntry` | @37 | WAL entry during catch-up streaming | 6 | + +**Key struct definitions** (to add to controller.capnp): +``` +struct RequestVoteRequest { + term @0 :UInt64; + candidateId @1 :Text; + configTag @2 :UInt64; + lastLogLsn @3 :UInt64; +} +struct RequestVoteReply { + term @0 :UInt64; + voteGranted @1 :Bool; + peerConfigTag @2 :UInt64; + peerLastLogLsn @3 :UInt64; +} +struct ProbeConfigTagRequest { name @0 :Text; } +struct ProbeConfigTagReply { configTag @0 :UInt64; } +struct WalAckRequest { term @0 :UInt64; lsn @1 :UInt64; } +struct WalAckReply { term @0 :UInt64; lsn @1 :UInt64; success @2 :Bool; } +struct UpgradeConfigAck { term @0 :UInt64; ackedTag @1 :UInt64; success @2 :Bool; } +struct SnapshotTransfer { snapshotLsn @0 :UInt64; payload @1 :Data; } +struct WalCatchupEntry { lsn @0 :UInt64; entryType @1 :UInt8; payload @2 :Data; } +``` + +Also extend: +- `CtrlHeartbeat`: add `committedLsn @N :UInt64` and `committedConfigTag @M :UInt64` for piggybacking (req 5.6, 8.4) +- `RequestReply`: add `redirectHost @N :Text` and `redirectPort @M :UInt16` for standby redirect (req 7.2) + +--- + +## Data Models + +### New Persistent State + +**`ha-term.bin`** — fixed-size binary, atomic write+rename: +```c +typedef struct { + uint64_t magic; // 0xDEADBEEFCAFEBABE + uint64_t term; + uint64_t voted_for_term; + char voted_for_name[XCE_MAXNAME_LEN]; + uint32_t crc32; // CRC32 of all preceding fields +} ha_term_file_t; +``` +- Read in `ha_init()` at startup; missing file → term=0, no prior vote (first boot) +- CRC mismatch → log WARN, initialize to term=0 (conservative: safe to vote again) + +### New In-Memory State + +| Struct | Field | Type | Invariant | +|--------|-------|------|-----------| +| `persist_t` | `committed_lsn` | `_Atomic uint64_t` | `committed_lsn <= applied_lsn` always | +| `ha_state_t` | 8 new fields | various | See Components section | +| `standby_conn_t` | `pending_ack_count` | `int` | `<= max_pending_acks` while in active set | + +### Consistency Guarantees + +- WAL trim guard: threshold = `committed_lsn` not `applied_lsn`; prevents trimming entries a promoted standby may need for divergence resolution +- `committed_lsn <= applied_lsn` asserted at promotion time; violated invariant is a programming error +- `ha-term.bin` is written atomically before any vote reply is sent; a crash between write and reply means the term is persisted but the vote may not have been counted — conservative and safe + +--- + +## Error Handling + +### Election Errors + +| Error | Condition | Response | +|-------|-----------|----------| +| `config_tag=0` and any peer has `tag > 0` | Hard gate A | Fatal exit `-ENODATA`; log ERROR with peer's tag value | +| Quorum not reached after 2 rounds | Normal election result | Enter `CTRL_STANDBY`; log WARN with vote tally | +| Fresher responding peer yield gate B | Winner yields to peer with higher tag or LSN | Enter `CTRL_STANDBY`; log INFO with peer tag/lsn seen | +| `ha-term.bin` CRC mismatch at startup | File corruption or incomplete write | Log WARN; initialize term=0; safe to proceed | +| Split vote after 2nd round | Split-vote safety net | Enter `CTRL_STANDBY`; log WARN | + +### WAL and Config Commit Errors + +| Error | Condition | Response | +|-------|-----------|----------| +| `-ETIMEDOUT` from `repl_wait_majority_ack` | No quorum within `commit_timeout_ms` | Return to caller; entry persisted locally but not committed; log WARN | +| Standby backpressure exceeded | `pending_ack_count > max_pending_acks` | Remove from active set; trigger catch-up reconnect; log WARN | +| Standby send failure in `repl_broadcast` | TCP error on control channel fd | Same as backpressure — remove and trigger reconnect | + +### Startup Validation Errors + +| Error | Condition | Response | +|-------|-----------|----------| +| Even peer count | `n % 2 == 0 && n > 1` | Fatal `-EINVAL`; operator-friendly stderr with correct format | +| Cert missing or unreadable | `access(path, R_OK) != 0` | Fatal `-ENOENT`; print full path | +| `--vcpus < 1` | CLI argument | Fatal `-EINVAL`; print message | +| `vcpus` key in config file | Deprecation | Log WARN; do not fail startup | + +### Operational Constraint — WAL Stall Window + +Between primary winning election and first standby completing `repl_complete_catchup()`, the active replication set has 0 standbys. Every `persist_wal_sync_append()` call blocks for `commit_timeout_ms` and returns `-ETIMEDOUT`. This is correct CP behavior. Libxce retry (req 7) handles this transparently for workers and schedulers. Operators must be warned in deployment documentation. + +### Monitoring (Req 10) + +All new paths emit structured `xcelog.h` log entries: +```c +// Election completion (10.1) +INFO("election complete: role=%s term=%lu votes=%d/%d quorum=%d", + role_str, term, granted, n_peers, quorum); +// Bootstrap probe result (10.2) +INFO("config_bootstrap: outcome=defer best_peer_tag=%016llx", best_tag); +// WAL commit quorum (10.3) +INFO("wal_commit: lsn=%lu acked=%d/%d latency_ms=%d", lsn, acked, n_active, ms); +// Config push quorum (10.4) +WARN("config_push_timeout: tag=%016llx timeout_ms=%d", tag, commit_timeout_ms); +// FSM transition (10.5) +INFO("ha_role: %s -> %s term=%lu", from_str, to_str, term); +``` + +--- + +## Testing Strategy + +### Unit Tests (`src/controller/testsuite/`) + +| File | Verifies | Req | +|------|---------|-----| +| `test_ha_election.c` | Quorum met on 2/3 votes; voted_for enforcement across terms; freshness gate yields to fresher peer; split-vote triggers retry; single-node self-elects immediately | 2.1–2.12 | +| `test_ha_bootstrap.c` | All-zero first-boot proceeds; peer-has-config sets defer_config_fetch; no-peer-reachable retries with backoff; retry succeeds on 3rd attempt | 3.1–3.5 | +| `test_ha_durability.c` | ha_persist_term writes then ha_init restores term and voted_for; CRC corruption zeroes state safely; missing file zeroes state | 9.1–9.3 | +| `test_persist_truncate.c` | Truncation at segment boundary deletes whole segment; truncation mid-segment scans to correct offset; calling twice with same cutoff is idempotent | 5.6, 5.7 | +| `test_repl_majority.c` | Majority reached on 2/3 ACKs; timeout returns -ETIMEDOUT; single-node returns 0 immediately; in_catch_up standby excluded from quorum count | 5.1–5.8, 8.1–8.5 | +| `test_ha_startup.c` | Even peer count rejected; cert missing returns -ENOENT with path; --vcpus 0 rejected; vcpus in config logs deprecation warning but does not fail | 4.1–4.4 | + +### Unit Tests (`src/lib/testsuite/`) + +| File | Verifies | Req | +|------|---------|-----| +| `test_libxce_retry.c` | NOT_READY triggers backoff and retry same host; STANDBY with redirect connects to redirected host; STANDBY without redirect falls back to backoff; AUTH error surfaces immediately; max_retries exhausted surfaces last error; 50 concurrent clients with backoff do not starve | 7.1–7.6 | + +### Integration Tests (multi-node, `make test-integration`) + +| Scenario | Verifies | Req | +|----------|---------|-----| +| Concurrent-startup: both nodes start within 3s | Only one node wins election; no false self-election | 1.1, 2.1 | +| Config-fresh node vs stale node election | Stale node yields; fresh node wins | 2.7 | +| Unconfigured node joins live cluster | defer_config_fetch set; deferred fetch succeeds; node enters standby with correct config | 3.2, 3.5 | +| Primary crash → standby promotion | Sub-second EPOLLRDHUP detection; promoted standby has all committed WAL entries | 5.6, 6.2 | +| WAL commit with all standbys down | Returns -ETIMEDOUT per entry; single-standby quorum resumes on reconnect | 5.5 | +| Config upgrade under network partition | Operator receives -ETIMEDOUT; retry after heal succeeds | 8.3 | +| Restart after participating in term N | Restored voted_for prevents double-vote | 9.1, 9.2 | + +### Performance Tests + +| Test | Target | Req | +|------|--------|-----| +| Election round-trip latency (3-node LAN) | < 100ms p99 | 2.1 | +| WAL majority-commit overhead (3-node LAN) | < 5ms p99 per WAL_JOB_SUBMIT | 5.1 | +| Failover detection latency | < 1s from TCP disconnect to election start | 6.2 | +| 50 concurrent libxce clients post-failover | All succeed within 30s; no starvation | 7.6 | + +--- + +## Security Considerations + +- **Cert pre-flight** (4.2): Controller refuses to start with missing or unreadable TLS cert/key. Prevents silent unencrypted operation. +- **Cluster auth key in `OP_ctrlHello`** (6.1): `key = SHA-256(cluster_name + shared_secret)` gates standby join. `cluster_key` is a new field read from the existing config file (not CLI, to avoid shell history exposure). Primary rejects `OP_ctrlHello` where HMAC does not match. +- **`voted_for` persistence** (9.1): Prevents double-vote within a term across restarts. A node that crashes after voting and restarts must not vote for a second candidate in the same term. +- **Term-based fencing**: A primary that receives any reply with `term > my_term` immediately steps down (`ha_do_demote()`). Prevents a reconnected stale primary from continuing after partition heal. +- **No priority tiebreak**: Raft freshness (config_tag + LSN) fully determines winner. The `ha.my_priority` field is preserved in `ha_state_t` for backward compatibility but plays no role in election outcome. + +## Performance & Scalability + +- **Write latency**: 1 LAN RTT + max(fsync_primary, fsync_standby) per majority-commit entry. At 0.1–0.5ms LAN RTT and 0.05–0.2ms NVMe: ~0.2–0.7ms commit latency. Applies only to `WAL_JOB_SUBMIT`, `WAL_JOB_START`, `WAL_JOB_FINISH`, `WAL_QUEUE_CHANGE`. +- **Throughput ceiling**: 1,000–20,000 job submits/sec before jobmgr locking dominates. Raft overhead is not the bottleneck at HPC-typical rates. +- **5-node vs 3-node write latency**: negligible. 5-node waits for 2nd-fastest of 4 standbys vs 1st of 2; sub-millisecond difference on uniform LAN. +- **Single-node shortcircuit**: all majority-commit paths check `n_peers == 1` first; self-only write immediately satisfies quorum. +- **Backpressure**: `max_pending_acks = 64` (configurable) prevents any single slow standby from stalling primary fanout. +- **`commit_timeout_ms`**: configurable, default 5000ms; tune per deployment network latency. diff --git a/.kiro/specs/ha-config-bootstrap/spec.json b/.kiro/specs/ha-config-bootstrap/spec.json new file mode 100644 index 0000000..fc3d56b --- /dev/null +++ b/.kiro/specs/ha-config-bootstrap/spec.json @@ -0,0 +1,23 @@ +{ + "feature_name": "ha-config-bootstrap", + "description": "Replace LSF-style priority election with Raft-style quorum voting, add majority-commit WAL and config replication, establish persistent control channels, and harden the controller startup sequence for full CP high-availability.", + "phase": "tasks-generated", + "created_at": "2026-05-27T01:24:00-07:00", + "updated_at": "2026-05-27T17:35:00-07:00", + "language": "en", + "status": "active", + "approvals": { + "requirements": { + "generated": true, + "approved": true + }, + "design": { + "generated": true, + "approved": true + }, + "tasks": { + "generated": true, + "approved": true + } + } +} diff --git a/.kiro/specs/ha-config-bootstrap/tasks.md b/.kiro/specs/ha-config-bootstrap/tasks.md new file mode 100644 index 0000000..96b4123 --- /dev/null +++ b/.kiro/specs/ha-config-bootstrap/tasks.md @@ -0,0 +1,281 @@ +# Implementation Plan: HA Config Bootstrap & Raft-Style Quorum Election + +--- + +## Task Overview + +10 requirements → 8 implementation changes (D→A→B→C→E→H→F→G) + observability woven throughout. +Dependency order: startup validation (D) unblocks vtaskmgr before election (A); listener before election (A) unblocks quorum FSM (B); quorum FSM (B) unblocks config bootstrap (C); majority-commit WAL infrastructure (E) unblocks majority-commit config (H) and persistent channel (F); libxce retry (G) is independent of all controller changes. + +--- + +- [ ] 1. Foundation: Wire protocol extensions and shared infrastructure +- [ ] 1.1 Add 6 new opcodes and message structs to controller.capnp + - Append `requestVote @32`, `probeConfigTag @33`, `walAck @34`, `upgradeConfigAck @35`, `snapshotTransfer @36`, `walCatchupEntry @37` after existing `upgradeConfig @31` + - Add structs: `RequestVoteRequest`, `RequestVoteReply`, `ProbeConfigTagRequest`, `ProbeConfigTagReply`, `WalAckRequest`, `WalAckReply`, `UpgradeConfigAck`, `SnapshotTransfer`, `WalCatchupEntry` + - Extend `CtrlHeartbeat`: add `committedLsn :UInt64` and `committedConfigTag :UInt64` fields + - Extend `RequestReply`: add `redirectHost :Text` and `redirectPort :UInt16` fields for standby redirect + - Regenerate capnp bindings (`nix build .#xcecore_debug` must compile clean) + - `make test` passes; existing decode paths unaffected + - _Requirements: 2, 3, 5, 6, 7, 8_ + - _Boundary: Wire Protocol (controller.capnp)_ + +- [ ] 1.2 Add new enum values and types to ha.h + - Add `CTRL_INIT=2`, `CTRL_ELECTING=3` to `ctrl_role_t` (values above existing 0/1) + - Add `ha_vote_entry_t` struct with fields: `peer`, `my_term`, `my_tag`, `my_lsn`, `vote_granted`, `replied`, `peer_tag`, `peer_lsn` + - Add `ha_elect_ctx_t` struct with C99 flexible array `ha_vote_entry_t votes[]`; allocation comment: `calloc(1, sizeof(*ctx) + (n_peers-1) * sizeof(ctx->votes[0]))` + - Add 8 new `ha_state_t` fields: `voted_for_term`, `voted_for_name[XCE_MAXNAME_LEN]`, `primary_ctrl_fd` (int, -1=none), `defer_config_fetch` (bool), `elect_ctx` (_Atomic void*), `last_known_committed_lsn` (_Atomic uint64_t), `heartbeat_seen` (sem_t), `last_known_committed_config_tag` (uint64_t) + - Declare: `ha_persist_term()`, `bootstrap_config_if_needed()`, `xce_probe_config_tag()` + - `make test` passes; no new functions yet — declarations only + - _Requirements: 1, 2, 3, 5, 6, 9_ + - _Boundary: Controller FSM (ha.h)_ + +- [ ] 2. Change D: Startup validation and ASAN fix +- [ ] 2.1 Write failing tests for startup validation + - In `src/controller/testsuite/test_ha_startup.c`: test even-N peer count returns `-EINVAL`; test cert missing returns `-ENOENT` with full path in message; test `--vcpus 0` returns `-EINVAL`; test `vcpus` key in config file logs WARN but does not fail + - `make test` fails (functions under test do not exist yet) + - _Requirements: 4.1, 4.2, 4.3, 4.4_ + +- [ ] 2.2 Implement --vcpus CLI arg and remove vcpus from cluster config + - Add `--vcpus N` argp option in `main.c`; store in a new startup parameter struct passed to `myapp_run()` + - Remove `vcpus @7` field from `ClusterConfig` in `cluster.h` / `cluster.c` + - Update `vtaskmgr_init()` call site to use the CLI-provided vcpus value directly + - Log WARN when config file contains `vcpus` key (4.4) + - `make test` GREEN on vcpus tests + - _Requirements: 4.3, 4.4_ + - _Boundary: Startup Validation (main.c, cluster.h, cluster.c)_ + +- [ ] 2.3 Add peer count validation and cert pre-flight checks + - In `main.c` after argument parsing: if `n_peers % 2 == 0 && n_peers > 1` → `fprintf(stderr, ...)` + `exit(1)` with message explaining odd-N requirement + - Call `access(cert_path, R_OK)` and `access(key_path, R_OK)` in `main.c` before any SSL init; fatal exit with full path on failure + - Fix ASAN leak: add `SSL_CTX_free(ha->ssl_ctx)` in `ha_fini()` + - `make test` GREEN on all startup validation tests + - _Requirements: 4.1, 4.2_ + - _Boundary: Startup Validation (main.c, ha.c)_ + +- [ ] 3. Change A: Listener and iomgr/vtaskmgr before election +- [ ] 3.1 Write failing tests for dispatch gate and FSM init state + - In `src/controller/testsuite/test_ha_startup.c` (extend): test that requests other than `OP_requestVote` and `OP_probeConfigTag` receive `XCE_ERR_NOT_READY` when role is `CTRL_INIT`; test `CTRL_STANDBY` returns `XCE_ERR_STANDBY` for primary-only ops; test `heartbeat_seen` semaphore initialized in `ha_init()` + - `make test` fails + - _Requirements: 1.1, 1.2, 1.3, 1.4_ + +- [ ] 3.2 Extract cluster_open_listener() from cluster_init_phase2() + - Move `bind()+listen()` + socket setup into new `cluster_open_listener(cluster_t *c)` in `cluster.c` + - `cluster_init_phase2()` calls `cluster_open_listener()` for compatibility + - `myapp.c` startup sequence calls `cluster_open_listener()` before election (after `cluster_init_phase1()`) + - `iomgr_init()` and `vtaskmgr_init(vcpus)` also moved before election + - `make test` GREEN; listener is open in startup sequence before any election probe + - _Requirements: 1.1, 1.2_ + - _Boundary: Startup Sequence (cluster.c, myapp.c)_ + +- [ ] 3.3 Implement dispatch gate for CTRL_INIT and CTRL_ELECTING states + - Initialize `ha.heartbeat_seen` semaphore in `ha_init()`; initialize `ha.primary_ctrl_fd = -1` + - Add role-check at vtaskmgr dispatch entry: `CTRL_INIT` and `CTRL_ELECTING` pass only `OP_requestVote` and `OP_probeConfigTag`; all others return `XCE_ERR_NOT_READY` + - `CTRL_STANDBY` returns `XCE_ERR_STANDBY` with `redirect_host/port` for primary-only ops + - `make test` GREEN on all dispatch gate tests + - _Requirements: 1.3, 1.4_ + - _Boundary: Controller FSM (ha.c, cluster.c)_ + +- [ ] 3.4 Implement ha_wait_for_heartbeat_or_timeout() + - `sem_timedwait(&c->ha.heartbeat_seen, &deadline)` where `deadline = now + election_timeout_ms + xorshift64_jitter()` + - Heartbeat handler posts `heartbeat_seen` only when `ha.role == CTRL_INIT`; receiving a heartbeat transitions to `CTRL_STANDBY` and returns + - Election timeout fires → returns without setting standby; caller proceeds to `am_i_primary_quorum()` + - Called from `myapp.c` after `bootstrap_config_if_needed()` (see task 5) + - `make test` GREEN + - _Requirements: 1.1, 1.2_ + - _Boundary: Controller FSM (ha.c)_ + +- [ ] 4. Change B: Quorum-based controller election (replaces am_i_primary) +- [ ] 4.1 Write failing tests for quorum election logic + - In `src/controller/testsuite/test_ha_election.c`: test 2/3 votes → `CTRL_PRIMARY`; test only 1/3 vote → `CTRL_STANDBY`; test `voted_for` enforcement (deny second candidate in same term); test freshness gate (higher peer tag yields to peer → `CTRL_STANDBY`); test fresher candidate wins over stale peer; test split vote triggers retry with incremented term; test single-node self-elects immediately without network; test `in_catch_up=1` standby votes are denied + - `make test` fails + - _Requirements: 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 2.10, 2.11, 2.12_ + +- [ ] 4.2 Implement handle_request_vote() and ha_persist_term() + - `ha_persist_term()`: write `ha_term_file_t` (magic + term + voted_for_term + voted_for_name + CRC32) atomically to `{workdir}/ha-term.bin` via write+rename + - `handle_request_vote()`: if `req->term < my_term` → deny; if `req->term > my_term` → update term, clear voted_for, persist; if already voted for different candidate → deny; if `req->config_tag < my_tag || (eq_tag && req->last_lsn < my_lsn)` → deny; otherwise → grant, set `voted_for_name`, persist, reply + - `handle_request_vote_reply()`: update `ha_elect_ctx_t` entry; post `ctx->done` when quorum reached or all replied + - `make test` GREEN on all vote handler sub-cases + - _Requirements: 2.2, 2.3, 2.4, 2.5, 2.6, 9.1_ + - _Boundary: Controller FSM (ha.c)_ + +- [ ] 4.3 Implement ha_elect_evaluate_and_signal() and am_i_primary_quorum() + - `ha_elect_evaluate_and_signal()`: count votes_granted; check freshness gate (any responding peer has higher config_tag or equal tag + higher LSN → yield); if `votes_granted >= quorum && !fresher_peer` → `CTRL_PRIMARY`; else → `CTRL_STANDBY` + - `am_i_primary_quorum()`: single-node shortcircuit (n_peers==1 → `CTRL_PRIMARY`); allocate `ha_elect_ctx_t`; spawn per-peer pthreads that call `connect_ssl_raw()` + send `OP_requestVote`; `sem_wait(&ctx->done)` for result; handle split-vote: one retry with incremented term + `xorshift64_jitter()` delay; second split → `CTRL_STANDBY` + - Replace `am_i_primary()` call in `myapp.c` with `am_i_primary_quorum()` + - Delete `ha_check_priority_stepdown()` and `ha_check_heartbeat_nocluster()` + - Log INFO: election outcome, term, votes granted, quorum size (10.1); log INFO: FSM transitions (10.5) + - `make test` GREEN on all election tests + - _Requirements: 2.1, 2.7, 2.8, 2.9, 2.10, 2.11, 2.12, 10.1, 10.5_ + - _Boundary: Controller FSM (ha.c, myapp.c)_ + +- [ ] 5. Change C: Config bootstrap for unconfigured nodes +- [ ] 5.1 Write failing tests for config bootstrap + - In `src/controller/testsuite/test_ha_bootstrap.c`: test `config_tag != 0` → returns immediately; test all-zero peers → proceed to election; test any peer has `tag > 0` → `defer_config_fetch = true`; test no peer reachable → retries with exponential backoff (mock clock); test deferred fetch called when election winner known and `defer_config_fetch` is set; test `xce_probe_config_tag()` returns 0 on success, `-ECONNREFUSED` on unreachable + - `make test` fails + - _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5_ + +- [ ] 5.2 Implement xce_probe_config_tag() + - Single-RTT SSL probe: `connect_ssl_raw()` + xce encode `OP_probeConfigTag` + decode `ProbeConfigTagReply` + - Returns 0 on success with `*tag_out` set; returns `-ECONNREFUSED` / `-ETIMEDOUT` on unreachable peer + - Peer dispatch gate exempts `OP_probeConfigTag` in `CTRL_INIT` and `CTRL_ELECTING` states + - Log INFO: probe outcome per peer (10.2) + - `make test` GREEN on probe unit tests + - _Requirements: 3.1, 1.3_ + - _Boundary: Config Bootstrap (config_repl.c)_ + +- [ ] 5.3 Implement bootstrap_config_if_needed() and wire into startup + - Returns immediately if `config_tag != 0` + - Probes all peers; three outcomes: all-zero → proceed; any non-zero → set `defer_config_fetch`, return; no peer reachable → retry with backoff 500ms→10ms→...→10s cap; log WARN each failed attempt + - Integrate into `myapp.c`: call after `cluster_open_listener()` + `iomgr_init()` + `vtaskmgr_init()`, before `ha_wait_for_heartbeat_or_timeout()` + - After election: if `defer_config_fetch && role == CTRL_STANDBY` → `do_one_config_fetch()` from proven winner + - Log INFO: bootstrap outcome (10.2) + - `make test` GREEN on all bootstrap tests + - _Requirements: 3.2, 3.3, 3.4, 3.5, 10.2_ + - _Boundary: Config Bootstrap (config_repl.c, myapp.c)_ + +- [ ] 6. Change 9: Election state durability (ha-term.bin) +- [ ] 6.1 Write failing tests for ha-term.bin persist and restore + - In `src/controller/testsuite/test_ha_durability.c`: test `ha_persist_term()` writes file; test `ha_init()` restores term and voted_for from file; test CRC mismatch → zeroes state safely with WARN log; test missing file → term=0, no prior vote; test write+rename atomicity (partial write leaves old file intact) + - `make test` fails + - _Requirements: 9.1, 9.2, 9.3_ + +- [ ] 6.2 Implement ha_init() restore path for ha-term.bin + - Read `{workdir}/ha-term.bin` in `ha_init()` at startup; verify magic and CRC32; on match → restore `ha.term` and `ha.voted_for_*`; on missing → term=0 (3rd boot); on CRC mismatch → log WARN, initialize to term=0 + - `ha_persist_term()` already implemented in task 4.2; this task only adds the read-path to `ha_init()` + - `make test` GREEN on all durability tests + - _Requirements: 9.1, 9.2, 9.3_ + - _Boundary: Term Durability (ha.c)_ + +- [ ] 7. Change E: Majority-commit WAL replication +- [ ] 7.1 Write failing tests for repl_broadcast and repl_wait_majority_ack + - In `src/controller/testsuite/test_repl_majority.c`: test 2/3 ACKs → `repl_wait_majority_ack` returns 0; test timeout returns `-ETIMEDOUT`; test single-node (`n_active == 0`) returns 0 immediately; test `in_catch_up=1` standby excluded from quorum denominator; test `pending_ack_count > max` removes standby from active set; test `OP_walAck` vs `OP_upgradeConfigAck` dispatched separately via `ack_op` + - `make test` fails + - _Requirements: 5.1, 5.4, 5.5, 5.8, 8.2_ + +- [ ] 7.2 Implement repl_broadcast() (replace stub) + - Add `pending_ack_count` field to `standby_conn_t` + - `repl_broadcast(cluster_t *c, uint8_t op, const void *payload, size_t len)`: iterate active standbys; skip `in_catch_up==1`; if `pending_ack_count > max_pending_acks` → remove from active set, trigger catch-up reconnect, log WARN; else send payload over `standby.ctrl_fd`, increment `pending_ack_count`; returns 0 after fanout initiated + - `make test` GREEN on broadcast fanout tests + - _Requirements: 5.1, 5.4_ + - _Boundary: Replication Layer (repl.c, repl.h)_ + +- [ ] 7.3 Implement repl_wait_majority_ack() + - `repl_wait_majority_ack(cluster_t *c, uint32_t timeout_ms, uint8_t ack_op)`: `cond_timedwait` until `acked_count + 1 >= (n_active + 1) / 2`; single-node shortcircuit; `ack_op` disambiguates WAL vs config ACKs; returns 0 on quorum, `-ETIMEDOUT` on timeout + - `repl_on_ack()` extended to decrement `pending_ack_count` and signal condition + - Log INFO on quorum (10.3); log WARN on timeout (10.3) + - `make test` GREEN on all majority-ack tests + - _Requirements: 5.1, 5.5, 5.8, 10.3_ + - _Boundary: Replication Layer (repl.c)_ + +- [ ] 7.4 Write failing tests for persist_truncate_wal_above() + - In `src/controller/testsuite/test_persist_truncate.c`: test truncation at segment boundary deletes whole segment; test truncation mid-segment scans sequentially to correct byte offset; test idempotency (calling twice with same `cutoff_lsn` is safe); test `applied_lsn` updated atomically; test no-op when `cutoff_lsn == applied_lsn` + - `make test` fails + - _Requirements: 5.6, 5.7_ + +- [ ] 7.5 Implement committed_lsn in persist_t and persist_truncate_wal_above() + - Add `_Atomic uint64_t committed_lsn` to `persist_t` (invariant: `committed_lsn <= applied_lsn`) + - `persist_truncate_wal_above(persist_t *p, uint64_t cutoff_lsn)`: scan segment files named `seg-{LSN_hex}.wal`; find segment containing `cutoff_lsn`; sequential scan of entries until LSN > cutoff_lsn; truncate at that byte offset; delete later segments; atomically update `p->applied_lsn = cutoff_lsn`; idempotent + - Update WAL trim guard: use `committed_lsn` not `applied_lsn` as threshold + - `make test` GREEN on all truncation tests + - _Requirements: 5.6, 5.7_ + - _Boundary: Persistence Layer (persist.c, persist.h)_ + +- [ ] 7.6 Wire majority-commit into persist_wal_sync_append() + - In `persist_wal_sync_append()`: after local write + fsync for `WAL_JOB_SUBMIT`, `WAL_JOB_START`, `WAL_JOB_FINISH`, `WAL_QUEUE_CHANGE` → call `repl_broadcast(c, OP_walAck, payload, len)` + `repl_wait_majority_ack(c, commit_timeout_ms, OP_walAck)` → on quorum advance `committed_lsn` + - `WAL_HOST_STATE`, `WAL_CHECKPOINT_REF` bypass to `persist_wal_async()` (unchanged) + - `make test` passes; integration test verifies committed WAL entries survive failover + - _Requirements: 5.1, 5.2, 5.3_ + - _Boundary: Persistence Layer (persist.c), Replication Layer (repl.c)_ + +- [ ] 8. Change H: Majority-commit config replication (P) +- [ ] 8.1 Implement OP_upgradeConfigAck handler and committed_config_tag + - Add `committed_config_tag` field to cluster config state (ha_state_t via `last_known_committed_config_tag`); piggybacked on `CtrlHeartbeat` via `committedConfigTag` + - Standby handler for `OP_upgradeConfig`: write config blob locally + reply `OP_upgradeConfigAck` with `ackedTag` + - `push_config_to_standbys()` upgraded: write locally → `repl_broadcast(c, OP_upgradeConfig, blob, len)` → `repl_wait_majority_ack(c, commit_timeout_ms, OP_upgradeConfigAck)` → on quorum `atomic_store(&committed_config_tag, tag)` → return; on timeout return `-ETIMEDOUT` + - Log INFO on quorum (10.4); log WARN on timeout (10.4) + - `make test` GREEN using `test_repl_majority.c` with `OP_upgradeConfigAck` + - _Requirements: 8.1, 8.2, 8.3, 8.4, 8.5, 10.4_ + - _Boundary: Config Replication (config_repl.c)_ + - _Depends: 7.3_ + +- [ ] 9. Change F: Persistent standby-to-primary control channel +- [ ] 9.1 Extract apply_wal_entry() prerequisite + - Extract the WAL replay loop body in the startup path into standalone `apply_wal_entry(cluster_t *c, wal_entry_t *e)` function; existing startup replay calls it in a loop; behavior unchanged + - `make test` GREEN; this is a preparatory refactor enabling standby catch-up streaming + - _Requirements: 6.5_ + - _Boundary: Persistence Layer (persist.c) / WAL replay_ + +- [ ] 9.2 Implement ha_find_and_connect_to_primary() with three-tier discovery + - Tier 1: if `ha.voted_for_name` set → direct connect; on `XCE_ERR_NOT_READY` → backoff, retry Tier 1 + - Tier 2: probe remaining peers; follow `XCE_ERR_STANDBY` `redirect_host/port` immediately + - Tier 3: blind sequential scan with 100ms→5s exponential backoff + - Sets `ha.primary_ctrl_fd` on success; registers fd with iomgr `EPOLLIN | EPOLLRDHUP` + - Called after standby wins no election; also called by re-election loser + - `make test` GREEN (mock endpoints for each tier) + - _Requirements: 6.1_ + - _Boundary: Persistent Control Channel (ha.c)_ + +- [ ] 9.3 Implement extended OP_ctrlHello handler and state sync sequence + - Extend `CtrlHelloRequest` with `applied_lsn` and `config_tag`; extend `CtrlHelloReply` with `committed_lsn`, `needs_snapshot`, `config_tag` + - Primary side `handle_ctrlhello()`: `repl_add_standby()` → `repl_start_catchup()`; reply; if `needs_snapshot` → `persist_checkpoint()` + send `OP_snapshotTransfer`; stream `OP_walCatchupEntry` from `[standby.applied_lsn+1 → committed_lsn]`; on `OP_walAck` reaching `committed_lsn` → `repl_complete_catchup()`; if config_tag mismatch → call `push_config_to_standbys()` + - Standby side: store `last_known_committed_lsn`; call `persist_truncate_wal_above(primary_committed_lsn)`; set `in_catch_up=1`; apply `OP_snapshotTransfer` if present; apply each `OP_walCatchupEntry` via `apply_wal_entry()`; send `OP_walAck` after each; on catch-up end: `in_catch_up=0` + - `make test` GREEN; integration test: fresh standby joins and reaches parity with primary committed_lsn + - _Requirements: 6.5, 6.6, 6.7_ + - _Boundary: Persistent Control Channel (ha.c, cluster.c), Replication Layer (repl.c)_ + - _Depends: 7.5, 9.1_ + +- [ ] 9.4 Implement EPOLLRDHUP failover and heartbeat timeout vtask + - `EPOLLRDHUP` event on `primary_ctrl_fd` → close fd, set `primary_ctrl_fd = -1`, transition to `CTRL_ELECTING`, call `persist_truncate_wal_above(last_known_committed_lsn)`, fire `am_i_primary_quorum()` + - Heartbeat timeout vtask: armed on each heartbeat receipt; fires after `election_timeout_ms + rand_jitter` of silence; same action as EPOLLRDHUP path + - On `CTRL_STANDBY` after new election → close outbound primary fd; on `CTRL_PRIMARY` after new election → accept inbound `OP_ctrlHello` connections; reuse per-standby fd table + - Log INFO: FSM transitions (10.5) + - `make test` GREEN; integration test: TCP disconnect → sub-second election start + - _Requirements: 6.2, 6.3, 6.4, 6.8, 10.5_ + - _Boundary: Persistent Control Channel (ha.c), Controller FSM (ha.c)_ + - _Depends: 9.2_ + +- [ ] 10. Change G: libxce client-side transparent retry (P) +- [ ] 10.1 Write failing tests for xce_request_with_retry() + - In `src/lib/testsuite/test_libxce_retry.c`: test `XCE_ERR_NOT_READY` → backoff + retry same host; test `XCE_ERR_STANDBY` with `redirect_host/port` → reconnect to redirect + retry; test `XCE_ERR_STANDBY` without redirect → fallback to backoff; test `XCE_ERR_AUTH` → surface immediately (no retry); test `max_retries` exhausted → surface last error; test 50 concurrent simulated clients → all succeed within 30s (backoff prevents starvation) + - `make test` fails + - _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6_ + +- [ ] 10.2 Implement xce_retry_config_t and xce_request_with_retry() + - Add `xce_retry_config_t` to `request.h` with defaults: `max_retries=20`, `initial_retry_ms=50`, `max_retry_ms=5000`, `backoff_multiplier=2.0` + - `xce_request_with_retry(endpoint_t *ep, xce_request_t *req, const xce_retry_config_t *cfg)`: NULL cfg → use defaults; `XCE_ERR_NOT_READY` → backoff, retry; `XCE_ERR_STANDBY` with redirect → reconnect, retry; `ECONNREFUSED`/`ETIMEDOUT` → reconnect, retry; `XCE_ERR_AUTH`/`XCE_ERR_PERM`/`XCE_ERR_INVALID`/`XCE_ERR_NOENT` → surface immediately; exhausted → surface last error + - Jitter via `rand_r()` to prevent thundering herd (7.6) + - `make test` GREEN on all retry tests; 50-client concurrency test passes + - _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6_ + - _Boundary: Client Retry (request.c, request.h)_ + +- [ ] 11. Integration validation and cleanup +- [ ] 11.1 End-to-end: concurrent startup and config-fresh election + - Integration test: 2 nodes start within 3s — only one wins election (no false self-election); config-fresh node beats stale node; stale node enters standby + - `make test-integration` GREEN on these scenarios + - _Requirements: 1.1, 2.1, 2.7_ + - _Depends: 4.3, 9.4_ + +- [ ] 11.2 End-to-end: failover and WAL durability + - Integration test: primary crash → standby detects via `EPOLLRDHUP` within 1s → promotes → all committed WAL entries present; integration test: WAL commit with all standbys down returns `-ETIMEDOUT`; reconnected standby resumes replication + - `make test-integration` GREEN + - _Requirements: 5.6, 6.2, 5.5_ + - _Depends: 7.6, 9.4_ + +- [ ] 11.3 End-to-end: unconfigured node bootstrap + - Integration test: unconfigured node (`config_tag=0`) joins live cluster; `defer_config_fetch` set; after election resolves, `do_one_config_fetch()` succeeds; node enters standby with correct config + - `make test-integration` GREEN + - _Requirements: 3.2, 3.5_ + - _Depends: 5.3, 9.3_ + +- [ ] 11.4 End-to-end: config upgrade majority-commit and term durability + - Integration test: config upgrade under simulated network partition returns `-ETIMEDOUT`; retry after heal succeeds; integration test: restart after term N participation → `ha-term.bin` restores `voted_for`, prevents double-vote + - `make test-integration` GREEN + - _Requirements: 8.3, 9.1, 9.2_ + - _Depends: 8.1, 6.2_ + +- [ ] 11.5 Full ASAN build verification + - `nix build --option sandbox false .#xcecore_debug` clean (no leaks, no memory errors) + - Confirm `SSL_CTX_free` fix (task 2.3) resolves ASAN leak; confirm no new leaks from `ha_elect_ctx_t` allocation paths + - _Requirements: 4_ + - _Depends: 2.3, 4.3_ diff --git a/src/schema/controller.capnp b/src/schema/controller.capnp index 9d7d028..fc1c007 100644 --- a/src/schema/controller.capnp +++ b/src/schema/controller.capnp @@ -40,6 +40,12 @@ enum OPCode { replConfig @29; replConfigPush @30; upgradeConfig @31; + requestVote @32; + probeConfigTag @33; + walAck @34; + upgradeConfigAck @35; + snapshotTransfer @36; + walCatchupEntry @37; } struct PeerInfo $C.mapname("xce_peerinfo_t") { @@ -552,6 +558,9 @@ struct RequestReply $C.mapname("xce_reply_t"){ replconfigpushreplymsg @9: ReplConfigPushReply; upgradeconfigreplymsg @10: UpgradeConfigReply; } + + redirectHost @11: Text $C.mapname("redirect_host"); + redirectPort @12: UInt16 $C.mapname("redirect_port"); } struct TaskHost $C.mapname("xce_taskhost_t") { @@ -716,8 +725,10 @@ struct SnapshotHeader $C.mapname("capnp_snap_hdr_t") { # ── HA heartbeat ──────────────────────────────────────────────────── struct HeartbeatMsg $C.mapname("heartbeat_msg_t") { - senderPriority @0: UInt32 $C.mapname("sender_priority"); - term @1: UInt64 $C.mapname("term"); + senderPriority @0: UInt32 $C.mapname("sender_priority"); + term @1: UInt64 $C.mapname("term"); + committedLsn @2: UInt64 $C.mapname("committed_lsn"); + committedConfigTag @3: UInt64 $C.mapname("committed_config_tag"); } # ── Config replication message structs ────────────────────────────── @@ -764,3 +775,55 @@ struct ReplEnvelope $C.mapname("capnp_repl_envelope_t") { crc @4 :UInt32; prevCrc @5 :UInt32; } + +# ── HA election and replication message structs (Task 1.1) ─────────── + +struct RequestVoteRequest $C.mapname("request_vote_req_t") { + term @0 :UInt64 $C.mapname("term"); + candidateId @1 :Text $C.mapname("candidate_id"); + configTag @2 :UInt64 $C.mapname("config_tag"); + lastLogLsn @3 :UInt64 $C.mapname("last_log_lsn"); +} + +struct RequestVoteReply $C.mapname("request_vote_reply_t") { + term @0 :UInt64 $C.mapname("term"); + voteGranted @1 :Bool $C.mapname("vote_granted"); + peerConfigTag @2 :UInt64 $C.mapname("peer_config_tag"); + peerLastLogLsn @3 :UInt64 $C.mapname("peer_last_log_lsn"); +} + +struct ProbeConfigTagRequest $C.mapname("probe_config_tag_req_t") { + name @0 :Text $C.mapname("name"); +} + +struct ProbeConfigTagReply $C.mapname("probe_config_tag_reply_t") { + configTag @0 :UInt64 $C.mapname("config_tag"); +} + +struct WalAckRequest $C.mapname("wal_ack_req_t") { + term @0 :UInt64 $C.mapname("term"); + lsn @1 :UInt64 $C.mapname("lsn"); +} + +struct WalAckReply $C.mapname("wal_ack_reply_t") { + term @0 :UInt64 $C.mapname("term"); + lsn @1 :UInt64 $C.mapname("lsn"); + success @2 :Bool $C.mapname("success"); +} + +struct UpgradeConfigAck $C.mapname("upgrade_config_ack_t") { + term @0 :UInt64 $C.mapname("term"); + ackedTag @1 :UInt64 $C.mapname("acked_tag"); + success @2 :Bool $C.mapname("success"); +} + +struct SnapshotTransfer $C.mapname("snapshot_transfer_t") { + snapshotLsn @0 :UInt64 $C.mapname("snapshot_lsn"); + payload @1 :Data $C.mapname("payload"); +} + +struct WalCatchupEntry $C.mapname("wal_catchup_entry_t") { + lsn @0 :UInt64 $C.mapname("lsn"); + entryType @1 :UInt8 $C.mapname("entry_type"); + payload @2 :Data $C.mapname("payload"); +}