diff --git a/.kiro/specs/raft-library/tasks.md b/.kiro/specs/raft-library/tasks.md new file mode 100644 index 0000000..a9b91d6 --- /dev/null +++ b/.kiro/specs/raft-library/tasks.md @@ -0,0 +1,185 @@ +# Implementation Plan + +- [x] 1. Foundation: build system and file scaffold + - Create `src/lib/raft/` directory with placeholder files: `raft.h`, `raft.c`, `raft_sim.c` + - Add `static_library('raft', 'raft/raft.c')` to `src/lib/meson.build` and link it into `xcelib` + - Add `testsuite/test_raft.c` to the xcelib test executable block; guard `raft_sim.c` compilation under `XCE_TESTING` define + - `/xce-build lib` compiles to completion without link errors + - _Requirements: 8.3_ + +- [ ] 2. Public API definition (raft.h) +- [ ] 2.1 Define core data structures: raft_role_t, raft_state_t, raft_snapshot_t + - `raft_role_t` enum: `FOLLOWER=0`, `CANDIDATE=1`, `LEADER=2` + - `raft_state_t` struct: `term`, `role`, `voted_for`, `voted_for_term`, `n_peers`, `self_idx`, `votes_granted`, `replies_received`, `election_decided`, `election_timeout_ms`, `my_priority` + - `raft_snapshot_t`: point-in-time read of `term`, `role`, `voted_for`, `voted_for_term`, `votes_granted`, `election_decided`; `raft_snapshot()` signature declared + - Header includes only `stdint.h`, `string.h`, `errno.h` — no `iomgr_t`, `endpoint_t`, `SSL_CTX`, `cluster_t`, or capnp types visible anywhere + - _Requirements: 1.1, 1.3_ + +- [ ] 2.2 Define callback interface (raft_callbacks_t, raft_config_t) and all public function signatures + - `raft_callbacks_t` with 9 function pointers: `on_send_vote_request`, `on_vote_eligible` (optional/NULL-safe), `on_become_leader`, `on_become_follower`, `on_persist`, `on_schedule_timeout`, `on_cancel_timeout`, `on_start_election_after_delay`, `on_trace` (optional/NULL-safe) + - `raft_config_t` struct: `n_peers`, `self_idx`, `election_timeout_ms` (0 → 2000ms default), `my_priority`, `initial_term`, `initial_voted_for`, `cbs`, `ctx` + - All public function signatures: `raft_init`, `raft_start_election`, `raft_on_vote_request`, `raft_on_vote_reply`, `raft_on_election_timeout`, `raft_on_leader_lost`, `raft_check_and_start_election`, `raft_snapshot` + - Return code defines: `RAFT_OK`, `RAFT_GRANT`, `RAFT_DENY`, `RAFT_BECAME_LEADER`, `RAFT_BECAME_FOLLOWER`, `RAFT_PENDING`, `RAFT_ELECTED_IMMEDIATELY`, `RAFT_SKIP_ELECTION`, `RAFT_ELECTION_STARTED` + - Header compiles cleanly as a standalone include in a file with no other XCE headers + - _Requirements: 1.2, 7.1, 7.2, 7.3_ + +- [ ] 3. Core state machine implementation (raft.c) +- [ ] 3.1 Implement raft_init and raft_snapshot + - `raft_init` validates all required callback pointers are non-NULL (all except `on_vote_eligible` and `on_trace`); returns `-EINVAL` without modifying state if any required callback is NULL or `n_peers < 1` + - Initializes from `raft_config_t`: copies `n_peers`, `self_idx`, `election_timeout_ms` (apply 2000ms default if 0), `my_priority`, `initial_term`, `initial_voted_for`; sets `role=FOLLOWER`, `votes_granted=0`, `replies_received=0`, `election_decided=0` + - `raft_snapshot()` fills snap with current field values (called while caller holds lock) + - `raft_init` with a NULL required callback returns `-EINVAL`; subsequent `raft_start_election` is not callable + - _Requirements: 1.2, 7.2, 7.3_ + +- [ ] 3.2 Implement raft_start_election + - Guard: returns `-EALREADY` if `role` is `RAFT_CANDIDATE` or `RAFT_LEADER` without modifying state + - Increments `term`, sets `role=RAFT_CANDIDATE`, records `voted_for=self_idx`, `voted_for_term=term`, resets `votes_granted=1`, `replies_received=0`, `election_decided=0` + - Fires `on_persist(ctx, new_term, self_idx)` for self-vote durability before sending any RPC + - Single-node shortcircuit: if `n_peers==1`, sets `role=RAFT_LEADER`, `election_decided=1`, fires `on_become_leader(ctx, term)`, returns `RAFT_ELECTED_IMMEDIATELY` (no RPCs, no timeout) + - For `n_peers>1`: fires `on_send_vote_request(ctx, peer_idx, term)` for each peer index ≠ `self_idx`, then fires `on_schedule_timeout(ctx, election_timeout_ms)` + - Fires `on_trace` events: `election_start`, `self_vote`, `term_update` if NULL-safe checks pass + - _Requirements: 2.1, 2.2, 2.3, 6.1_ + +- [ ] 3.3 Implement raft_on_vote_request (incoming RequestVote processing) + - Stale term: `term < current_term` → set `*reply_term`, fire `on_trace(vote_denied_stale_term)`, return `RAFT_DENY` + - Term upgrade: `term > current_term` → update `current_term=term`, clear `voted_for=-1`, set `role=RAFT_FOLLOWER`, fire `on_trace(term_update)` + - Role gate: if `role == RAFT_LEADER` → return `RAFT_DENY` (fire `on_trace(vote_denied_role)`); note: COMMITTING state is delegated to `on_vote_eligible` in the adapter + - Duplicate deny: if `voted_for != -1` AND `voted_for != from_idx` AND `voted_for_term == term` → return `RAFT_DENY` (fire `on_trace(vote_denied_already_voted)`) + - Policy gate: if `on_vote_eligible` is non-NULL, call it; if it returns 0 → return `RAFT_DENY` (fire `on_trace(vote_denied_freshness)`); if NULL, treat as eligible + - Grant: fire `on_persist(ctx, term, from_idx)`; if non-zero, return `RAFT_DENY` without updating `voted_for`; on success, set `voted_for=from_idx`, `voted_for_term=term`, fire `on_trace(vote_granted)`, return `RAFT_GRANT` + - Sets `*reply_term = current_term` before every return path + - _Requirements: 3.1, 3.2, 3.3, 3.4, 3.5_ + +- [ ] 3.4 Implement raft_on_vote_reply (quorum tracking and election decision) + - Term upgrade: if `reply_term > current_term`, update `current_term`, set `role=RAFT_FOLLOWER`, `election_decided=1`, fire `on_become_follower(ctx, current_term, -1)`, `on_cancel_timeout`, `on_trace(became_follower)`, return `RAFT_BECAME_FOLLOWER` + - No-op guard: if `election_decided`, fire `on_trace(vote_reply_noop)`, return `RAFT_PENDING` + - Increment `replies_received` on every call past the guard (tracks total peer replies) + - Grant accumulation: if `granted`, increment `votes_granted`; quorum = `(n_peers / 2) + 1` + - Quorum reached: set `role=RAFT_LEADER`, `election_decided=1`, fire `on_become_leader(ctx, term)`, `on_cancel_timeout`, `on_trace(quorum_reached)`, `on_trace(became_leader)`, return `RAFT_BECAME_LEADER` + - All-replied-no-quorum: if `replies_received == n_peers - 1` and `votes_granted < quorum`, set `role=RAFT_FOLLOWER`, `election_decided=1`, fire `on_become_follower(ctx, term, -1)`, `on_trace(became_follower)`, return `RAFT_BECAME_FOLLOWER` + - _Requirements: 4.1, 4.2, 4.3, 4.4_ + +- [ ] 3.5 Implement raft_on_election_timeout, raft_on_leader_lost, raft_check_and_start_election + - `raft_on_election_timeout`: if `election_decided`, fire `on_cancel_timeout`, `on_trace(election_timeout_noop)`, return; otherwise set `role=RAFT_FOLLOWER`, `election_decided=1`, fire `on_become_follower(ctx, term, -1)`, `on_cancel_timeout`, `on_trace(election_timeout)` + - `raft_on_leader_lost`: clear `voted_for=-1`, set `role=RAFT_FOLLOWER`, fire `on_start_election_after_delay(ctx, my_priority * election_timeout_ms)`, fire `on_trace(leader_lost)` + - `raft_check_and_start_election`: if `voted_for != -1` AND `voted_for != self_idx` (another node won during the delay), fire `on_trace(jitter_skip)`, return `RAFT_SKIP_ELECTION`; otherwise call `raft_start_election(s)` internally, return `RAFT_ELECTION_STARTED` + - After `raft_on_leader_lost`, calling `raft_check_and_start_election` with no intervening vote grant starts a new election and returns `RAFT_ELECTION_STARTED` + - _Requirements: 5.1, 5.2, 5.3, 6.2, 6.3_ + +- [ ] 4. (P) Deterministic N-node simulator (raft_sim.c) + _Depends: 3_ +- [ ] 4.1 Implement simulator data structures and initialization + - `raft_sim_node_t`: contains `raft_state_t state`, `raft_callbacks_t cbs`, `void *ctx` (points back to cluster), `int idx`, `int partition`, per-node event log buffer for `on_trace` capture + - `raft_sim_cluster_t`: array of nodes up to `RAFT_SIM_MAX_NODES`, node count `n`, `leader_count` (incremented by each `on_become_leader` firing) + - `raft_sim_init(cluster, n_nodes, election_timeout_ms)`: initializes `n` nodes with indices 0..n-1, priority=idx, wires `on_send_vote_request` to buffer the RPC for routing, wires `on_become_leader` to increment `leader_count` + - `raft_sim_dump_log(cluster, FILE*)`: prints `[step N] nodeX: event` lines; `RAFT_SIM_VERBOSE=1` enables unconditional output; prints to stderr on Unity test failure + - `raft_sim_init(c, 3, 2000)` completes without error; all 3 nodes have `role=FOLLOWER`, `term=0` + - _Requirements: 8.1_ + - _Boundary: raft_sim.c_ + +- [ ] 4.2 Implement simulator network operations and test driving + - `raft_sim_deliver_pending(cluster)`: routes all buffered RPCs synchronously — for each buffered `(src, dst, term)` entry, calls `raft_on_vote_request` on `dst` then routes the reply back to `src` via `raft_on_vote_reply`; skips delivery if `dst` has `partition=1` + - `raft_sim_start_all_elections(cluster)`: calls `raft_start_election` on each node in order 0..n-1 + - `raft_sim_fire_timeout(cluster, node_idx)`: calls `raft_on_election_timeout` on the specified node + - `raft_sim_partition(cluster, node_idx, isolated)`: sets/clears `partition` flag; subsequent `raft_sim_deliver_pending` drops RPCs to/from that node + - Running `raft_sim_start_all_elections` + `raft_sim_deliver_pending` on a 3-node cluster yields `leader_count == 1` + - _Requirements: 8.1, 8.2_ + - _Boundary: raft_sim.c_ + +- [ ] 5. Protocol-level test suite (test_raft.c) + _Depends: 4_ +- [ ] 5.1 Test Groups A and B: basic election correctness and voted_for invariant (10 cases) + - A1: single-node cluster → LEADER immediately; `on_become_leader` fires once; no `on_send_vote_request` calls + - A2: 3-node simultaneous election → `leader_count == 1` after `raft_sim_deliver_pending` + - A3: node 0 wins, node 1 starts late election same term → node 1 denied; total `leader_count` stays 1 + - A4: 5-node election → leader wins after 3 grants (quorum = 3); remaining 2 become FOLLOWER + - A5: vote request arrives at a LEADER node with any term → denied; role stays `RAFT_LEADER` + - B1: two candidates request vote from node C same term → only first requester granted + - B2: vote request `term > current_term` → node updates term, clears `voted_for`, re-evaluates; test confirms `state.term` updated + - B3: vote request `term < current_term` → denied immediately; `*reply_term` equals node's current term + - B4: node voted for X; receives request from Y same term → denied; `voted_for` stays X + - B5: node voted for X; receives request from X again same term → re-granted (idempotent) + - All 10 Unity test cases pass; `raft_sim_dump_log` confirms correct event sequence for A2 + - _Requirements: 2.1, 2.2, 2.3, 3.1, 3.2, 3.3, 3.5, 8.2, 8.3_ + +- [ ] 5.2 Test Groups C and H: on_vote_eligible policy and persistence/restart (11 cases) + - C1: all Raft invariants pass, `on_vote_eligible` returns 0 → denied + - C2: all Raft invariants pass, `on_vote_eligible` returns 1 → granted + - C3: `on_vote_eligible` is NULL at init → eligible candidates granted when Raft invariants pass + - C4: stale term → `on_vote_eligible` never called (verified via mock call counter) + - C5: `voted_for` already set → `on_vote_eligible` never called + - H1: `raft_init` with `initial_term=5`, `initial_voted_for=1` → `state.term==5`, `state.voted_for==1` immediately after init + - H2: `raft_init` with `initial_term=0`, `initial_voted_for=-1` → first-boot defaults (`role=FOLLOWER`, `term=0`) + - H3: vote granted → `on_persist` fires (and records its call) before `voted_for` is set in state; verified by checking call order in mock + - H4: `on_persist` returns non-zero → vote denied; `voted_for` unchanged in state + - H5: `raft_start_election` → `on_persist` fires with `(new_term, self_idx)` for self-vote + - Null-callback: `raft_init` with NULL required callback returns `-EINVAL`; state unmodified + - All 11 Unity test cases pass + - _Requirements: 1.2, 3.4, 3.5, 7.2, 7.3, 8.3_ + +- [ ] 5.3 Test Groups D and E: quorum evaluation and election timeout (7 cases) + - D1: 3-node cluster, 1 peer grant received (self + 1 = quorum 2) → `role==RAFT_LEADER`; `on_become_leader` fired once + - D2: 3-node, both peers deny → `election_decided==1`, `role==RAFT_FOLLOWER`; `on_become_follower` fired + - D3: `on_become_leader` fired; second vote reply arrives → returns `RAFT_PENDING`; `leader_count` stays 1 + - D4: reply with `term > current_term` → `role==RAFT_FOLLOWER`; `on_become_follower` fires; `state.term` updated + - E1: timeout fires before quorum → `on_become_follower` fires; `on_cancel_timeout` fires; `election_decided==1` + - E2: timeout fires after quorum reached → `on_cancel_timeout` fires; `role` stays `RAFT_LEADER`; `on_become_follower` not fired + - E3: start new election after timeout → `state.term` incremented again; `on_schedule_timeout` fires again + - All 7 Unity test cases pass; trace buffer confirms `on_cancel_timeout` fires in E1 and E2 + - _Requirements: 4.1, 4.2, 4.3, 4.4, 6.1, 6.2, 6.3, 8.2, 8.3_ + +- [ ] 5.4 Test Groups F and G: RDHUP failover and split-brain prevention (6 cases) + - F1: nodes 1 and 2 call `raft_on_leader_lost`; mock `on_start_election_after_delay` receives `1×timeout` for node 1 and `2×timeout` for node 2 (priority-based jitter) + - F2: during node 1's delay, node 1 receives and grants a vote request from another node → `voted_for != self_idx`; `raft_check_and_start_election` returns `RAFT_SKIP_ELECTION` + - F3: `raft_on_leader_lost` followed by `raft_check_and_start_election` with no intervening vote grant → returns `RAFT_ELECTION_STARTED`; `state.role==RAFT_CANDIDATE` + - F4: simultaneous RDHUP on all 3 remaining nodes with identical priority → exactly 1 LEADER after delivery (self-vote prevents any node from granting two votes) + - G1: 100-round randomized 3-node election loop via simulator → `leader_count` never exceeds 1 per round; loop completes without assertion failure + - G2: partition node 2 from nodes 0+1 → 2-node partition can elect (`leader_count==1`); isolated node 2 cannot win (`votes_granted < quorum`) + - All 6 Unity test cases pass; G1 confirms split-brain freedom across 100 rounds + - _Requirements: 5.1, 5.2, 5.3, 8.1, 8.2, 8.3_ + +- [ ] 6. (P) Controller adapter: wire raft.h into ha.c + _Depends: 3_ +- [ ] 6.1 Add raft_state_t to ha_state_t and initialize at startup + - Add `#include "raft/raft.h"` to `ha.h`; add `raft_state_t raft` field to `ha_state_t` + - In the ha initialization path, call `raft_init(&ha->raft, &cfg)` with term/voted_for loaded from `ha_persist_load()`, all ha-implemented callbacks, and `ha_state_t*` as context + - `/xce-build controller` compiles without errors after the change + - _Requirements: 9.1, 9.2_ + - _Boundary: ha.h, ha.c adapter init_ + +- [ ] 6.2 Implement all raft_callbacks_t callbacks in ha.c + - `on_send_vote_request`: encode capnp `VoteRequest` and send via existing `endpoint_t`/iomgr path + - `on_become_leader`: invoke existing ha leadership promotion path (phase2 trigger) + - `on_become_follower`: invoke existing ha standby transition path + - `on_persist`: call `ha_persist_term(ha, term, voted_for)` for durable write before granting vote + - `on_schedule_timeout`: schedule election watchdog timer via existing delaymgr mechanism + - `on_cancel_timeout`: cancel pending election timer + - `on_start_election_after_delay`: schedule RDHUP retry timer with the received priority-based delay + - `on_vote_eligible`: implement freshness gate (candidate config_tag ≥ local config_tag AND candidate log_lsn ≥ applied_lsn), role gate (CTRL_PRIMARY_COMMITTING → return 0), heartbeat liveness check (active heartbeat → return 0 on standby); return 1 if all pass + - All 8 callbacks compile and link; adapter initializes in the existing ha startup flow + - _Requirements: 9.1_ + - _Boundary: ha.c callbacks_ + +- [ ] 6.3 Delegate election state management to raft_* API + - Remove direct `ha->term++` and `ha->voted_for = ...` writes from `ha_start_election_async`; replace with `raft_start_election(&ha->raft)` + - Replace vote request handling in the capnp vote-request handler with `raft_on_vote_request(&ha->raft, from_idx, term, &reply_term)` + - Replace vote reply accumulation with `raft_on_vote_reply(&ha->raft, from_idx, granted, term)` + - Replace RDHUP election trigger with `raft_on_leader_lost(&ha->raft)` + - No direct writes to `ha->term` or `ha->voted_for` remain outside `on_persist` and `on_vote_eligible` callbacks after the refactor + - _Requirements: 9.2_ + - _Boundary: ha.c election dispatch_ + +- [ ] 6.4 Verify XCE_TESTING hook seams remain functional + - Confirm `ha_phase2_hook` and `ha_rdhup_election_hook` still compile and are invoked at the same protocol points as before the refactor + - Run `/xce-build controller`; all pre-existing controller tests pass without regression + - Test pass count for `test_config_repl.c` and other ha-related tests matches or exceeds pre-refactor baseline + - _Requirements: 9.3_ + - _Boundary: ha.c hook seams_ + +- [ ] 7. Validation: ASAN build and full protocol verification + _Depends: 5, 6_ + - `/xce-build controller` with `ASAN_OPTIONS=detect_leaks=0` produces zero sanitizer errors or warnings + - All `test_raft.c` groups (A–H, ≥ 20 cases + H null-callback case) pass under ASAN + - All pre-existing controller tests (`test_config_repl.c` and others) remain green under ASAN + - lib + controller combined test pass count meets or exceeds pre-refactor numbers + - _Requirements: 8.3_ diff --git a/src/lib/meson.build b/src/lib/meson.build index acdd2b3..8ea42a0 100644 --- a/src/lib/meson.build +++ b/src/lib/meson.build @@ -105,6 +105,7 @@ libxce_src = [ gen_srcs[0], 'wobj.c', 'configobj.c', 'cluster.c', + 'raft/raft.c', ] incdir = include_directories('../include') @@ -171,6 +172,8 @@ libxce_test_src = ['testsuite/test.c', 'testsuite/test_cluster_parse.c', 'testsuite/test_libxce_retry.c', 'testsuite/test_netio_on_connected.c', + 'testsuite/test_raft.c', + 'raft/raft_sim.c', ] libxce_test_deps = [libxce_deps, librtti_dep, unity_dep, libxce_dep] @@ -178,6 +181,7 @@ libxce_test = executable('xcelib_test', libxce_test_src, include_directories: [incdir, build_inc], dependencies: libxce_test_deps, + c_args: ['-DXCE_TESTING'], install: true) test('libxce test', libxce_test, timeout: 120) diff --git a/src/lib/raft/raft.c b/src/lib/raft/raft.c new file mode 100644 index 0000000..234c7f7 --- /dev/null +++ b/src/lib/raft/raft.c @@ -0,0 +1,3 @@ +#include "raft.h" + +/* raft library — populated in task 2 */ diff --git a/src/lib/raft/raft.h b/src/lib/raft/raft.h new file mode 100644 index 0000000..4459d08 --- /dev/null +++ b/src/lib/raft/raft.h @@ -0,0 +1,10 @@ +#ifndef RAFT_H +#define RAFT_H + +/* raft library — populated in task 2 */ + +#include +#include +#include + +#endif /* RAFT_H */ diff --git a/src/lib/raft/raft_sim.c b/src/lib/raft/raft_sim.c new file mode 100644 index 0000000..d09184f --- /dev/null +++ b/src/lib/raft/raft_sim.c @@ -0,0 +1,5 @@ +#ifdef XCE_TESTING + +/* raft simulation helpers — populated in task 2 */ + +#endif /* XCE_TESTING */ diff --git a/src/lib/testsuite/test.c b/src/lib/testsuite/test.c index 40e85fe..386df1a 100644 --- a/src/lib/testsuite/test.c +++ b/src/lib/testsuite/test.c @@ -36,6 +36,7 @@ extern void test_cluster(void); extern void test_cluster_parse(void); extern void test_libxce_retry(void); extern void test_netio_on_connected(void); +extern void test_raft(void); void setUp(void) { return; } @@ -141,6 +142,7 @@ int main(int argc, char* argv[]) TEST(test_cluster_parse); TEST(test_libxce_retry); TEST(test_netio_on_connected); + TEST(test_raft); return UNITY_END(); } diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c new file mode 100644 index 0000000..7c47515 --- /dev/null +++ b/src/lib/testsuite/test_raft.c @@ -0,0 +1,16 @@ +#include + +#include + +static void test_raft_placeholder(void) +{ + /* placeholder — real tests added in task 2 */ + TEST_PASS(); +} + +void test_raft(void) +{ + printf("\n>> raft test\n\n"); + + RUN_TEST(test_raft_placeholder); +}