#ifndef RAFT_H #define RAFT_H #include #include #include /* raft_role_t: self-contained role enum (does not depend on ctrl_role_t) */ typedef enum { RAFT_FOLLOWER = 0, RAFT_CANDIDATE = 1, RAFT_LEADER = 2, } raft_role_t; /* Upper bound on cluster size for the fixed-size pre-vote reply-tracking * array. Matches the simulator bound (RAFT_SIM_MAX_NODES); real clusters * are far smaller. */ #define RAFT_MAX_PEERS 8 /* Forward declaration — full definition follows raft_state_t */ typedef struct raft_callbacks_t raft_callbacks_t; /* raft_state_t: pure Raft consensus state for one node. * * Contains only the Raft invariant state. Application-specific fields * (config_tag, log_lsn, heartbeat timestamps, ctrl_role) stay in ha_state_t; * they are accessed by the caller via the on_vote_eligible callback. */ typedef struct { uint64_t term; /* current term (monotonically increasing) */ raft_role_t role; /* current role */ int voted_for; /* peer index voted for this term; -1=none; self_idx=self-vote */ uint64_t voted_for_term; /* term in which voted_for was set */ int n_peers; /* total cluster size (including self) */ int self_idx; /* this node's index in [0, n_peers) */ int votes_granted; /* votes collected in current election */ int replies_received; /* total peer replies received (for all-replied-no-quorum, Req 4.4) */ int election_decided; /* 1 once on_become_leader/follower fired */ uint32_t election_timeout_ms; int my_priority; /* priority index for jitter (0 = highest) */ /* ---- PreVote round tally (Req 1.2) ---- * Self-contained pre-vote bookkeeping. A pre-vote round mutates NONE of * the persistent Raft state above (term, voted_for, role); these fields * only track the in-flight straw poll and are reset at each round start. */ int pre_vote_in_progress; /* 1 while a pre-vote round is active */ int pre_votes_granted; /* grants collected this round (incl. self) */ int pre_vote_replies[RAFT_MAX_PEERS]; /* per-peer replied flag; avoids double-count */ /* ---- Log replication state (raft-log-replication, raft.tla logVars/leaderVars) ---- * The library stores NO entry bytes and NO per-entry term array — entry terms * are obtained via on_get_entry_term; bytes live in the adapter/simulator. This * keeps raft_state_t fixed-size and the core pure (same discipline as not * storing (term, voted_for) bytes). Index is 1-based (raft.tla) and maps 1:1 * to the WAL LSN. */ uint64_t log_len; /* highest local log index (0 = empty) */ uint64_t next_index[RAFT_MAX_PEERS]; /* leader: next index to send each peer */ uint64_t match_index[RAFT_MAX_PEERS]; /* leader: highest index known replicated to each peer */ /* set by raft_init — needed by the state machine to fire callbacks */ const raft_callbacks_t *cbs; void *ctx; } raft_state_t; /* raft_snapshot_t: point-in-time read of all Raft state for logging/debugging */ typedef struct { uint64_t term; raft_role_t role; int voted_for; uint64_t voted_for_term; int votes_granted; int election_decided; } raft_snapshot_t; /* raft_callbacks_t: all side effects the library fires. * * The library owns Raft invariants (term ordering, one-vote-per-term, quorum). * All application-specific policy (freshness gate, role gate, catch-up state, * heartbeat liveness) is delegated to on_vote_eligible so the library is * fully independent of XCE domain logic. * * REQUIRED callbacks (must be non-NULL at raft_init): * on_send_vote_request, on_become_leader, on_become_follower, * on_persist, on_schedule_timeout, on_cancel_timeout, * on_start_election_after_delay, * on_persist_entry, on_get_entry_term, on_send_append_entries, on_truncate_from * * OPTIONAL callbacks (may be NULL): * on_vote_eligible — if NULL, all Raft-eligible candidates are granted * on_send_pre_vote_request — if NULL, pre-vote emits are suppressed * on_trace — if NULL, tracing is suppressed */ typedef struct raft_callbacks_t { /* REQUIRED: Send a RequestVote RPC to peer peer_idx for the given term. * The caller encodes config_tag, log_lsn, and any other fields in the wire RPC. */ int (*on_send_vote_request)(void *ctx, int peer_idx, uint64_t term); /* OPTIONAL: Send a PreVote (straw-poll) request to peer peer_idx for the * prospective term (current term + 1). Unlike on_send_vote_request, this * MUST NOT cause the peer to persist or mutate term/voted_for — it is a * non-binding probe. If NULL, the raft library suppresses the emit and * the caller is responsible for any pre-vote transport itself. */ int (*on_send_pre_vote_request)(void *ctx, int peer_idx, uint64_t prospective_term); /* OPTIONAL: Application-specific vote eligibility gate. * Called after Raft invariants pass (correct term, voted_for unset). * Returns 1 to grant, 0 to deny. Receives the candidate's peer_idx and * the incoming term so the caller can check freshness (config_tag, log_lsn), * role gate (LEADER/COMMITTING deny), catch-up status, heartbeat liveness. * If NULL, the library grants the vote whenever Raft invariants pass. */ int (*on_vote_eligible)(void *ctx, int from_idx, uint64_t term); /* REQUIRED: Node has won the election */ void (*on_become_leader)(void *ctx, uint64_t term); /* REQUIRED: Node lost election or saw higher term; leader_idx = -1 if unknown */ void (*on_become_follower)(void *ctx, uint64_t term, int leader_idx); /* REQUIRED: Persist term + voted_for before granting vote (durability guarantee) */ int (*on_persist)(void *ctx, uint64_t term, int voted_for); /* REQUIRED: Schedule election timeout (fires raft_on_election_timeout after delay_ms) */ void (*on_schedule_timeout)(void *ctx, uint32_t delay_ms); /* REQUIRED: Cancel pending election timeout */ void (*on_cancel_timeout)(void *ctx); /* REQUIRED: Request delayed election start after RDHUP (priority-based jitter) */ void (*on_start_election_after_delay)(void *ctx, uint32_t delay_ms); /* ---- Log replication callbacks (raft-log-replication) ---- * REQUIRED (all four non-NULL at raft_init). The library owns the protocol * decisions; these perform the I/O. The library stores no entry bytes, so it * reads entry terms only via on_get_entry_term and writes/truncates only via * these callbacks. */ /* REQUIRED: durably append/overwrite the log entry at 1-based `index` with * `term` (persist-before-act: called before an append/accept is reported). */ int (*on_persist_entry)(void *ctx, uint64_t index, uint64_t term, const void *entry, size_t len); /* REQUIRED: term of the local entry at 1-based `index` (0 if absent). */ uint64_t (*on_get_entry_term)(void *ctx, uint64_t index); /* REQUIRED: transmit one AppendEntries to peer_idx with prevLog{Index,Term} * and leader_commit. `n_entries` is 0 (heartbeat) or 1 (this spec, like * raft.tla); when 1 the adapter sources the single entry at prev_log_index+1 * from its own log (the library holds no entry bytes). [Concretizes the * design's varargs "entries handle" placeholder into an explicit count.] */ int (*on_send_append_entries)(void *ctx, int peer_idx, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, int n_entries); /* REQUIRED: delete local entries at `index`..end. */ int (*on_truncate_from)(void *ctx, uint64_t index); /* OPTIONAL: Trace callback for observability. * event: short string literal, e.g. "vote_granted", "became_leader" * detail: formatted human-readable context string * If NULL, tracing is suppressed. */ void (*on_trace)(void *ctx, const char *event, const char *detail); } raft_callbacks_t; /* * raft_config_t: initialization context for a Raft node. * * Encapsulates all configuration so raft_init() has a single, extensible * parameter rather than a flat list. Matches the existing XCE pattern * (ha_config_t, persist_config_t). Zero-initialize and set only the * fields you need — the library fills in safe defaults for unset fields. * * Persistence model: * initial_term / initial_voted_for are restored by the caller from disk * before raft_init(). The library never reads storage; it only writes * via on_persist callbacks. Pass {0, -1} on first boot. */ typedef struct { int n_peers; /* total cluster size (including self) */ int self_idx; /* this node's index in [0, n_peers) */ uint32_t election_timeout_ms; /* base election timeout; 0 → default 2000ms */ int my_priority; /* priority index for RDHUP jitter (0 = highest) */ uint64_t initial_term; /* term restored from disk; 0 on first boot */ int initial_voted_for; /* voted_for restored from disk; -1 if none */ uint64_t initial_log_len; /* WAL length restored from disk; 0 on first boot (library never reads storage) */ const raft_callbacks_t *cbs; /* all callbacks; required callbacks must be non-NULL */ void *ctx; /* opaque caller context passed to every callback */ } raft_config_t; /* Return codes * * Note: RAFT_GRANT and RAFT_BECAME_LEADER share value 1; RAFT_DENY and RAFT_OK * share value 0. This is intentional — different return values are used by * different functions (raft_on_vote_request vs raft_on_vote_reply) in different * call contexts. */ #define RAFT_OK 0 #define RAFT_GRANT 1 #define RAFT_DENY 0 #define RAFT_BECAME_LEADER 1 #define RAFT_BECAME_FOLLOWER 2 #define RAFT_PENDING 3 #define RAFT_ELECTED_IMMEDIATELY 4 #define RAFT_SKIP_ELECTION 5 #define RAFT_ELECTION_STARTED 6 /* PreVote (straw-poll) outcomes returned by raft_on_pre_vote_reply. * Distinct values from the election codes above; a pre-vote round mutates * no persistent state, so these never imply a term/role transition. */ #define RAFT_PRE_VOTE_WON 7 /* quorum of grants reached this round */ #define RAFT_PRE_VOTE_LOST 8 /* all peers replied, no quorum */ #define RAFT_PRE_VOTE_PENDING 9 /* still awaiting replies */ /* * raft_init: initialize raft_state_t from a raft_config_t. * on_vote_eligible and on_trace may be NULL (optional policy / observability). * All other callbacks in cbs must be non-NULL. * Returns 0 on success; -EINVAL if a required callback is NULL or n_peers < 1. */ int raft_init(raft_state_t *s, const raft_config_t *cfg); /* * raft_start_election: begin a new election. * Increments term, records self-vote (voted_for=self_idx), transitions to * CANDIDATE, fires on_send_vote_request for each peer, fires on_schedule_timeout. * Returns 0 on success; -EALREADY if already CANDIDATE/LEADER; * RAFT_ELECTED_IMMEDIATELY for single-node (n_peers==1). */ int raft_start_election(raft_state_t *s); /* * raft_on_vote_request: process an incoming RequestVote RPC. * Applies Raft invariants (term comparison, one-vote-per-term). * If invariants pass, calls on_vote_eligible to allow application policy. * Fires on_persist before granting. * Returns RAFT_GRANT (1) or RAFT_DENY (0); sets *reply_term to current term. */ int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term, uint64_t *reply_term); /* * raft_on_vote_reply: process a reply to our RequestVote. * Returns RAFT_BECAME_LEADER, RAFT_BECAME_FOLLOWER, or RAFT_PENDING. */ int raft_on_vote_reply(raft_state_t *s, int from_idx, int granted, uint64_t term); /* * raft_on_election_timeout: called by the timer when no quorum was reached. * Transitions CANDIDATE → FOLLOWER; fires on_become_follower, on_cancel_timeout. */ void raft_on_election_timeout(raft_state_t *s); /* * raft_on_leader_lost: primary control channel dropped (RDHUP). * Clears voted_for, transitions to FOLLOWER, fires on_start_election_after_delay. */ void raft_on_leader_lost(raft_state_t *s); /* * raft_check_and_start_election: call after the delay from raft_on_leader_lost. * Starts election only if we have not already voted for a non-self candidate * in the new term (i.e., another node won during the jitter window). * Returns RAFT_SKIP_ELECTION or RAFT_ELECTION_STARTED. */ int raft_check_and_start_election(raft_state_t *s); /* ---------- PreVote (straw poll before a binding election) ---------- * * The pre-vote round is a non-binding probe: it asks peers whether they WOULD * grant a vote at term+1, without anyone incrementing term, setting voted_for, * or persisting. Only on RAFT_PRE_VOTE_WON does the caller proceed to the * unchanged raft_start_election (which performs the real term bump + persist). * * INVARIANT (Req 1.2, 6.2): none of these three functions mutate s->term, * s->voted_for, s->voted_for_term, or s->role, and none invoke on_persist. */ /* * raft_begin_pre_vote: start a pre-vote round. * Computes prospective term = term + 1 (NOT assigned to s->term), sets * pre_vote_in_progress, pre_votes_granted = 1 (self), and fires * on_send_pre_vote_request(ctx, peer_idx, prospective_term) for each peer * (NULL-checked — suppressed if the callback is unset). * Precondition: role == RAFT_FOLLOWER. */ int raft_begin_pre_vote(raft_state_t *s); /* * raft_on_pre_vote_request: answer an incoming PreVote probe. * Denies if cand_term < s->term; otherwise returns on_vote_eligible(ctx, * from_idx, cand_term) (granting when the gate is NULL). Never touches term, * voted_for, role, or fires on_persist. Freshness is applied by the * dispatcher before this call (same contract as the real vote path). * Returns 1 = grant, 0 = deny. */ int raft_on_pre_vote_request(raft_state_t *s, int from_idx, uint64_t cand_term); /* * raft_on_pre_vote_reply: tally a reply to our PreVote. * Owns the pre-vote tally in raft_state_t (pre_votes_granted, * pre_vote_replies), mirroring raft_on_vote_reply but WITHOUT firing * on_become_leader/on_become_follower or making any role/term change. * Returns RAFT_PRE_VOTE_WON at quorum, RAFT_PRE_VOTE_LOST when all peers * replied without quorum, else RAFT_PRE_VOTE_PENDING. */ int raft_on_pre_vote_reply(raft_state_t *s, int peer_idx, int granted); /* ---------- Log replication (raft-log-replication) ---------- * * The library owns the log protocol decisions (append, AppendEntries logOk + * conflict truncation, UpdateTerm-on-any-RPC) and drives I/O through the four * log callbacks. Index is 1-based (raft.tla) == WAL LSN. Commit-index advance * and (lastLogTerm,lastLogIndex) vote freshness are NOT here — they are * raft-commit-safety. */ /* Result of handling an AppendEntries request — the adapter replies with this. */ typedef struct { uint64_t reply_term; /* current term to return to the leader */ int success; /* 1 = accepted, 0 = rejected */ uint64_t match_index; /* on success: prev_log_index + entries_len; else 0 */ } raft_append_result_t; /* * raft_log_append: leader appends one client entry (raft.tla ClientRequest, 246). * Stamps the entry with s->term, assigns index log_len+1, persists it via * on_persist_entry BEFORE returning success (persist-before-act). Does NOT * advance any commit index (owned by raft-commit-safety). * out_index receives the assigned 1-based index (== WAL LSN) on success. * Returns RAFT_OK; -EPERM if role != RAFT_LEADER; propagates a non-zero * on_persist_entry return on persist failure (no log mutation on failure). */ int raft_log_append(raft_state_t *s, const void *entry, size_t len, uint64_t *out_index); /* * raft_send_append_entries: emit one AppendEntries to peer_idx (raft.tla * AppendEntries, 204). Computes prev_log_index = next_index[peer]-1 and * prev_log_term via on_get_entry_term, sends at most one entry (n_entries 0/1), * via on_send_append_entries. Leader-only (R3.5); leader_commit is opaque. * Returns RAFT_OK. */ int raft_send_append_entries(raft_state_t *s, int peer_idx, uint64_t leader_commit); /* * raft_on_append_entries_request: handle an incoming AppendEntries / heartbeat * (raft.tla HandleAppendEntriesRequest 327 + UpdateTerm 406). Step order is * fixed: UpdateTerm first (G5, every RPC incl. heartbeat) → logOk (G2) → * reject / candidate-step-down / accept (already-have / conflict-truncate (G3) / * append). entries_len is 0 (heartbeat) or 1. Fills *out. leader_commit is * accepted but NOT acted on here (raft-commit-safety). Returns RAFT_OK. */ int raft_on_append_entries_request(raft_state_t *s, int leader_idx, uint64_t term, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, const void *entry, size_t entry_len, uint64_t entry_term, int entries_len, raft_append_result_t *out); /* * raft_on_append_entries_reply: process a follower's AppendEntries response * (raft.tla HandleAppendEntriesResponse 393). Applied only when * reply_term == s->term (stale-term drop, 415). success: next_index[peer] = * match_index+1, match_index[peer] = match_index. failure: next_index[peer] = * Max(next_index[peer]-1, 1). Does NOT advance commit. Returns RAFT_OK. */ int raft_on_append_entries_reply(raft_state_t *s, int peer_idx, int success, uint64_t match_index, uint64_t reply_term); /* Fills snap with current state. Caller must hold lock. */ void raft_snapshot(const raft_state_t *s, raft_snapshot_t *snap); /* ---------- deterministic simulator (test builds only) ---------- */ #ifdef XCE_TESTING #include #define RAFT_SIM_MAX_NODES 8 /* Buffered RPC entry: one pending vote-request to route */ typedef struct { int src; int dst; uint64_t term; } raft_sim_rpc_t; /* One simulated node */ typedef struct { raft_state_t state; raft_callbacks_t cbs; /* wired to sim routing functions */ void *ctx; /* points back to raft_sim_cluster_t */ int idx; int partition; /* 1 = isolated from other nodes */ /* per-node trace event log for test assertions */ char trace_buf[4096]; int trace_len; } raft_sim_node_t; /* N-node simulation cluster */ typedef struct { raft_sim_node_t nodes[RAFT_SIM_MAX_NODES]; int n; int leader_count; /* incremented by each on_become_leader */ /* buffered RPC queue */ raft_sim_rpc_t pending[64]; int pending_count; } raft_sim_cluster_t; /* * raft_sim_init: initialize an n-node simulation cluster. * Each node gets index 0..n-1, priority=idx, election_timeout_ms as given. * Wires on_send_vote_request to buffer RPCs into c->pending. * Wires on_become_leader to increment c->leader_count. * All other required callbacks get no-op implementations. * on_trace appends to the node's trace_buf. * Returns 0 on success. */ int raft_sim_init(raft_sim_cluster_t *c, int n_nodes, uint32_t election_timeout_ms); /* * raft_sim_dump_log: print trace logs for all nodes to fp. * If RAFT_SIM_VERBOSE env var is set, always prints. */ void raft_sim_dump_log(raft_sim_cluster_t *c, FILE *fp); /* * raft_sim_deliver_pending: route all buffered RPCs synchronously. * For each buffered (src, dst, term), calls raft_on_vote_request on dst * then routes the reply back to src via raft_on_vote_reply. * Skips delivery if dst has partition=1. */ void raft_sim_deliver_pending(raft_sim_cluster_t *c); /* * raft_sim_start_all_elections: call raft_start_election on each node 0..n-1. */ void raft_sim_start_all_elections(raft_sim_cluster_t *c); /* * raft_sim_fire_timeout: call raft_on_election_timeout on node at node_idx. */ void raft_sim_fire_timeout(raft_sim_cluster_t *c, int node_idx); /* * raft_sim_partition: set/clear partition flag for node_idx. * Subsequent raft_sim_deliver_pending drops RPCs to/from that node. */ void raft_sim_partition(raft_sim_cluster_t *c, int node_idx, int isolated); #endif /* XCE_TESTING */ #endif /* RAFT_H */