wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/lib/raft/raft.h
Rongsong Shen 6b5b201a64 feat(raft-log-replication): header contract — log model, callbacks, entry points (tasks 1.1-1.3) 
Extend src/lib/raft/raft.h for the TLA+ log half (raft.tla logVars/leaderVars):
- raft_state_t: log_len, next_index[], match_index[] (no entry bytes stored —
  terms via on_get_entry_term, bytes in the adapter); raft_config_t.initial_log_len.
- raft_callbacks_t: 4 REQUIRED log callbacks on_persist_entry, on_get_entry_term,
  on_send_append_entries, on_truncate_from.
- Entry points + raft_append_result_t: raft_log_append (ClientRequest),
  raft_send_append_entries (AppendEntries emit), raft_on_append_entries_request
  (Handle + UpdateTerm-first), raft_on_append_entries_reply.

Concretized on_send_append_entries's varargs "entries handle" placeholder to an
explicit n_entries (0/1); design.md updated to match. Header compiles clean with
and without XCE_TESTING (-Wall -Wextra -fsyntax-only).
2026-06-21 14:15:15 +00:00

446 lines
20 KiB
C
Raw Blame History

#ifndef RAFT_H
#define RAFT_H
#include <errno.h>
#include <stdint.h>
#include <string.h>
/* raft_role_t: self-contained role enum (does not depend on ctrl_role_t) */
typedef enum {
RAFT_FOLLOWER = 0,
RAFT_CANDIDATE = 1,
RAFT_LEADER = 2,
} raft_role_t;
/* Upper bound on cluster size for the fixed-size pre-vote reply-tracking
* array. Matches the simulator bound (RAFT_SIM_MAX_NODES); real clusters
* are far smaller. */
#define RAFT_MAX_PEERS 8
/* Forward declaration — full definition follows raft_state_t */
typedef struct raft_callbacks_t raft_callbacks_t;
/* raft_state_t: pure Raft consensus state for one node.
*
* Contains only the Raft invariant state. Application-specific fields
* (config_tag, log_lsn, heartbeat timestamps, ctrl_role) stay in ha_state_t;
* they are accessed by the caller via the on_vote_eligible callback. */
typedef struct {
uint64_t term; /* current term (monotonically increasing) */
raft_role_t role; /* current role */
int voted_for; /* peer index voted for this term; -1=none; self_idx=self-vote */
uint64_t voted_for_term; /* term in which voted_for was set */
int n_peers; /* total cluster size (including self) */
int self_idx; /* this node's index in [0, n_peers) */
int votes_granted; /* votes collected in current election */
int replies_received; /* total peer replies received (for all-replied-no-quorum, Req 4.4) */
int election_decided; /* 1 once on_become_leader/follower fired */
uint32_t election_timeout_ms;
int my_priority; /* priority index for jitter (0 = highest) */
/* ---- PreVote round tally (Req 1.2) ----
* Self-contained pre-vote bookkeeping. A pre-vote round mutates NONE of
* the persistent Raft state above (term, voted_for, role); these fields
* only track the in-flight straw poll and are reset at each round start. */
int pre_vote_in_progress; /* 1 while a pre-vote round is active */
int pre_votes_granted; /* grants collected this round (incl. self) */
int pre_vote_replies[RAFT_MAX_PEERS]; /* per-peer replied flag; avoids double-count */
/* ---- Log replication state (raft-log-replication, raft.tla logVars/leaderVars) ----
* The library stores NO entry bytes and NO per-entry term array — entry terms
* are obtained via on_get_entry_term; bytes live in the adapter/simulator. This
* keeps raft_state_t fixed-size and the core pure (same discipline as not
* storing (term, voted_for) bytes). Index is 1-based (raft.tla) and maps 1:1
* to the WAL LSN. */
uint64_t log_len; /* highest local log index (0 = empty) */
uint64_t next_index[RAFT_MAX_PEERS]; /* leader: next index to send each peer */
uint64_t match_index[RAFT_MAX_PEERS]; /* leader: highest index known replicated to each peer */
/* set by raft_init — needed by the state machine to fire callbacks */
const raft_callbacks_t *cbs;
void *ctx;
} raft_state_t;
/* raft_snapshot_t: point-in-time read of all Raft state for logging/debugging */
typedef struct {
uint64_t term;
raft_role_t role;
int voted_for;
uint64_t voted_for_term;
int votes_granted;
int election_decided;
} raft_snapshot_t;
/* raft_callbacks_t: all side effects the library fires.
*
* The library owns Raft invariants (term ordering, one-vote-per-term, quorum).
* All application-specific policy (freshness gate, role gate, catch-up state,
* heartbeat liveness) is delegated to on_vote_eligible so the library is
* fully independent of XCE domain logic.
*
* REQUIRED callbacks (must be non-NULL at raft_init):
* on_send_vote_request, on_become_leader, on_become_follower,
* on_persist, on_schedule_timeout, on_cancel_timeout,
* on_start_election_after_delay,
* on_persist_entry, on_get_entry_term, on_send_append_entries, on_truncate_from
*
* OPTIONAL callbacks (may be NULL):
* on_vote_eligible — if NULL, all Raft-eligible candidates are granted
* on_send_pre_vote_request — if NULL, pre-vote emits are suppressed
* on_trace — if NULL, tracing is suppressed
*/
typedef struct raft_callbacks_t {
/* REQUIRED: Send a RequestVote RPC to peer peer_idx for the given term.
* The caller encodes config_tag, log_lsn, and any other fields in the wire RPC. */
int (*on_send_vote_request)(void *ctx, int peer_idx, uint64_t term);
/* OPTIONAL: Send a PreVote (straw-poll) request to peer peer_idx for the
* prospective term (current term + 1). Unlike on_send_vote_request, this
* MUST NOT cause the peer to persist or mutate term/voted_for — it is a
* non-binding probe. If NULL, the raft library suppresses the emit and
* the caller is responsible for any pre-vote transport itself. */
int (*on_send_pre_vote_request)(void *ctx, int peer_idx,
uint64_t prospective_term);
/* OPTIONAL: Application-specific vote eligibility gate.
* Called after Raft invariants pass (correct term, voted_for unset).
* Returns 1 to grant, 0 to deny. Receives the candidate's peer_idx and
* the incoming term so the caller can check freshness (config_tag, log_lsn),
* role gate (LEADER/COMMITTING deny), catch-up status, heartbeat liveness.
* If NULL, the library grants the vote whenever Raft invariants pass. */
int (*on_vote_eligible)(void *ctx, int from_idx, uint64_t term);
/* REQUIRED: Node has won the election */
void (*on_become_leader)(void *ctx, uint64_t term);
/* REQUIRED: Node lost election or saw higher term; leader_idx = -1 if unknown */
void (*on_become_follower)(void *ctx, uint64_t term, int leader_idx);
/* REQUIRED: Persist term + voted_for before granting vote (durability guarantee) */
int (*on_persist)(void *ctx, uint64_t term, int voted_for);
/* REQUIRED: Schedule election timeout (fires raft_on_election_timeout after delay_ms) */
void (*on_schedule_timeout)(void *ctx, uint32_t delay_ms);
/* REQUIRED: Cancel pending election timeout */
void (*on_cancel_timeout)(void *ctx);
/* REQUIRED: Request delayed election start after RDHUP (priority-based jitter) */
void (*on_start_election_after_delay)(void *ctx, uint32_t delay_ms);
/* ---- Log replication callbacks (raft-log-replication) ----
* REQUIRED (all four non-NULL at raft_init). The library owns the protocol
* decisions; these perform the I/O. The library stores no entry bytes, so it
* reads entry terms only via on_get_entry_term and writes/truncates only via
* these callbacks. */
/* REQUIRED: durably append/overwrite the log entry at 1-based `index` with
* `term` (persist-before-act: called before an append/accept is reported). */
int (*on_persist_entry)(void *ctx, uint64_t index, uint64_t term,
const void *entry, size_t len);
/* REQUIRED: term of the local entry at 1-based `index` (0 if absent). */
uint64_t (*on_get_entry_term)(void *ctx, uint64_t index);
/* REQUIRED: transmit one AppendEntries to peer_idx with prevLog{Index,Term}
* and leader_commit. `n_entries` is 0 (heartbeat) or 1 (this spec, like
* raft.tla); when 1 the adapter sources the single entry at prev_log_index+1
* from its own log (the library holds no entry bytes). [Concretizes the
* design's varargs "entries handle" placeholder into an explicit count.] */
int (*on_send_append_entries)(void *ctx, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries);
/* REQUIRED: delete local entries at `index`..end. */
int (*on_truncate_from)(void *ctx, uint64_t index);
/* OPTIONAL: Trace callback for observability.
* event: short string literal, e.g. "vote_granted", "became_leader"
* detail: formatted human-readable context string
* If NULL, tracing is suppressed. */
void (*on_trace)(void *ctx, const char *event, const char *detail);
} raft_callbacks_t;
/*
* raft_config_t: initialization context for a Raft node.
*
* Encapsulates all configuration so raft_init() has a single, extensible
* parameter rather than a flat list. Matches the existing XCE pattern
* (ha_config_t, persist_config_t). Zero-initialize and set only the
* fields you need — the library fills in safe defaults for unset fields.
*
* Persistence model:
* initial_term / initial_voted_for are restored by the caller from disk
* before raft_init(). The library never reads storage; it only writes
* via on_persist callbacks. Pass {0, -1} on first boot.
*/
typedef struct {
int n_peers; /* total cluster size (including self) */
int self_idx; /* this node's index in [0, n_peers) */
uint32_t election_timeout_ms; /* base election timeout; 0 → default 2000ms */
int my_priority; /* priority index for RDHUP jitter (0 = highest) */
uint64_t initial_term; /* term restored from disk; 0 on first boot */
int initial_voted_for; /* voted_for restored from disk; -1 if none */
uint64_t initial_log_len; /* WAL length restored from disk; 0 on first boot (library never reads storage) */
const raft_callbacks_t *cbs; /* all callbacks; required callbacks must be non-NULL */
void *ctx; /* opaque caller context passed to every callback */
} raft_config_t;
/* Return codes
*
* Note: RAFT_GRANT and RAFT_BECAME_LEADER share value 1; RAFT_DENY and RAFT_OK
* share value 0. This is intentional — different return values are used by
* different functions (raft_on_vote_request vs raft_on_vote_reply) in different
* call contexts.
*/
#define RAFT_OK 0
#define RAFT_GRANT 1
#define RAFT_DENY 0
#define RAFT_BECAME_LEADER 1
#define RAFT_BECAME_FOLLOWER 2
#define RAFT_PENDING 3
#define RAFT_ELECTED_IMMEDIATELY 4
#define RAFT_SKIP_ELECTION 5
#define RAFT_ELECTION_STARTED 6
/* PreVote (straw-poll) outcomes returned by raft_on_pre_vote_reply.
* Distinct values from the election codes above; a pre-vote round mutates
* no persistent state, so these never imply a term/role transition. */
#define RAFT_PRE_VOTE_WON 7 /* quorum of grants reached this round */
#define RAFT_PRE_VOTE_LOST 8 /* all peers replied, no quorum */
#define RAFT_PRE_VOTE_PENDING 9 /* still awaiting replies */
/*
* raft_init: initialize raft_state_t from a raft_config_t.
* on_vote_eligible and on_trace may be NULL (optional policy / observability).
* All other callbacks in cbs must be non-NULL.
* Returns 0 on success; -EINVAL if a required callback is NULL or n_peers < 1.
*/
int raft_init(raft_state_t *s, const raft_config_t *cfg);
/*
* raft_start_election: begin a new election.
* Increments term, records self-vote (voted_for=self_idx), transitions to
* CANDIDATE, fires on_send_vote_request for each peer, fires on_schedule_timeout.
* Returns 0 on success; -EALREADY if already CANDIDATE/LEADER;
* RAFT_ELECTED_IMMEDIATELY for single-node (n_peers==1).
*/
int raft_start_election(raft_state_t *s);
/*
* raft_on_vote_request: process an incoming RequestVote RPC.
* Applies Raft invariants (term comparison, one-vote-per-term).
* If invariants pass, calls on_vote_eligible to allow application policy.
* Fires on_persist before granting.
* Returns RAFT_GRANT (1) or RAFT_DENY (0); sets *reply_term to current term.
*/
int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term,
uint64_t *reply_term);
/*
* raft_on_vote_reply: process a reply to our RequestVote.
* Returns RAFT_BECAME_LEADER, RAFT_BECAME_FOLLOWER, or RAFT_PENDING.
*/
int raft_on_vote_reply(raft_state_t *s, int from_idx, int granted, uint64_t term);
/*
* raft_on_election_timeout: called by the timer when no quorum was reached.
* Transitions CANDIDATE → FOLLOWER; fires on_become_follower, on_cancel_timeout.
*/
void raft_on_election_timeout(raft_state_t *s);
/*
* raft_on_leader_lost: primary control channel dropped (RDHUP).
* Clears voted_for, transitions to FOLLOWER, fires on_start_election_after_delay.
*/
void raft_on_leader_lost(raft_state_t *s);
/*
* raft_check_and_start_election: call after the delay from raft_on_leader_lost.
* Starts election only if we have not already voted for a non-self candidate
* in the new term (i.e., another node won during the jitter window).
* Returns RAFT_SKIP_ELECTION or RAFT_ELECTION_STARTED.
*/
int raft_check_and_start_election(raft_state_t *s);
/* ---------- PreVote (straw poll before a binding election) ----------
*
* The pre-vote round is a non-binding probe: it asks peers whether they WOULD
* grant a vote at term+1, without anyone incrementing term, setting voted_for,
* or persisting. Only on RAFT_PRE_VOTE_WON does the caller proceed to the
* unchanged raft_start_election (which performs the real term bump + persist).
*
* INVARIANT (Req 1.2, 6.2): none of these three functions mutate s->term,
* s->voted_for, s->voted_for_term, or s->role, and none invoke on_persist.
*/
/*
* raft_begin_pre_vote: start a pre-vote round.
* Computes prospective term = term + 1 (NOT assigned to s->term), sets
* pre_vote_in_progress, pre_votes_granted = 1 (self), and fires
* on_send_pre_vote_request(ctx, peer_idx, prospective_term) for each peer
* (NULL-checked — suppressed if the callback is unset).
* Precondition: role == RAFT_FOLLOWER.
*/
int raft_begin_pre_vote(raft_state_t *s);
/*
* raft_on_pre_vote_request: answer an incoming PreVote probe.
* Denies if cand_term < s->term; otherwise returns on_vote_eligible(ctx,
* from_idx, cand_term) (granting when the gate is NULL). Never touches term,
* voted_for, role, or fires on_persist. Freshness is applied by the
* dispatcher before this call (same contract as the real vote path).
* Returns 1 = grant, 0 = deny.
*/
int raft_on_pre_vote_request(raft_state_t *s, int from_idx, uint64_t cand_term);
/*
* raft_on_pre_vote_reply: tally a reply to our PreVote.
* Owns the pre-vote tally in raft_state_t (pre_votes_granted,
* pre_vote_replies), mirroring raft_on_vote_reply but WITHOUT firing
* on_become_leader/on_become_follower or making any role/term change.
* Returns RAFT_PRE_VOTE_WON at quorum, RAFT_PRE_VOTE_LOST when all peers
* replied without quorum, else RAFT_PRE_VOTE_PENDING.
*/
int raft_on_pre_vote_reply(raft_state_t *s, int peer_idx, int granted);
/* ---------- Log replication (raft-log-replication) ----------
*
* The library owns the log protocol decisions (append, AppendEntries logOk +
* conflict truncation, UpdateTerm-on-any-RPC) and drives I/O through the four
* log callbacks. Index is 1-based (raft.tla) == WAL LSN. Commit-index advance
* and (lastLogTerm,lastLogIndex) vote freshness are NOT here — they are
* raft-commit-safety. */
/* Result of handling an AppendEntries request — the adapter replies with this. */
typedef struct {
uint64_t reply_term; /* current term to return to the leader */
int success; /* 1 = accepted, 0 = rejected */
uint64_t match_index; /* on success: prev_log_index + entries_len; else 0 */
} raft_append_result_t;
/*
* raft_log_append: leader appends one client entry (raft.tla ClientRequest, 246).
* Stamps the entry with s->term, assigns index log_len+1, persists it via
* on_persist_entry BEFORE returning success (persist-before-act). Does NOT
* advance any commit index (owned by raft-commit-safety).
* out_index receives the assigned 1-based index (== WAL LSN) on success.
* Returns RAFT_OK; -EPERM if role != RAFT_LEADER; propagates a non-zero
* on_persist_entry return on persist failure (no log mutation on failure).
*/
int raft_log_append(raft_state_t *s, const void *entry, size_t len,
uint64_t *out_index);
/*
* raft_send_append_entries: emit one AppendEntries to peer_idx (raft.tla
* AppendEntries, 204). Computes prev_log_index = next_index[peer]-1 and
* prev_log_term via on_get_entry_term, sends at most one entry (n_entries 0/1),
* via on_send_append_entries. Leader-only (R3.5); leader_commit is opaque.
* Returns RAFT_OK.
*/
int raft_send_append_entries(raft_state_t *s, int peer_idx,
uint64_t leader_commit);
/*
* raft_on_append_entries_request: handle an incoming AppendEntries / heartbeat
* (raft.tla HandleAppendEntriesRequest 327 + UpdateTerm 406). Step order is
* fixed: UpdateTerm first (G5, every RPC incl. heartbeat) → logOk (G2) →
* reject / candidate-step-down / accept (already-have / conflict-truncate (G3) /
* append). entries_len is 0 (heartbeat) or 1. Fills *out. leader_commit is
* accepted but NOT acted on here (raft-commit-safety). Returns RAFT_OK.
*/
int raft_on_append_entries_request(raft_state_t *s, int leader_idx,
uint64_t term, uint64_t prev_log_index,
uint64_t prev_log_term, uint64_t leader_commit,
const void *entry, size_t entry_len,
uint64_t entry_term, int entries_len,
raft_append_result_t *out);
/*
* raft_on_append_entries_reply: process a follower's AppendEntries response
* (raft.tla HandleAppendEntriesResponse 393). Applied only when
* reply_term == s->term (stale-term drop, 415). success: next_index[peer] =
* match_index+1, match_index[peer] = match_index. failure: next_index[peer] =
* Max(next_index[peer]-1, 1). Does NOT advance commit. Returns RAFT_OK.
*/
int raft_on_append_entries_reply(raft_state_t *s, int peer_idx, int success,
uint64_t match_index, uint64_t reply_term);
/* Fills snap with current state. Caller must hold lock. */
void raft_snapshot(const raft_state_t *s, raft_snapshot_t *snap);
/* ---------- deterministic simulator (test builds only) ---------- */
#ifdef XCE_TESTING
#include <stdio.h>
#define RAFT_SIM_MAX_NODES 8
/* Buffered RPC entry: one pending vote-request to route */
typedef struct {
int src;
int dst;
uint64_t term;
} raft_sim_rpc_t;
/* One simulated node */
typedef struct {
raft_state_t state;
raft_callbacks_t cbs; /* wired to sim routing functions */
void *ctx; /* points back to raft_sim_cluster_t */
int idx;
int partition; /* 1 = isolated from other nodes */
/* per-node trace event log for test assertions */
char trace_buf[4096];
int trace_len;
} raft_sim_node_t;
/* N-node simulation cluster */
typedef struct {
raft_sim_node_t nodes[RAFT_SIM_MAX_NODES];
int n;
int leader_count; /* incremented by each on_become_leader */
/* buffered RPC queue */
raft_sim_rpc_t pending[64];
int pending_count;
} raft_sim_cluster_t;
/*
* raft_sim_init: initialize an n-node simulation cluster.
* Each node gets index 0..n-1, priority=idx, election_timeout_ms as given.
* Wires on_send_vote_request to buffer RPCs into c->pending.
* Wires on_become_leader to increment c->leader_count.
* All other required callbacks get no-op implementations.
* on_trace appends to the node's trace_buf.
* Returns 0 on success.
*/
int raft_sim_init(raft_sim_cluster_t *c, int n_nodes,
uint32_t election_timeout_ms);
/*
* raft_sim_dump_log: print trace logs for all nodes to fp.
* If RAFT_SIM_VERBOSE env var is set, always prints.
*/
void raft_sim_dump_log(raft_sim_cluster_t *c, FILE *fp);
/*
* raft_sim_deliver_pending: route all buffered RPCs synchronously.
* For each buffered (src, dst, term), calls raft_on_vote_request on dst
* then routes the reply back to src via raft_on_vote_reply.
* Skips delivery if dst has partition=1.
*/
void raft_sim_deliver_pending(raft_sim_cluster_t *c);
/*
* raft_sim_start_all_elections: call raft_start_election on each node 0..n-1.
*/
void raft_sim_start_all_elections(raft_sim_cluster_t *c);
/*
* raft_sim_fire_timeout: call raft_on_election_timeout on node at node_idx.
*/
void raft_sim_fire_timeout(raft_sim_cluster_t *c, int node_idx);
/*
* raft_sim_partition: set/clear partition flag for node_idx.
* Subsequent raft_sim_deliver_pending drops RPCs to/from that node.
*/
void raft_sim_partition(raft_sim_cluster_t *c, int node_idx, int isolated);
#endif /* XCE_TESTING */
#endif /* RAFT_H */
View git blame Copy permalink