779 lines
39 KiB
C
779 lines
39 KiB
C
#ifndef RAFT_H
|
|
#define RAFT_H
|
|
|
|
#include <errno.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
|
|
/* raft_role_t: self-contained role enum (does not depend on ctrl_role_t) */
|
|
typedef enum {
|
|
RAFT_FOLLOWER = 0,
|
|
RAFT_CANDIDATE = 1,
|
|
RAFT_LEADER = 2,
|
|
} raft_role_t;
|
|
|
|
/* Upper bound on cluster size for the fixed-size pre-vote reply-tracking
|
|
* array. Matches the simulator bound (RAFT_SIM_MAX_NODES); real clusters
|
|
* are far smaller. */
|
|
#define RAFT_MAX_PEERS 8
|
|
|
|
/* Forward declaration — full definition follows raft_state_t */
|
|
typedef struct raft_callbacks_t raft_callbacks_t;
|
|
|
|
/* raft_state_t: pure Raft consensus state for one node.
|
|
*
|
|
* Contains only the Raft invariant state. Application-specific fields
|
|
* (config_tag, log_lsn, heartbeat timestamps, ctrl_role) stay in ha_state_t;
|
|
* they are accessed by the caller via the on_vote_eligible callback. */
|
|
typedef struct {
|
|
uint64_t term; /* current term (monotonically increasing) */
|
|
raft_role_t role; /* current role */
|
|
int voted_for; /* peer index voted for this term; -1=none; self_idx=self-vote */
|
|
uint64_t voted_for_term; /* term in which voted_for was set */
|
|
int n_peers; /* total cluster size (including self) */
|
|
int self_idx; /* this node's index in [0, n_peers) */
|
|
int votes_granted; /* votes collected in current election */
|
|
int replies_received; /* total peer replies received (for all-replied-no-quorum, Req 4.4) */
|
|
int election_decided; /* 1 once on_become_leader/follower fired */
|
|
uint32_t election_timeout_ms;
|
|
int my_priority; /* priority index for jitter (0 = highest) */
|
|
/* ---- PreVote round tally (Req 1.2) ----
|
|
* Self-contained pre-vote bookkeeping. A pre-vote round mutates NONE of
|
|
* the persistent Raft state above (term, voted_for, role); these fields
|
|
* only track the in-flight straw poll and are reset at each round start. */
|
|
int pre_vote_in_progress; /* 1 while a pre-vote round is active */
|
|
int pre_votes_granted; /* grants collected this round (incl. self) */
|
|
int pre_vote_replies[RAFT_MAX_PEERS]; /* per-peer replied flag; avoids double-count */
|
|
/* ---- Log replication state (raft-log-replication, raft.tla logVars/leaderVars) ----
|
|
* The library stores NO entry bytes and NO per-entry term array — entry terms
|
|
* are obtained via on_get_entry_term; bytes live in the adapter/simulator. This
|
|
* keeps raft_state_t fixed-size and the core pure (same discipline as not
|
|
* storing (term, voted_for) bytes). Index is 1-based (raft.tla) and maps 1:1
|
|
* to the WAL LSN. */
|
|
uint64_t log_len; /* highest local log index (0 = empty) */
|
|
uint64_t next_index[RAFT_MAX_PEERS]; /* leader: next index to send each peer */
|
|
uint64_t match_index[RAFT_MAX_PEERS]; /* leader: highest index known replicated to each peer */
|
|
/* ---- Commit index (raft-commit-safety, raft.tla commitIndex) ----
|
|
* Highest 1-based index (== WAL LSN) the library considers safely committed.
|
|
* Monotonic non-decreasing. Restored at init from initial_commit_index (the
|
|
* library never reads storage — same discipline as initial_term/log_len).
|
|
* Advanced ONLY through raft_advance_commit (the current-term Figure-8 rule);
|
|
* raft_on_append_entries_reply leaves it untouched. commit_index <= log_len. */
|
|
uint64_t commit_index;
|
|
/* set by raft_init — needed by the state machine to fire callbacks */
|
|
const raft_callbacks_t *cbs;
|
|
void *ctx;
|
|
} raft_state_t;
|
|
|
|
/* raft_snapshot_t: point-in-time read of all Raft state for logging/debugging */
|
|
typedef struct {
|
|
uint64_t term;
|
|
raft_role_t role;
|
|
int voted_for;
|
|
uint64_t voted_for_term;
|
|
int votes_granted;
|
|
int election_decided;
|
|
} raft_snapshot_t;
|
|
|
|
/* raft_callbacks_t: all side effects the library fires.
|
|
*
|
|
* The library owns Raft invariants (term ordering, one-vote-per-term, quorum).
|
|
* All application-specific policy (freshness gate, role gate, catch-up state,
|
|
* heartbeat liveness) is delegated to on_vote_eligible so the library is
|
|
* fully independent of XCE domain logic.
|
|
*
|
|
* REQUIRED callbacks (must be non-NULL at raft_init):
|
|
* on_send_vote_request, on_become_leader, on_become_follower,
|
|
* on_persist, on_schedule_timeout, on_cancel_timeout,
|
|
* on_start_election_after_delay,
|
|
* on_persist_entry, on_get_entry_term, on_send_append_entries, on_truncate_from,
|
|
* on_commit_advanced
|
|
*
|
|
* OPTIONAL callbacks (may be NULL):
|
|
* on_vote_eligible — if NULL, all Raft-eligible candidates are granted
|
|
* on_send_pre_vote_request — if NULL, pre-vote emits are suppressed
|
|
* on_trace — if NULL, tracing is suppressed
|
|
*/
|
|
typedef struct raft_callbacks_t {
|
|
/* REQUIRED: Send a RequestVote RPC to peer peer_idx for the given term.
|
|
* The caller encodes config_tag, log_lsn, and any other fields in the wire RPC. */
|
|
int (*on_send_vote_request)(void *ctx, int peer_idx, uint64_t term);
|
|
|
|
/* OPTIONAL: Send a PreVote (straw-poll) request to peer peer_idx for the
|
|
* prospective term (current term + 1). Unlike on_send_vote_request, this
|
|
* MUST NOT cause the peer to persist or mutate term/voted_for — it is a
|
|
* non-binding probe. If NULL, the raft library suppresses the emit and
|
|
* the caller is responsible for any pre-vote transport itself. */
|
|
int (*on_send_pre_vote_request)(void *ctx, int peer_idx,
|
|
uint64_t prospective_term);
|
|
|
|
/* OPTIONAL: Application-specific vote eligibility gate.
|
|
* Called after Raft invariants pass (correct term, voted_for unset).
|
|
* Returns 1 to grant, 0 to deny. Receives the candidate's peer_idx and
|
|
* the incoming term so the caller can check freshness (config_tag, log_lsn),
|
|
* role gate (LEADER/COMMITTING deny), catch-up status, heartbeat liveness.
|
|
* If NULL, the library grants the vote whenever Raft invariants pass. */
|
|
int (*on_vote_eligible)(void *ctx, int from_idx, uint64_t term);
|
|
|
|
/* REQUIRED: Node has won the election */
|
|
void (*on_become_leader)(void *ctx, uint64_t term);
|
|
/* REQUIRED: Node lost election or saw higher term; leader_idx = -1 if unknown */
|
|
void (*on_become_follower)(void *ctx, uint64_t term, int leader_idx);
|
|
/* REQUIRED: Persist term + voted_for before granting vote (durability guarantee) */
|
|
int (*on_persist)(void *ctx, uint64_t term, int voted_for);
|
|
/* REQUIRED: Schedule election timeout (fires raft_on_election_timeout after delay_ms) */
|
|
void (*on_schedule_timeout)(void *ctx, uint32_t delay_ms);
|
|
/* REQUIRED: Cancel pending election timeout */
|
|
void (*on_cancel_timeout)(void *ctx);
|
|
/* REQUIRED: Request delayed election start after RDHUP (priority-based jitter) */
|
|
void (*on_start_election_after_delay)(void *ctx, uint32_t delay_ms);
|
|
|
|
/* ---- Log replication callbacks (raft-log-replication) ----
|
|
* REQUIRED (all four non-NULL at raft_init). The library owns the protocol
|
|
* decisions; these perform the I/O. The library stores no entry bytes, so it
|
|
* reads entry terms only via on_get_entry_term and writes/truncates only via
|
|
* these callbacks. */
|
|
|
|
/* REQUIRED: durably append/overwrite the log entry at 1-based `index` with
|
|
* `term` (persist-before-act: called before an append/accept is reported). */
|
|
int (*on_persist_entry)(void *ctx, uint64_t index, uint64_t term,
|
|
const void *entry, size_t len);
|
|
|
|
/* REQUIRED: term of the local entry at 1-based `index` (0 if absent). */
|
|
uint64_t (*on_get_entry_term)(void *ctx, uint64_t index);
|
|
|
|
/* REQUIRED: transmit one AppendEntries to peer_idx with prevLog{Index,Term}
|
|
* and leader_commit. `n_entries` is 0 (heartbeat) or 1 (this spec, like
|
|
* raft.tla); when 1 the adapter sources the single entry at prev_log_index+1
|
|
* from its own log (the library holds no entry bytes). [Concretizes the
|
|
* design's varargs "entries handle" placeholder into an explicit count.] */
|
|
int (*on_send_append_entries)(void *ctx, int peer_idx,
|
|
uint64_t prev_log_index,
|
|
uint64_t prev_log_term,
|
|
uint64_t leader_commit, int n_entries);
|
|
|
|
/* REQUIRED: delete local entries at `index`..end. */
|
|
int (*on_truncate_from)(void *ctx, uint64_t index);
|
|
|
|
/* ---- Commit-advance trigger (raft-commit-safety) ----
|
|
* REQUIRED: fired AFTER commit_index strictly advances. The controller adapter
|
|
* feeds this into the EXISTING synchronous commit barrier / persist-before-ACK
|
|
* durability — this callback TRIGGERS commit-eval; it does NOT replace the
|
|
* barrier.
|
|
*
|
|
* Contract: the library decides WHETHER the commit index advanced; the callback
|
|
* only triggers the adapter's commit work. Fired exactly once per strict advance,
|
|
* NEVER on a held commit index (current-term Figure-8 rule), and NEVER from
|
|
* raft_on_append_entries_reply. NULL-checked / required in raft_init (-EINVAL if
|
|
* NULL). It NEVER performs durability itself. */
|
|
void (*on_commit_advanced)(void *ctx, uint64_t commit_index);
|
|
|
|
/* OPTIONAL: Trace callback for observability.
|
|
* event: short string literal, e.g. "vote_granted", "became_leader"
|
|
* detail: formatted human-readable context string
|
|
* If NULL, tracing is suppressed. */
|
|
void (*on_trace)(void *ctx, const char *event, const char *detail);
|
|
} raft_callbacks_t;
|
|
|
|
/*
|
|
* raft_config_t: initialization context for a Raft node.
|
|
*
|
|
* Encapsulates all configuration so raft_init() has a single, extensible
|
|
* parameter rather than a flat list. Matches the existing XCE pattern
|
|
* (ha_config_t, persist_config_t). Zero-initialize and set only the
|
|
* fields you need — the library fills in safe defaults for unset fields.
|
|
*
|
|
* Persistence model:
|
|
* initial_term / initial_voted_for are restored by the caller from disk
|
|
* before raft_init(). The library never reads storage; it only writes
|
|
* via on_persist callbacks. Pass {0, -1} on first boot.
|
|
*/
|
|
typedef struct {
|
|
int n_peers; /* total cluster size (including self) */
|
|
int self_idx; /* this node's index in [0, n_peers) */
|
|
uint32_t election_timeout_ms; /* base election timeout; 0 → default 2000ms */
|
|
int my_priority; /* priority index for RDHUP jitter (0 = highest) */
|
|
uint64_t initial_term; /* term restored from disk; 0 on first boot */
|
|
int initial_voted_for; /* voted_for restored from disk; -1 if none */
|
|
uint64_t initial_log_len; /* WAL length restored from disk; 0 on first boot (library never reads storage) */
|
|
uint64_t initial_commit_index; /* committed LSN restored from disk; 0 on first boot (library never reads storage) */
|
|
const raft_callbacks_t *cbs; /* all callbacks; required callbacks must be non-NULL */
|
|
void *ctx; /* opaque caller context passed to every callback */
|
|
} raft_config_t;
|
|
|
|
/* Return codes
|
|
*
|
|
* Note: RAFT_GRANT and RAFT_BECAME_LEADER share value 1; RAFT_DENY and RAFT_OK
|
|
* share value 0. This is intentional — different return values are used by
|
|
* different functions (raft_on_vote_request vs raft_on_vote_reply) in different
|
|
* call contexts.
|
|
*/
|
|
#define RAFT_OK 0
|
|
#define RAFT_GRANT 1
|
|
#define RAFT_DENY 0
|
|
#define RAFT_BECAME_LEADER 1
|
|
#define RAFT_BECAME_FOLLOWER 2
|
|
#define RAFT_PENDING 3
|
|
#define RAFT_ELECTED_IMMEDIATELY 4
|
|
#define RAFT_SKIP_ELECTION 5
|
|
#define RAFT_ELECTION_STARTED 6
|
|
|
|
/* PreVote (straw-poll) outcomes returned by raft_on_pre_vote_reply.
|
|
* Distinct values from the election codes above; a pre-vote round mutates
|
|
* no persistent state, so these never imply a term/role transition. */
|
|
#define RAFT_PRE_VOTE_WON 7 /* quorum of grants reached this round */
|
|
#define RAFT_PRE_VOTE_LOST 8 /* all peers replied, no quorum */
|
|
#define RAFT_PRE_VOTE_PENDING 9 /* still awaiting replies */
|
|
|
|
/*
|
|
* raft_init: initialize raft_state_t from a raft_config_t.
|
|
* on_vote_eligible and on_trace may be NULL (optional policy / observability).
|
|
* All other callbacks in cbs must be non-NULL.
|
|
* Returns 0 on success; -EINVAL if a required callback is NULL or n_peers < 1.
|
|
*/
|
|
int raft_init(raft_state_t *s, const raft_config_t *cfg);
|
|
|
|
/*
|
|
* raft_start_election: begin a new election.
|
|
* Increments term, records self-vote (voted_for=self_idx), transitions to
|
|
* CANDIDATE, fires on_send_vote_request for each peer, fires on_schedule_timeout.
|
|
* Returns 0 on success; -EALREADY if already CANDIDATE/LEADER;
|
|
* RAFT_ELECTED_IMMEDIATELY for single-node (n_peers==1).
|
|
*/
|
|
int raft_start_election(raft_state_t *s);
|
|
|
|
/*
|
|
* raft_on_vote_request: process an incoming RequestVote RPC.
|
|
* Applies Raft invariants (term comparison, one-vote-per-term), then the
|
|
* (lastLogTerm, lastLogIndex) freshness gate (raft.tla HandleRequestVoteRequest
|
|
* logOk, line 285, closes G6), then the application policy gate.
|
|
*
|
|
* The library owns the logOk decision (design "Option B: library owns logOk"):
|
|
* local_last_log_term = (log_len > 0) ? on_get_entry_term(log_len) : 0
|
|
* logOk = (cand_last_log_term > local_last_log_term)
|
|
* || (cand_last_log_term == local_last_log_term
|
|
* && cand_last_log_index >= log_len)
|
|
* The candidate advertises (cand_last_log_term, cand_last_log_index) in its wire
|
|
* RequestVote; the caller decodes and passes them here.
|
|
*
|
|
* Step order: stale-term deny → UpdateTerm/step-down → role gate →
|
|
* one-vote-per-term → logOk gate (NEW; precedes the application gate, so a
|
|
* !logOk denies regardless of on_vote_eligible) → on_vote_eligible →
|
|
* persist-before-grant.
|
|
*
|
|
* Fires on_persist before granting.
|
|
* Returns RAFT_GRANT (1) or RAFT_DENY (0); sets *reply_term to current term.
|
|
*/
|
|
int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term,
|
|
uint64_t cand_last_log_term,
|
|
uint64_t cand_last_log_index,
|
|
uint64_t *reply_term);
|
|
|
|
/*
|
|
* raft_on_vote_reply: process a reply to our RequestVote.
|
|
* Returns RAFT_BECAME_LEADER, RAFT_BECAME_FOLLOWER, or RAFT_PENDING.
|
|
*/
|
|
int raft_on_vote_reply(raft_state_t *s, int from_idx, int granted, uint64_t term);
|
|
|
|
/*
|
|
* raft_on_election_timeout: called by the timer when no quorum was reached.
|
|
* Transitions CANDIDATE → FOLLOWER; fires on_become_follower, on_cancel_timeout.
|
|
*/
|
|
void raft_on_election_timeout(raft_state_t *s);
|
|
|
|
/*
|
|
* raft_on_leader_lost: primary control channel dropped (RDHUP).
|
|
* Clears voted_for, transitions to FOLLOWER, fires on_start_election_after_delay.
|
|
*/
|
|
void raft_on_leader_lost(raft_state_t *s);
|
|
|
|
/*
|
|
* raft_check_and_start_election: call after the delay from raft_on_leader_lost.
|
|
* Starts election only if we have not already voted for a non-self candidate
|
|
* in the new term (i.e., another node won during the jitter window).
|
|
* Returns RAFT_SKIP_ELECTION or RAFT_ELECTION_STARTED.
|
|
*/
|
|
int raft_check_and_start_election(raft_state_t *s);
|
|
|
|
/* ---------- PreVote (straw poll before a binding election) ----------
|
|
*
|
|
* The pre-vote round is a non-binding probe: it asks peers whether they WOULD
|
|
* grant a vote at term+1, without anyone incrementing term, setting voted_for,
|
|
* or persisting. Only on RAFT_PRE_VOTE_WON does the caller proceed to the
|
|
* unchanged raft_start_election (which performs the real term bump + persist).
|
|
*
|
|
* INVARIANT (Req 1.2, 6.2): none of these three functions mutate s->term,
|
|
* s->voted_for, s->voted_for_term, or s->role, and none invoke on_persist.
|
|
*/
|
|
|
|
/*
|
|
* raft_begin_pre_vote: start a pre-vote round.
|
|
* Computes prospective term = term + 1 (NOT assigned to s->term), sets
|
|
* pre_vote_in_progress, pre_votes_granted = 1 (self), and fires
|
|
* on_send_pre_vote_request(ctx, peer_idx, prospective_term) for each peer
|
|
* (NULL-checked — suppressed if the callback is unset).
|
|
* Precondition: role == RAFT_FOLLOWER.
|
|
*/
|
|
int raft_begin_pre_vote(raft_state_t *s);
|
|
|
|
/*
|
|
* raft_on_pre_vote_request: answer an incoming PreVote probe.
|
|
* Denies if cand_term < s->term; then applies the IDENTICAL (lastLogTerm,
|
|
* lastLogIndex) logOk as raft_on_vote_request (the library owns the predicate —
|
|
* it is NO LONGER applied by the dispatcher before this call), then consults
|
|
* on_vote_eligible (granting when the gate is NULL). Mutates NOTHING — never
|
|
* touches term, voted_for, voted_for_term, role, or commit_index, and never
|
|
* fires on_persist — preserving the PreVote side-effect-free invariant.
|
|
* Returns 1 = grant, 0 = deny.
|
|
*/
|
|
int raft_on_pre_vote_request(raft_state_t *s, int from_idx, uint64_t cand_term,
|
|
uint64_t cand_last_log_term,
|
|
uint64_t cand_last_log_index);
|
|
|
|
/*
|
|
* raft_on_pre_vote_reply: tally a reply to our PreVote.
|
|
* Owns the pre-vote tally in raft_state_t (pre_votes_granted,
|
|
* pre_vote_replies), mirroring raft_on_vote_reply but WITHOUT firing
|
|
* on_become_leader/on_become_follower or making any role/term change.
|
|
* Returns RAFT_PRE_VOTE_WON at quorum, RAFT_PRE_VOTE_LOST when all peers
|
|
* replied without quorum, else RAFT_PRE_VOTE_PENDING.
|
|
*/
|
|
int raft_on_pre_vote_reply(raft_state_t *s, int peer_idx, int granted);
|
|
|
|
/* ---------- Log replication (raft-log-replication) ----------
|
|
*
|
|
* The library owns the log protocol decisions (append, AppendEntries logOk +
|
|
* conflict truncation, UpdateTerm-on-any-RPC) and drives I/O through the four
|
|
* log callbacks. Index is 1-based (raft.tla) == WAL LSN. Commit-index advance
|
|
* and (lastLogTerm,lastLogIndex) vote freshness are NOT here — they are
|
|
* raft-commit-safety. */
|
|
|
|
/* Result of handling an AppendEntries request — the adapter replies with this. */
|
|
typedef struct {
|
|
uint64_t reply_term; /* current term to return to the leader */
|
|
int success; /* 1 = accepted, 0 = rejected */
|
|
uint64_t match_index; /* on success: prev_log_index + entries_len; else 0 */
|
|
} raft_append_result_t;
|
|
|
|
/*
|
|
* raft_log_append: leader appends one client entry (raft.tla ClientRequest, 246).
|
|
* Stamps the entry with s->term, assigns index log_len+1, persists it via
|
|
* on_persist_entry BEFORE returning success (persist-before-act). Does NOT
|
|
* advance any commit index (owned by raft-commit-safety).
|
|
* out_index receives the assigned 1-based index (== WAL LSN) on success.
|
|
* Returns RAFT_OK; -EPERM if role != RAFT_LEADER; propagates a non-zero
|
|
* on_persist_entry return on persist failure (no log mutation on failure).
|
|
*/
|
|
int raft_log_append(raft_state_t *s, const void *entry, size_t len,
|
|
uint64_t *out_index);
|
|
|
|
/*
|
|
* raft_send_append_entries: emit one AppendEntries to peer_idx (raft.tla
|
|
* AppendEntries, 204). Computes prev_log_index = next_index[peer]-1 and
|
|
* prev_log_term via on_get_entry_term, sends at most one entry (n_entries 0/1),
|
|
* via on_send_append_entries. Leader-only (R3.5); leader_commit is opaque.
|
|
* Returns RAFT_OK.
|
|
*/
|
|
int raft_send_append_entries(raft_state_t *s, int peer_idx,
|
|
uint64_t leader_commit);
|
|
|
|
/*
|
|
* raft_on_append_entries_request: handle an incoming AppendEntries / heartbeat
|
|
* (raft.tla HandleAppendEntriesRequest 327 + UpdateTerm 406). Step order is
|
|
* fixed: UpdateTerm first (G5, every RPC incl. heartbeat) → logOk (G2) →
|
|
* reject / candidate-step-down / accept (already-have / conflict-truncate (G3) /
|
|
* append). entries_len is 0 (heartbeat) or 1. Fills *out. leader_commit is
|
|
* accepted but NOT acted on here (raft-commit-safety). Returns RAFT_OK.
|
|
*/
|
|
int raft_on_append_entries_request(raft_state_t *s, int leader_idx,
|
|
uint64_t term, uint64_t prev_log_index,
|
|
uint64_t prev_log_term, uint64_t leader_commit,
|
|
const void *entry, size_t entry_len,
|
|
uint64_t entry_term, int entries_len,
|
|
raft_append_result_t *out);
|
|
|
|
/*
|
|
* raft_on_append_entries_reply: process a follower's AppendEntries response
|
|
* (raft.tla HandleAppendEntriesResponse 393). Applied only when
|
|
* reply_term == s->term (stale-term drop, 415). success: next_index[peer] =
|
|
* match_index+1, match_index[peer] = match_index. failure: next_index[peer] =
|
|
* Max(next_index[peer]-1, 1). Does NOT advance commit. Returns RAFT_OK.
|
|
*/
|
|
int raft_on_append_entries_reply(raft_state_t *s, int peer_idx, int success,
|
|
uint64_t match_index, uint64_t reply_term);
|
|
|
|
/*
|
|
* raft_advance_commit: leader recomputes its commit index (raft.tla
|
|
* AdvanceCommitIndex, 259). A SEPARATE action from raft_on_append_entries_reply
|
|
* (which only updates match_index[]) — mirroring raft.tla, where
|
|
* AdvanceCommitIndex is distinct from HandleAppendEntriesResponse; this lets a
|
|
* single-node leader commit ({self} in agreeIndexes).
|
|
*
|
|
* agreeIndexes = { index in 1..log_len : {self} U {peer : match_index[peer] >=
|
|
* index} is a quorum }. When agreeIndexes is non-empty AND
|
|
* on_get_entry_term(Max(agreeIndexes)) == s->term, sets commit_index =
|
|
* Max(agreeIndexes) (the Figure-8 current-term rule, line 270) and, on a STRICT
|
|
* advance, fires on_commit_advanced(ctx, commit_index) exactly once. Otherwise
|
|
* leaves commit_index unchanged and does NOT fire on_commit_advanced.
|
|
*
|
|
* Monotonic: commit_index never decreases. Reads entry terms only via
|
|
* on_get_entry_term; performs NO storage I/O. on_commit_advanced only TRIGGERS
|
|
* the controller's existing synchronous commit barrier — the library performs
|
|
* no durability work here. Returns RAFT_OK; -EPERM if role != RAFT_LEADER
|
|
* (commit_index left unchanged, no callback).
|
|
*/
|
|
int raft_advance_commit(raft_state_t *s);
|
|
|
|
/* Fills snap with current state. Caller must hold lock. */
|
|
void raft_snapshot(const raft_state_t *s, raft_snapshot_t *snap);
|
|
|
|
/* ---------- deterministic simulator (test builds only) ---------- */
|
|
|
|
#ifdef XCE_TESTING
|
|
|
|
#include <stdio.h>
|
|
|
|
#define RAFT_SIM_MAX_NODES 8
|
|
|
|
/* Upper bound on a simulated node's in-memory log. Test entries are tiny and
|
|
* the sim drives short runs, so a small fixed capacity keeps the node struct
|
|
* ASAN-clean (no allocation) while comfortably exceeding any test's log depth. */
|
|
#define RAFT_SIM_MAX_LOG 64
|
|
/* Fixed entry payload size — test entries are a few bytes; bounded copies only. */
|
|
#define RAFT_SIM_ENTRY_LEN 16
|
|
/* Upper bound on buffered AppendEntries awaiting delivery (mirrors pending[]). */
|
|
#define RAFT_SIM_MAX_AE 64
|
|
|
|
/* ---- conformance harness bounds (raft-sim-conformance) ----
|
|
* Capacity of the unified message bag (cluster.bag[], mirrors raft.tla's
|
|
* `messages`): in-flight vote/AE requests and responses, the unit on which
|
|
* Drop/Duplicate/Reorder operate. Sized generously above RAFT_SIM_MAX_AE so a
|
|
* bounded run never overflows the bag (duplicates add members). */
|
|
#define RAFT_SIM_MAX_MSGS 256
|
|
/* Upper bound on recorded scheduler steps (cluster.trace[]) for replayable
|
|
* failure traces. One raft_sim_action_t is recorded per step. */
|
|
#define RAFT_SIM_MAX_STEPS 1024
|
|
|
|
/* Buffered RPC entry: one pending vote-request to route.
|
|
* last_log_term/last_log_index capture the CANDIDATE's advertised last-log tuple
|
|
* at SEND time (mirroring raft_sim_ae_t.leader_term) so the (lastLogTerm,
|
|
* lastLogIndex) freshness gate (raft-commit-safety) is evaluated against the
|
|
* candidate's send-epoch log, not a value re-read at delivery. */
|
|
typedef struct {
|
|
int src;
|
|
int dst;
|
|
uint64_t term;
|
|
uint64_t last_log_term; /* candidate's last-log term at send (0 if empty) */
|
|
uint64_t last_log_index; /* candidate's last-log index at send (== log_count) */
|
|
} raft_sim_rpc_t;
|
|
|
|
/* One in-memory log entry: term stamp + fixed-size payload bytes.
|
|
* term == 0 means "absent" (matches the 0-means-absent convention of
|
|
* on_get_entry_term); 1-based indexing maps slot[i-1] to log index i. */
|
|
typedef struct {
|
|
uint64_t term;
|
|
unsigned char bytes[RAFT_SIM_ENTRY_LEN];
|
|
size_t len;
|
|
} raft_sim_entry_t;
|
|
|
|
/* Buffered AppendEntries RPC awaiting synchronous delivery (task 8.2).
|
|
* For n_entries == 1 the single entry's index/term/bytes are captured here
|
|
* because the library holds no entry bytes — the sender sources them from its
|
|
* own in-memory log at prev_log_index + 1 when buffering. */
|
|
typedef struct {
|
|
int src;
|
|
int dst;
|
|
uint64_t leader_term; /* leader's term captured at SEND time (not re-read
|
|
* at delivery) — required so reorder/churn tests
|
|
* (9.3/9.4) deliver the send-epoch term, preserving
|
|
* UpdateTerm/stale-drop (G5) semantics. */
|
|
uint64_t prev_log_index;
|
|
uint64_t prev_log_term;
|
|
uint64_t leader_commit;
|
|
int n_entries; /* 0 (heartbeat) or 1 (this spec) */
|
|
uint64_t entry_index; /* valid when n_entries == 1 */
|
|
uint64_t entry_term; /* valid when n_entries == 1 */
|
|
unsigned char entry_bytes[RAFT_SIM_ENTRY_LEN];
|
|
size_t entry_len; /* valid when n_entries == 1 */
|
|
} raft_sim_ae_t;
|
|
|
|
/* ---- conformance harness data types (raft-sim-conformance) ----
|
|
*
|
|
* These mirror raft.tla's `messages` bag and per-node variables for the
|
|
* deterministic conformance/verification harness. They are SEPARATE from the
|
|
* byte-payload WAL model above (raft_sim_entry_t / raft_sim_ae_t / raft_sim_rpc_t):
|
|
* the conformance harness uses a lightweight (term, value) log entry and a single
|
|
* unified message type so a randomized, seeded action scheduler can drive N library
|
|
* instances purely synchronously and assert the four Raft safety invariants. */
|
|
|
|
/* Unified bag-message kind: a vote/AE request or its matching response. */
|
|
typedef enum {
|
|
RAFT_SIM_MSG_VOTE_REQ,
|
|
RAFT_SIM_MSG_VOTE_RSP,
|
|
RAFT_SIM_MSG_AE_REQ,
|
|
RAFT_SIM_MSG_AE_RSP,
|
|
} raft_sim_msg_kind_t;
|
|
|
|
/* One in-flight RPC in the message bag (mirrors raft.tla `messages`). Carries the
|
|
* union of vote/AE request and response fields; only the fields relevant to `kind`
|
|
* are meaningful. This is the unit Drop/Duplicate/Reorder operate on; a handler
|
|
* reply is pushed back as a *_RSP member so responses are first-class. */
|
|
typedef struct {
|
|
raft_sim_msg_kind_t kind;
|
|
int src, dst;
|
|
uint64_t term; /* RPC term (all kinds) */
|
|
/* vote request: candidate's advertised last-log tuple at SEND time, feeding the
|
|
* (lastLogTerm, lastLogIndex) freshness gate at delivery (mirrors
|
|
* raft_sim_rpc_t.last_log_*). Sourced from the candidate's own log at send. */
|
|
uint64_t last_log_term, last_log_index;
|
|
/* AE request */
|
|
uint64_t prev_log_index, prev_log_term, leader_commit;
|
|
uint64_t entry_term, entry_value; /* one entry per AE (raft.tla "<=1 entry") */
|
|
int entries_len; /* 0 = heartbeat, 1 = one entry */
|
|
/* responses */
|
|
int granted; /* vote rsp */
|
|
int success; /* AE rsp */
|
|
uint64_t match_index; /* AE rsp */
|
|
} raft_sim_msg_t;
|
|
|
|
/* One conformance-harness log entry: term stamp + synthetic monotone client value.
|
|
* The 1-based array position is the log index (= WAL LSN); `value` makes State
|
|
* Machine Safety divergence observable. Distinct from raft_sim_entry_t (which models
|
|
* the byte-payload WAL): this is the (term, value) bag/applied model. */
|
|
typedef struct {
|
|
uint64_t term;
|
|
uint64_t value;
|
|
} raft_sim_log_entry_t;
|
|
|
|
/* The Next-action set the scheduler chooses from; each maps to a raft.tla `Next`
|
|
* disjunct. Only *enabled* actions are eligible at a given step. */
|
|
typedef enum {
|
|
RAFT_SIM_ACT_RESTART, /* Restart: re-init preserving persisted state */
|
|
RAFT_SIM_ACT_TIMEOUT, /* Timeout: become candidate, bump term */
|
|
RAFT_SIM_ACT_REQUEST_VOTE, /* RequestVote: deliver a vote-request */
|
|
RAFT_SIM_ACT_BECOME_LEADER, /* BecomeLeader: deliver quorum-completing vote-rsp */
|
|
RAFT_SIM_ACT_CLIENT_REQUEST, /* ClientRequest: leader appends a client value */
|
|
RAFT_SIM_ACT_APPEND_ENTRIES, /* AppendEntries: leader pushes an AE-request */
|
|
RAFT_SIM_ACT_ADVANCE_COMMIT, /* AdvanceCommitIndex: leader advances commit */
|
|
RAFT_SIM_ACT_RECEIVE, /* Receive: deliver a bag message to its handler */
|
|
RAFT_SIM_ACT_DUPLICATE_MESSAGE, /* DuplicateMessage: re-push a bag member */
|
|
RAFT_SIM_ACT_DROP_MESSAGE, /* DropMessage: discard a bag member */
|
|
} raft_sim_action_kind_t;
|
|
|
|
/* One replayable trace unit: the action taken at a scheduler step plus the operands
|
|
* needed to reproduce it (actor node, target message index in the bag, and the
|
|
* (term, index) the action referenced). Recorded per step into cluster.trace[]. */
|
|
typedef struct {
|
|
raft_sim_action_kind_t kind;
|
|
int actor; /* node that took the action (-1 if N/A) */
|
|
int msg_index; /* bag index acted on (-1 if N/A) */
|
|
uint64_t term; /* term referenced by the action (0 if N/A) */
|
|
uint64_t index; /* log index referenced by the action (0 if N/A) */
|
|
} raft_sim_action_t;
|
|
|
|
/* One simulated node */
|
|
typedef struct {
|
|
raft_state_t state;
|
|
raft_callbacks_t cbs; /* wired to sim routing functions */
|
|
void *ctx; /* points back to raft_sim_cluster_t */
|
|
int idx;
|
|
int partition; /* 1 = isolated from other nodes */
|
|
/* in-memory replicated log backing the four log callbacks (1-based index;
|
|
* log[i-1] holds entry i; log_count is the highest occupied index). */
|
|
raft_sim_entry_t log[RAFT_SIM_MAX_LOG];
|
|
int log_count;
|
|
/* ---- commit-advance callback mirror (raft-commit-safety) ----
|
|
* Updated by the node's on_commit_advanced callback: commit_index records the
|
|
* last value the library reported (the strict-advance commit position), and
|
|
* commit_advanced_count counts how many times on_commit_advanced fired (so
|
|
* tests can assert fire-once / never-on-hold trigger semantics). The
|
|
* commit-advance DRIVER helper is a later task; this is only the callback half. */
|
|
uint64_t commit_index;
|
|
int commit_advanced_count;
|
|
/* ---- conformance harness per-node state (raft-sim-conformance) ----
|
|
* applied[k-1] holds the client value applied at committed index k (set by the
|
|
* node's on_commit_advanced as commit_index advances). State Machine Safety
|
|
* checks that applied[k] agrees across nodes. NOTE: log_count above already
|
|
* satisfies the design's `log_len` requirement for the in-memory log length;
|
|
* no separate log_len field is added (it would duplicate log_count). */
|
|
uint64_t applied[RAFT_SIM_MAX_LOG];
|
|
/* Persisted-(term, votedFor) shadow surviving a Restart, mirroring raft.tla's
|
|
* `Restart` (keeps currentTerm/votedFor/log, resets volatile state). The
|
|
* conformance restart action restores from this shadow rather than re-reading
|
|
* live volatile state. */
|
|
uint64_t persisted_term;
|
|
int persisted_voted_for;
|
|
/* per-node trace event log for test assertions */
|
|
char trace_buf[4096];
|
|
int trace_len;
|
|
} raft_sim_node_t;
|
|
|
|
/* N-node simulation cluster */
|
|
typedef struct {
|
|
raft_sim_node_t nodes[RAFT_SIM_MAX_NODES];
|
|
int n;
|
|
int leader_count; /* incremented by each on_become_leader */
|
|
/* buffered RPC queue */
|
|
raft_sim_rpc_t pending[64];
|
|
int pending_count;
|
|
/* buffered AppendEntries queue (delivered by raft_sim_deliver_append_entries,
|
|
* task 8.2); on_send_append_entries enqueues here. */
|
|
raft_sim_ae_t ae_pending[RAFT_SIM_MAX_AE];
|
|
int ae_pending_count;
|
|
/* ---- conformance harness cluster state (raft-sim-conformance) ----
|
|
* Unified message bag mirroring raft.tla `messages`: vote/AE requests and
|
|
* responses, the unit Drop/Duplicate/Reorder operate on. */
|
|
raft_sim_msg_t bag[RAFT_SIM_MAX_MSGS];
|
|
int bag_count;
|
|
/* Seeded deterministic PRNG: `seed` is the run's reproducibility key; `prng`
|
|
* is the evolving generator state (SplitMix64/xorshift, no libc rand()). */
|
|
uint64_t seed;
|
|
uint64_t prng;
|
|
/* Replayable per-step action trace for failure dumps. */
|
|
raft_sim_action_t trace[RAFT_SIM_MAX_STEPS];
|
|
int trace_len;
|
|
} raft_sim_cluster_t;
|
|
|
|
/*
|
|
* raft_sim_init: initialize an n-node simulation cluster.
|
|
* Each node gets index 0..n-1, priority=idx, election_timeout_ms as given.
|
|
* Wires on_send_vote_request to buffer RPCs into c->pending.
|
|
* Wires on_become_leader to increment c->leader_count.
|
|
* All other required callbacks get no-op implementations.
|
|
* on_trace appends to the node's trace_buf.
|
|
* Returns 0 on success.
|
|
*/
|
|
int raft_sim_init(raft_sim_cluster_t *c, int n_nodes,
|
|
uint32_t election_timeout_ms);
|
|
|
|
/*
|
|
* raft_sim_conformance_init: initialize an n-node CONFORMANCE cluster
|
|
* (raft-sim-conformance task 1.2). Modeled on raft_sim_init but wires the
|
|
* conformance adapter callback set and seeds the deterministic PRNG:
|
|
* - log storage callbacks (on_persist_entry / on_get_entry_term /
|
|
* on_truncate_from) reuse the in-memory node->log[] backing,
|
|
* - on_commit_advanced sets commit_index AND applies each newly-committed
|
|
* entry's decoded value into node->applied[] (State Machine Safety oracle),
|
|
* - on_send_vote_request / on_send_append_entries push a raft_sim_msg_t onto
|
|
* the unified cluster bag[] (mirroring raft.tla `messages`).
|
|
* `seed` is the run's reproducibility key; c->prng is initialized from it so
|
|
* every later nondeterministic choice flows from the seed (R1.2). Returns 0 on
|
|
* success, -1 on an invalid node count. Does NOT deliver bag messages (that is
|
|
* task 2.1).
|
|
*/
|
|
int raft_sim_conformance_init(raft_sim_cluster_t *c, int n_nodes,
|
|
uint32_t election_timeout_ms, uint64_t seed);
|
|
|
|
/*
|
|
* raft_sim_prng_next: advance the cluster's seeded SplitMix64 generator and return
|
|
* a well-mixed 64-bit value. The single source of nondeterminism for the scheduler
|
|
* (R1.2): for a fixed seed the value sequence is byte-identical across runs and
|
|
* platforms (no libc rand()). Exposed so tests can assert deterministic replay.
|
|
*/
|
|
uint64_t raft_sim_prng_next(raft_sim_cluster_t *c);
|
|
|
|
/*
|
|
* raft_sim_bag_push: append one message to the unified cluster bag[] (mirrors
|
|
* raft.tla `messages`). Fails loud (returns -1) on overflow rather than silently
|
|
* truncating; returns 0 on success. Exposed so tests (and later scheduler tasks)
|
|
* can seed the bag directly.
|
|
*/
|
|
int raft_sim_bag_push(raft_sim_cluster_t *c, const raft_sim_msg_t *msg);
|
|
|
|
/*
|
|
* raft_sim_bag_deliver: route the bag message at index `i` to its matching library
|
|
* handler (vote/AE request or response) and push any handler reply back onto the
|
|
* bag as a *_RSP message (src/dst swapped). Mirrors raft.tla Receive: delivering a
|
|
* request consumes it (swap-with-last removal, so bag indices stay compact for the
|
|
* Drop/Duplicate/Reorder actions in task 2.2) and produces a response member.
|
|
* UpdateTerm higher-term adoption and the stale-response drop live INSIDE the
|
|
* handlers (R2.4) — this only routes. Delivery honors node.partition: if either
|
|
* endpoint is partitioned the message is NOT delivered (no removal, no reply),
|
|
* mirroring a dropped network edge.
|
|
* Returns 0 on a delivered message, -1 on an out-of-range index, -2 when the
|
|
* endpoint is partitioned (distinct non-zero status, message left in the bag).
|
|
*/
|
|
int raft_sim_bag_deliver(raft_sim_cluster_t *c, int i);
|
|
|
|
/*
|
|
* raft_sim_dump_log: print trace logs for all nodes to fp.
|
|
* If RAFT_SIM_VERBOSE env var is set, always prints.
|
|
*/
|
|
void raft_sim_dump_log(raft_sim_cluster_t *c, FILE *fp);
|
|
|
|
/*
|
|
* raft_sim_deliver_pending: route all buffered RPCs synchronously.
|
|
* For each buffered (src, dst, term), calls raft_on_vote_request on dst
|
|
* then routes the reply back to src via raft_on_vote_reply.
|
|
* Skips delivery if dst has partition=1.
|
|
*/
|
|
void raft_sim_deliver_pending(raft_sim_cluster_t *c);
|
|
|
|
/*
|
|
* raft_sim_deliver_append_entries: route all buffered AppendEntries synchronously.
|
|
* For each buffered AE (src, dst, ...), calls raft_on_append_entries_request on dst
|
|
* then routes the reply back to src via raft_on_append_entries_reply, mirroring the
|
|
* vote-RPC routing in raft_sim_deliver_pending. Skips delivery if either endpoint
|
|
* has partition=1. Drains the queue (snapshots the count, since handling may buffer
|
|
* more AEs). The single entry (n_entries == 1) is sourced from the buffered AE.
|
|
*/
|
|
void raft_sim_deliver_append_entries(raft_sim_cluster_t *c);
|
|
|
|
/*
|
|
* raft_sim_start_all_elections: call raft_start_election on each node 0..n-1.
|
|
*/
|
|
void raft_sim_start_all_elections(raft_sim_cluster_t *c);
|
|
|
|
/*
|
|
* raft_sim_fire_timeout: call raft_on_election_timeout on node at node_idx.
|
|
*/
|
|
void raft_sim_fire_timeout(raft_sim_cluster_t *c, int node_idx);
|
|
|
|
/*
|
|
* raft_sim_partition: set/clear partition flag for node_idx.
|
|
* Subsequent raft_sim_deliver_pending drops RPCs to/from that node.
|
|
*/
|
|
void raft_sim_partition(raft_sim_cluster_t *c, int node_idx, int isolated);
|
|
|
|
/*
|
|
* raft_sim_check_log_matching: assert the Raft Log Matching property over the
|
|
* cluster's in-memory logs (Req 8.4).
|
|
*
|
|
* For every pair of nodes (a, b) and every index i present in BOTH logs
|
|
* (1 <= i <= min(log_count_a, log_count_b)), if the entries at i carry the
|
|
* SAME term, then the logs must be identical for all indices 1..i — same term
|
|
* AND same payload (length + bytes) at every k in 1..i.
|
|
*
|
|
* Returns 1 if Log Matching holds across the whole cluster, 0 if any pair/index
|
|
* violates it. Pure read-only over c (no I/O, no Unity assertions) so the lib
|
|
* stays test-framework-agnostic; the caller asserts on the returned value.
|
|
*/
|
|
int raft_sim_check_log_matching(const raft_sim_cluster_t *c);
|
|
|
|
/*
|
|
* raft_sim_advance_commit: drive raft_advance_commit on node_idx (task 6.1).
|
|
*
|
|
* Recomputes the node's commit index from its current match_index[] / log_len /
|
|
* term (the leader-only AdvanceCommitIndex + current-term Figure-8 rule). On a
|
|
* strict advance the node's on_commit_advanced fires, updating the per-node
|
|
* commit_index mirror and commit_advanced_count. No-op if node_idx is out of
|
|
* range. Off-leader nodes return -EPERM internally (the return is discarded;
|
|
* tests read the per-node mirror).
|
|
*/
|
|
void raft_sim_advance_commit(raft_sim_cluster_t *c, int node_idx);
|
|
|
|
/*
|
|
* raft_sim_restart_node: model a process restart of node_idx (Req 8.3).
|
|
*
|
|
* Durable state survives a restart: the in-memory log[] (which models the WAL on
|
|
* disk) and the persisted (term, voted_for) are retained. Volatile consensus
|
|
* state (role, votes, election bookkeeping, leader progress) is cleared by
|
|
* re-running raft_init with the restored (term, voted_for, log_count), returning
|
|
* the node to FOLLOWER. The four callbacks, ctx, idx, and partition flag are
|
|
* preserved. Returns raft_init's status (0 on success).
|
|
*/
|
|
int raft_sim_restart_node(raft_sim_cluster_t *c, int node_idx);
|
|
|
|
#endif /* XCE_TESTING */
|
|
|
|
#endif /* RAFT_H */
|