wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/lib/raft/raft.h

898 lines
45 KiB
C
Raw Blame History

#ifndef RAFT_H
#define RAFT_H
#include <errno.h>
#include <stdint.h>
#include <string.h>
/* raft_role_t: self-contained role enum (does not depend on ctrl_role_t) */
typedef enum {
RAFT_FOLLOWER = 0,
RAFT_CANDIDATE = 1,
RAFT_LEADER = 2,
} raft_role_t;
/* Upper bound on cluster size for the fixed-size pre-vote reply-tracking
* array. Matches the simulator bound (RAFT_SIM_MAX_NODES); real clusters
* are far smaller. */
#define RAFT_MAX_PEERS 8
/* Forward declaration — full definition follows raft_state_t */
typedef struct raft_callbacks_t raft_callbacks_t;
/* raft_state_t: pure Raft consensus state for one node.
*
* Contains only the Raft invariant state. Application-specific fields
* (config_tag, log_lsn, heartbeat timestamps, ctrl_role) stay in ha_state_t;
* they are accessed by the caller via the on_vote_eligible callback. */
typedef struct {
uint64_t term; /* current term (monotonically increasing) */
raft_role_t role; /* current role */
int voted_for; /* peer index voted for this term; -1=none; self_idx=self-vote */
uint64_t voted_for_term; /* term in which voted_for was set */
int n_peers; /* total cluster size (including self) */
int self_idx; /* this node's index in [0, n_peers) */
int votes_granted; /* votes collected in current election */
int replies_received; /* total peer replies received (for all-replied-no-quorum, Req 4.4) */
int election_decided; /* 1 once on_become_leader/follower fired */
uint32_t election_timeout_ms;
int my_priority; /* priority index for jitter (0 = highest) */
/* ---- PreVote round tally (Req 1.2) ----
* Self-contained pre-vote bookkeeping. A pre-vote round mutates NONE of
* the persistent Raft state above (term, voted_for, role); these fields
* only track the in-flight straw poll and are reset at each round start. */
int pre_vote_in_progress; /* 1 while a pre-vote round is active */
int pre_votes_granted; /* grants collected this round (incl. self) */
int pre_vote_replies[RAFT_MAX_PEERS]; /* per-peer replied flag; avoids double-count */
/* ---- Log replication state (raft-log-replication, raft.tla logVars/leaderVars) ----
* The library stores NO entry bytes and NO per-entry term array — entry terms
* are obtained via on_get_entry_term; bytes live in the adapter/simulator. This
* keeps raft_state_t fixed-size and the core pure (same discipline as not
* storing (term, voted_for) bytes). Index is 1-based (raft.tla) and maps 1:1
* to the WAL LSN. */
uint64_t log_len; /* highest local log index (0 = empty) */
uint64_t next_index[RAFT_MAX_PEERS]; /* leader: next index to send each peer */
uint64_t match_index[RAFT_MAX_PEERS]; /* leader: highest index known replicated to each peer */
/* ---- Commit index (raft-commit-safety, raft.tla commitIndex) ----
* Highest 1-based index (== WAL LSN) the library considers safely committed.
* Monotonic non-decreasing. Restored at init from initial_commit_index (the
* library never reads storage — same discipline as initial_term/log_len).
* Advanced ONLY through raft_advance_commit (the current-term Figure-8 rule);
* raft_on_append_entries_reply leaves it untouched. commit_index <= log_len. */
uint64_t commit_index;
/* set by raft_init — needed by the state machine to fire callbacks */
const raft_callbacks_t *cbs;
void *ctx;
} raft_state_t;
/* raft_snapshot_t: point-in-time read of all Raft state for logging/debugging */
typedef struct {
uint64_t term;
raft_role_t role;
int voted_for;
uint64_t voted_for_term;
int votes_granted;
int election_decided;
} raft_snapshot_t;
/* raft_callbacks_t: all side effects the library fires.
*
* The library owns Raft invariants (term ordering, one-vote-per-term, quorum).
* All application-specific policy (freshness gate, role gate, catch-up state,
* heartbeat liveness) is delegated to on_vote_eligible so the library is
* fully independent of XCE domain logic.
*
* REQUIRED callbacks (must be non-NULL at raft_init):
* on_send_vote_request, on_become_leader, on_become_follower,
* on_persist, on_schedule_timeout, on_cancel_timeout,
* on_start_election_after_delay,
* on_persist_entry, on_get_entry_term, on_send_append_entries, on_truncate_from,
* on_commit_advanced
*
* OPTIONAL callbacks (may be NULL):
* on_vote_eligible — if NULL, all Raft-eligible candidates are granted
* on_send_pre_vote_request — if NULL, pre-vote emits are suppressed
* on_trace — if NULL, tracing is suppressed
*/
typedef struct raft_callbacks_t {
/* REQUIRED: Send a RequestVote RPC to peer peer_idx for the given term.
* The caller encodes config_tag, log_lsn, and any other fields in the wire RPC. */
int (*on_send_vote_request)(void *ctx, int peer_idx, uint64_t term);
/* OPTIONAL: Send a PreVote (straw-poll) request to peer peer_idx for the
* prospective term (current term + 1). Unlike on_send_vote_request, this
* MUST NOT cause the peer to persist or mutate term/voted_for — it is a
* non-binding probe. If NULL, the raft library suppresses the emit and
* the caller is responsible for any pre-vote transport itself. */
int (*on_send_pre_vote_request)(void *ctx, int peer_idx,
uint64_t prospective_term);
/* OPTIONAL: Application-specific vote eligibility gate.
* Called after Raft invariants pass (correct term, voted_for unset).
* Returns 1 to grant, 0 to deny. Receives the candidate's peer_idx and
* the incoming term so the caller can check freshness (config_tag, log_lsn),
* role gate (LEADER/COMMITTING deny), catch-up status, heartbeat liveness.
* If NULL, the library grants the vote whenever Raft invariants pass. */
int (*on_vote_eligible)(void *ctx, int from_idx, uint64_t term);
/* REQUIRED: Node has won the election */
void (*on_become_leader)(void *ctx, uint64_t term);
/* REQUIRED: Node lost election or saw higher term; leader_idx = -1 if unknown */
void (*on_become_follower)(void *ctx, uint64_t term, int leader_idx);
/* REQUIRED: Persist term + voted_for before granting vote (durability guarantee) */
int (*on_persist)(void *ctx, uint64_t term, int voted_for);
/* REQUIRED: Schedule election timeout (fires raft_on_election_timeout after delay_ms) */
void (*on_schedule_timeout)(void *ctx, uint32_t delay_ms);
/* REQUIRED: Cancel pending election timeout */
void (*on_cancel_timeout)(void *ctx);
/* REQUIRED: Request delayed election start after RDHUP (priority-based jitter) */
void (*on_start_election_after_delay)(void *ctx, uint32_t delay_ms);
/* ---- Log replication callbacks (raft-log-replication) ----
* REQUIRED (all four non-NULL at raft_init). The library owns the protocol
* decisions; these perform the I/O. The library stores no entry bytes, so it
* reads entry terms only via on_get_entry_term and writes/truncates only via
* these callbacks. */
/* REQUIRED: durably append/overwrite the log entry at 1-based `index` with
* `term` (persist-before-act: called before an append/accept is reported). */
int (*on_persist_entry)(void *ctx, uint64_t index, uint64_t term,
const void *entry, size_t len);
/* REQUIRED: term of the local entry at 1-based `index` (0 if absent). */
uint64_t (*on_get_entry_term)(void *ctx, uint64_t index);
/* REQUIRED: transmit one AppendEntries to peer_idx with prevLog{Index,Term}
* and leader_commit. `n_entries` is 0 (heartbeat) or 1 (this spec, like
* raft.tla); when 1 the adapter sources the single entry at prev_log_index+1
* from its own log (the library holds no entry bytes). [Concretizes the
* design's varargs "entries handle" placeholder into an explicit count.] */
int (*on_send_append_entries)(void *ctx, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries);
/* REQUIRED: delete local entries at `index`..end. */
int (*on_truncate_from)(void *ctx, uint64_t index);
/* ---- Commit-advance trigger (raft-commit-safety) ----
* REQUIRED: fired AFTER commit_index strictly advances. The controller adapter
* feeds this into the EXISTING synchronous commit barrier / persist-before-ACK
* durability — this callback TRIGGERS commit-eval; it does NOT replace the
* barrier.
*
* Contract: the library decides WHETHER the commit index advanced; the callback
* only triggers the adapter's commit work. Fired exactly once per strict advance,
* NEVER on a held commit index (current-term Figure-8 rule), and NEVER from
* raft_on_append_entries_reply. NULL-checked / required in raft_init (-EINVAL if
* NULL). It NEVER performs durability itself. */
void (*on_commit_advanced)(void *ctx, uint64_t commit_index);
/* OPTIONAL: Trace callback for observability.
* event: short string literal, e.g. "vote_granted", "became_leader"
* detail: formatted human-readable context string
* If NULL, tracing is suppressed. */
void (*on_trace)(void *ctx, const char *event, const char *detail);
} raft_callbacks_t;
/*
* raft_config_t: initialization context for a Raft node.
*
* Encapsulates all configuration so raft_init() has a single, extensible
* parameter rather than a flat list. Matches the existing XCE pattern
* (ha_config_t, persist_config_t). Zero-initialize and set only the
* fields you need — the library fills in safe defaults for unset fields.
*
* Persistence model:
* initial_term / initial_voted_for are restored by the caller from disk
* before raft_init(). The library never reads storage; it only writes
* via on_persist callbacks. Pass {0, -1} on first boot.
*/
typedef struct {
int n_peers; /* total cluster size (including self) */
int self_idx; /* this node's index in [0, n_peers) */
uint32_t election_timeout_ms; /* base election timeout; 0 → default 2000ms */
int my_priority; /* priority index for RDHUP jitter (0 = highest) */
uint64_t initial_term; /* term restored from disk; 0 on first boot */
int initial_voted_for; /* voted_for restored from disk; -1 if none */
uint64_t initial_log_len; /* WAL length restored from disk; 0 on first boot (library never reads storage) */
uint64_t initial_commit_index; /* committed LSN restored from disk; 0 on first boot (library never reads storage) */
const raft_callbacks_t *cbs; /* all callbacks; required callbacks must be non-NULL */
void *ctx; /* opaque caller context passed to every callback */
} raft_config_t;
/* Return codes
*
* Note: RAFT_GRANT and RAFT_BECAME_LEADER share value 1; RAFT_DENY and RAFT_OK
* share value 0. This is intentional — different return values are used by
* different functions (raft_on_vote_request vs raft_on_vote_reply) in different
* call contexts.
*/
#define RAFT_OK 0
#define RAFT_GRANT 1
#define RAFT_DENY 0
#define RAFT_BECAME_LEADER 1
#define RAFT_BECAME_FOLLOWER 2
#define RAFT_PENDING 3
#define RAFT_ELECTED_IMMEDIATELY 4
#define RAFT_SKIP_ELECTION 5
#define RAFT_ELECTION_STARTED 6
/* PreVote (straw-poll) outcomes returned by raft_on_pre_vote_reply.
* Distinct values from the election codes above; a pre-vote round mutates
* no persistent state, so these never imply a term/role transition. */
#define RAFT_PRE_VOTE_WON 7 /* quorum of grants reached this round */
#define RAFT_PRE_VOTE_LOST 8 /* all peers replied, no quorum */
#define RAFT_PRE_VOTE_PENDING 9 /* still awaiting replies */
/*
* raft_init: initialize raft_state_t from a raft_config_t.
* on_vote_eligible and on_trace may be NULL (optional policy / observability).
* All other callbacks in cbs must be non-NULL.
* Returns 0 on success; -EINVAL if a required callback is NULL or n_peers < 1.
*/
int raft_init(raft_state_t *s, const raft_config_t *cfg);
/*
* raft_start_election: begin a new election.
* Increments term, records self-vote (voted_for=self_idx), transitions to
* CANDIDATE, fires on_send_vote_request for each peer, fires on_schedule_timeout.
* Returns 0 on success; -EALREADY if already CANDIDATE/LEADER;
* RAFT_ELECTED_IMMEDIATELY for single-node (n_peers==1).
*/
int raft_start_election(raft_state_t *s);
/*
* raft_on_vote_request: process an incoming RequestVote RPC.
* Applies Raft invariants (term comparison, one-vote-per-term), then the
* (lastLogTerm, lastLogIndex) freshness gate (raft.tla HandleRequestVoteRequest
* logOk, line 285, closes G6), then the application policy gate.
*
* The library owns the logOk decision (design "Option B: library owns logOk"):
* local_last_log_term = (log_len > 0) ? on_get_entry_term(log_len) : 0
* logOk = (cand_last_log_term > local_last_log_term)
* || (cand_last_log_term == local_last_log_term
* && cand_last_log_index >= log_len)
* The candidate advertises (cand_last_log_term, cand_last_log_index) in its wire
* RequestVote; the caller decodes and passes them here.
*
* Step order: stale-term deny → UpdateTerm/step-down → role gate →
* one-vote-per-term → logOk gate (NEW; precedes the application gate, so a
* !logOk denies regardless of on_vote_eligible) → on_vote_eligible →
* persist-before-grant.
*
* Fires on_persist before granting.
* Returns RAFT_GRANT (1) or RAFT_DENY (0); sets *reply_term to current term.
*/
int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term,
uint64_t cand_last_log_term,
uint64_t cand_last_log_index,
uint64_t *reply_term);
/*
* raft_on_vote_reply: process a reply to our RequestVote.
* Returns RAFT_BECAME_LEADER, RAFT_BECAME_FOLLOWER, or RAFT_PENDING.
*/
int raft_on_vote_reply(raft_state_t *s, int from_idx, int granted, uint64_t term);
/*
* raft_on_election_timeout: called by the timer when no quorum was reached.
* Transitions CANDIDATE → FOLLOWER; fires on_become_follower, on_cancel_timeout.
*/
void raft_on_election_timeout(raft_state_t *s);
/*
* raft_on_leader_lost: primary control channel dropped (RDHUP).
* Clears voted_for, transitions to FOLLOWER, fires on_start_election_after_delay.
*/
void raft_on_leader_lost(raft_state_t *s);
/*
* raft_check_and_start_election: call after the delay from raft_on_leader_lost.
* Starts election only if we have not already voted for a non-self candidate
* in the new term (i.e., another node won during the jitter window).
* Returns RAFT_SKIP_ELECTION or RAFT_ELECTION_STARTED.
*/
int raft_check_and_start_election(raft_state_t *s);
/* ---------- PreVote (straw poll before a binding election) ----------
*
* The pre-vote round is a non-binding probe: it asks peers whether they WOULD
* grant a vote at term+1, without anyone incrementing term, setting voted_for,
* or persisting. Only on RAFT_PRE_VOTE_WON does the caller proceed to the
* unchanged raft_start_election (which performs the real term bump + persist).
*
* INVARIANT (Req 1.2, 6.2): none of these three functions mutate s->term,
* s->voted_for, s->voted_for_term, or s->role, and none invoke on_persist.
*/
/*
* raft_begin_pre_vote: start a pre-vote round.
* Computes prospective term = term + 1 (NOT assigned to s->term), sets
* pre_vote_in_progress, pre_votes_granted = 1 (self), and fires
* on_send_pre_vote_request(ctx, peer_idx, prospective_term) for each peer
* (NULL-checked — suppressed if the callback is unset).
* Precondition: role == RAFT_FOLLOWER.
*/
int raft_begin_pre_vote(raft_state_t *s);
/*
* raft_on_pre_vote_request: answer an incoming PreVote probe.
* Denies if cand_term < s->term; then applies the IDENTICAL (lastLogTerm,
* lastLogIndex) logOk as raft_on_vote_request (the library owns the predicate —
* it is NO LONGER applied by the dispatcher before this call), then consults
* on_vote_eligible (granting when the gate is NULL). Mutates NOTHING — never
* touches term, voted_for, voted_for_term, role, or commit_index, and never
* fires on_persist — preserving the PreVote side-effect-free invariant.
* Returns 1 = grant, 0 = deny.
*/
int raft_on_pre_vote_request(raft_state_t *s, int from_idx, uint64_t cand_term,
uint64_t cand_last_log_term,
uint64_t cand_last_log_index);
/*
* raft_on_pre_vote_reply: tally a reply to our PreVote.
* Owns the pre-vote tally in raft_state_t (pre_votes_granted,
* pre_vote_replies), mirroring raft_on_vote_reply but WITHOUT firing
* on_become_leader/on_become_follower or making any role/term change.
* Returns RAFT_PRE_VOTE_WON at quorum, RAFT_PRE_VOTE_LOST when all peers
* replied without quorum, else RAFT_PRE_VOTE_PENDING.
*/
int raft_on_pre_vote_reply(raft_state_t *s, int peer_idx, int granted);
/* ---------- Log replication (raft-log-replication) ----------
*
* The library owns the log protocol decisions (append, AppendEntries logOk +
* conflict truncation, UpdateTerm-on-any-RPC) and drives I/O through the four
* log callbacks. Index is 1-based (raft.tla) == WAL LSN. Commit-index advance
* and (lastLogTerm,lastLogIndex) vote freshness are NOT here — they are
* raft-commit-safety. */
/* Result of handling an AppendEntries request — the adapter replies with this. */
typedef struct {
uint64_t reply_term; /* current term to return to the leader */
int success; /* 1 = accepted, 0 = rejected */
uint64_t match_index; /* on success: prev_log_index + entries_len; else 0 */
} raft_append_result_t;
/*
* raft_log_append: leader appends one client entry (raft.tla ClientRequest, 246).
* Stamps the entry with s->term, assigns index log_len+1, persists it via
* on_persist_entry BEFORE returning success (persist-before-act). Does NOT
* advance any commit index (owned by raft-commit-safety).
* out_index receives the assigned 1-based index (== WAL LSN) on success.
* Returns RAFT_OK; -EPERM if role != RAFT_LEADER; propagates a non-zero
* on_persist_entry return on persist failure (no log mutation on failure).
*/
int raft_log_append(raft_state_t *s, const void *entry, size_t len,
uint64_t *out_index);
/*
* raft_send_append_entries: emit one AppendEntries to peer_idx (raft.tla
* AppendEntries, 204). Computes prev_log_index = next_index[peer]-1 and
* prev_log_term via on_get_entry_term, sends at most one entry (n_entries 0/1),
* via on_send_append_entries. Leader-only (R3.5); leader_commit is opaque.
* Returns RAFT_OK.
*/
int raft_send_append_entries(raft_state_t *s, int peer_idx,
uint64_t leader_commit);
/*
* raft_on_append_entries_request: handle an incoming AppendEntries / heartbeat
* (raft.tla HandleAppendEntriesRequest 327 + UpdateTerm 406). Step order is
* fixed: UpdateTerm first (G5, every RPC incl. heartbeat) → logOk (G2) →
* reject / candidate-step-down / accept (already-have / conflict-truncate (G3) /
* append). entries_len is 0 (heartbeat) or 1. Fills *out. leader_commit is
* accepted but NOT acted on here (raft-commit-safety). Returns RAFT_OK.
*/
int raft_on_append_entries_request(raft_state_t *s, int leader_idx,
uint64_t term, uint64_t prev_log_index,
uint64_t prev_log_term, uint64_t leader_commit,
const void *entry, size_t entry_len,
uint64_t entry_term, int entries_len,
raft_append_result_t *out);
/*
* raft_on_append_entries_reply: process a follower's AppendEntries response
* (raft.tla HandleAppendEntriesResponse 393). Applied only when
* reply_term == s->term (stale-term drop, 415). success: next_index[peer] =
* match_index+1, match_index[peer] = match_index. failure: next_index[peer] =
* Max(next_index[peer]-1, 1). Does NOT advance commit. Returns RAFT_OK.
*/
int raft_on_append_entries_reply(raft_state_t *s, int peer_idx, int success,
uint64_t match_index, uint64_t reply_term);
/*
* raft_advance_commit: leader recomputes its commit index (raft.tla
* AdvanceCommitIndex, 259). A SEPARATE action from raft_on_append_entries_reply
* (which only updates match_index[]) — mirroring raft.tla, where
* AdvanceCommitIndex is distinct from HandleAppendEntriesResponse; this lets a
* single-node leader commit ({self} in agreeIndexes).
*
* agreeIndexes = { index in 1..log_len : {self} U {peer : match_index[peer] >=
* index} is a quorum }. When agreeIndexes is non-empty AND
* on_get_entry_term(Max(agreeIndexes)) == s->term, sets commit_index =
* Max(agreeIndexes) (the Figure-8 current-term rule, line 270) and, on a STRICT
* advance, fires on_commit_advanced(ctx, commit_index) exactly once. Otherwise
* leaves commit_index unchanged and does NOT fire on_commit_advanced.
*
* Monotonic: commit_index never decreases. Reads entry terms only via
* on_get_entry_term; performs NO storage I/O. on_commit_advanced only TRIGGERS
* the controller's existing synchronous commit barrier — the library performs
* no durability work here. Returns RAFT_OK; -EPERM if role != RAFT_LEADER
* (commit_index left unchanged, no callback).
*/
int raft_advance_commit(raft_state_t *s);
/* Fills snap with current state. Caller must hold lock. */
void raft_snapshot(const raft_state_t *s, raft_snapshot_t *snap);
/* ---------- deterministic simulator (test builds only) ---------- */
#ifdef XCE_TESTING
#include <stdio.h>
#define RAFT_SIM_MAX_NODES 8
/* Upper bound on a simulated node's in-memory log. Test entries are tiny and
* the sim drives short runs, so a small fixed capacity keeps the node struct
* ASAN-clean (no allocation) while comfortably exceeding any test's log depth. */
#define RAFT_SIM_MAX_LOG 64
/* Fixed entry payload size — test entries are a few bytes; bounded copies only. */
#define RAFT_SIM_ENTRY_LEN 16
/* Upper bound on buffered AppendEntries awaiting delivery (mirrors pending[]). */
#define RAFT_SIM_MAX_AE 64
/* ---- conformance harness bounds (raft-sim-conformance) ----
* Capacity of the unified message bag (cluster.bag[], mirrors raft.tla's
* `messages`): in-flight vote/AE requests and responses, the unit on which
* Drop/Duplicate/Reorder operate. Sized generously above RAFT_SIM_MAX_AE so a
* bounded run never overflows the bag (duplicates add members). */
#define RAFT_SIM_MAX_MSGS 256
/* Upper bound on recorded scheduler steps (cluster.trace[]) for replayable
* failure traces. One raft_sim_action_t is recorded per step. */
#define RAFT_SIM_MAX_STEPS 1024
/* Buffered RPC entry: one pending vote-request to route.
* last_log_term/last_log_index capture the CANDIDATE's advertised last-log tuple
* at SEND time (mirroring raft_sim_ae_t.leader_term) so the (lastLogTerm,
* lastLogIndex) freshness gate (raft-commit-safety) is evaluated against the
* candidate's send-epoch log, not a value re-read at delivery. */
typedef struct {
int src;
int dst;
uint64_t term;
uint64_t last_log_term; /* candidate's last-log term at send (0 if empty) */
uint64_t last_log_index; /* candidate's last-log index at send (== log_count) */
} raft_sim_rpc_t;
/* One in-memory log entry: term stamp + fixed-size payload bytes.
* term == 0 means "absent" (matches the 0-means-absent convention of
* on_get_entry_term); 1-based indexing maps slot[i-1] to log index i. */
typedef struct {
uint64_t term;
unsigned char bytes[RAFT_SIM_ENTRY_LEN];
size_t len;
} raft_sim_entry_t;
/* Buffered AppendEntries RPC awaiting synchronous delivery (task 8.2).
* For n_entries == 1 the single entry's index/term/bytes are captured here
* because the library holds no entry bytes — the sender sources them from its
* own in-memory log at prev_log_index + 1 when buffering. */
typedef struct {
int src;
int dst;
uint64_t leader_term; /* leader's term captured at SEND time (not re-read
* at delivery) — required so reorder/churn tests
* (9.3/9.4) deliver the send-epoch term, preserving
* UpdateTerm/stale-drop (G5) semantics. */
uint64_t prev_log_index;
uint64_t prev_log_term;
uint64_t leader_commit;
int n_entries; /* 0 (heartbeat) or 1 (this spec) */
uint64_t entry_index; /* valid when n_entries == 1 */
uint64_t entry_term; /* valid when n_entries == 1 */
unsigned char entry_bytes[RAFT_SIM_ENTRY_LEN];
size_t entry_len; /* valid when n_entries == 1 */
} raft_sim_ae_t;
/* ---- conformance harness data types (raft-sim-conformance) ----
*
* These mirror raft.tla's `messages` bag and per-node variables for the
* deterministic conformance/verification harness. They are SEPARATE from the
* byte-payload WAL model above (raft_sim_entry_t / raft_sim_ae_t / raft_sim_rpc_t):
* the conformance harness uses a lightweight (term, value) log entry and a single
* unified message type so a randomized, seeded action scheduler can drive N library
* instances purely synchronously and assert the four Raft safety invariants. */
/* Unified bag-message kind: a vote/AE request or its matching response. */
typedef enum {
RAFT_SIM_MSG_VOTE_REQ,
RAFT_SIM_MSG_VOTE_RSP,
RAFT_SIM_MSG_AE_REQ,
RAFT_SIM_MSG_AE_RSP,
} raft_sim_msg_kind_t;
/* One in-flight RPC in the message bag (mirrors raft.tla `messages`). Carries the
* union of vote/AE request and response fields; only the fields relevant to `kind`
* are meaningful. This is the unit Drop/Duplicate/Reorder operate on; a handler
* reply is pushed back as a *_RSP member so responses are first-class. */
typedef struct {
raft_sim_msg_kind_t kind;
int src, dst;
uint64_t term; /* RPC term (all kinds) */
/* vote request: candidate's advertised last-log tuple at SEND time, feeding the
* (lastLogTerm, lastLogIndex) freshness gate at delivery (mirrors
* raft_sim_rpc_t.last_log_*). Sourced from the candidate's own log at send. */
uint64_t last_log_term, last_log_index;
/* AE request */
uint64_t prev_log_index, prev_log_term, leader_commit;
uint64_t entry_term, entry_value; /* one entry per AE (raft.tla "<=1 entry") */
int entries_len; /* 0 = heartbeat, 1 = one entry */
/* responses */
int granted; /* vote rsp */
int success; /* AE rsp */
uint64_t match_index; /* AE rsp */
} raft_sim_msg_t;
/* One conformance-harness log entry: term stamp + synthetic monotone client value.
* The 1-based array position is the log index (= WAL LSN); `value` makes State
* Machine Safety divergence observable. Distinct from raft_sim_entry_t (which models
* the byte-payload WAL): this is the (term, value) bag/applied model. */
typedef struct {
uint64_t term;
uint64_t value;
} raft_sim_log_entry_t;
/* The Next-action set the scheduler chooses from; each maps to a raft.tla `Next`
* disjunct. Only *enabled* actions are eligible at a given step. */
typedef enum {
RAFT_SIM_ACT_RESTART, /* Restart: re-init preserving persisted state */
RAFT_SIM_ACT_TIMEOUT, /* Timeout: become candidate, bump term */
RAFT_SIM_ACT_REQUEST_VOTE, /* RequestVote: deliver a vote-request */
RAFT_SIM_ACT_BECOME_LEADER, /* BecomeLeader: deliver quorum-completing vote-rsp */
RAFT_SIM_ACT_CLIENT_REQUEST, /* ClientRequest: leader appends a client value */
RAFT_SIM_ACT_APPEND_ENTRIES, /* AppendEntries: leader pushes an AE-request */
RAFT_SIM_ACT_ADVANCE_COMMIT, /* AdvanceCommitIndex: leader advances commit */
RAFT_SIM_ACT_RECEIVE, /* Receive: deliver a bag message to its handler */
RAFT_SIM_ACT_DUPLICATE_MESSAGE, /* DuplicateMessage: re-push a bag member */
RAFT_SIM_ACT_DROP_MESSAGE, /* DropMessage: discard a bag member */
} raft_sim_action_kind_t;
/* One replayable trace unit: the action taken at a scheduler step plus the operands
* needed to reproduce it (actor node, target message index in the bag, and the
* (term, index) the action referenced). Recorded per step into cluster.trace[]. */
typedef struct {
raft_sim_action_kind_t kind;
int actor; /* node that took the action (-1 if N/A) */
int msg_index; /* bag index acted on (-1 if N/A) */
uint64_t term; /* term referenced by the action (0 if N/A) */
uint64_t index; /* log index referenced by the action (0 if N/A) */
} raft_sim_action_t;
/* One simulated node */
typedef struct {
raft_state_t state;
raft_callbacks_t cbs; /* wired to sim routing functions */
void *ctx; /* points back to raft_sim_cluster_t */
int idx;
int partition; /* 1 = isolated from other nodes */
/* in-memory replicated log backing the four log callbacks (1-based index;
* log[i-1] holds entry i; log_count is the highest occupied index). */
raft_sim_entry_t log[RAFT_SIM_MAX_LOG];
int log_count;
/* ---- commit-advance callback mirror (raft-commit-safety) ----
* Updated by the node's on_commit_advanced callback: commit_index records the
* last value the library reported (the strict-advance commit position), and
* commit_advanced_count counts how many times on_commit_advanced fired (so
* tests can assert fire-once / never-on-hold trigger semantics). The
* commit-advance DRIVER helper is a later task; this is only the callback half. */
uint64_t commit_index;
int commit_advanced_count;
/* ---- conformance harness per-node state (raft-sim-conformance) ----
* applied[k-1] holds the client value applied at committed index k (set by the
* node's on_commit_advanced as commit_index advances). State Machine Safety
* checks that applied[k] agrees across nodes. NOTE: log_count above already
* satisfies the design's `log_len` requirement for the in-memory log length;
* no separate log_len field is added (it would duplicate log_count). */
uint64_t applied[RAFT_SIM_MAX_LOG];
/* Persisted-(term, votedFor) shadow surviving a Restart, mirroring raft.tla's
* `Restart` (keeps currentTerm/votedFor/log, resets volatile state). The
* conformance restart action restores from this shadow rather than re-reading
* live volatile state. */
uint64_t persisted_term;
int persisted_voted_for;
/* per-node trace event log for test assertions */
char trace_buf[4096];
int trace_len;
} raft_sim_node_t;
/* N-node simulation cluster */
typedef struct {
raft_sim_node_t nodes[RAFT_SIM_MAX_NODES];
int n;
int leader_count; /* incremented by each on_become_leader */
/* buffered RPC queue */
raft_sim_rpc_t pending[64];
int pending_count;
/* buffered AppendEntries queue (delivered by raft_sim_deliver_append_entries,
* task 8.2); on_send_append_entries enqueues here. */
raft_sim_ae_t ae_pending[RAFT_SIM_MAX_AE];
int ae_pending_count;
/* ---- conformance harness cluster state (raft-sim-conformance) ----
* Unified message bag mirroring raft.tla `messages`: vote/AE requests and
* responses, the unit Drop/Duplicate/Reorder operate on. */
raft_sim_msg_t bag[RAFT_SIM_MAX_MSGS];
int bag_count;
/* Seeded deterministic PRNG: `seed` is the run's reproducibility key; `prng`
* is the evolving generator state (SplitMix64/xorshift, no libc rand()). */
uint64_t seed;
uint64_t prng;
/* Replayable per-step action trace for failure dumps. */
raft_sim_action_t trace[RAFT_SIM_MAX_STEPS];
int trace_len;
} raft_sim_cluster_t;
/*
* raft_sim_init: initialize an n-node simulation cluster.
* Each node gets index 0..n-1, priority=idx, election_timeout_ms as given.
* Wires on_send_vote_request to buffer RPCs into c->pending.
* Wires on_become_leader to increment c->leader_count.
* All other required callbacks get no-op implementations.
* on_trace appends to the node's trace_buf.
* Returns 0 on success.
*/
int raft_sim_init(raft_sim_cluster_t *c, int n_nodes,
uint32_t election_timeout_ms);
/*
* raft_sim_conformance_init: initialize an n-node CONFORMANCE cluster
* (raft-sim-conformance task 1.2). Modeled on raft_sim_init but wires the
* conformance adapter callback set and seeds the deterministic PRNG:
* - log storage callbacks (on_persist_entry / on_get_entry_term /
* on_truncate_from) reuse the in-memory node->log[] backing,
* - on_commit_advanced sets commit_index AND applies each newly-committed
* entry's decoded value into node->applied[] (State Machine Safety oracle),
* - on_send_vote_request / on_send_append_entries push a raft_sim_msg_t onto
* the unified cluster bag[] (mirroring raft.tla `messages`).
* `seed` is the run's reproducibility key; c->prng is initialized from it so
* every later nondeterministic choice flows from the seed (R1.2). Returns 0 on
* success, -1 on an invalid node count. Does NOT deliver bag messages (that is
* task 2.1).
*/
int raft_sim_conformance_init(raft_sim_cluster_t *c, int n_nodes,
uint32_t election_timeout_ms, uint64_t seed);
/*
* raft_sim_prng_next: advance the cluster's seeded SplitMix64 generator and return
* a well-mixed 64-bit value. The single source of nondeterminism for the scheduler
* (R1.2): for a fixed seed the value sequence is byte-identical across runs and
* platforms (no libc rand()). Exposed so tests can assert deterministic replay.
*/
uint64_t raft_sim_prng_next(raft_sim_cluster_t *c);
/*
* raft_sim_bag_push: append one message to the unified cluster bag[] (mirrors
* raft.tla `messages`). Fails loud (returns -1) on overflow rather than silently
* truncating; returns 0 on success. Exposed so tests (and later scheduler tasks)
* can seed the bag directly.
*/
int raft_sim_bag_push(raft_sim_cluster_t *c, const raft_sim_msg_t *msg);
/*
* raft_sim_bag_deliver: route the bag message at index `i` to its matching library
* handler (vote/AE request or response) and push any handler reply back onto the
* bag as a *_RSP message (src/dst swapped). Mirrors raft.tla Receive: delivering a
* request consumes it (swap-with-last removal, so bag indices stay compact for the
* Drop/Duplicate/Reorder actions in task 2.2) and produces a response member.
* UpdateTerm higher-term adoption and the stale-response drop live INSIDE the
* handlers (R2.4) — this only routes. Delivery honors node.partition: if either
* endpoint is partitioned the message is NOT delivered (no removal, no reply),
* mirroring a dropped network edge.
* Returns 0 on a delivered message, -1 on an out-of-range index, -2 when the
* endpoint is partitioned (distinct non-zero status, message left in the bag).
*/
int raft_sim_bag_deliver(raft_sim_cluster_t *c, int i);
/*
* raft_sim_bag_drop: DropMessage (raft.tla:448). Discard the bag member at index
* `i` (swap-with-last removal, bag_count -= 1) without mutating ANY node — mirrors
* the TLA+ UNCHANGED of all non-message vars. Returns 0 on a discarded member, -1
* (fail loud, bag untouched) on an out-of-range index.
*/
int raft_sim_bag_drop(raft_sim_cluster_t *c, int i);
/*
* raft_sim_bag_duplicate: DuplicateMessage (raft.tla:443). Re-enqueue a copy of the
* bag member at index `i` onto the tail (multiplicity += 1) without mutating ANY
* node. Honors RAFT_SIM_MAX_MSGS (fail loud on overflow). Returns -1 on an
* out-of-range index, otherwise the push status (0 on success, -1 on bag overflow).
*/
int raft_sim_bag_duplicate(raft_sim_cluster_t *c, int i);
/*
* raft_sim_bag_pick: Reorder source. Return a PRNG-chosen in-flight bag index in
* [0, bag_count) so delivery order is seed-deterministic (replayable) rather than
* FIFO. Pure read: advances only c->prng, never any node state. Returns -1 when the
* bag is empty. The caller delivers the chosen index via raft_sim_bag_deliver.
*/
int raft_sim_bag_pick(raft_sim_cluster_t *c);
/*
* raft_sim_dump_log: print trace logs for all nodes to fp.
* If RAFT_SIM_VERBOSE env var is set, always prints.
*/
void raft_sim_dump_log(raft_sim_cluster_t *c, FILE *fp);
/*
* raft_sim_deliver_pending: route all buffered RPCs synchronously.
* For each buffered (src, dst, term), calls raft_on_vote_request on dst
* then routes the reply back to src via raft_on_vote_reply.
* Skips delivery if dst has partition=1.
*/
void raft_sim_deliver_pending(raft_sim_cluster_t *c);
/*
* raft_sim_deliver_append_entries: route all buffered AppendEntries synchronously.
* For each buffered AE (src, dst, ...), calls raft_on_append_entries_request on dst
* then routes the reply back to src via raft_on_append_entries_reply, mirroring the
* vote-RPC routing in raft_sim_deliver_pending. Skips delivery if either endpoint
* has partition=1. Drains the queue (snapshots the count, since handling may buffer
* more AEs). The single entry (n_entries == 1) is sourced from the buffered AE.
*/
void raft_sim_deliver_append_entries(raft_sim_cluster_t *c);
/*
* raft_sim_start_all_elections: call raft_start_election on each node 0..n-1.
*/
void raft_sim_start_all_elections(raft_sim_cluster_t *c);
/*
* raft_sim_fire_timeout: call raft_on_election_timeout on node at node_idx.
*/
void raft_sim_fire_timeout(raft_sim_cluster_t *c, int node_idx);
/*
* raft_sim_partition: set/clear partition flag for node_idx.
* Subsequent raft_sim_deliver_pending drops RPCs to/from that node.
*/
void raft_sim_partition(raft_sim_cluster_t *c, int node_idx, int isolated);
/*
* raft_sim_check_log_matching: assert the Raft Log Matching property over the
* cluster's in-memory logs (Req 8.4).
*
* For every pair of nodes (a, b) and every index i present in BOTH logs
* (1 <= i <= min(log_count_a, log_count_b)), if the entries at i carry the
* SAME term, then the logs must be identical for all indices 1..i — same term
* AND same payload (length + bytes) at every k in 1..i.
*
* Returns 1 if Log Matching holds across the whole cluster, 0 if any pair/index
* violates it. Pure read-only over c (no I/O, no Unity assertions) so the lib
* stays test-framework-agnostic; the caller asserts on the returned value.
*/
int raft_sim_check_log_matching(const raft_sim_cluster_t *c);
/*
* raft_sim_check_election_safety: assert the Raft Election Safety property over
* the cluster's observable per-node state (Req 3.1, 3.6).
*
* For each term t, at most one node may hold the LEADER role: no two distinct
* nodes may both be (role == RAFT_LEADER) at the SAME term. Computed only from
* observable state (each node's role + term, read via raft_snapshot) — never
* from internal protocol bookkeeping (Req 3.6).
*
* Returns 1 if Election Safety holds across the whole cluster, 0 if any
* same-term LEADER pair exists. Pure read-only over c (no I/O, no Unity
* assertions) so the lib stays test-framework-agnostic; the caller asserts on
* the returned value (same convention as raft_sim_check_log_matching).
*/
int raft_sim_check_election_safety(const raft_sim_cluster_t *c);
/*
* raft_sim_check_leader_completeness: assert the Raft Leader Completeness property
* over the cluster's observable per-node state (Req 3.3, 3.6).
*
* Leader Completeness: if an entry is committed in term t, then that entry is
* present (same index, same term) in the logs of all leaders of terms > t.
*
* Computed purely from observable state (Req 3.6): the committed set is every
* index k in 1..node->commit_index for each node (carrying that node's
* log[k-1].term — the per-node mirror filled by on_commit_advanced); the current
* leaders are the nodes with role == RAFT_LEADER (read via raft_snapshot). For
* each leader L at term T_L and each committed entry (k, t) with t < T_L, L's log
* MUST contain index k at term t; otherwise the property is violated. Presence is
* keyed on (index, term) — Log Matching covers payload identity.
*
* Returns 1 if Leader Completeness holds across the whole cluster, 0 if any
* higher-term leader is missing (or carries a different term for) a lower-term
* committed entry. Pure read-only over c (no I/O, no Unity assertions) so the lib
* stays test-framework-agnostic; the caller asserts on the returned value (same
* convention as raft_sim_check_log_matching).
*/
int raft_sim_check_leader_completeness(const raft_sim_cluster_t *c);
/*
* raft_sim_check_state_machine_safety: assert the Raft State Machine Safety property
* over the cluster's observable per-node applied state (Req 3.4, 3.6).
*
* State Machine Safety: no two nodes have applied DIFFERENT values at the same log
* index. Computed purely from observable state (Req 3.6): each node's applied extent
* is 1..commit_index (sim_conf_on_commit_advanced fills applied[k-1] with the value
* applied at 1-based committed index k). For every pair (a < b) and every index k in
* the shared applied prefix 1..min(a->commit_index, b->commit_index), the property is
* violated if applied[k-1] differs between the two nodes. Pairs are symmetric, so
* a < b suffices.
*
* Returns 1 if State Machine Safety holds across the whole cluster, 0 if any pair of
* nodes disagrees on the value applied at a commonly-applied index. Pure read-only
* over c (reads applied[] and commit_index only; no I/O, no Unity assertions) so the
* lib stays test-framework-agnostic; the caller asserts on the returned value (same
* convention as raft_sim_check_log_matching).
*/
int raft_sim_check_state_machine_safety(const raft_sim_cluster_t *c);
/*
* raft_sim_invariant_t: identifier for the four Raft safety invariants checked by
* the conformance harness, in the FIXED evaluation order used by
* raft_sim_check_all (Req 3.5). RAFT_SIM_INV_NONE (0) means "all invariants hold".
* The ordinal order mirrors the requirements/design listing so the trace dumper
* (task 4.1) and scheduler (task 4.2) can report the failing invariant by name.
*/
typedef enum {
RAFT_SIM_INV_NONE = 0, /* all four invariants hold */
RAFT_SIM_INV_ELECTION_SAFETY, /* 1: raft_sim_check_election_safety */
RAFT_SIM_INV_LOG_MATCHING, /* 2: raft_sim_check_log_matching */
RAFT_SIM_INV_LEADER_COMPLETENESS, /* 3: raft_sim_check_leader_completeness */
RAFT_SIM_INV_STATE_MACHINE_SAFETY,/* 4: raft_sim_check_state_machine_safety */
} raft_sim_invariant_t;
/*
* raft_sim_check_all: evaluate the four Raft safety invariants over the cluster in
* the FIXED order (1) Election Safety, (2) Log Matching, (3) Leader Completeness,
* (4) State Machine Safety, and return the identifier of the FIRST one that is
* violated (Req 3.5 — "halt the run immediately and report the failing invariant").
*
* Short-circuits: once a checker reports a violation, later checkers are NOT
* evaluated, so the reported "first failing invariant" is deterministic. Returns
* RAFT_SIM_INV_NONE (0) when every invariant holds. Pure read-only over c (no I/O,
* no Unity assertions) so the lib stays test-framework-agnostic; the caller (trace
* dumper / scheduler) consumes the returned identity.
*/
raft_sim_invariant_t raft_sim_check_all(const raft_sim_cluster_t *c);
/*
* raft_sim_invariant_name: human-readable name for a raft_sim_invariant_t, for the
* trace dumper (task 4.1) and scheduler diagnostics. Returns a static string; never
* NULL (an out-of-range value maps to "unknown").
*/
const char *raft_sim_invariant_name(raft_sim_invariant_t inv);
/*
* raft_sim_advance_commit: drive raft_advance_commit on node_idx (task 6.1).
*
* Recomputes the node's commit index from its current match_index[] / log_len /
* term (the leader-only AdvanceCommitIndex + current-term Figure-8 rule). On a
* strict advance the node's on_commit_advanced fires, updating the per-node
* commit_index mirror and commit_advanced_count. No-op if node_idx is out of
* range. Off-leader nodes return -EPERM internally (the return is discarded;
* tests read the per-node mirror).
*/
void raft_sim_advance_commit(raft_sim_cluster_t *c, int node_idx);
/*
* raft_sim_restart_node: model a process restart of node_idx (Req 8.3).
*
* Durable state survives a restart: the in-memory log[] (which models the WAL on
* disk) and the persisted (term, voted_for) are retained. Volatile consensus
* state (role, votes, election bookkeeping, leader progress) is cleared by
* re-running raft_init with the restored (term, voted_for, log_count), returning
* the node to FOLLOWER. The four callbacks, ctx, idx, and partition flag are
* preserved. Returns raft_init's status (0 on success).
*/
int raft_sim_restart_node(raft_sim_cluster_t *c, int node_idx);
#endif /* XCE_TESTING */
#endif /* RAFT_H */
View git blame Copy permalink