wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/lib/raft/raft_sim.c
Rongsong Shen 6563adf8c6 docs(raft-sim-conformance): cosmetic cleanups (LOW review nits) 
- sim_tla_action_name: drop brittle per-case 'raft.tla:NNN' line-number comments
  (some had drifted a few lines off; the return string IS the stable raft.tla
  Next-disjunct name, and the function header already cites raft.tla:454).
- test_sim_5_1_partition_heal: the per-follower convergence loop was vacuous —
  it bounded by commit_index, which stays 0 for followers under the leader-only
  commit model, so the loop never ran. Bound by log_count instead (followers DO
  advance the replicated log); Log Matching still guarantees the shared-prefix
  terms agree. Now non-vacuous; still passes.

Test/comment-only; no behavior change. Lib suite Ok:1 Fail:0.
2026-06-25 04:38:43 +00:00

2510 lines
100 KiB
C
Raw Blame History

#ifdef XCE_TESTING
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "raft.h"
/* ---------- no-op callbacks ---------- */
static void sim_noop_become_follower(void *ctx, uint64_t term, int leader_idx)
{
(void)ctx; (void)term; (void)leader_idx;
}
static int sim_noop_persist(void *ctx, uint64_t term, int voted_for)
{
(void)ctx; (void)term; (void)voted_for;
return 0;
}
static void sim_noop_schedule_timeout(void *ctx, uint32_t delay_ms)
{
(void)ctx; (void)delay_ms;
}
static void sim_noop_cancel_timeout(void *ctx)
{
(void)ctx;
}
static void sim_noop_start_election_after_delay(void *ctx, uint32_t delay_ms)
{
(void)ctx; (void)delay_ms;
}
/* ---------- wired callbacks ---------- */
/*
* sim_send_vote_request: buffer a pending RPC into the cluster queue.
*
* ctx is raft_sim_node_t * (the source node).
* The node's ctx field points to raft_sim_cluster_t *.
*/
static int sim_send_vote_request(void *ctx, int peer_idx, uint64_t term)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
if (c->pending_count >= 64)
return -1; /* queue full — drop */
raft_sim_rpc_t *rpc = &c->pending[c->pending_count++];
rpc->src = node->idx;
rpc->dst = peer_idx;
rpc->term = term;
/* Capture the CANDIDATE's last-log tuple at SEND time from its own in-memory
* log (the library holds no entry bytes): last_log_index == log_count,
* last_log_term == term of the tail entry (0 when the log is empty). This
* feeds the (lastLogTerm, lastLogIndex) freshness gate at delivery. */
rpc->last_log_index = (uint64_t)node->log_count;
rpc->last_log_term = (node->log_count > 0)
? node->log[node->log_count - 1].term
: 0;
return 0;
}
/*
* sim_become_leader: increment cluster-level leader_count.
*
* ctx is raft_sim_node_t *.
*/
static void sim_become_leader(void *ctx, uint64_t term)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
(void)term;
c->leader_count++;
}
/* ---------- log-replication callbacks (raft-log-replication) ----------
*
* These back the four required log callbacks with the node's in-memory log.
* ctx is raft_sim_node_t * (same convention as the wired election callbacks).
* The library holds no entry bytes, so the simulator owns storage here:
* - on_persist_entry: append/overwrite at 1-based index (term + bytes)
* - on_get_entry_term: read the stored term (0 = absent)
* - on_truncate_from: drop entries index..end
* - on_send_append_entries: buffer one AE (sourcing the single entry from the
* sender's own log at prev_log_index+1 when n_entries == 1)
* No AppendEntries delivery happens here — that is raft_sim_deliver_append_entries
* (task 8.2). These callbacks only do storage/term-read/truncate/buffer.
*/
/*
* sim_persist_entry: durably append or overwrite the entry at 1-based `index`.
* Sets the slot's term and copies up to RAFT_SIM_ENTRY_LEN bytes; extends
* log_count when appending past the current tail.
*/
static int sim_persist_entry(void *ctx, uint64_t index, uint64_t term,
const void *entry, size_t len)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
if (index < 1 || index > RAFT_SIM_MAX_LOG)
return -1; /* out of fixed sim bounds */
/* Deliberately-broken-build fault switch (task 7.1): when RAFT_SIM_BREAK_TERM_STAMP
* is set on the cluster, stamp the entry with term 0 instead of `term`. This
* emulates bug-851 G1 (entries carry term 0 so on_get_entry_term can't distinguish
* same-index/different-term entries). Purely additive — DEFAULT OFF, so legacy and
* conformance runs that never set the flag are byte-identical to before. */
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
if (c && (c->fault_inject & RAFT_SIM_BREAK_TERM_STAMP))
term = 0;
raft_sim_entry_t *slot = &node->log[index - 1];
slot->term = term;
size_t n = len;
if (n > RAFT_SIM_ENTRY_LEN)
n = RAFT_SIM_ENTRY_LEN; /* bounded copy */
if (entry && n > 0)
memcpy(slot->bytes, entry, n);
slot->len = n;
if ((int)index > node->log_count)
node->log_count = (int)index;
return 0;
}
/*
* sim_get_entry_term: return the stored term at 1-based `index`, or 0 if absent
* (0-means-absent convention).
*/
static uint64_t sim_get_entry_term(void *ctx, uint64_t index)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
if (index < 1 || index > (uint64_t)node->log_count)
return 0;
return node->log[index - 1].term;
}
/*
* sim_truncate_from: drop entries at `index`..end (clear term so they read as
* absent) and shrink log_count.
*/
static int sim_truncate_from(void *ctx, uint64_t index)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
if (index < 1)
return -1;
/* Deliberately-broken-build fault switch (task 7.1): when RAFT_SIM_BREAK_TRUNCATE
* is set on the cluster, make truncation a NO-OP so the divergent suffix is never
* deleted (emulates bug-851 G3). Returns 0 (success) without touching the log, so
* the library believes the truncate succeeded while the stale suffix lingers.
* Purely additive — DEFAULT OFF; runs that never set the flag are unaffected. */
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
if (c && (c->fault_inject & RAFT_SIM_BREAK_TRUNCATE))
return 0;
for (uint64_t i = index; i <= (uint64_t)node->log_count; i++) {
if (i <= RAFT_SIM_MAX_LOG) {
raft_sim_entry_t *slot = &node->log[i - 1];
memset(slot, 0, sizeof(*slot)); /* term=0 => absent */
}
}
if (node->log_count >= (int)index)
node->log_count = (int)index - 1;
return 0;
}
/*
* sim_send_append_entries: buffer one AppendEntries into the cluster AE queue.
* For n_entries == 1 the single entry's index/term/bytes are sourced from the
* sender's own in-memory log at prev_log_index + 1 (the library holds no bytes).
*/
static int sim_send_append_entries(void *ctx, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
if (c->ae_pending_count >= RAFT_SIM_MAX_AE)
return -1; /* queue full — drop */
raft_sim_ae_t *ae = &c->ae_pending[c->ae_pending_count++];
memset(ae, 0, sizeof(*ae));
ae->src = node->idx;
ae->dst = peer_idx;
ae->leader_term = node->state.term; /* capture SEND-time term (see raft.h) */
ae->prev_log_index = prev_log_index;
ae->prev_log_term = prev_log_term;
ae->leader_commit = leader_commit;
ae->n_entries = n_entries;
if (n_entries == 1) {
uint64_t idx = prev_log_index + 1;
ae->entry_index = idx;
if (idx >= 1 && idx <= (uint64_t)node->log_count) {
raft_sim_entry_t *slot = &node->log[idx - 1];
ae->entry_term = slot->term;
ae->entry_len = slot->len;
if (slot->len > 0)
memcpy(ae->entry_bytes, slot->bytes, slot->len);
}
}
return 0;
}
/*
* sim_on_commit_advanced: record the library-reported commit_index and bump the
* fire counter so tests can assert the strict-advance / fire-once semantics. The
* library fires this ONLY on a strict commit_index advance (never on a held
* index), so the count == number of strict advances. No durability here — the
* sim has no commit barrier; this is the trigger mirror only.
*/
static void sim_on_commit_advanced(void *ctx, uint64_t commit_index)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
node->commit_index = commit_index;
node->commit_advanced_count++;
}
/*
* sim_on_trace: append the event string to the node's trace_buf.
*
* ctx is raft_sim_node_t *.
*/
static void sim_on_trace(void *ctx, const char *event, const char *detail)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
(void)detail;
if (!event)
return;
int remaining = (int)sizeof(node->trace_buf) - node->trace_len - 1;
if (remaining <= 0)
return;
int written = snprintf(node->trace_buf + node->trace_len, (size_t)remaining,
"%s\n", event);
if (written > 0 && written < remaining)
node->trace_len += written;
else if (written >= remaining)
node->trace_len += remaining - 1; /* fill without null — snprintf already null-terminated */
}
/* ---------- conformance harness adapter (raft-sim-conformance task 1.2) ----------
*
* The conformance init wires a SEPARATE callback set alongside the legacy ones.
* Log storage REUSES sim_persist_entry / sim_get_entry_term / sim_truncate_from
* (they already back node->log[] = on_persist_entry / on_get_entry_term /
* on_truncate_from). Only the commit + send callbacks differ:
* - sim_conf_on_commit_advanced fills node->applied[] (the legacy variant does
* not), so State Machine Safety can observe value divergence,
* - sim_conf_send_vote_request / sim_conf_send_append_entries push a unified
* raft_sim_msg_t onto cluster.bag[] (mirroring raft.tla `messages`) instead
* of the legacy pending[] / ae_pending[] queues.
* Bag DELIVERY/routing is task 2.1 — not implemented here.
*/
/*
* raft_sim_prng_next: SplitMix64 step. Advances c->prng and returns a well-mixed
* 64-bit value. Deterministic and reproducible (no libc rand()): seeded from
* c->seed in raft_sim_conformance_init, this is the single source for every
* later nondeterministic choice the scheduler makes (R1.2).
*/
uint64_t raft_sim_prng_next(raft_sim_cluster_t *c)
{
uint64_t z = (c->prng += 0x9E3779B97F4A7C15ULL);
z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL;
z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL;
return z ^ (z >> 31);
}
/*
* sim_conf_decode_value: deterministic decode of a committed entry's payload into
* the applied state-machine value. Interprets the first 8 bytes (little-endian, as
* stored by memcpy on the same host) as a uint64_t; shorter payloads zero-extend.
* This is the stable decode applied[] uses so State Machine Safety divergence (two
* nodes applying different values at the same index) is observable.
*/
static uint64_t sim_conf_decode_value(const raft_sim_entry_t *slot)
{
uint64_t v = 0;
size_t n = slot->len;
if (n > sizeof(v))
n = sizeof(v);
if (n > 0)
memcpy(&v, slot->bytes, n);
return v;
}
/*
* sim_conf_on_commit_advanced: conformance commit callback. Records the
* library-reported commit_index and applies each newly-committed entry's decoded
* value into node->applied[] (for committed indexes prev_commit+1 .. commit_index).
* applied[k-1] holds the value applied at 1-based committed index k. Also bumps the
* fire counter (mirrors the legacy variant's count semantics).
*/
static void sim_conf_on_commit_advanced(void *ctx, uint64_t commit_index)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
uint64_t prev = node->commit_index;
if (commit_index > prev) {
for (uint64_t k = prev + 1; k <= commit_index; k++) {
if (k >= 1 && k <= (uint64_t)node->log_count && k <= RAFT_SIM_MAX_LOG)
node->applied[k - 1] = sim_conf_decode_value(&node->log[k - 1]);
}
}
node->commit_index = commit_index;
node->commit_advanced_count++;
}
/*
* raft_sim_bag_push: append one message to the unified cluster bag. Fails loud
* (returns -1) on overflow rather than silently truncating, so a bounds bug is
* visible (design Error Handling: bag overflow fails loud). Returns 0 on success.
*/
int raft_sim_bag_push(raft_sim_cluster_t *c, const raft_sim_msg_t *msg)
{
if (c->bag_count >= RAFT_SIM_MAX_MSGS)
return -1; /* fail loud — do not silently drop */
c->bag[c->bag_count++] = *msg;
return 0;
}
/*
* sim_conf_send_vote_request: conformance vote-request send. Builds a
* RAFT_SIM_MSG_VOTE_REQ from the callback args and pushes it onto the bag.
*/
static int sim_conf_send_vote_request(void *ctx, int peer_idx, uint64_t term)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
raft_sim_msg_t msg;
memset(&msg, 0, sizeof(msg));
msg.kind = RAFT_SIM_MSG_VOTE_REQ;
msg.src = node->idx;
msg.dst = peer_idx;
msg.term = term;
/* Capture the CANDIDATE's last-log tuple at SEND time from its own in-memory log
* (the library holds no entry bytes): last_log_index == log_count, last_log_term
* == the tail entry's term (0 when empty). This feeds the (lastLogTerm,
* lastLogIndex) freshness gate when raft_sim_bag_deliver routes the request —
* exactly as the legacy sim_send_vote_request does for the pending[] path. */
msg.last_log_index = (uint64_t)node->log_count;
msg.last_log_term = (node->log_count > 0)
? node->log[node->log_count - 1].term
: 0;
return raft_sim_bag_push(c, &msg);
}
/*
* sim_conf_send_append_entries: conformance AppendEntries send. Builds a
* RAFT_SIM_MSG_AE_REQ from the callback args and pushes it onto the bag. For
* n_entries == 1 the single entry's (term, value) is sourced from the SENDER's own
* in-memory log at prev_log_index + 1 (the library holds no entry bytes), decoded
* via sim_conf_decode_value — exactly as the legacy sim_send_append_entries sources
* bytes.
*/
static int sim_conf_send_append_entries(void *ctx, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries)
{
raft_sim_node_t *node = (raft_sim_node_t *)ctx;
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
raft_sim_msg_t msg;
memset(&msg, 0, sizeof(msg));
msg.kind = RAFT_SIM_MSG_AE_REQ;
msg.src = node->idx;
msg.dst = peer_idx;
msg.term = node->state.term; /* send-time leader term */
msg.prev_log_index = prev_log_index;
msg.prev_log_term = prev_log_term;
msg.leader_commit = leader_commit;
msg.entries_len = n_entries;
if (n_entries == 1) {
uint64_t idx = prev_log_index + 1;
if (idx >= 1 && idx <= (uint64_t)node->log_count) {
raft_sim_entry_t *slot = &node->log[idx - 1];
msg.entry_term = slot->term;
msg.entry_value = sim_conf_decode_value(slot);
}
}
return raft_sim_bag_push(c, &msg);
}
/* ---------- public API ---------- */
int raft_sim_init(raft_sim_cluster_t *c, int n_nodes, uint32_t election_timeout_ms)
{
memset(c, 0, sizeof(*c));
if (n_nodes < 1 || n_nodes > RAFT_SIM_MAX_NODES)
return -1;
c->n = n_nodes;
c->leader_count = 0;
c->pending_count = 0;
for (int i = 0; i < n_nodes; i++) {
raft_sim_node_t *node = &c->nodes[i];
/* Zero the node struct */
memset(node, 0, sizeof(*node));
node->idx = i;
node->partition = 0;
node->ctx = c; /* points back to the cluster */
/* Wire callbacks */
node->cbs.on_send_vote_request = sim_send_vote_request;
node->cbs.on_become_leader = sim_become_leader;
node->cbs.on_become_follower = sim_noop_become_follower;
node->cbs.on_persist = sim_noop_persist;
node->cbs.on_schedule_timeout = sim_noop_schedule_timeout;
node->cbs.on_cancel_timeout = sim_noop_cancel_timeout;
node->cbs.on_start_election_after_delay = sim_noop_start_election_after_delay;
/* log-replication callbacks backed by the node's in-memory log */
node->cbs.on_persist_entry = sim_persist_entry;
node->cbs.on_get_entry_term = sim_get_entry_term;
node->cbs.on_send_append_entries = sim_send_append_entries;
node->cbs.on_truncate_from = sim_truncate_from;
node->cbs.on_commit_advanced = sim_on_commit_advanced;
node->cbs.on_vote_eligible = NULL; /* optional — treat all as eligible */
node->cbs.on_trace = sim_on_trace;
/* Initialize raft state — pass node as ctx so callbacks can find both
* node->idx (source) and node->ctx (cluster). */
raft_config_t cfg;
memset(&cfg, 0, sizeof(cfg));
cfg.n_peers = n_nodes;
cfg.self_idx = i;
cfg.my_priority = i; /* priority = index: node 0 has highest priority */
cfg.election_timeout_ms = election_timeout_ms;
cfg.initial_term = 0;
cfg.initial_voted_for = -1;
cfg.initial_commit_index = 0; /* first boot — library never reads storage */
cfg.cbs = &node->cbs;
cfg.ctx = node; /* node* as ctx — callbacks cast back to raft_sim_node_t* */
int rc = raft_init(&node->state, &cfg);
if (rc != 0)
return rc;
}
return 0;
}
int raft_sim_conformance_init(raft_sim_cluster_t *c, int n_nodes,
uint32_t election_timeout_ms, uint64_t seed)
{
memset(c, 0, sizeof(*c));
if (n_nodes < 1 || n_nodes > RAFT_SIM_MAX_NODES)
return -1;
c->n = n_nodes;
c->seed = seed;
c->prng = seed; /* deterministic PRNG state seeded from the run key */
for (int i = 0; i < n_nodes; i++) {
raft_sim_node_t *node = &c->nodes[i];
memset(node, 0, sizeof(*node));
node->idx = i;
node->partition = 0;
node->ctx = c; /* points back to the cluster */
/* Conformance adapter callback set. Election/persist/timeout callbacks
* reuse the existing sim implementations; the conformance variants differ
* only for commit (fills applied[]) and the send paths (push to bag[]). */
node->cbs.on_send_vote_request = sim_conf_send_vote_request;
node->cbs.on_become_leader = sim_become_leader;
node->cbs.on_become_follower = sim_noop_become_follower;
node->cbs.on_persist = sim_noop_persist;
node->cbs.on_schedule_timeout = sim_noop_schedule_timeout;
node->cbs.on_cancel_timeout = sim_noop_cancel_timeout;
node->cbs.on_start_election_after_delay = sim_noop_start_election_after_delay;
/* log storage reused from the legacy in-memory log callbacks */
node->cbs.on_persist_entry = sim_persist_entry;
node->cbs.on_get_entry_term = sim_get_entry_term;
node->cbs.on_send_append_entries = sim_conf_send_append_entries;
node->cbs.on_truncate_from = sim_truncate_from;
node->cbs.on_commit_advanced = sim_conf_on_commit_advanced;
node->cbs.on_vote_eligible = NULL; /* all eligible */
node->cbs.on_trace = sim_on_trace;
raft_config_t cfg;
memset(&cfg, 0, sizeof(cfg));
cfg.n_peers = n_nodes;
cfg.self_idx = i;
cfg.my_priority = i; /* priority = index */
cfg.election_timeout_ms = election_timeout_ms;
cfg.initial_term = 0;
cfg.initial_voted_for = -1;
cfg.initial_commit_index = 0;
cfg.cbs = &node->cbs;
cfg.ctx = node;
int rc = raft_init(&node->state, &cfg);
if (rc != 0)
return rc;
}
return 0;
}
void raft_sim_dump_log(raft_sim_cluster_t *c, FILE *fp)
{
for (int i = 0; i < c->n; i++) {
raft_sim_node_t *node = &c->nodes[i];
if (node->trace_len > 0) {
fprintf(fp, "[node%d trace]\n%.*s\n",
i, node->trace_len, node->trace_buf);
}
}
}
void raft_sim_deliver_pending(raft_sim_cluster_t *c)
{
/* Snapshot current count — callbacks may add more entries */
int count = c->pending_count;
c->pending_count = 0;
for (int i = 0; i < count; i++) {
raft_sim_rpc_t *rpc = &c->pending[i];
int src = rpc->src;
int dst = rpc->dst;
uint64_t term = rpc->term;
/* Drop if either endpoint is partitioned */
if (c->nodes[src].partition || c->nodes[dst].partition)
continue;
/* Deliver vote request to dst, passing the candidate's send-time last-log
* tuple so dst evaluates the (lastLogTerm, lastLogIndex) freshness gate. */
uint64_t reply_term = 0;
int granted = raft_on_vote_request(&c->nodes[dst].state, src, term,
rpc->last_log_term,
rpc->last_log_index,
&reply_term);
/* Route reply back to src */
raft_on_vote_reply(&c->nodes[src].state, dst, granted, reply_term);
}
}
void raft_sim_deliver_append_entries(raft_sim_cluster_t *c)
{
/* Snapshot current count — handling may buffer more AEs (mirrors vote path) */
int count = c->ae_pending_count;
c->ae_pending_count = 0;
for (int i = 0; i < count; i++) {
raft_sim_ae_t *ae = &c->ae_pending[i];
int src = ae->src;
int dst = ae->dst;
/* Drop if either endpoint is partitioned */
if (c->nodes[src].partition || c->nodes[dst].partition)
continue;
/* The single entry's bytes/term were captured into the AE at buffer
* time (library holds no entry bytes); pass them through for n_entries==1. */
const void *entry = (ae->n_entries == 1) ? ae->entry_bytes : NULL;
size_t entry_len = (ae->n_entries == 1) ? ae->entry_len : 0;
uint64_t entry_term = (ae->n_entries == 1) ? ae->entry_term : 0;
/* Deliver AppendEntries request to dst using the SEND-time leader term
* captured into the AE buffer at sim_send_append_entries (NOT the live
* source term). This mirrors the vote path's buffered rpc->term and keeps
* reorder/churn delivery (9.3/9.4) honest about the sending epoch so the
* follower's UpdateTerm/stale-drop decisions match the real protocol. */
uint64_t leader_term = ae->leader_term;
raft_append_result_t result;
memset(&result, 0, sizeof(result));
raft_on_append_entries_request(&c->nodes[dst].state,
src, leader_term,
ae->prev_log_index, ae->prev_log_term,
ae->leader_commit, entry, entry_len,
entry_term, ae->n_entries, &result);
/* Route reply back to src */
raft_on_append_entries_reply(&c->nodes[src].state, dst,
result.success, result.match_index,
result.reply_term);
}
}
/*
* raft_sim_bag_remove_at: swap-with-last removal of bag member `i`.
*
* Bag removal model (consumed on delivery, shared with task 2.2 Drop/Duplicate/
* Reorder): the message at `i` is overwritten by the last member and bag_count is
* decremented. This keeps the bag a compact dense array with no holes, so an index
* chosen by the PRNG always names a live member. Delivery removes the request
* FIRST, then pushes the reply, so the reply lands at a fresh tail slot and never
* collides with the just-vacated index.
*/
static void raft_sim_bag_remove_at(raft_sim_cluster_t *c, int i)
{
/* Caller guarantees 0 <= i < bag_count. */
int last = c->bag_count - 1;
if (i != last)
c->bag[i] = c->bag[last];
c->bag_count = last;
}
int raft_sim_bag_deliver(raft_sim_cluster_t *c, int i)
{
if (i < 0 || i >= c->bag_count)
return -1; /* out of range — nothing to deliver */
/* Snapshot the message by value before any removal/push reshuffles the bag. */
raft_sim_msg_t m = c->bag[i];
int src = m.src;
int dst = m.dst;
/* Honor partitions: a partitioned endpoint's message is not delivered, mirroring
* a dropped network edge. Leave it in the bag (a later heal can deliver it). */
if (c->nodes[src].partition || c->nodes[dst].partition)
return -2;
/* Consume the request/response FIRST (swap-with-last), then push any reply so
* indices stay sane and the reply occupies a fresh tail slot. */
raft_sim_bag_remove_at(c, i);
switch (m.kind) {
case RAFT_SIM_MSG_VOTE_REQ: {
/* Route to dst's vote handler, passing the candidate's send-time last-log
* tuple so dst evaluates the (lastLogTerm, lastLogIndex) freshness gate. */
uint64_t reply_term = 0;
int granted = raft_on_vote_request(&c->nodes[dst].state, src, m.term,
m.last_log_term, m.last_log_index,
&reply_term);
raft_sim_msg_t rsp;
memset(&rsp, 0, sizeof(rsp));
rsp.kind = RAFT_SIM_MSG_VOTE_RSP;
rsp.src = dst; /* voter replies */
rsp.dst = src; /* back to candidate */
rsp.term = reply_term;
rsp.granted = granted;
return raft_sim_bag_push(c, &rsp);
}
case RAFT_SIM_MSG_AE_REQ: {
/* For n_entries == 1 the single entry's bytes come from the carried value
* (the library holds none); re-encode the same little-endian uint64_t the
* SEND callback decoded, so the receiver's sim_persist_entry / decode round-
* trips it. Use the SEND-time leader term carried in the message (m.term),
* NOT the live source term. */
unsigned char entry_bytes[sizeof(uint64_t)];
const void *entry = NULL;
size_t entry_len = 0;
uint64_t entry_term = 0;
if (m.entries_len == 1) {
memcpy(entry_bytes, &m.entry_value, sizeof(entry_bytes));
entry = entry_bytes;
entry_len = sizeof(entry_bytes);
entry_term = m.entry_term;
}
raft_append_result_t result;
memset(&result, 0, sizeof(result));
raft_on_append_entries_request(&c->nodes[dst].state, src, m.term,
m.prev_log_index, m.prev_log_term,
m.leader_commit, entry, entry_len,
entry_term, m.entries_len, &result);
raft_sim_msg_t rsp;
memset(&rsp, 0, sizeof(rsp));
rsp.kind = RAFT_SIM_MSG_AE_RSP;
rsp.src = dst; /* follower replies */
rsp.dst = src; /* back to leader */
rsp.term = result.reply_term;
rsp.success = result.success;
rsp.match_index = result.match_index;
return raft_sim_bag_push(c, &rsp);
}
case RAFT_SIM_MSG_VOTE_RSP:
/* Responses are first-class bag members: route to the candidate's tally. */
raft_on_vote_reply(&c->nodes[dst].state, src, m.granted, m.term);
return 0;
case RAFT_SIM_MSG_AE_RSP:
raft_on_append_entries_reply(&c->nodes[dst].state, src, m.success,
m.match_index, m.term);
return 0;
}
return 0; /* unreachable — enum is exhaustive */
}
/*
* raft_sim_bag_drop: DropMessage (raft.tla:448). Discard the bag member at `i`
* (swap-with-last removal, bag_count -= 1) without touching ANY node. Mirrors the
* TLA+ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>: this is a
* pure bag mutation. Bounds-checked; returns -1 (fail loud, bag untouched) on an
* out-of-range index, 0 on a discarded member.
*/
int raft_sim_bag_drop(raft_sim_cluster_t *c, int i)
{
if (i < 0 || i >= c->bag_count)
return -1; /* fail loud — no silent no-op */
raft_sim_bag_remove_at(c, i); /* swap-with-last; decrements bag_count */
return 0;
}
/*
* raft_sim_bag_duplicate: DuplicateMessage (raft.tla:443). Re-enqueue a COPY of the
* bag member at `i` onto the tail (multiplicity += 1) without touching ANY node —
* mirrors the TLA+ UNCHANGED of all non-message vars. Honors RAFT_SIM_MAX_MSGS via
* raft_sim_bag_push (fail loud on overflow). The original at `i` is left intact.
* Returns -1 on an out-of-range index, the push status otherwise (0 on success,
* -1 on bag overflow).
*/
int raft_sim_bag_duplicate(raft_sim_cluster_t *c, int i)
{
if (i < 0 || i >= c->bag_count)
return -1; /* fail loud — no silent no-op */
/* Copy by value FIRST: raft_sim_bag_push may grow the array, but the source is
* already an independent local, so a tail push can never alias the source. */
raft_sim_msg_t copy = c->bag[i];
return raft_sim_bag_push(c, &copy);
}
/*
* raft_sim_bag_pick: Reorder source. Return a PRNG-chosen in-flight bag index in
* [0, bag_count) so the scheduler delivers in a deterministic, seed-replayable
* order rather than FIFO. Pure read of the bag: advances only the cluster PRNG, never
* any node's (term, role, votedFor, log). Returns -1 when the bag is empty (nothing
* to deliver). The subsequent raft_sim_bag_deliver does the node-side work.
*/
int raft_sim_bag_pick(raft_sim_cluster_t *c)
{
if (c->bag_count <= 0)
return -1; /* empty bag — nothing to pick */
return (int)(raft_sim_prng_next(c) % (uint64_t)c->bag_count);
}
void raft_sim_start_all_elections(raft_sim_cluster_t *c)
{
for (int i = 0; i < c->n; i++)
raft_start_election(&c->nodes[i].state);
}
void raft_sim_fire_timeout(raft_sim_cluster_t *c, int node_idx)
{
if (node_idx >= 0 && node_idx < c->n)
raft_on_election_timeout(&c->nodes[node_idx].state);
}
void raft_sim_partition(raft_sim_cluster_t *c, int node_idx, int isolated)
{
if (node_idx >= 0 && node_idx < c->n)
c->nodes[node_idx].partition = isolated;
}
void raft_sim_advance_commit(raft_sim_cluster_t *c, int node_idx)
{
if (node_idx < 0 || node_idx >= c->n)
return;
/* Library decides whether the commit index advanced; on a strict advance the
* node's on_commit_advanced (sim_on_commit_advanced) updates commit_index /
* commit_advanced_count. The -EPERM off-leader return is intentionally
* discarded — tests read the per-node mirror, not this status. */
(void)raft_advance_commit(&c->nodes[node_idx].state);
}
/* Two log entries are identical iff their term, length, and payload bytes
* (over the recorded length) all match. */
static int sim_entry_identical(const raft_sim_entry_t *a,
const raft_sim_entry_t *b)
{
if (a->term != b->term || a->len != b->len)
return 0;
return memcmp(a->bytes, b->bytes, a->len) == 0;
}
int raft_sim_check_log_matching(const raft_sim_cluster_t *c)
{
/* For every ordered-pair-free pair (a < b) and every index i present in
* both logs: if the entries at i carry the same term, the prefixes 1..i
* must be entry-identical. Pairs are symmetric, so a < b suffices. */
for (int a = 0; a < c->n; a++) {
const raft_sim_node_t *na = &c->nodes[a];
for (int b = a + 1; b < c->n; b++) {
const raft_sim_node_t *nb = &c->nodes[b];
int shared = na->log_count < nb->log_count
? na->log_count : nb->log_count;
for (int i = 1; i <= shared; i++) {
const raft_sim_entry_t *ea = &na->log[i - 1];
const raft_sim_entry_t *eb = &nb->log[i - 1];
/* Same (index, term) ⇒ prefixes through i must be identical. */
if (ea->term != eb->term)
continue;
for (int k = 1; k <= i; k++) {
if (!sim_entry_identical(&na->log[k - 1], &nb->log[k - 1]))
return 0; /* Log Matching violated */
}
}
}
}
return 1; /* holds */
}
int raft_sim_check_election_safety(const raft_sim_cluster_t *c)
{
/* For every distinct pair (a < b): fail if BOTH are LEADER at the SAME term.
* Pairs are symmetric, so a < b suffices. Role/term are read from each node's
* observable snapshot (Req 3.6) — no internal protocol bookkeeping. */
for (int a = 0; a < c->n; a++) {
raft_snapshot_t sa;
raft_snapshot(&c->nodes[a].state, &sa);
if (sa.role != RAFT_LEADER)
continue;
for (int b = a + 1; b < c->n; b++) {
raft_snapshot_t sb;
raft_snapshot(&c->nodes[b].state, &sb);
if (sb.role == RAFT_LEADER && sb.term == sa.term)
return 0; /* two LEADERs at the same term — Election Safety violated */
}
}
return 1; /* holds */
}
int raft_sim_check_leader_completeness(const raft_sim_cluster_t *c)
{
/* Leader Completeness: an entry committed in term t must be present (same
* index, same term) in the log of every leader of a strictly-higher term.
*
* Pure per-snapshot observer (Req 3.6): the committed set is every index k in
* 1..commit_index over ALL nodes (carrying that node's log[k-1].term); the
* current leaders are the role==RAFT_LEADER nodes. For each leader L at term
* T_L and each committed (k, t) with t < T_L, L's log must hold index k at
* term t — else the property is violated. (index, term) presence suffices;
* Log Matching covers payload identity. */
for (int L = 0; L < c->n; L++) {
raft_snapshot_t sl;
raft_snapshot(&c->nodes[L].state, &sl);
if (sl.role != RAFT_LEADER)
continue;
const raft_sim_node_t *leader = &c->nodes[L];
/* Scan every node's committed prefix for entries committed under a term
* strictly lower than this leader's term. */
for (int d = 0; d < c->n; d++) {
const raft_sim_node_t *src = &c->nodes[d];
int committed = (int)src->commit_index;
if (committed > src->log_count)
committed = src->log_count;
for (int k = 1; k <= committed; k++) {
uint64_t t = src->log[k - 1].term;
if (t >= sl.term)
continue; /* only lower-term committed entries bind */
/* Leader L must contain index k at term t. */
if (k > leader->log_count)
return 0; /* leader missing the committed index */
if (leader->log[k - 1].term != t)
return 0; /* leader has a different term at index k */
}
}
}
return 1; /* holds */
}
int raft_sim_check_state_machine_safety(const raft_sim_cluster_t *c)
{
/* State Machine Safety: no two nodes have applied a different value at the same
* (1-based) log index. Observable-only (Req 3.6): each node's applied extent is
* 1..commit_index (on_commit_advanced fills applied[k-1] for committed index k).
* For every pair (a < b) and every index k in the shared applied prefix
* 1..min(commit_a, commit_b), fail if applied[k-1] differs. Pairs are symmetric,
* so a < b suffices. */
for (int a = 0; a < c->n; a++) {
const raft_sim_node_t *na = &c->nodes[a];
for (int b = a + 1; b < c->n; b++) {
const raft_sim_node_t *nb = &c->nodes[b];
uint64_t shared = na->commit_index < nb->commit_index
? na->commit_index : nb->commit_index;
if (shared > RAFT_SIM_MAX_LOG)
shared = RAFT_SIM_MAX_LOG;
for (uint64_t k = 1; k <= shared; k++) {
if (na->applied[k - 1] != nb->applied[k - 1])
return 0; /* divergent applied value — State Machine Safety violated */
}
}
}
return 1; /* holds */
}
raft_sim_invariant_t raft_sim_check_all(const raft_sim_cluster_t *c)
{
/* Evaluate the four invariants in the FIXED order required by Req 3.5 and
* short-circuit on the FIRST violation, so the reported failing invariant is
* deterministic. Each checker returns 1=holds / 0=violated. */
if (!raft_sim_check_election_safety(c))
return RAFT_SIM_INV_ELECTION_SAFETY;
if (!raft_sim_check_log_matching(c))
return RAFT_SIM_INV_LOG_MATCHING;
if (!raft_sim_check_leader_completeness(c))
return RAFT_SIM_INV_LEADER_COMPLETENESS;
if (!raft_sim_check_state_machine_safety(c))
return RAFT_SIM_INV_STATE_MACHINE_SAFETY;
return RAFT_SIM_INV_NONE; /* all four hold */
}
const char *raft_sim_invariant_name(raft_sim_invariant_t inv)
{
switch (inv) {
case RAFT_SIM_INV_NONE: return "none";
case RAFT_SIM_INV_ELECTION_SAFETY: return "election-safety";
case RAFT_SIM_INV_LOG_MATCHING: return "log-matching";
case RAFT_SIM_INV_LEADER_COMPLETENESS: return "leader-completeness";
case RAFT_SIM_INV_STATE_MACHINE_SAFETY: return "state-machine-safety";
}
return "unknown";
}
const char *raft_sim_action_kind_name(raft_sim_action_kind_t kind)
{
switch (kind) {
case RAFT_SIM_ACT_RESTART: return "restart";
case RAFT_SIM_ACT_TIMEOUT: return "timeout";
case RAFT_SIM_ACT_REQUEST_VOTE: return "request-vote";
case RAFT_SIM_ACT_BECOME_LEADER: return "become-leader";
case RAFT_SIM_ACT_CLIENT_REQUEST: return "client-request";
case RAFT_SIM_ACT_APPEND_ENTRIES: return "append-entries";
case RAFT_SIM_ACT_ADVANCE_COMMIT: return "advance-commit";
case RAFT_SIM_ACT_RECEIVE: return "receive";
case RAFT_SIM_ACT_DUPLICATE_MESSAGE: return "duplicate-message";
case RAFT_SIM_ACT_DROP_MESSAGE: return "drop-message";
}
return "unknown";
}
/*
* raft_sim_trace_record: append the chosen action to c->trace[] at trace_len and
* advance trace_len (Req 1.4 ordering — the scheduler calls this BEFORE applying the
* action). RAFT_SIM_MAX_STEPS bounds the buffer: an overflow is a harness bug (the
* step budget is fixed), so fail loud via assert rather than writing out of bounds
* or silently dropping the action.
*/
void raft_sim_trace_record(raft_sim_cluster_t *c, raft_sim_action_t action)
{
/* Fail loud on overflow — a full trace means the run exceeded its fixed step
* budget, which is a harness bug, not a silently-droppable condition. */
assert(c->trace_len < RAFT_SIM_MAX_STEPS);
if (c->trace_len >= RAFT_SIM_MAX_STEPS)
return; /* belt-and-suspenders: never write OOB if assert is compiled out */
c->trace[c->trace_len++] = action;
}
void raft_sim_dump_trace(const raft_sim_cluster_t *c, FILE *fp,
raft_sim_invariant_t failing)
{
/* Quiet-on-success (Req 5.4): a clean run dumps nothing unless verbose tracing
* was explicitly requested. A real failure always dumps. */
if (failing == RAFT_SIM_INV_NONE && getenv("RAFT_SIM_VERBOSE") == NULL)
return;
fprintf(fp, "raft-sim trace: seed=%llu n=%d failing-invariant=%s steps=%d\n",
(unsigned long long)c->seed, c->n,
raft_sim_invariant_name(failing), c->trace_len);
for (int i = 0; i < c->trace_len; i++) {
const raft_sim_action_t *a = &c->trace[i];
/* One human-readable line per action (Req 5.2): kind name, actor,
* msg_index, term, index. */
fprintf(fp,
" [%d] %s actor=%d msg_index=%d term=%llu index=%llu\n",
i, raft_sim_action_kind_name(a->kind), a->actor, a->msg_index,
(unsigned long long)a->term, (unsigned long long)a->index);
}
}
/* ---------- optional TLC-consumable trace export (task 8.1) ---------- */
/*
* sim_tla_action_name: map a harness action kind to the corresponding raft.tla
* `Next` disjunct name (R8.2), so the export reads in the spec's own vocabulary
* (raft.tla:454). Distinct from raft_sim_action_kind_name (kebab-case, human dump):
* this is the PascalCase spec-action name a maintainer maps directly to raft.tla.
*/
static const char *sim_tla_action_name(raft_sim_action_kind_t kind)
{
/* Each return string is verbatim the raft.tla `Next` disjunct name (raft.tla:454);
* no per-case line numbers — they drift as raft.tla evolves and the name IS the
* stable spec reference. */
switch (kind) {
case RAFT_SIM_ACT_RESTART: return "Restart";
case RAFT_SIM_ACT_TIMEOUT: return "Timeout";
case RAFT_SIM_ACT_REQUEST_VOTE: return "RequestVote";
case RAFT_SIM_ACT_BECOME_LEADER: return "BecomeLeader";
case RAFT_SIM_ACT_CLIENT_REQUEST: return "ClientRequest";
case RAFT_SIM_ACT_APPEND_ENTRIES: return "AppendEntries";
case RAFT_SIM_ACT_ADVANCE_COMMIT: return "AdvanceCommitIndex";
case RAFT_SIM_ACT_RECEIVE: return "Receive";
case RAFT_SIM_ACT_DUPLICATE_MESSAGE: return "DuplicateMessage";
case RAFT_SIM_ACT_DROP_MESSAGE: return "DropMessage";
}
return "Unknown";
}
/*
* sim_tla_state_name: map the library role enum to the raft.tla `state` value name
* (Follower/Candidate/Leader, raft.tla:17/144) so the per-node export uses the spec
* name for `state[i]` directly.
*/
static const char *sim_tla_state_name(raft_role_t role)
{
switch (role) {
case RAFT_FOLLOWER: return "Follower";
case RAFT_CANDIDATE: return "Candidate";
case RAFT_LEADER: return "Leader";
}
return "Unknown";
}
/*
* sim_tla_msg_kind_name: map a bag message kind to the raft.tla message `mtype`
* (RequestVoteRequest/RequestVoteResponse/AppendEntriesRequest/AppendEntriesResponse,
* raft.tla:23-24) for the `messages` bag summary.
*/
static const char *sim_tla_msg_kind_name(raft_sim_msg_kind_t kind)
{
switch (kind) {
case RAFT_SIM_MSG_VOTE_REQ: return "RequestVoteRequest";
case RAFT_SIM_MSG_VOTE_RSP: return "RequestVoteResponse";
case RAFT_SIM_MSG_AE_REQ: return "AppendEntriesRequest";
case RAFT_SIM_MSG_AE_RSP: return "AppendEntriesResponse";
}
return "Unknown";
}
/*
* raft_sim_export_trace: STABLE, line-oriented, machine-readable export of the
* harness's tracked raft.tla variables (R8.1) using raft.tla-consistent names (R8.2),
* gated OFF by default behind RAFT_SIM_EXPORT (R8.3 — mirrors the RAFT_SIM_VERBOSE
* idiom in raft_sim_dump_trace). Documented format (one record per line; fields are
* `key=value`; deterministic — no addresses, no wall-clock):
*
* trace-export seed=<u64> n=<int> steps=<trace_len>
* action <i> name=<RaftTlaActionName> actor=<int> msg_index=<int> term=<u64> index=<u64>
* ... one per recorded c->trace[] step, in order ...
* node <i> currentTerm=<u64> state=<Follower|Candidate|Leader> commitIndex=<u64> \
* log=[(<term>,<value>);...]
* ... one per node ...
* messages count=<int> [<i>]=<MsgType>{src=..,dst=..,term=..} ...
*
* The action lines ARE the per-step sequence (R8.1's "per step"); a deterministic
* replay from (seed, n, steps) reconstructs the per-step intermediate state, while the
* node/messages lines pin the resulting tracked variables. `log` decodes each entry's
* client value with the same little-endian decode applied[] uses, so the (term, value)
* sequence matches raft.tla `log[i]`.
*/
void raft_sim_export_trace(const raft_sim_cluster_t *c, FILE *fp)
{
/* Default-OFF gate (R8.3): emit NOTHING unless explicitly enabled. */
if (getenv("RAFT_SIM_EXPORT") == NULL)
return;
/* Header: run key + step count (R8.1). */
fprintf(fp, "trace-export seed=%llu n=%d steps=%d\n",
(unsigned long long)c->seed, c->n, c->trace_len);
/* Ordered action trace — each named per raft.tla's Next disjuncts (R8.2). */
for (int i = 0; i < c->trace_len; i++) {
const raft_sim_action_t *a = &c->trace[i];
fprintf(fp,
"action %d name=%s actor=%d msg_index=%d term=%llu index=%llu\n",
i, sim_tla_action_name(a->kind), a->actor, a->msg_index,
(unsigned long long)a->term, (unsigned long long)a->index);
}
/* Per-node resulting tracked variables: currentTerm, state, commitIndex, log
* (the (term, value) sequence) — raft.tla variable names (R8.1, R8.2). */
for (int i = 0; i < c->n; i++) {
const raft_sim_node_t *node = &c->nodes[i];
raft_snapshot_t snap;
raft_snapshot(&node->state, &snap);
fprintf(fp, "node %d currentTerm=%llu state=%s commitIndex=%llu log=[",
i, (unsigned long long)snap.term,
sim_tla_state_name(snap.role),
(unsigned long long)node->commit_index);
for (int k = 1; k <= node->log_count; k++) {
const raft_sim_entry_t *slot = &node->log[k - 1];
uint64_t value = sim_conf_decode_value(slot);
fprintf(fp, "%s(%llu,%llu)", (k > 1) ? ";" : "",
(unsigned long long)slot->term,
(unsigned long long)value);
}
fprintf(fp, "]\n");
}
/* `messages` bag summary: count + per-member (type, src, dst, term). Mirrors
* raft.tla `messages` (the in-flight RPC bag). */
fprintf(fp, "messages count=%d", c->bag_count);
for (int i = 0; i < c->bag_count; i++) {
const raft_sim_msg_t *m = &c->bag[i];
fprintf(fp, " [%d]=%s{src=%d,dst=%d,term=%llu}",
i, sim_tla_msg_kind_name(m->kind), m->src, m->dst,
(unsigned long long)m->term);
}
fprintf(fp, "\n");
}
int raft_sim_restart_node(raft_sim_cluster_t *c, int node_idx)
{
if (node_idx < 0 || node_idx >= c->n)
return -1;
raft_sim_node_t *node = &c->nodes[node_idx];
/* Snapshot the DURABLE state that survives a process restart: the persisted
* (term, voted_for) and the log length (the in-memory log[] models the WAL on
* disk and is left untouched). Volatile consensus state is discarded by the
* re-init below. */
uint64_t restored_term = node->state.term;
int restored_voted_for = node->state.voted_for;
uint64_t restored_log_len = (uint64_t)node->log_count;
/* The committed LSN is durable too: on recovery the controller restores it
* into initial_commit_index. Model that here so a restarted node retains its
* committed position (never regresses). */
uint64_t restored_commit_index = node->state.commit_index;
raft_config_t cfg;
memset(&cfg, 0, sizeof(cfg));
cfg.n_peers = node->state.n_peers;
cfg.self_idx = node->idx;
cfg.my_priority = node->idx;
cfg.election_timeout_ms = node->state.election_timeout_ms;
cfg.initial_term = restored_term;
cfg.initial_voted_for = restored_voted_for;
cfg.initial_log_len = restored_log_len;
cfg.initial_commit_index = restored_commit_index;
cfg.cbs = &node->cbs;
cfg.ctx = node; /* same ctx convention as raft_sim_init */
/* raft_init returns the node to FOLLOWER and clears votes/election/leader
* progress while restoring (term, voted_for, log_len). The log[] array and
* the partition flag are preserved. */
return raft_init(&node->state, &cfg);
}
/* ---------- scheduler step (raft-sim-conformance task 4.2) ---------- */
/*
* sim_find_leader: return the index of a node currently in role RAFT_LEADER (read
* via raft_snapshot — observable state only), or -1 if none. When more than one
* node reports LEADER (only possible AFTER an invariant violation, which the
* post-step check then catches), the lowest index is returned deterministically.
*/
static int sim_find_leader(const raft_sim_cluster_t *c)
{
for (int i = 0; i < c->n; i++) {
raft_snapshot_t snap;
raft_snapshot(&c->nodes[i].state, &snap);
if (snap.role == RAFT_LEADER)
return i;
}
return -1;
}
int raft_sim_step(raft_sim_cluster_t *c)
{
/* ---- 1. Compute the ENABLED action set (weighted) ----
* Build the eligible-action multiset for the CURRENT state. Only enabled
* actions are eligible (design "Action scheduler"); weighting is achieved by
* how many slots each kind occupies in `enabled[]`, while every enabled kind
* keeps a NON-ZERO weight (so the whole action space stays reachable). The
* weights bias the run toward LIVENESS — delivery and leader progress dominate,
* faults (drop/duplicate) and restart remain occasional perturbations — so a
* fresh cluster reliably elects a leader, replicates, and ADVANCES COMMIT within
* a bounded run, while still exercising every fault kind. RESTART and TIMEOUT are
* always enabled (n >= 1) so the cluster stays live; leader-gated actions
* (CLIENT_REQUEST, APPEND_ENTRIES, ADVANCE_COMMIT) require a LEADER; bag-gated
* actions (RECEIVE, DROP, DUPLICATE) require a non-empty bag. */
enum { SIM_W_RESTART = 1, SIM_W_TIMEOUT = 2, SIM_W_CLIENT = 3,
SIM_W_ADVANCE = 3, SIM_W_APPEND = 4, SIM_W_RECEIVE = 6,
SIM_W_FAULT = 1 };
/* Upper bound: sum of all weights above (every kind enabled at once). */
raft_sim_action_kind_t enabled[SIM_W_RESTART + SIM_W_TIMEOUT + SIM_W_CLIENT +
SIM_W_ADVANCE + SIM_W_APPEND + SIM_W_RECEIVE +
2 * SIM_W_FAULT];
int n_enabled = 0;
for (int w = 0; w < SIM_W_RESTART; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_RESTART; /* occasional process restart */
for (int w = 0; w < SIM_W_TIMEOUT; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_TIMEOUT; /* (re)start an election */
int leader = sim_find_leader(c);
int has_leader = (leader >= 0);
/* A leader needs at least one OTHER node to replicate to for AppendEntries. */
int has_peer = (c->n > 1);
if (has_leader) {
for (int w = 0; w < SIM_W_CLIENT; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_CLIENT_REQUEST;
for (int w = 0; w < SIM_W_ADVANCE; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_ADVANCE_COMMIT;
if (has_peer)
for (int w = 0; w < SIM_W_APPEND; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_APPEND_ENTRIES;
}
if (c->bag_count > 0) {
for (int w = 0; w < SIM_W_RECEIVE; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_RECEIVE;
for (int w = 0; w < SIM_W_FAULT; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_DROP_MESSAGE;
for (int w = 0; w < SIM_W_FAULT; w++)
enabled[n_enabled++] = RAFT_SIM_ACT_DUPLICATE_MESSAGE;
}
/* ---- 2. PRNG-pick one enabled action ---- */
raft_sim_action_kind_t kind =
enabled[raft_sim_prng_next(c) % (uint64_t)n_enabled];
/* The action's operands (actor / message index / referenced term+index) are
* filled per-kind below, recorded BEFORE applying, then applied. */
raft_sim_action_t action;
memset(&action, 0, sizeof(action));
action.kind = kind;
action.actor = -1;
action.msg_index = -1;
action.term = 0;
action.index = 0;
switch (kind) {
case RAFT_SIM_ACT_RESTART: {
int idx = (int)(raft_sim_prng_next(c) % (uint64_t)c->n);
action.actor = idx;
action.term = c->nodes[idx].state.term;
raft_sim_trace_record(c, action);
(void)raft_sim_restart_node(c, idx);
break;
}
case RAFT_SIM_ACT_TIMEOUT: {
int idx = (int)(raft_sim_prng_next(c) % (uint64_t)c->n);
action.actor = idx;
action.term = c->nodes[idx].state.term;
raft_sim_trace_record(c, action);
/* TLA+ Timeout: a node becomes candidate and campaigns. Drive the published
* raft_start_election entry point — the library performs the term bump,
* self-vote, persist, and emits vote requests onto the bag (R2.1). (The
* separate raft_on_election_timeout is the give-up/step-down path, not the
* campaign-start action.) */
(void)raft_start_election(&c->nodes[idx].state);
break;
}
case RAFT_SIM_ACT_CLIENT_REQUEST: {
/* Append a synthetic monotone client value on the leader so State Machine
* Safety can observe divergence. Derive a distinguishable value from the
* PRNG (R2.1 ClientRequest via raft_log_append). */
uint64_t value = raft_sim_prng_next(c);
uint64_t out_index = 0;
action.actor = leader;
/* Record the leader's NEXT index as the referenced operand BEFORE applying
* (R1.4): the entry will land at the current log tail + 1. */
action.index = (uint64_t)c->nodes[leader].log_count + 1;
raft_sim_trace_record(c, action);
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value),
&out_index);
break;
}
case RAFT_SIM_ACT_APPEND_ENTRIES: {
/* Pick a peer != leader and emit one AppendEntries. raft_send_append_entries
* computes prev_log_index/prev_log_term from the library's per-peer progress
* (next_index[peer]); the harness only supplies leader_commit (R2.2). */
int peer = (int)(raft_sim_prng_next(c) % (uint64_t)c->n);
if (peer == leader)
peer = (peer + 1) % c->n;
action.actor = leader;
action.msg_index = peer; /* record the target peer in the operand slot */
raft_sim_trace_record(c, action);
(void)raft_send_append_entries(&c->nodes[leader].state, peer,
c->nodes[leader].state.commit_index);
break;
}
case RAFT_SIM_ACT_ADVANCE_COMMIT: {
/* The library applies the Figure-8 current-term commit rule; on a strict
* advance the leader's on_commit_advanced applies committed values into
* applied[] (R2.3). The harness only triggers the recompute. */
action.actor = leader;
raft_sim_trace_record(c, action);
(void)raft_advance_commit(&c->nodes[leader].state);
break;
}
case RAFT_SIM_ACT_RECEIVE: {
/* Deliver a PRNG-picked bag message through its library handler. This
* organically covers RequestVote propagation, BecomeLeader (a quorum-
* completing vote-rsp makes raft_on_vote_reply elect the candidate), AE
* request/response, and UpdateTerm/stale-drop — all INSIDE the handlers,
* never pre-filtered (R2.4/R2.6). */
int i = raft_sim_bag_pick(c);
action.msg_index = i;
if (i >= 0 && i < c->bag_count) {
action.actor = c->bag[i].dst;
action.term = c->bag[i].term;
}
raft_sim_trace_record(c, action);
if (i >= 0)
(void)raft_sim_bag_deliver(c, i);
break;
}
case RAFT_SIM_ACT_DUPLICATE_MESSAGE: {
/* Bag-only fault: re-enqueue a copy of a PRNG-picked member (R2.5). The bag
* is treated uniformly (raft.tla DuplicateMessage operates on the whole
* `messages` bag); the harness never pre-filters by kind. */
int i = raft_sim_bag_pick(c);
action.msg_index = i;
raft_sim_trace_record(c, action);
if (i >= 0)
(void)raft_sim_bag_duplicate(c, i);
break;
}
case RAFT_SIM_ACT_DROP_MESSAGE: {
/* Bag-only fault: discard a PRNG-picked member (R2.5). */
int i = raft_sim_bag_pick(c);
action.msg_index = i;
raft_sim_trace_record(c, action);
if (i >= 0)
(void)raft_sim_bag_drop(c, i);
break;
}
case RAFT_SIM_ACT_REQUEST_VOTE:
case RAFT_SIM_ACT_BECOME_LEADER:
/* Never SELECTED as standalone scheduler actions: RequestVote and
* BecomeLeader are realized organically by the RECEIVE path (delivering a
* vote-request / the quorum-completing vote-response through the library
* handlers — design "Action scheduler"). Listed here only so the dispatch is
* exhaustive over raft_sim_action_kind_t; reached only if a future caller
* forges one, in which case it is a recorded no-op. */
raft_sim_trace_record(c, action);
break;
}
/* ---- 5. Check all four invariants AFTER applying (Req 3.5) ----
* On a violation, dump the replayable trace and return < 0 (the negated failing
* invariant). A clean step returns 0. */
raft_sim_invariant_t failing = raft_sim_check_all(c);
if (failing != RAFT_SIM_INV_NONE) {
raft_sim_dump_trace(c, stderr, failing);
return -(int)failing;
}
return 0;
}
/* ---------- multi-step run driver (raft-sim-conformance task 4.4) ---------- */
int raft_sim_run(raft_sim_cluster_t *c, int max_steps)
{
/* Bounded loop of single steps. raft_sim_step is the SOLE consumer of c->prng,
* so a run is a pure deterministic function of (n, seed) and the step count —
* two same-keyed clusters produce identical traces and final state (R1.5/1.6/
* 5.3). Halt on the FIRST violation: raft_sim_step returns < 0 and has already
* dumped the replayable trace, so propagate that value unchanged (R3.5). A clean
* exhaustion of the step budget returns 0. */
for (int step = 0; step < max_steps; step++) {
int rc = raft_sim_step(c);
if (rc < 0)
return rc; /* first invariant violation — stop and surface it */
}
return 0;
}
/* ---------- partition/heal scenario (raft-sim-conformance task 5.1) ---------- */
/*
* sim_scen_check: run all four invariants over `c`. On a violation, dump the
* replayable trace to stderr and return the negated failing invariant (matching the
* raft_sim_step / raft_sim_run contract); 0 when all hold. The scenario calls this
* after EVERY action it applies so a violation is surfaced the moment it appears
* (Req 4.1 — invariants asserted THROUGHOUT, not only at the end).
*/
static int sim_scen_check(raft_sim_cluster_t *c)
{
raft_sim_invariant_t failing = raft_sim_check_all(c);
if (failing != RAFT_SIM_INV_NONE) {
raft_sim_dump_trace(c, stderr, failing);
return -(int)failing;
}
return 0;
}
/*
* sim_scen_drain_bag: deliver every in-flight bag message (PRNG-picked order, so a
* partition leaves partitioned-endpoint messages parked rather than delivered),
* running the invariant check after EACH delivery. `cap` bounds the work so a
* duplicate/heal storm can never spin unbounded. Stops early and propagates < 0 on
* the first violation. Returns 0 when the bag drains (or only undeliverable,
* partitioned messages remain) with no violation.
*/
static int sim_scen_drain_bag(raft_sim_cluster_t *c, int cap)
{
for (int g = 0; g < cap && c->bag_count > 0; g++) {
int i = raft_sim_bag_pick(c);
if (i < 0)
break;
int rc = raft_sim_bag_deliver(c, i);
/* rc == -2 means the picked message targets a partitioned endpoint and was
* left in the bag (a heal can deliver it later). To avoid re-picking the same
* parked message forever, sweep for ANY deliverable message; if none remain,
* the bag holds only partitioned traffic and we are done draining for now. */
if (rc == -2) {
int progressed = 0;
for (int j = 0; j < c->bag_count; j++) {
int s = c->bag[j].src, d = c->bag[j].dst;
if (!c->nodes[s].partition && !c->nodes[d].partition) {
if (raft_sim_bag_deliver(c, j) >= 0)
progressed = 1;
break;
}
}
if (!progressed)
break; /* only partitioned traffic left */
}
int chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
return 0;
}
/*
* sim_scen_replicate_round: on the current leader, append one client value, send
* AppendEntries to every other node, drain the bag, then recompute commit — checking
* invariants after every action. Non-leaders and partitioned peers are handled by the
* library / bag delivery (a partitioned peer's AE simply never delivers). Returns 0 on
* a clean round, < 0 on the first invariant violation.
*/
static int sim_scen_replicate_round(raft_sim_cluster_t *c, int leader, uint64_t value)
{
uint64_t out_index = 0;
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
int chk = sim_scen_check(c);
if (chk < 0)
return chk;
for (int p = 0; p < c->n; p++) {
if (p == leader)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
(void)raft_advance_commit(&c->nodes[leader].state);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* Commit advance can release a higher leader_commit; replicate it out so followers
* apply the newly-committed entries, then re-check. */
for (int p = 0; p < c->n; p++) {
if (p == leader)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
}
return sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
}
int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed)
{
/* A minority partition with a surviving majority needs n >= 3 (floor((n-1)/2) >= 1
* isolated, with a strict majority still connected). */
if (n < 3 || n > RAFT_SIM_MAX_NODES)
return -1;
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
return -1;
int chk;
/* ---- 1. Elect a leader: node 0 campaigns; drain the vote traffic. ---- */
(void)raft_start_election(&c->nodes[0].state);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
int leader = sim_find_leader(c);
if (leader < 0) {
/* No leader emerged — retry a couple of campaigns (still all via the library).
* If the cluster genuinely cannot elect, that is observable, not a violation;
* the post-scenario single-leader assertion in the test will catch it. */
for (int retry = 0; retry < 3 && leader < 0; retry++) {
(void)raft_start_election(&c->nodes[retry % n].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
}
if (leader < 0)
return 0; /* no violation occurred; nothing more to drive */
}
/* ---- 2. Replicate + commit a few entries with the cluster fully connected. ---- */
for (int r = 0; r < 3; r++) {
chk = sim_scen_replicate_round(c, leader, 0x100ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
if (leader < 0)
return 0;
}
/* ---- 3. Isolate a MINORITY (highest indices); majority keeps progressing. ---- */
int minority = (n - 1) / 2; /* < n/2, so the remaining nodes are a strict majority */
for (int m = 0; m < minority; m++)
raft_sim_partition(c, n - 1 - m, 1);
/* If the isolated set happened to include the leader, re-elect within the majority
* so the partitioned phase still makes progress on the connected side. */
{
int leader_isolated = c->nodes[leader].partition;
if (leader_isolated) {
leader = -1;
for (int cand = 0; cand < n - minority && leader < 0; cand++) {
if (c->nodes[cand].partition)
continue;
(void)raft_start_election(&c->nodes[cand].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
/* Only accept a CONNECTED leader (a stale isolated leader does not count). */
if (leader >= 0 && c->nodes[leader].partition)
leader = -1;
}
if (leader < 0)
return 0;
}
}
/* Drive a phase of client appends/replication while partitioned. The majority must
* keep committing; AE to isolated peers simply never delivers. A minority node also
* times out and campaigns — it must NOT be able to win a higher-term election (the
* library enforces quorum; the harness just drives + checks). */
for (int r = 0; r < 4; r++) {
chk = sim_scen_replicate_round(c, leader, 0x200ULL + (uint64_t)r);
if (chk < 0)
return chk;
/* A minority node campaigns mid-partition (cannot reach a majority of votes). */
if (minority > 0) {
(void)raft_start_election(&c->nodes[n - 1].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
}
leader = sim_find_leader(c);
if (leader < 0 || c->nodes[leader].partition)
return 0;
}
/* ---- 4. HEAL the partition and drain everything so the minority catches up. ---- */
for (int m = 0; m < minority; m++)
raft_sim_partition(c, n - 1 - m, 0);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* Draining the parked minority traffic (those nodes campaigned at higher terms while
* isolated) can step the connected leader down via UpdateTerm — correct library
* behavior. Re-establish a single leader over the now-fully-connected cluster before
* catch-up: campaign one node at a time, draining each round, until exactly one
* leader holds. This is the natural Raft recovery, driven only via library entry
* points. */
leader = sim_find_leader(c);
for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) {
(void)raft_start_election(&c->nodes[attempt % n].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
}
if (leader < 0)
return 0; /* no leader emerged — observable, not a violation; test asserts it */
/* Catch-up rounds: re-replicate to ALL nodes (now reachable) until the formerly
* isolated followers converge to the leader's committed prefix. Re-establish the
* leader inside the loop if a late vote-request steps it down again. Bounded. */
for (int r = 0; r < n + 4; r++) {
chk = sim_scen_replicate_round(c, leader, 0x300ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) {
(void)raft_start_election(&c->nodes[attempt % n].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
}
if (leader < 0)
return 0;
}
/* ---- 5. Final convergence check (also covered after every step above). ---- */
return sim_scen_check(c);
}
/* ---------- restart-churn scenario (raft-sim-conformance task 5.2) ---------- */
/*
* sim_scen_elect: campaign one node at a time (raft_start_election), draining the bag
* after each, until a single LEADER emerges or the bounded attempts are exhausted.
* Returns the leader index (>= 0), -1 if none emerged, or < 0 propagated from
* sim_scen_check on an invariant violation (callers distinguish: a violation value is
* always <= -(int)RAFT_SIM_INV_NONE - 1 < -1, but to keep this simple the caller treats
* any returned violation by re-checking; here we surface the violation via *viol). On a
* violation *viol holds the negated invariant and the return is -1.
*/
static int sim_scen_elect(raft_sim_cluster_t *c, int *viol)
{
*viol = 0;
int leader = sim_find_leader(c);
for (int attempt = 0; attempt < 2 * c->n && leader < 0; attempt++) {
(void)raft_start_election(&c->nodes[attempt % c->n].state);
int chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0) {
*viol = chk;
return -1;
}
leader = sim_find_leader(c);
}
return leader;
}
int raft_sim_scenario_restart_churn(raft_sim_cluster_t *c, int n, uint64_t seed)
{
if (n < 3 || n > RAFT_SIM_MAX_NODES)
return -1;
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
return -1;
int chk, viol;
/* ---- 1. Elect a leader and replicate a committed prefix. ---- */
int leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0; /* no leader emerged — observable, not a violation; test asserts it */
for (int r = 0; r < 3; r++) {
chk = sim_scen_replicate_round(c, leader, 0x500ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
}
/* ---- 2. Restart a FOLLOWER mid-replication (it holds a partial log). ---- */
int follower = -1;
for (int i = 0; i < n; i++) {
if (i != leader) {
follower = i;
break;
}
}
/* Append one more entry on the leader and send it out, but DO NOT fully drain —
* leave replication in flight, then restart the follower so its restart lands
* mid-replication (it may hold only a partial prefix). */
{
uint64_t value = 0x5A0ULL;
uint64_t out_index = 0;
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
for (int p = 0; p < n; p++) {
if (p == leader)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
}
(void)raft_sim_restart_node(c, follower); /* preserves persisted (term, votedFor, log) */
chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* Drain in-flight traffic; the restarted follower (now a fresh FOLLOWER) re-syncs
* via AppendEntries. */
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* Re-establish a leader if the in-flight churn stepped the current one down. */
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
/* ---- 3. Restart the LEADER mid-replication, forcing a re-election. ---- */
{
uint64_t value = 0x5B0ULL;
uint64_t out_index = 0;
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
for (int p = 0; p < n; p++) {
if (p == leader)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
}
(void)raft_sim_restart_node(c, leader); /* persisted state survives; volatile reset */
chk = sim_scen_check(c);
if (chk < 0)
return chk;
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* The old leader is now a FOLLOWER; a survivor must win a fresh election. */
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
/* ---- 4. Catch-up rounds: replicate + propagate commit so EVERY node (including the
* two formerly-restarted ones) converges to the leader's log and advances its
* committed/applied prefix. Re-establish the leader inside the loop if a late
* vote-request steps it down. Bounded. ---- */
for (int r = 0; r < n + 4; r++) {
chk = sim_scen_replicate_round(c, leader, 0x600ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
}
/* A final heartbeat round so any follower lagging on leader_commit observes the
* latest committed position and advances its applied prefix (commit propagation). */
for (int p = 0; p < n; p++) {
if (p == leader)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
}
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* ---- 5. Final convergence check (also covered after every step above). ---- */
return sim_scen_check(c);
}
/* ---------- Figure-8 scenario (raft-sim-conformance task 5.4) ---------- */
/*
* sim_scen_quorum_replicate_one: on `leader`, send ONE AppendEntries to enough
* distinct peers that, together with the leader's self-match, a strict majority
* physically holds the leader's tail entry — WITHOUT advancing commit. Each AE is
* delivered (request + reply) so the entry lands in the peer's log via the real
* library handler. Invariants are checked after every action. Returns 0 on a clean
* round, < 0 on the first violation.
*
* This is the load-bearing primitive for Figure-8: it puts a previous-term entry on
* a quorum of LOGS while deliberately NOT calling raft_advance_commit, so the
* scenario can then prove the library refuses to commit it by replica count under a
* new term.
*/
static int sim_scen_quorum_replicate_one(raft_sim_cluster_t *c, int leader)
{
/* A strict majority of n includes the leader itself, so it needs floor(n/2)
* OTHER peers to physically hold the tail. Send + deliver to the lowest-indexed
* peers != leader until that many have acked. */
int needed = c->n / 2; /* other peers required for a majority with self */
int acked = 0;
for (int p = 0; p < c->n && acked < needed; p++) {
if (p == leader)
continue;
if (c->nodes[p].partition)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
int chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* Drain just this exchange (request -> reply) so the peer's log gains the
* entry. Bounded by a small cap; a single AE produces a single reply. */
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
acked++;
}
return 0;
}
/*
* sim_scen_catch_up_commit: drive bounded AppendEntries-to-all-connected-peers rounds
* (NO new client appends) interleaved with raft_advance_commit, until the leader's
* commit_index reaches `target` or the bound is exhausted. Each round retransmits the
* leader's tail to every connected peer; a peer behind the prev-log point rejects, the
* library backs its next_index off, and the next round backfills it — so a connected
* follower lacking the prefix is brought current over a few rounds. Invariants are
* checked after every action. Returns 0 on a clean run (committed-or-not), < 0 on the
* first violation. Used to commit the CURRENT-term entry E2 to a connected quorum that
* may need backfill of the prior-term prefix first.
*/
static int sim_scen_catch_up_commit(raft_sim_cluster_t *c, int leader, uint64_t target)
{
for (int r = 0; r < c->n + 4 && c->nodes[leader].commit_index < target; r++) {
for (int p = 0; p < c->n; p++) {
if (p == leader || c->nodes[p].partition)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
int chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
int chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
(void)raft_advance_commit(&c->nodes[leader].state);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
return 0;
}
int raft_sim_scenario_figure8(raft_sim_cluster_t *c, int n, uint64_t seed)
{
/* Figure-8 needs a real quorum AND a leader change to a node that holds the
* prior-term entry, so n >= 3. */
if (n < 3 || n > RAFT_SIM_MAX_NODES)
return -1;
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
return -1;
int chk, viol;
/* ---- 1. Elect leader L1 at term T1; replicate a PREVIOUS-term entry E1 to a
* QUORUM of logs WITHOUT committing it. ---- */
int l1 = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (l1 < 0)
return 0; /* no leader emerged — observable, not a violation; test asserts it */
/* Append E1 on L1 (a term-T1 entry) and physically replicate it to a quorum of
* logs via real AppendEntries — but do NOT call raft_advance_commit, so E1 stays
* UNcommitted even though a majority holds it. */
uint64_t e1_out = 0;
(void)raft_log_append(&c->nodes[l1].state, &(uint64_t){0xF8E1ULL},
sizeof(uint64_t), &e1_out);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
uint64_t e1_index = e1_out; /* the prior-term entry's 1-based index */
uint64_t e1_term = c->nodes[l1].state.term;
chk = sim_scen_quorum_replicate_one(c, l1);
if (chk < 0)
return chk;
/* E1 must NOT yet be committed on L1 (we never advanced commit over it). This is a
* harness precondition, not the safety property: if it were already committed the
* construction is wrong, so surface it as a violation-style failure. */
if (c->nodes[l1].commit_index >= e1_index)
return -(int)RAFT_SIM_INV_STATE_MACHINE_SAFETY;
/* ---- 2. FORCE A LEADER CHANGE to a higher term T2. ---- */
/* Kill L1 (partition it away) so a survivor must campaign at a higher term. A
* winner must be at least as up-to-date as a majority, and since E1 is on a
* quorum the new leader L2 necessarily HOLDS E1 in its log (Leader Completeness
* is what we are stress-testing; the freshness gate makes a winner hold E1). */
raft_sim_partition(c, l1, 1);
/* The new leader MUST hold E1 (Leader Completeness is what we stress-test). Find a
* CONNECTED peer that physically holds E1 (sim_scen_quorum_replicate_one put it on
* the lowest-indexed peers) and campaign ONLY that node — campaigning a single
* deterministic up-to-date candidate avoids term-bumping interleaving that prevents
* any candidate from consolidating a quorum. The freshness gate guarantees a node
* lacking E1 cannot win against this up-to-date candidate. */
int e1_holder = -1;
for (int p = 0; p < n; p++) {
if (p == l1 || c->nodes[p].partition)
continue;
if ((uint64_t)c->nodes[p].log_count >= e1_index &&
c->nodes[p].log[e1_index - 1].term == e1_term) {
e1_holder = p;
break;
}
}
if (e1_holder < 0)
return 0; /* no connected E1-holder survived — observable, not a violation */
int l2 = -1;
for (int attempt = 0; attempt < 2 * n && l2 < 0; attempt++) {
(void)raft_start_election(&c->nodes[e1_holder].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* Check the E1-holder's OWN role directly — sim_find_leader returns the lowest
* index, which is the stale partitioned-away L1 (still role==LEADER at the old
* term until it sees a higher term), not the freshly-elected connected leader. */
raft_snapshot_t snap;
raft_snapshot(&c->nodes[e1_holder].state, &snap);
if (snap.role == RAFT_LEADER)
l2 = e1_holder;
}
if (l2 < 0)
return 0; /* no new leader emerged — observable, not a violation */
/* L2 must be at a strictly-higher term than E1, and must HOLD E1 (it won a quorum
* that holds E1). If L2 lacked E1 the construction failed; surface it. */
if (c->nodes[l2].state.term <= e1_term)
return -(int)RAFT_SIM_INV_ELECTION_SAFETY;
if ((uint64_t)c->nodes[l2].log_count < e1_index ||
c->nodes[l2].log[e1_index - 1].term != e1_term)
return -(int)RAFT_SIM_INV_LEADER_COMPLETENESS;
/* ---- 3. SAFETY PROPERTY (a): under term-T2 leader L2, E1 (a term-T1 entry) sits
* on a CONNECTED quorum, yet raft_advance_commit must NOT commit it by replica
* count alone — the current-term Figure-8 rule (raft.c G4) forbids committing a
* prior-term entry without a current-term entry above it.
*
* First backfill E1 onto a connected majority under L2: drive AE rounds so the
* connected peers (which may have lacked E1 after the leader change) physically
* hold it, making the "E1 looks committable by count" danger REAL. Interleaved
* raft_advance_commit is harmless here — it is the very rule under test and must
* refuse to commit prior-term E1 (so commit_index stays below e1_index, which the
* loop's target never reaches; the bound then exits cleanly). ---- */
chk = sim_scen_catch_up_commit(c, l2, e1_index); /* target deliberately unreachable */
if (chk < 0)
return chk;
/* A final explicit advance attempt on the now-quorum-replicated prior-term E1. */
(void)raft_advance_commit(&c->nodes[l2].state);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* The marquee assertion: E1 is NOT covered by L2's commit_index despite quorum
* replication. A regression to term-blind commit would advance past e1_index here. */
if (c->nodes[l2].commit_index >= e1_index)
return -(int)RAFT_SIM_INV_STATE_MACHINE_SAFETY;
/* No node may report E1 committed either (observable State Machine Safety). */
for (int i = 0; i < n; i++) {
if (c->nodes[i].commit_index >= e1_index)
return -(int)RAFT_SIM_INV_STATE_MACHINE_SAFETY;
}
/* ---- 4. L2 appends a CURRENT-term (T2) entry E2 and replicates it to a quorum;
* NOW raft_advance_commit commits E2 and TRANSITIVELY commits E1 below it. ---- */
uint64_t e2_out = 0;
(void)raft_log_append(&c->nodes[l2].state, &(uint64_t){0xF8E2ULL},
sizeof(uint64_t), &e2_out);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
uint64_t e2_index = e2_out;
/* Replicate + commit the current-term E2 to a connected quorum (backfilling the
* prefix on any lagging connected peer as needed). */
chk = sim_scen_catch_up_commit(c, l2, e2_index);
if (chk < 0)
return chk;
/* SAFETY PROPERTY (b): committing the current-term entry E2 advances commit_index
* to cover BOTH E2 and (transitively) the prior-term E1 below it. */
if (c->nodes[l2].commit_index < e1_index)
return -(int)RAFT_SIM_INV_STATE_MACHINE_SAFETY;
if (c->nodes[l2].commit_index < e2_index)
return -(int)RAFT_SIM_INV_STATE_MACHINE_SAFETY;
/* ---- 5. HEAL the old leader L1 back so the cluster converges to a SINGLE leader
* (the stale isolated L1 still reports LEADER at the old term T1 until it observes
* T2). On heal it sees L2's higher term via the drained traffic / a replicate round
* and steps down to FOLLOWER, catching up the committed prefix. This leaves the
* post-scenario cluster with exactly one leader (L2) holding the committed E1+E2 —
* matching the other 5.x scenarios' converged end-state. ---- */
raft_sim_partition(c, l1, 0);
/* Re-establish a single leader and replicate/propagate commit so L1 (now reachable)
* converges. Bounded catch-up rounds; re-elect inside the loop if a late vote steps
* the leader down. Invariants checked after every action (via the helpers). */
int leader = sim_find_leader(c);
for (int r = 0; r < n + 4; r++) {
if (leader < 0 || c->nodes[leader].partition) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0; /* no leader emerged — observable, not a violation */
}
chk = sim_scen_replicate_round(c, leader, 0xF800ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
}
/* ---- 6. Final invariant check (also covered after every step above). ---- */
return sim_scen_check(c);
}
/* ---------- drop/duplicate/reorder scenario (raft-sim-conformance task 5.3) ---------- */
/*
* sim_scen_logs_converged: 1 iff every node's log has caught up to the leader's tip
* (same length AND every entry's term matches over the FULL log). Pure observer over
* node->log[] / log_count — the convergence target the retransmit loop drives toward.
* (Followers never advance commit_index from AE in this library, so convergence is
* asserted over the LOG, not commit_index — see task 5.2.)
*/
static int sim_scen_logs_converged(const raft_sim_cluster_t *c, int leader)
{
const raft_sim_node_t *L = &c->nodes[leader];
for (int i = 0; i < c->n; i++) {
if (i == leader)
continue;
const raft_sim_node_t *f = &c->nodes[i];
if (f->log_count != L->log_count)
return 0;
for (int e = 1; e <= L->log_count; e++) {
if (f->log[e - 1].term != L->log[e - 1].term)
return 0;
}
}
return 1;
}
/*
* sim_scen_perturb_bag: apply one round of bag faults BEFORE delivery so the subsequent
* drain sees a dropped/duplicated/reordered message set. Mirrors raft.tla DropMessage /
* DuplicateMessage as pure bag mutations (no node touched). Uses the cluster PRNG so the
* perturbation is seed-replayable: a PRNG-chosen fraction of in-flight members are
* DUPLICATEd (re-enqueued — multiplicity += 1) and a PRNG-chosen fraction are DROPped.
* Bounded by the live bag size; never drains the bag empty (drops at most ~half) so
* retransmit + drain can still converge. Returns 0 (it performs only bag ops, which the
* scenario invariant-checks after).
*/
static void sim_scen_perturb_bag(raft_sim_cluster_t *c)
{
/* DUPLICATE pass: re-enqueue a PRNG-chosen subset (delivered more than once). Snapshot
* the count first so freshly-pushed copies are not themselves re-duplicated. */
int n_live = c->bag_count;
for (int i = 0; i < n_live; i++) {
if ((raft_sim_prng_next(c) & 1ULL) == 0)
(void)raft_sim_bag_duplicate(c, i); /* fail-loud on overflow; ignored here */
}
/* DROP pass: discard at most half of the CURRENT members so the bag never empties.
* Pick by PRNG index (swap-with-last removal keeps the array dense). */
int max_drop = c->bag_count / 2;
for (int d = 0; d < max_drop && c->bag_count > 1; d++) {
if ((raft_sim_prng_next(c) & 1ULL) == 0) {
int idx = raft_sim_bag_pick(c); /* PRNG-chosen live index */
if (idx >= 0)
(void)raft_sim_bag_drop(c, idx);
}
}
}
int raft_sim_scenario_drop_dup_reorder(raft_sim_cluster_t *c, int n, uint64_t seed)
{
if (n < 3 || n > RAFT_SIM_MAX_NODES)
return -1;
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
return -1;
int chk, viol;
/* ---- 1. Elect a leader and replicate a committed prefix (clean rounds). ---- */
int leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0; /* no leader emerged — observable, not a violation; test asserts it */
for (int r = 0; r < 3; r++) {
chk = sim_scen_replicate_round(c, leader, 0x700ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
}
/* ---- 2. PERTURBED rounds: append + send AE to every peer, then DROP/DUPLICATE/
* REORDER the in-flight bag BEFORE delivering it in PRNG order. Invariants are
* checked after the append, after each AE send, after the perturbation, and after
* every delivery (sim_scen_drain_bag delivers PRNG-picked = reordered, checking each
* step). A duplicated AE that lands twice hits the library's already-have branch (a
* no-op); a dropped AE leaves a follower behind, recovered by the retransmit phase. */
for (int r = 0; r < 4; r++) {
uint64_t value = 0x800ULL + (uint64_t)r;
uint64_t out_index = 0;
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
for (int p = 0; p < n; p++) {
if (p == leader)
continue;
(void)raft_send_append_entries(&c->nodes[leader].state, p,
c->nodes[leader].state.commit_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
}
/* Fault injection on the in-flight bag (drop + duplicate). */
sim_scen_perturb_bag(c);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* Reordered delivery: sim_scen_drain_bag picks PRNG order, checking each step. */
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* The duplicate/reorder churn may have stepped the leader down via a stray
* higher-term message; re-establish one before the next round. */
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
}
/* ---- 3. RETRANSMIT to convergence: dropped entries left followers behind, so drive
* clean replicate rounds (the leader re-sends from its per-peer next_index) until
* every log converges to the leader's tip. Bounded — a leader's log holds at most
* RAFT_SIM_MAX_LOG entries, so n + log_count + 4 rounds is ample. ---- */
int max_rounds = n + c->nodes[leader].log_count + 4;
for (int r = 0; r < max_rounds && !sim_scen_logs_converged(c, leader); r++) {
chk = sim_scen_replicate_round(c, leader, 0x900ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
}
/* ---- 4. Final convergence check (also covered after every step above). ---- */
return sim_scen_check(c);
}
/* ---------- stale-rejoin scenario (raft-sim-conformance task 5.5; bug-851 G5) ---------- */
/*
* sim_scen_elect_connected: campaign one CONNECTED node at a time (raft_start_election),
* draining the bag after each, until a CONNECTED node holds role==RAFT_LEADER or the
* bounded attempts are exhausted. Differs from sim_scen_elect in two ways the stale-
* rejoin needs: (1) it skips partitioned candidates, and (2) it returns the index of a
* CONNECTED leader (a stale, partitioned-away node that still reports LEADER at an old
* term is NOT accepted — sim_find_leader would return that lowest-index stale leader).
* On an invariant violation *viol holds the negated invariant and the return is -1;
* otherwise *viol is 0 and the return is the connected-leader index (>= 0) or -1.
*/
static int sim_scen_elect_connected(raft_sim_cluster_t *c, int *viol)
{
*viol = 0;
for (int attempt = 0; attempt < 4 * c->n; attempt++) {
int cand = attempt % c->n;
if (c->nodes[cand].partition)
continue; /* never campaign an isolated node */
(void)raft_start_election(&c->nodes[cand].state);
int chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0) {
*viol = chk;
return -1;
}
/* Accept only a CONNECTED leader — check candidate roles directly rather than
* sim_find_leader (which returns the lowest index, possibly a stale isolated
* leader still at an old term). */
for (int p = 0; p < c->n; p++) {
if (c->nodes[p].partition)
continue;
raft_snapshot_t snap;
raft_snapshot(&c->nodes[p].state, &snap);
if (snap.role == RAFT_LEADER)
return p;
}
}
return -1; /* no connected leader emerged */
}
int raft_sim_scenario_stale_rejoin(raft_sim_cluster_t *c, int n, uint64_t seed)
{
/* A surviving connected majority while one node is isolated needs n >= 3. */
if (n < 3 || n > RAFT_SIM_MAX_NODES)
return -1;
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
return -1;
int chk, viol;
/* ---- 1. Elect a leader and replicate a committed prefix (fully connected). ---- */
int leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0; /* no leader emerged — observable, not a violation; test asserts it */
for (int r = 0; r < 3; r++) {
chk = sim_scen_replicate_round(c, leader, 0xA00ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
if (leader < 0) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
}
/* ---- 2. Pick N_stale = a node != leader; ISOLATE it. While isolated it keeps its
* (now stale) term. The connected majority will advance its term below. ---- */
int n_stale = (leader == 0) ? 1 : 0; /* any node other than the current leader */
raft_sim_partition(c, n_stale, 1);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* The isolated node times out and campaigns — it can bump its OWN term but can NEVER
* win (no quorum reachable). Drive a couple of its campaigns so the run exercises the
* realistic "stale node spins elections in isolation" path; its messages park in the
* bag (partitioned endpoint) and never grant it leadership. */
for (int e = 0; e < 2; e++) {
(void)raft_start_election(&c->nodes[n_stale].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
if (chk < 0)
return chk;
}
/* ---- 3. Advance the CONNECTED majority's TERM via fresh elections until a connected
* leader sits at a STRICTLY-HIGHER term than N_stale. The connected leader from phase
* 1 may have stepped down (or N_stale's isolated campaigns left stale parked traffic);
* re-establish a connected leader, then keep campaigning connected nodes until the
* connected leader's term exceeds N_stale's term. Bounded. ---- */
int conn_leader = sim_scen_elect_connected(c, &viol);
if (viol < 0)
return viol;
if (conn_leader < 0)
return 0; /* connected majority could not elect — observable, not a violation */
for (int bump = 0; bump < 4 * n &&
c->nodes[conn_leader].state.term <= c->nodes[n_stale].state.term;
bump++) {
/* Force another term bump on the connected side: campaign a different connected
* node (a higher term steps the current connected leader down, and the survivors
* re-elect at a strictly-higher term). */
conn_leader = sim_scen_elect_connected(c, &viol);
if (viol < 0)
return viol;
if (conn_leader < 0)
return 0;
if (c->nodes[conn_leader].state.term > c->nodes[n_stale].state.term)
break;
/* Not yet higher — bump the connected leader's term explicitly by campaigning a
* connected follower (its prevote/term bump raises the connected epoch). */
for (int p = 0; p < n; p++) {
if (p == conn_leader || c->nodes[p].partition)
continue;
(void)raft_start_election(&c->nodes[p].state);
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
break;
}
conn_leader = sim_scen_elect_connected(c, &viol);
if (viol < 0)
return viol;
if (conn_leader < 0)
return 0;
}
/* The connected leader MUST now outrank N_stale; otherwise the construction failed to
* create the term gap (observable, not a violation). */
if (c->nodes[conn_leader].state.term <= c->nodes[n_stale].state.term)
return 0;
/* Replicate a current-term entry on the connected leader so its log tip carries the
* higher term, then drain — this gives the rejoin AE real content to carry. */
chk = sim_scen_replicate_round(c, conn_leader, 0xA50ULL);
if (chk < 0)
return chk;
/* sim_scen_replicate_round can step the leader down via a late vote; re-establish a
* connected leader still outranking N_stale. */
{
raft_snapshot_t lsnap;
raft_snapshot(&c->nodes[conn_leader].state, &lsnap);
if (lsnap.role != RAFT_LEADER) {
conn_leader = sim_scen_elect_connected(c, &viol);
if (viol < 0)
return viol;
if (conn_leader < 0)
return 0;
}
}
if (c->nodes[conn_leader].state.term <= c->nodes[n_stale].state.term)
return 0;
/* ---- 4. NON-VACUITY: capture the leader's term and N_stale's term BEFORE the rejoin
* AE. The adoption assertion is real only if N_stale is STRICTLY behind here. ---- */
uint64_t leader_term_pre = c->nodes[conn_leader].state.term;
uint64_t stale_term_pre = c->nodes[n_stale].state.term;
if (stale_term_pre >= leader_term_pre)
return 0; /* no term gap — construction failed; not a safety violation */
/* ---- 5. HEAL the partition and drive ONE AppendEntries from the higher-term leader
* to N_stale, then DELIVER it through the bag. The library's
* raft_on_append_entries_request UpdateTerm path adopts the leader's higher term and
* steps N_stale down to FOLLOWER — the term is NOT set by the harness. ---- */
raft_sim_partition(c, n_stale, 0);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
(void)raft_send_append_entries(&c->nodes[conn_leader].state, n_stale,
c->nodes[conn_leader].state.commit_index);
chk = sim_scen_check(c);
if (chk < 0)
return chk;
/* Deliver the just-sent AE (and its reply) to N_stale via the bag. Each delivery is
* invariant-checked so an Election-Safety violation surfaces the moment it appears. */
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
if (chk < 0)
return chk;
/* ---- 6. ADOPTION ASSERTIONS (R4.5), all observable via raft_snapshot:
* (a) N_stale adopted the leader's higher term via the AE path,
* (b) N_stale is a FOLLOWER (it did NOT re-elect at a reused term). ---- */
raft_snapshot_t stale_snap;
raft_snapshot(&c->nodes[n_stale].state, &stale_snap);
/* (a) Term adopted from the AE path: N_stale's term advanced to AT LEAST the leader's
* pre-rejoin term (UpdateTerm adopts the higher term). It was strictly lower before
* (asserted at phase 4), so this is a genuine library-driven adoption, not a no-op. */
if (stale_snap.term < leader_term_pre)
return -(int)RAFT_SIM_INV_ELECTION_SAFETY;
/* (b) N_stale stepped to FOLLOWER on adopting the higher term — it did not become a
* leader at a reused/duplicate term. */
if (stale_snap.role == RAFT_LEADER)
return -(int)RAFT_SIM_INV_ELECTION_SAFETY;
/* ---- 7. Let the cluster converge to a single leader and re-replicate so N_stale
* catches up the committed prefix. Bounded; invariants checked after every action via
* the helpers (Election Safety asserted THROUGHOUT). ---- */
leader = sim_find_leader(c);
for (int r = 0; r < n + 4; r++) {
if (leader < 0 || c->nodes[leader].partition) {
leader = sim_scen_elect(c, &viol);
if (viol < 0)
return viol;
if (leader < 0)
return 0;
}
chk = sim_scen_replicate_round(c, leader, 0xA80ULL + (uint64_t)r);
if (chk < 0)
return chk;
leader = sim_find_leader(c);
}
/* ---- 8. Final invariant check (also covered after every step above). ---- */
return sim_scen_check(c);
}
/*
* raft_sim_set_fault_inject: set the cluster's deliberately-broken-build fault mask
* (task 7.1). Consulted only by the harness adapter callbacks (sim_persist_entry /
* sim_truncate_from); raft.c is never touched. DEFAULT OFF — only a test that calls
* this enables the corruption, and passing 0 clears it again within the same binary.
*/
void raft_sim_set_fault_inject(raft_sim_cluster_t *c, unsigned flags)
{
if (c)
c->fault_inject = flags;
}
#endif /* XCE_TESTING */
View git blame Copy permalink