diff --git a/.kiro/specs/raft-sim-conformance/tasks.md b/.kiro/specs/raft-sim-conformance/tasks.md index 6b5c7dd..e87fce5 100644 --- a/.kiro/specs/raft-sim-conformance/tasks.md +++ b/.kiro/specs/raft-sim-conformance/tasks.md @@ -138,7 +138,7 @@ - _Boundary: raft_sim.c, test_raft.c_ - _Depends: 4.2_ -- [ ] 5.2 Restart-churn scenario (P) +- [x] 5.2 Restart-churn scenario (P) - Implement `raft_sim_scenario_restart_churn`: restart nodes mid-replication preserving persisted state; assert re-convergence with no violation - Observable completion: scenario passes; restarted nodes converge to the leader's prefix - _Requirements: 4.2_ @@ -249,3 +249,17 @@ `min(commit_index)` when followers are at 0 (it silently passes). 5.1 was accepted as-is (meets R4.1 via throughout-checking + log matching); 5.2–5.5 should propagate commit so their commit assertions are non-vacuous. +- **CORRECTION (task 5.2, confirmed): this library uses a LEADER-ONLY commit model.** + `raft_on_append_entries_request` does `(void)leader_commit;` (raft.c:646-655) — a FOLLOWER + never advances its `commit_index`/`applied[]` from AppendEntries; `on_commit_advanced` + fires ONLY on the leader via `raft_advance_commit` (raft.c:816). This is intended XCE + design (follower commit visibility is a controller/heartbeat concern — see + ha-heartbeat-committed-propagation — not the raft library). CONSEQUENCE for scenarios: a + follow-up AE round does NOT advance follower commit; do NOT assert follower + `commit_index == leader commit_index`. Assert re-convergence over the LOG instead + (every node's `log_count` reaches the leader's tip + per-entry term match / Log Matching), + and assert leader-side commit via `leader.commit_index`. For 5.4 (Figure-8), the property + under test is LEADER-side (the leader must not mark a previous-term entry committed until a + current-term entry commits) — driven by `raft_advance_commit` on the leader, which works. + applied[]/SM-Safety is exercised across leader CHANGES (each node accumulates applied[] + while it is leader). This SUPERSEDES the follower-commit-propagation guidance above. diff --git a/src/lib/raft/raft.h b/src/lib/raft/raft.h index ad2608a..8fbef19 100644 --- a/src/lib/raft/raft.h +++ b/src/lib/raft/raft.h @@ -992,6 +992,27 @@ int raft_sim_run(raft_sim_cluster_t *c, int max_steps); */ int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed); +/* + * raft_sim_scenario_restart_churn: process-restart scenario (raft-sim-conformance + * task 5.2; Req 4.2). Initializes the caller's cluster from (n, election_timeout=2000, + * seed), then: + * 1. drives it to a single leader + a committed prefix, + * 2. MID-replication, restarts one or more nodes (raft_sim_restart_node) — a follower + * holding a partial log AND the leader (forcing a re-election). The persisted + * (term, votedFor, log) survives each restart (task 4.3 guarantees this); the + * restart resets only volatile state to a fresh FOLLOWER, + * 3. re-elects if the leader was restarted, replicates more, and PROPAGATES COMMIT + * (a follow-up AppendEntries round carrying the higher leader_commit) so the + * restarted nodes catch up and their committed/applied prefix advances. + * raft_sim_check_all is run after EVERY action it applies — DURING and after the churn — + * so a violation is caught the moment it appears. Returns 0 if the scenario completes + * with ZERO invariant violations; < 0 (the negated first failing invariant, matching + * raft_sim_step) on any violation, after dumping the replayable trace to stderr. Returns + * -1 on an invalid node count (n < 3). Drives only via published library entry points + + * harness primitives — no protocol re-implementation. + */ +int raft_sim_scenario_restart_churn(raft_sim_cluster_t *c, int n, uint64_t seed); + #endif /* XCE_TESTING */ #endif /* RAFT_H */ diff --git a/src/lib/raft/raft_sim.c b/src/lib/raft/raft_sim.c index d095c4e..bb5267f 100644 --- a/src/lib/raft/raft_sim.c +++ b/src/lib/raft/raft_sim.c @@ -1518,4 +1518,181 @@ int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed return sim_scen_check(c); } +/* ---------- restart-churn scenario (raft-sim-conformance task 5.2) ---------- */ + +/* + * sim_scen_elect: campaign one node at a time (raft_start_election), draining the bag + * after each, until a single LEADER emerges or the bounded attempts are exhausted. + * Returns the leader index (>= 0), -1 if none emerged, or < 0 propagated from + * sim_scen_check on an invariant violation (callers distinguish: a violation value is + * always <= -(int)RAFT_SIM_INV_NONE - 1 < -1, but to keep this simple the caller treats + * any returned violation by re-checking; here we surface the violation via *viol). On a + * violation *viol holds the negated invariant and the return is -1. + */ +static int sim_scen_elect(raft_sim_cluster_t *c, int *viol) +{ + *viol = 0; + int leader = sim_find_leader(c); + for (int attempt = 0; attempt < 2 * c->n && leader < 0; attempt++) { + (void)raft_start_election(&c->nodes[attempt % c->n].state); + int chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) { + *viol = chk; + return -1; + } + leader = sim_find_leader(c); + } + return leader; +} + +int raft_sim_scenario_restart_churn(raft_sim_cluster_t *c, int n, uint64_t seed) +{ + if (n < 3 || n > RAFT_SIM_MAX_NODES) + return -1; + + if (raft_sim_conformance_init(c, n, 2000, seed) != 0) + return -1; + + int chk, viol; + + /* ---- 1. Elect a leader and replicate a committed prefix. ---- */ + int leader = sim_scen_elect(c, &viol); + if (viol < 0) + return viol; + if (leader < 0) + return 0; /* no leader emerged — observable, not a violation; test asserts it */ + + for (int r = 0; r < 3; r++) { + chk = sim_scen_replicate_round(c, leader, 0x500ULL + (uint64_t)r); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + if (leader < 0) { + leader = sim_scen_elect(c, &viol); + if (viol < 0) + return viol; + if (leader < 0) + return 0; + } + } + + /* ---- 2. Restart a FOLLOWER mid-replication (it holds a partial log). ---- */ + int follower = -1; + for (int i = 0; i < n; i++) { + if (i != leader) { + follower = i; + break; + } + } + /* Append one more entry on the leader and send it out, but DO NOT fully drain — + * leave replication in flight, then restart the follower so its restart lands + * mid-replication (it may hold only a partial prefix). */ + { + uint64_t value = 0x5A0ULL; + uint64_t out_index = 0; + (void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + for (int p = 0; p < n; p++) { + if (p == leader) + continue; + (void)raft_send_append_entries(&c->nodes[leader].state, p, + c->nodes[leader].state.commit_index); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + } + } + + (void)raft_sim_restart_node(c, follower); /* preserves persisted (term, votedFor, log) */ + chk = sim_scen_check(c); + if (chk < 0) + return chk; + + /* Drain in-flight traffic; the restarted follower (now a fresh FOLLOWER) re-syncs + * via AppendEntries. */ + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) + return chk; + + /* Re-establish a leader if the in-flight churn stepped the current one down. */ + leader = sim_find_leader(c); + if (leader < 0) { + leader = sim_scen_elect(c, &viol); + if (viol < 0) + return viol; + if (leader < 0) + return 0; + } + + /* ---- 3. Restart the LEADER mid-replication, forcing a re-election. ---- */ + { + uint64_t value = 0x5B0ULL; + uint64_t out_index = 0; + (void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + for (int p = 0; p < n; p++) { + if (p == leader) + continue; + (void)raft_send_append_entries(&c->nodes[leader].state, p, + c->nodes[leader].state.commit_index); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + } + } + + (void)raft_sim_restart_node(c, leader); /* persisted state survives; volatile reset */ + chk = sim_scen_check(c); + if (chk < 0) + return chk; + + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) + return chk; + + /* The old leader is now a FOLLOWER; a survivor must win a fresh election. */ + leader = sim_scen_elect(c, &viol); + if (viol < 0) + return viol; + if (leader < 0) + return 0; + + /* ---- 4. Catch-up rounds: replicate + propagate commit so EVERY node (including the + * two formerly-restarted ones) converges to the leader's log and advances its + * committed/applied prefix. Re-establish the leader inside the loop if a late + * vote-request steps it down. Bounded. ---- */ + for (int r = 0; r < n + 4; r++) { + chk = sim_scen_replicate_round(c, leader, 0x600ULL + (uint64_t)r); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + if (leader < 0) { + leader = sim_scen_elect(c, &viol); + if (viol < 0) + return viol; + if (leader < 0) + return 0; + } + } + + /* A final heartbeat round so any follower lagging on leader_commit observes the + * latest committed position and advances its applied prefix (commit propagation). */ + for (int p = 0; p < n; p++) { + if (p == leader) + continue; + (void)raft_send_append_entries(&c->nodes[leader].state, p, + c->nodes[leader].state.commit_index); + } + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) + return chk; + + /* ---- 5. Final convergence check (also covered after every step above). ---- */ + return sim_scen_check(c); +} + #endif /* XCE_TESTING */ diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c index 3700e29..fe41d11 100644 --- a/src/lib/testsuite/test_raft.c +++ b/src/lib/testsuite/test_raft.c @@ -8157,6 +8157,96 @@ static void test_sim_5_1_partition_heal(void) } } } + +/* + * test_sim_5_2_restart_churn (raft-sim-conformance task 5.2; Req 4.2) + * + * Restart-churn scenario: drive a cluster to a leader + committed entries, then + * restart nodes MID-replication (a follower holding a partial log AND the leader, + * forcing a re-election) — each restart preserving persisted (term, votedFor, log) + * while resetting volatile state to a fresh FOLLOWER. Continue driving: re-elect, + * replicate more, and propagate commit (a follow-up AppendEntries round) so the + * restarted nodes catch up. raft_sim_scenario_restart_churn asserts all four + * invariants THROUGHOUT (after every action it applies — not only at the end) and + * after convergence; it returns 0 on zero violations and < 0 (the negated first + * failing invariant) on any violation. + * + * The library is correct (bug-1095/1096 fixed), so a faithful restart-churn MUST + * complete with zero violations: a return < 0 here is a REAL divergence the scenario + * surfaced (the step already dumped the replayable trace to stderr) — NOT something + * to mask by hand-picking a seed. Several (n, seed) combinations are exercised so the + * property is not a single-seed accident. + */ +static void test_sim_5_2_restart_churn(void) +{ + /* (n, seed) pairs across 3- and 5-node clusters. The scenario inits the caller's + * cluster internally from (n, seed). */ + struct { int n; uint64_t seed; } cases[] = { + { 3, 5ULL }, { 3, 0x4A33ULL }, { 3, 2ULL }, + { 5, 11ULL }, { 5, 0xBEEFULL }, + }; + + for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) { + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + int rc = raft_sim_scenario_restart_churn(&c, cases[k].n, cases[k].seed); + + /* Zero invariant violations throughout the restart churn + catch-up. */ + TEST_ASSERT_TRUE_MESSAGE(rc == 0, + "restart-churn scenario surfaced an invariant violation on the correct library"); + + /* Post-convergence: the cluster still satisfies all four invariants and has a + * single leader. */ + TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c)); + + int leaders = 0; + int leader_idx = -1; + for (int i = 0; i < c.n; i++) { + raft_snapshot_t snap; + raft_snapshot(&c.nodes[i].state, &snap); + if (snap.role == RAFT_LEADER) { + leaders++; + leader_idx = i; + } + } + TEST_ASSERT_EQUAL_INT_MESSAGE(1, leaders, + "post-churn cluster does not have exactly one leader"); + + /* Commit progress happened despite the churn: the leader holds a non-trivial + * committed prefix (its commit_index advanced through the restarts via + * raft_advance_commit). In this conformance model commit_index is advanced ONLY + * on the leader by the library (followers never act on leader_commit in + * raft_on_append_entries_request — see raft.c), so a follower's commit_index + * legitimately stays at its last-leader value; convergence is therefore asserted + * over the LOG, which is how followers actually catch up. */ + TEST_ASSERT_TRUE_MESSAGE(c.nodes[leader_idx].commit_index > 0, + "leader never committed any entry across the scenario"); + + for (int i = 0; i < c.n; i++) { + if (i == leader_idx) + continue; + + /* Re-convergence is observable AND non-vacuous: every other node's log + * caught up to the leader's tip (length AND every entry's term match over + * the FULL log via Log Matching — not just a committed prefix that could be + * empty). This includes BOTH formerly-restarted nodes (the partial-log + * follower and the old leader), proving they re-synced their persisted log to + * the new leader's. A node stuck behind (shorter log or a divergent term at + * any index) would FAIL here — so the assertion never passes vacuously. */ + TEST_ASSERT_EQUAL_INT_MESSAGE( + c.nodes[leader_idx].log_count, c.nodes[i].log_count, + "follower log did not catch up to the leader's tip after restart churn"); + + for (int e = 1; e <= c.nodes[leader_idx].log_count; e++) { + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + c.nodes[leader_idx].log[e - 1].term, + c.nodes[i].log[e - 1].term, + "restarted node diverges from leader log after re-convergence"); + } + } + } +} #endif /* XCE_TESTING */ /* ---------- test_raft entry point ---------- */ @@ -8368,6 +8458,7 @@ void test_raft(void) RUN_TEST(test_sim_4_3_restart_preserves_persistence); RUN_TEST(test_sim_4_4_run_deterministic_replay); RUN_TEST(test_sim_5_1_partition_heal); + RUN_TEST(test_sim_5_2_restart_churn); /* Group A: basic election correctness (task 5.1) */ RUN_TEST(test_proto_A1_single_node_self_elects);