feat(raft-sim-conformance): restart-churn adversarial scenario (task 5.2)

src/lib/testsuite/test_raft.c

@@ -8157,6 +8157,96 @@ static void test_sim_5_1_partition_heal(void)
         }
     }
 }
+
+/*
+ * test_sim_5_2_restart_churn (raft-sim-conformance task 5.2; Req 4.2)
+ *
+ * Restart-churn scenario: drive a cluster to a leader + committed entries, then
+ * restart nodes MID-replication (a follower holding a partial log AND the leader,
+ * forcing a re-election) — each restart preserving persisted (term, votedFor, log)
+ * while resetting volatile state to a fresh FOLLOWER. Continue driving: re-elect,
+ * replicate more, and propagate commit (a follow-up AppendEntries round) so the
+ * restarted nodes catch up. raft_sim_scenario_restart_churn asserts all four
+ * invariants THROUGHOUT (after every action it applies — not only at the end) and
+ * after convergence; it returns 0 on zero violations and < 0 (the negated first
+ * failing invariant) on any violation.
+ *
+ * The library is correct (bug-1095/1096 fixed), so a faithful restart-churn MUST
+ * complete with zero violations: a return < 0 here is a REAL divergence the scenario
+ * surfaced (the step already dumped the replayable trace to stderr) — NOT something
+ * to mask by hand-picking a seed. Several (n, seed) combinations are exercised so the
+ * property is not a single-seed accident.
+ */
+static void test_sim_5_2_restart_churn(void)
+{
+    /* (n, seed) pairs across 3- and 5-node clusters. The scenario inits the caller's
+     * cluster internally from (n, seed). */
+    struct { int n; uint64_t seed; } cases[] = {
+        { 3, 5ULL }, { 3, 0x4A33ULL }, { 3, 2ULL },
+        { 5, 11ULL }, { 5, 0xBEEFULL },
+    };
+
+    for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
+        raft_sim_cluster_t c;
+        memset(&c, 0, sizeof(c));
+
+        int rc = raft_sim_scenario_restart_churn(&c, cases[k].n, cases[k].seed);
+
+        /* Zero invariant violations throughout the restart churn + catch-up. */
+        TEST_ASSERT_TRUE_MESSAGE(rc == 0,
+            "restart-churn scenario surfaced an invariant violation on the correct library");
+
+        /* Post-convergence: the cluster still satisfies all four invariants and has a
+         * single leader. */
+        TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c));
+
+        int leaders = 0;
+        int leader_idx = -1;
+        for (int i = 0; i < c.n; i++) {
+            raft_snapshot_t snap;
+            raft_snapshot(&c.nodes[i].state, &snap);
+            if (snap.role == RAFT_LEADER) {
+                leaders++;
+                leader_idx = i;
+            }
+        }
+        TEST_ASSERT_EQUAL_INT_MESSAGE(1, leaders,
+            "post-churn cluster does not have exactly one leader");
+
+        /* Commit progress happened despite the churn: the leader holds a non-trivial
+         * committed prefix (its commit_index advanced through the restarts via
+         * raft_advance_commit). In this conformance model commit_index is advanced ONLY
+         * on the leader by the library (followers never act on leader_commit in
+         * raft_on_append_entries_request — see raft.c), so a follower's commit_index
+         * legitimately stays at its last-leader value; convergence is therefore asserted
+         * over the LOG, which is how followers actually catch up. */
+        TEST_ASSERT_TRUE_MESSAGE(c.nodes[leader_idx].commit_index > 0,
+            "leader never committed any entry across the scenario");
+
+        for (int i = 0; i < c.n; i++) {
+            if (i == leader_idx)
+                continue;
+
+            /* Re-convergence is observable AND non-vacuous: every other node's log
+             * caught up to the leader's tip (length AND every entry's term match over
+             * the FULL log via Log Matching — not just a committed prefix that could be
+             * empty). This includes BOTH formerly-restarted nodes (the partial-log
+             * follower and the old leader), proving they re-synced their persisted log to
+             * the new leader's. A node stuck behind (shorter log or a divergent term at
+             * any index) would FAIL here — so the assertion never passes vacuously. */
+            TEST_ASSERT_EQUAL_INT_MESSAGE(
+                c.nodes[leader_idx].log_count, c.nodes[i].log_count,
+                "follower log did not catch up to the leader's tip after restart churn");
+
+            for (int e = 1; e <= c.nodes[leader_idx].log_count; e++) {
+                TEST_ASSERT_EQUAL_UINT64_MESSAGE(
+                    c.nodes[leader_idx].log[e - 1].term,
+                    c.nodes[i].log[e - 1].term,
+                    "restarted node diverges from leader log after re-convergence");
+            }
+        }
+    }
+}
 #endif /* XCE_TESTING */
 
 /* ---------- test_raft entry point ---------- */
@@ -8368,6 +8458,7 @@ void test_raft(void)
     RUN_TEST(test_sim_4_3_restart_preserves_persistence);
     RUN_TEST(test_sim_4_4_run_deterministic_replay);
     RUN_TEST(test_sim_5_1_partition_heal);
+    RUN_TEST(test_sim_5_2_restart_churn);
 
     /* Group A: basic election correctness (task 5.1) */
     RUN_TEST(test_proto_A1_single_node_self_elects);