feat(raft-sim-conformance): partition/heal adversarial scenario (task 5.1)

.kiro/specs/raft-sim-conformance/tasks.md

@@ -131,7 +131,7 @@
 
 ## 5. Targeted adversarial scenarios
 
-- [ ] 5.1 Partition/heal scenario (P)
+- [x] 5.1 Partition/heal scenario (P)
   - Implement `raft_sim_scenario_partition_heal`: isolate a minority during appends, heal, drain the bag; assert all four invariants throughout and after convergence
   - Observable completion: scenario passes with zero violations on the correct build
   - _Requirements: 4.1_
@@ -235,3 +235,17 @@
   correct-direction) CAN and MUST assert zero invariant violations on the correct build.
   The user authorized this cross-boundary fix. The deliberately-broken-build fault switch
   (task 7) remains the way to deliberately re-introduce a divergence for the negative test.
+- **Task 5.1 — scenario commit-propagation pattern (applies to 5.2–5.5).** A follower's
+  `commit_index`/`applied[]` only advance when it RECEIVES an AppendEntries carrying a
+  `leader_commit` greater than its own (the library applies it). A scenario that replicates
+  entries and calls `raft_advance_commit` on the LEADER, but never sends a follow-up
+  AE/heartbeat round afterward, leaves followers at `commit_index=0` — making any per-follower
+  commit-prefix / State-Machine-Safety / Leader-Completeness assertion VACUOUS on the follower
+  side (5.1 has this weakness; Log Matching still validates log convergence there). For
+  commit-critical scenarios — ESPECIALLY 5.4 (Figure-8) where follower commit advancement is
+  the whole point — after `raft_advance_commit` on the leader, drive ANOTHER AppendEntries
+  round so followers observe the new `leader_commit` and advance, then assert follower
+  `commit_index`/`applied[]` actually moved. Do NOT assert convergence via a loop bounded by
+  `min(commit_index)` when followers are at 0 (it silently passes). 5.1 was accepted as-is
+  (meets R4.1 via throughout-checking + log matching); 5.2–5.5 should propagate commit so
+  their commit assertions are non-vacuous.

src/lib/raft/raft.h

@@ -971,6 +971,27 @@ int raft_sim_step(raft_sim_cluster_t *c);
  */
 int raft_sim_run(raft_sim_cluster_t *c, int max_steps);
 
+/*
+ * raft_sim_scenario_partition_heal: network-partition scenario (raft-sim-conformance
+ * task 5.1; Req 4.1). Initializes the caller's cluster from (n, election_timeout=2000,
+ * seed), then:
+ *   1. drives it to a single leader + a committed prefix,
+ *   2. isolates a MINORITY of nodes (floor((n-1)/2), the highest indices) via
+ *      raft_sim_partition(..., 1) during a phase of client appends/replication while
+ *      the MAJORITY keeps making progress (the minority cannot win a new-term
+ *      election — the library enforces it),
+ *   3. HEALS the partition (raft_sim_partition(..., 0)) and drains the bag (bounded)
+ *      so the formerly-isolated nodes catch up.
+ * raft_sim_check_all is run after EVERY action it applies — DURING the partitioned
+ * phase and after heal/convergence — so a violation is caught the moment it appears.
+ * Returns 0 if the scenario completes with ZERO invariant violations; < 0 (the negated
+ * first failing invariant, matching raft_sim_step) on any violation, after dumping the
+ * replayable trace to stderr. Returns -1 on an invalid node count (n < 3 cannot host a
+ * minority partition with a surviving majority). Drives only via published library
+ * entry points + harness primitives — no protocol re-implementation.
+ */
+int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed);
+
 #endif /* XCE_TESTING */
 
 #endif /* RAFT_H */

src/lib/raft/raft_sim.c

@@ -1268,4 +1268,254 @@ int raft_sim_run(raft_sim_cluster_t *c, int max_steps)
     return 0;
 }
 
+/* ---------- partition/heal scenario (raft-sim-conformance task 5.1) ---------- */
+
+/*
+ * sim_scen_check: run all four invariants over `c`. On a violation, dump the
+ * replayable trace to stderr and return the negated failing invariant (matching the
+ * raft_sim_step / raft_sim_run contract); 0 when all hold. The scenario calls this
+ * after EVERY action it applies so a violation is surfaced the moment it appears
+ * (Req 4.1 — invariants asserted THROUGHOUT, not only at the end).
+ */
+static int sim_scen_check(raft_sim_cluster_t *c)
+{
+    raft_sim_invariant_t failing = raft_sim_check_all(c);
+    if (failing != RAFT_SIM_INV_NONE) {
+        raft_sim_dump_trace(c, stderr, failing);
+        return -(int)failing;
+    }
+    return 0;
+}
+
+/*
+ * sim_scen_drain_bag: deliver every in-flight bag message (PRNG-picked order, so a
+ * partition leaves partitioned-endpoint messages parked rather than delivered),
+ * running the invariant check after EACH delivery. `cap` bounds the work so a
+ * duplicate/heal storm can never spin unbounded. Stops early and propagates < 0 on
+ * the first violation. Returns 0 when the bag drains (or only undeliverable,
+ * partitioned messages remain) with no violation.
+ */
+static int sim_scen_drain_bag(raft_sim_cluster_t *c, int cap)
+{
+    for (int g = 0; g < cap && c->bag_count > 0; g++) {
+        int i = raft_sim_bag_pick(c);
+        if (i < 0)
+            break;
+
+        int rc = raft_sim_bag_deliver(c, i);
+        /* rc == -2 means the picked message targets a partitioned endpoint and was
+         * left in the bag (a heal can deliver it later). To avoid re-picking the same
+         * parked message forever, sweep for ANY deliverable message; if none remain,
+         * the bag holds only partitioned traffic and we are done draining for now. */
+        if (rc == -2) {
+            int progressed = 0;
+            for (int j = 0; j < c->bag_count; j++) {
+                int s = c->bag[j].src, d = c->bag[j].dst;
+                if (!c->nodes[s].partition && !c->nodes[d].partition) {
+                    if (raft_sim_bag_deliver(c, j) >= 0)
+                        progressed = 1;
+                    break;
+                }
+            }
+            if (!progressed)
+                break; /* only partitioned traffic left */
+        }
+
+        int chk = sim_scen_check(c);
+        if (chk < 0)
+            return chk;
+    }
+    return 0;
+}
+
+/*
+ * sim_scen_replicate_round: on the current leader, append one client value, send
+ * AppendEntries to every other node, drain the bag, then recompute commit — checking
+ * invariants after every action. Non-leaders and partitioned peers are handled by the
+ * library / bag delivery (a partitioned peer's AE simply never delivers). Returns 0 on
+ * a clean round, < 0 on the first invariant violation.
+ */
+static int sim_scen_replicate_round(raft_sim_cluster_t *c, int leader, uint64_t value)
+{
+    uint64_t out_index = 0;
+    (void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
+    int chk = sim_scen_check(c);
+    if (chk < 0)
+        return chk;
+
+    for (int p = 0; p < c->n; p++) {
+        if (p == leader)
+            continue;
+        (void)raft_send_append_entries(&c->nodes[leader].state, p,
+                                       c->nodes[leader].state.commit_index);
+        chk = sim_scen_check(c);
+        if (chk < 0)
+            return chk;
+    }
+
+    chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+    if (chk < 0)
+        return chk;
+
+    (void)raft_advance_commit(&c->nodes[leader].state);
+    chk = sim_scen_check(c);
+    if (chk < 0)
+        return chk;
+
+    /* Commit advance can release a higher leader_commit; replicate it out so followers
+     * apply the newly-committed entries, then re-check. */
+    for (int p = 0; p < c->n; p++) {
+        if (p == leader)
+            continue;
+        (void)raft_send_append_entries(&c->nodes[leader].state, p,
+                                       c->nodes[leader].state.commit_index);
+    }
+    return sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+}
+
+int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed)
+{
+    /* A minority partition with a surviving majority needs n >= 3 (floor((n-1)/2) >= 1
+     * isolated, with a strict majority still connected). */
+    if (n < 3 || n > RAFT_SIM_MAX_NODES)
+        return -1;
+
+    if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
+        return -1;
+
+    int chk;
+
+    /* ---- 1. Elect a leader: node 0 campaigns; drain the vote traffic. ---- */
+    (void)raft_start_election(&c->nodes[0].state);
+    chk = sim_scen_check(c);
+    if (chk < 0)
+        return chk;
+    chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+    if (chk < 0)
+        return chk;
+
+    int leader = sim_find_leader(c);
+    if (leader < 0) {
+        /* No leader emerged — retry a couple of campaigns (still all via the library).
+         * If the cluster genuinely cannot elect, that is observable, not a violation;
+         * the post-scenario single-leader assertion in the test will catch it. */
+        for (int retry = 0; retry < 3 && leader < 0; retry++) {
+            (void)raft_start_election(&c->nodes[retry % n].state);
+            chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+            if (chk < 0)
+                return chk;
+            leader = sim_find_leader(c);
+        }
+        if (leader < 0)
+            return 0; /* no violation occurred; nothing more to drive */
+    }
+
+    /* ---- 2. Replicate + commit a few entries with the cluster fully connected. ---- */
+    for (int r = 0; r < 3; r++) {
+        chk = sim_scen_replicate_round(c, leader, 0x100ULL + (uint64_t)r);
+        if (chk < 0)
+            return chk;
+        leader = sim_find_leader(c);
+        if (leader < 0)
+            return 0;
+    }
+
+    /* ---- 3. Isolate a MINORITY (highest indices); majority keeps progressing. ---- */
+    int minority = (n - 1) / 2; /* < n/2, so the remaining nodes are a strict majority */
+    for (int m = 0; m < minority; m++)
+        raft_sim_partition(c, n - 1 - m, 1);
+
+    /* If the isolated set happened to include the leader, re-elect within the majority
+     * so the partitioned phase still makes progress on the connected side. */
+    {
+        int leader_isolated = c->nodes[leader].partition;
+        if (leader_isolated) {
+            leader = -1;
+            for (int cand = 0; cand < n - minority && leader < 0; cand++) {
+                if (c->nodes[cand].partition)
+                    continue;
+                (void)raft_start_election(&c->nodes[cand].state);
+                chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+                if (chk < 0)
+                    return chk;
+                leader = sim_find_leader(c);
+                /* Only accept a CONNECTED leader (a stale isolated leader does not count). */
+                if (leader >= 0 && c->nodes[leader].partition)
+                    leader = -1;
+            }
+            if (leader < 0)
+                return 0;
+        }
+    }
+
+    /* Drive a phase of client appends/replication while partitioned. The majority must
+     * keep committing; AE to isolated peers simply never delivers. A minority node also
+     * times out and campaigns — it must NOT be able to win a higher-term election (the
+     * library enforces quorum; the harness just drives + checks). */
+    for (int r = 0; r < 4; r++) {
+        chk = sim_scen_replicate_round(c, leader, 0x200ULL + (uint64_t)r);
+        if (chk < 0)
+            return chk;
+
+        /* A minority node campaigns mid-partition (cannot reach a majority of votes). */
+        if (minority > 0) {
+            (void)raft_start_election(&c->nodes[n - 1].state);
+            chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+            if (chk < 0)
+                return chk;
+        }
+
+        leader = sim_find_leader(c);
+        if (leader < 0 || c->nodes[leader].partition)
+            return 0;
+    }
+
+    /* ---- 4. HEAL the partition and drain everything so the minority catches up. ---- */
+    for (int m = 0; m < minority; m++)
+        raft_sim_partition(c, n - 1 - m, 0);
+
+    chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
+    if (chk < 0)
+        return chk;
+
+    /* Draining the parked minority traffic (those nodes campaigned at higher terms while
+     * isolated) can step the connected leader down via UpdateTerm — correct library
+     * behavior. Re-establish a single leader over the now-fully-connected cluster before
+     * catch-up: campaign one node at a time, draining each round, until exactly one
+     * leader holds. This is the natural Raft recovery, driven only via library entry
+     * points. */
+    leader = sim_find_leader(c);
+    for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) {
+        (void)raft_start_election(&c->nodes[attempt % n].state);
+        chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
+        if (chk < 0)
+            return chk;
+        leader = sim_find_leader(c);
+    }
+    if (leader < 0)
+        return 0; /* no leader emerged — observable, not a violation; test asserts it */
+
+    /* Catch-up rounds: re-replicate to ALL nodes (now reachable) until the formerly
+     * isolated followers converge to the leader's committed prefix. Re-establish the
+     * leader inside the loop if a late vote-request steps it down again. Bounded. */
+    for (int r = 0; r < n + 4; r++) {
+        chk = sim_scen_replicate_round(c, leader, 0x300ULL + (uint64_t)r);
+        if (chk < 0)
+            return chk;
+        leader = sim_find_leader(c);
+        for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) {
+            (void)raft_start_election(&c->nodes[attempt % n].state);
+            chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
+            if (chk < 0)
+                return chk;
+            leader = sim_find_leader(c);
+        }
+        if (leader < 0)
+            return 0;
+    }
+
+    /* ---- 5. Final convergence check (also covered after every step above). ---- */
+    return sim_scen_check(c);
+}
+
 #endif /* XCE_TESTING */

src/lib/testsuite/test_raft.c

@@ -8080,6 +8080,83 @@ static void test_sim_4_4_run_deterministic_replay(void)
         TEST_ASSERT_TRUE(a.trace_len > 0);
     }
 }
+
+/*
+ * test_sim_5_1_partition_heal (raft-sim-conformance task 5.1; Req 4.1)
+ *
+ * Network-partition scenario: drive a cluster to a leader + committed entries,
+ * isolate a MINORITY of nodes during a phase of client appends/replication, keep
+ * the MAJORITY making progress, then HEAL and drain the bag so the formerly-isolated
+ * nodes catch up. raft_sim_scenario_partition_heal asserts all four invariants
+ * THROUGHOUT (after every action it applies — not only at the end) and after
+ * convergence; it returns 0 on zero violations and < 0 (the negated first failing
+ * invariant) on any violation.
+ *
+ * The library is correct (bug-1095/1096 fixed), so a faithful partition/heal MUST
+ * complete with zero violations: a return < 0 here is a REAL divergence the scenario
+ * surfaced (the step already dumped the replayable trace to stderr) — NOT something
+ * to mask by hand-picking a seed. Several (n, seed) combinations are exercised
+ * (minority isolation in both 3-node and 5-node clusters) so the property is not a
+ * single-seed accident.
+ */
+static void test_sim_5_1_partition_heal(void)
+{
+    /* (n, seed) pairs: 3-node clusters isolate 1 node; 5-node clusters isolate 2. The
+     * scenario inits the caller's cluster internally from (n, seed). */
+    struct { int n; uint64_t seed; } cases[] = {
+        { 3, 3ULL }, { 3, 0x4A33ULL }, { 3, 1ULL },
+        { 5, 7ULL }, { 5, 0xBEEFULL },
+    };
+
+    for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
+        raft_sim_cluster_t c;
+        memset(&c, 0, sizeof(c));
+
+        int rc = raft_sim_scenario_partition_heal(&c, cases[k].n, cases[k].seed);
+
+        /* Zero invariant violations throughout the partition + heal + convergence. */
+        TEST_ASSERT_TRUE_MESSAGE(rc == 0,
+            "partition/heal scenario surfaced an invariant violation on the correct library");
+
+        /* Post-convergence: the cluster still satisfies all four invariants, has a
+         * single leader, and the formerly-isolated minority caught up to the leader's
+         * committed prefix (Log Matching + Leader Completeness over a converged log). */
+        TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c));
+
+        int leaders = 0;
+        int leader_idx = -1;
+        for (int i = 0; i < c.n; i++) {
+            raft_snapshot_t snap;
+            raft_snapshot(&c.nodes[i].state, &snap);
+            if (snap.role == RAFT_LEADER) {
+                leaders++;
+                leader_idx = i;
+            }
+        }
+        TEST_ASSERT_EQUAL_INT_MESSAGE(1, leaders,
+            "post-heal cluster does not have exactly one leader");
+
+        /* Convergence is observable: the leader committed a non-trivial prefix, and
+         * every (formerly-isolated) node's committed entries match the leader's at the
+         * same index/term over their shared committed prefix. */
+        TEST_ASSERT_TRUE_MESSAGE(c.nodes[leader_idx].commit_index > 0,
+            "leader never committed any entry across the scenario");
+
+        for (int i = 0; i < c.n; i++) {
+            if (i == leader_idx)
+                continue;
+            uint64_t shared = c.nodes[i].commit_index;
+            if (shared > c.nodes[leader_idx].commit_index)
+                shared = c.nodes[leader_idx].commit_index;
+            for (uint64_t e = 1; e <= shared; e++) {
+                TEST_ASSERT_EQUAL_UINT64_MESSAGE(
+                    c.nodes[leader_idx].log[e - 1].term,
+                    c.nodes[i].log[e - 1].term,
+                    "formerly-isolated node diverges from leader committed prefix");
+            }
+        }
+    }
+}
 #endif /* XCE_TESTING */
 
 /* ---------- test_raft entry point ---------- */
@@ -8290,6 +8367,7 @@ void test_raft(void)
     RUN_TEST(test_sim_4_2_step_halts_on_forced_violation);
     RUN_TEST(test_sim_4_3_restart_preserves_persistence);
     RUN_TEST(test_sim_4_4_run_deterministic_replay);
+    RUN_TEST(test_sim_5_1_partition_heal);
 
     /* Group A: basic election correctness (task 5.1) */
     RUN_TEST(test_proto_A1_single_node_self_elects);