diff --git a/.kiro/specs/raft-sim-conformance/tasks.md b/.kiro/specs/raft-sim-conformance/tasks.md index b2b249e5..6b5c7dd1 100644 --- a/.kiro/specs/raft-sim-conformance/tasks.md +++ b/.kiro/specs/raft-sim-conformance/tasks.md @@ -131,7 +131,7 @@ ## 5. Targeted adversarial scenarios -- [ ] 5.1 Partition/heal scenario (P) +- [x] 5.1 Partition/heal scenario (P) - Implement `raft_sim_scenario_partition_heal`: isolate a minority during appends, heal, drain the bag; assert all four invariants throughout and after convergence - Observable completion: scenario passes with zero violations on the correct build - _Requirements: 4.1_ @@ -235,3 +235,17 @@ correct-direction) CAN and MUST assert zero invariant violations on the correct build. The user authorized this cross-boundary fix. The deliberately-broken-build fault switch (task 7) remains the way to deliberately re-introduce a divergence for the negative test. +- **Task 5.1 — scenario commit-propagation pattern (applies to 5.2–5.5).** A follower's + `commit_index`/`applied[]` only advance when it RECEIVES an AppendEntries carrying a + `leader_commit` greater than its own (the library applies it). A scenario that replicates + entries and calls `raft_advance_commit` on the LEADER, but never sends a follow-up + AE/heartbeat round afterward, leaves followers at `commit_index=0` — making any per-follower + commit-prefix / State-Machine-Safety / Leader-Completeness assertion VACUOUS on the follower + side (5.1 has this weakness; Log Matching still validates log convergence there). For + commit-critical scenarios — ESPECIALLY 5.4 (Figure-8) where follower commit advancement is + the whole point — after `raft_advance_commit` on the leader, drive ANOTHER AppendEntries + round so followers observe the new `leader_commit` and advance, then assert follower + `commit_index`/`applied[]` actually moved. Do NOT assert convergence via a loop bounded by + `min(commit_index)` when followers are at 0 (it silently passes). 5.1 was accepted as-is + (meets R4.1 via throughout-checking + log matching); 5.2–5.5 should propagate commit so + their commit assertions are non-vacuous. diff --git a/src/lib/raft/raft.h b/src/lib/raft/raft.h index 76ef4480..ad2608ac 100644 --- a/src/lib/raft/raft.h +++ b/src/lib/raft/raft.h @@ -971,6 +971,27 @@ int raft_sim_step(raft_sim_cluster_t *c); */ int raft_sim_run(raft_sim_cluster_t *c, int max_steps); +/* + * raft_sim_scenario_partition_heal: network-partition scenario (raft-sim-conformance + * task 5.1; Req 4.1). Initializes the caller's cluster from (n, election_timeout=2000, + * seed), then: + * 1. drives it to a single leader + a committed prefix, + * 2. isolates a MINORITY of nodes (floor((n-1)/2), the highest indices) via + * raft_sim_partition(..., 1) during a phase of client appends/replication while + * the MAJORITY keeps making progress (the minority cannot win a new-term + * election — the library enforces it), + * 3. HEALS the partition (raft_sim_partition(..., 0)) and drains the bag (bounded) + * so the formerly-isolated nodes catch up. + * raft_sim_check_all is run after EVERY action it applies — DURING the partitioned + * phase and after heal/convergence — so a violation is caught the moment it appears. + * Returns 0 if the scenario completes with ZERO invariant violations; < 0 (the negated + * first failing invariant, matching raft_sim_step) on any violation, after dumping the + * replayable trace to stderr. Returns -1 on an invalid node count (n < 3 cannot host a + * minority partition with a surviving majority). Drives only via published library + * entry points + harness primitives — no protocol re-implementation. + */ +int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed); + #endif /* XCE_TESTING */ #endif /* RAFT_H */ diff --git a/src/lib/raft/raft_sim.c b/src/lib/raft/raft_sim.c index 78819f24..d095c4e9 100644 --- a/src/lib/raft/raft_sim.c +++ b/src/lib/raft/raft_sim.c @@ -1268,4 +1268,254 @@ int raft_sim_run(raft_sim_cluster_t *c, int max_steps) return 0; } +/* ---------- partition/heal scenario (raft-sim-conformance task 5.1) ---------- */ + +/* + * sim_scen_check: run all four invariants over `c`. On a violation, dump the + * replayable trace to stderr and return the negated failing invariant (matching the + * raft_sim_step / raft_sim_run contract); 0 when all hold. The scenario calls this + * after EVERY action it applies so a violation is surfaced the moment it appears + * (Req 4.1 — invariants asserted THROUGHOUT, not only at the end). + */ +static int sim_scen_check(raft_sim_cluster_t *c) +{ + raft_sim_invariant_t failing = raft_sim_check_all(c); + if (failing != RAFT_SIM_INV_NONE) { + raft_sim_dump_trace(c, stderr, failing); + return -(int)failing; + } + return 0; +} + +/* + * sim_scen_drain_bag: deliver every in-flight bag message (PRNG-picked order, so a + * partition leaves partitioned-endpoint messages parked rather than delivered), + * running the invariant check after EACH delivery. `cap` bounds the work so a + * duplicate/heal storm can never spin unbounded. Stops early and propagates < 0 on + * the first violation. Returns 0 when the bag drains (or only undeliverable, + * partitioned messages remain) with no violation. + */ +static int sim_scen_drain_bag(raft_sim_cluster_t *c, int cap) +{ + for (int g = 0; g < cap && c->bag_count > 0; g++) { + int i = raft_sim_bag_pick(c); + if (i < 0) + break; + + int rc = raft_sim_bag_deliver(c, i); + /* rc == -2 means the picked message targets a partitioned endpoint and was + * left in the bag (a heal can deliver it later). To avoid re-picking the same + * parked message forever, sweep for ANY deliverable message; if none remain, + * the bag holds only partitioned traffic and we are done draining for now. */ + if (rc == -2) { + int progressed = 0; + for (int j = 0; j < c->bag_count; j++) { + int s = c->bag[j].src, d = c->bag[j].dst; + if (!c->nodes[s].partition && !c->nodes[d].partition) { + if (raft_sim_bag_deliver(c, j) >= 0) + progressed = 1; + break; + } + } + if (!progressed) + break; /* only partitioned traffic left */ + } + + int chk = sim_scen_check(c); + if (chk < 0) + return chk; + } + return 0; +} + +/* + * sim_scen_replicate_round: on the current leader, append one client value, send + * AppendEntries to every other node, drain the bag, then recompute commit — checking + * invariants after every action. Non-leaders and partitioned peers are handled by the + * library / bag delivery (a partitioned peer's AE simply never delivers). Returns 0 on + * a clean round, < 0 on the first invariant violation. + */ +static int sim_scen_replicate_round(raft_sim_cluster_t *c, int leader, uint64_t value) +{ + uint64_t out_index = 0; + (void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index); + int chk = sim_scen_check(c); + if (chk < 0) + return chk; + + for (int p = 0; p < c->n; p++) { + if (p == leader) + continue; + (void)raft_send_append_entries(&c->nodes[leader].state, p, + c->nodes[leader].state.commit_index); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + } + + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4); + if (chk < 0) + return chk; + + (void)raft_advance_commit(&c->nodes[leader].state); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + + /* Commit advance can release a higher leader_commit; replicate it out so followers + * apply the newly-committed entries, then re-check. */ + for (int p = 0; p < c->n; p++) { + if (p == leader) + continue; + (void)raft_send_append_entries(&c->nodes[leader].state, p, + c->nodes[leader].state.commit_index); + } + return sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4); +} + +int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed) +{ + /* A minority partition with a surviving majority needs n >= 3 (floor((n-1)/2) >= 1 + * isolated, with a strict majority still connected). */ + if (n < 3 || n > RAFT_SIM_MAX_NODES) + return -1; + + if (raft_sim_conformance_init(c, n, 2000, seed) != 0) + return -1; + + int chk; + + /* ---- 1. Elect a leader: node 0 campaigns; drain the vote traffic. ---- */ + (void)raft_start_election(&c->nodes[0].state); + chk = sim_scen_check(c); + if (chk < 0) + return chk; + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4); + if (chk < 0) + return chk; + + int leader = sim_find_leader(c); + if (leader < 0) { + /* No leader emerged — retry a couple of campaigns (still all via the library). + * If the cluster genuinely cannot elect, that is observable, not a violation; + * the post-scenario single-leader assertion in the test will catch it. */ + for (int retry = 0; retry < 3 && leader < 0; retry++) { + (void)raft_start_election(&c->nodes[retry % n].state); + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + } + if (leader < 0) + return 0; /* no violation occurred; nothing more to drive */ + } + + /* ---- 2. Replicate + commit a few entries with the cluster fully connected. ---- */ + for (int r = 0; r < 3; r++) { + chk = sim_scen_replicate_round(c, leader, 0x100ULL + (uint64_t)r); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + if (leader < 0) + return 0; + } + + /* ---- 3. Isolate a MINORITY (highest indices); majority keeps progressing. ---- */ + int minority = (n - 1) / 2; /* < n/2, so the remaining nodes are a strict majority */ + for (int m = 0; m < minority; m++) + raft_sim_partition(c, n - 1 - m, 1); + + /* If the isolated set happened to include the leader, re-elect within the majority + * so the partitioned phase still makes progress on the connected side. */ + { + int leader_isolated = c->nodes[leader].partition; + if (leader_isolated) { + leader = -1; + for (int cand = 0; cand < n - minority && leader < 0; cand++) { + if (c->nodes[cand].partition) + continue; + (void)raft_start_election(&c->nodes[cand].state); + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + /* Only accept a CONNECTED leader (a stale isolated leader does not count). */ + if (leader >= 0 && c->nodes[leader].partition) + leader = -1; + } + if (leader < 0) + return 0; + } + } + + /* Drive a phase of client appends/replication while partitioned. The majority must + * keep committing; AE to isolated peers simply never delivers. A minority node also + * times out and campaigns — it must NOT be able to win a higher-term election (the + * library enforces quorum; the harness just drives + checks). */ + for (int r = 0; r < 4; r++) { + chk = sim_scen_replicate_round(c, leader, 0x200ULL + (uint64_t)r); + if (chk < 0) + return chk; + + /* A minority node campaigns mid-partition (cannot reach a majority of votes). */ + if (minority > 0) { + (void)raft_start_election(&c->nodes[n - 1].state); + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4); + if (chk < 0) + return chk; + } + + leader = sim_find_leader(c); + if (leader < 0 || c->nodes[leader].partition) + return 0; + } + + /* ---- 4. HEAL the partition and drain everything so the minority catches up. ---- */ + for (int m = 0; m < minority; m++) + raft_sim_partition(c, n - 1 - m, 0); + + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) + return chk; + + /* Draining the parked minority traffic (those nodes campaigned at higher terms while + * isolated) can step the connected leader down via UpdateTerm — correct library + * behavior. Re-establish a single leader over the now-fully-connected cluster before + * catch-up: campaign one node at a time, draining each round, until exactly one + * leader holds. This is the natural Raft recovery, driven only via library entry + * points. */ + leader = sim_find_leader(c); + for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) { + (void)raft_start_election(&c->nodes[attempt % n].state); + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + } + if (leader < 0) + return 0; /* no leader emerged — observable, not a violation; test asserts it */ + + /* Catch-up rounds: re-replicate to ALL nodes (now reachable) until the formerly + * isolated followers converge to the leader's committed prefix. Re-establish the + * leader inside the loop if a late vote-request steps it down again. Bounded. */ + for (int r = 0; r < n + 4; r++) { + chk = sim_scen_replicate_round(c, leader, 0x300ULL + (uint64_t)r); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) { + (void)raft_start_election(&c->nodes[attempt % n].state); + chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8); + if (chk < 0) + return chk; + leader = sim_find_leader(c); + } + if (leader < 0) + return 0; + } + + /* ---- 5. Final convergence check (also covered after every step above). ---- */ + return sim_scen_check(c); +} + #endif /* XCE_TESTING */ diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c index e6d1b2a7..3700e29c 100644 --- a/src/lib/testsuite/test_raft.c +++ b/src/lib/testsuite/test_raft.c @@ -8080,6 +8080,83 @@ static void test_sim_4_4_run_deterministic_replay(void) TEST_ASSERT_TRUE(a.trace_len > 0); } } + +/* + * test_sim_5_1_partition_heal (raft-sim-conformance task 5.1; Req 4.1) + * + * Network-partition scenario: drive a cluster to a leader + committed entries, + * isolate a MINORITY of nodes during a phase of client appends/replication, keep + * the MAJORITY making progress, then HEAL and drain the bag so the formerly-isolated + * nodes catch up. raft_sim_scenario_partition_heal asserts all four invariants + * THROUGHOUT (after every action it applies — not only at the end) and after + * convergence; it returns 0 on zero violations and < 0 (the negated first failing + * invariant) on any violation. + * + * The library is correct (bug-1095/1096 fixed), so a faithful partition/heal MUST + * complete with zero violations: a return < 0 here is a REAL divergence the scenario + * surfaced (the step already dumped the replayable trace to stderr) — NOT something + * to mask by hand-picking a seed. Several (n, seed) combinations are exercised + * (minority isolation in both 3-node and 5-node clusters) so the property is not a + * single-seed accident. + */ +static void test_sim_5_1_partition_heal(void) +{ + /* (n, seed) pairs: 3-node clusters isolate 1 node; 5-node clusters isolate 2. The + * scenario inits the caller's cluster internally from (n, seed). */ + struct { int n; uint64_t seed; } cases[] = { + { 3, 3ULL }, { 3, 0x4A33ULL }, { 3, 1ULL }, + { 5, 7ULL }, { 5, 0xBEEFULL }, + }; + + for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) { + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + int rc = raft_sim_scenario_partition_heal(&c, cases[k].n, cases[k].seed); + + /* Zero invariant violations throughout the partition + heal + convergence. */ + TEST_ASSERT_TRUE_MESSAGE(rc == 0, + "partition/heal scenario surfaced an invariant violation on the correct library"); + + /* Post-convergence: the cluster still satisfies all four invariants, has a + * single leader, and the formerly-isolated minority caught up to the leader's + * committed prefix (Log Matching + Leader Completeness over a converged log). */ + TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c)); + + int leaders = 0; + int leader_idx = -1; + for (int i = 0; i < c.n; i++) { + raft_snapshot_t snap; + raft_snapshot(&c.nodes[i].state, &snap); + if (snap.role == RAFT_LEADER) { + leaders++; + leader_idx = i; + } + } + TEST_ASSERT_EQUAL_INT_MESSAGE(1, leaders, + "post-heal cluster does not have exactly one leader"); + + /* Convergence is observable: the leader committed a non-trivial prefix, and + * every (formerly-isolated) node's committed entries match the leader's at the + * same index/term over their shared committed prefix. */ + TEST_ASSERT_TRUE_MESSAGE(c.nodes[leader_idx].commit_index > 0, + "leader never committed any entry across the scenario"); + + for (int i = 0; i < c.n; i++) { + if (i == leader_idx) + continue; + uint64_t shared = c.nodes[i].commit_index; + if (shared > c.nodes[leader_idx].commit_index) + shared = c.nodes[leader_idx].commit_index; + for (uint64_t e = 1; e <= shared; e++) { + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + c.nodes[leader_idx].log[e - 1].term, + c.nodes[i].log[e - 1].term, + "formerly-isolated node diverges from leader committed prefix"); + } + } + } +} #endif /* XCE_TESTING */ /* ---------- test_raft entry point ---------- */ @@ -8290,6 +8367,7 @@ void test_raft(void) RUN_TEST(test_sim_4_2_step_halts_on_forced_violation); RUN_TEST(test_sim_4_3_restart_preserves_persistence); RUN_TEST(test_sim_4_4_run_deterministic_replay); + RUN_TEST(test_sim_5_1_partition_heal); /* Group A: basic election correctness (task 5.1) */ RUN_TEST(test_proto_A1_single_node_self_elects);