From e445404439c8082201168b2a3f45e0612c7c916f Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Wed, 24 Jun 2026 15:09:31 +0000 Subject: [PATCH] feat(raft-sim-conformance): drop/duplicate/reorder adversarial scenario (task 5.3) --- .kiro/specs/raft-sim-conformance/tasks.md | 2 +- src/lib/raft/raft.h | 24 ++++ src/lib/testsuite/test_raft.c | 161 ++++++++++++++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) diff --git a/.kiro/specs/raft-sim-conformance/tasks.md b/.kiro/specs/raft-sim-conformance/tasks.md index e87fce5..fcabaf6 100644 --- a/.kiro/specs/raft-sim-conformance/tasks.md +++ b/.kiro/specs/raft-sim-conformance/tasks.md @@ -145,7 +145,7 @@ - _Boundary: raft_sim.c, test_raft.c_ - _Depends: 4.3_ -- [ ] 5.3 Drop/duplicate/reorder scenario (P) +- [x] 5.3 Drop/duplicate/reorder scenario (P) - Implement `raft_sim_scenario_drop_dup_reorder`: perturb bag delivery order + multiplicity; assert idempotent convergence with all four invariants held - Observable completion: scenario passes; duplicated AE is a no-op (already-have branch) - _Requirements: 4.3_ diff --git a/src/lib/raft/raft.h b/src/lib/raft/raft.h index 8fbef19..78c3986 100644 --- a/src/lib/raft/raft.h +++ b/src/lib/raft/raft.h @@ -1013,6 +1013,30 @@ int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed */ int raft_sim_scenario_restart_churn(raft_sim_cluster_t *c, int n, uint64_t seed); +/* + * raft_sim_scenario_drop_dup_reorder: drop/duplicate/reorder scenario (raft-sim- + * conformance task 5.3; Req 4.3). Initializes the caller's cluster from (n, + * election_timeout=2000, seed), then: + * 1. drives it to a single leader + replicated entries, + * 2. PERTURBS the unified message bag — DROPs some in-flight messages + * (raft_sim_bag_drop), DUPLICATEs some (raft_sim_bag_duplicate, so an + * AppendEntries / vote response is delivered MORE THAN ONCE), and REORDERs + * delivery (raft_sim_bag_pick PRNG order, not FIFO) through the real handlers + * (raft_sim_bag_deliver), + * 3. because dropped messages leave followers behind, the leader RETRANSMITs over + * additional replicate rounds until the logs converge (bounded). + * raft_sim_check_all is run after EVERY action it applies — DURING perturbation and + * after each retransmit — so a violation is caught the moment it appears (Req 4.1). A + * duplicated AppendEntries is a no-op (the library's already-have branch): delivering + * the same AE twice never grows or corrupts the follower's log. Returns 0 if the + * scenario completes with ZERO invariant violations AND every node's log converged to + * the leader's tip (idempotent convergence); < 0 (the negated first failing invariant, + * matching raft_sim_step) on any violation, after dumping the replayable trace to + * stderr. Returns -1 on an invalid node count (n < 3). Drives only via published + * library entry points + harness bag primitives — no protocol re-implementation. + */ +int raft_sim_scenario_drop_dup_reorder(raft_sim_cluster_t *c, int n, uint64_t seed); + #endif /* XCE_TESTING */ #endif /* RAFT_H */ diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c index fe41d11..7d4b6f7 100644 --- a/src/lib/testsuite/test_raft.c +++ b/src/lib/testsuite/test_raft.c @@ -8247,6 +8247,166 @@ static void test_sim_5_2_restart_churn(void) } } } + +/* + * test_sim_5_3_drop_dup_reorder (raft-sim-conformance task 5.3; Req 4.3) + * + * Drop/duplicate/reorder scenario: drive a cluster to a leader + replicate entries, + * then PERTURB the unified conformance bag — DROP some in-flight messages, DUPLICATE + * some (so an AppendEntries / vote response is delivered MORE THAN ONCE), and REORDER + * delivery (PRNG-picked order, not FIFO). Because messages are dropped, the leader must + * RETRANSMIT, so the scenario drives additional replicate rounds until convergence. + * raft_sim_scenario_drop_dup_reorder asserts all four invariants THROUGHOUT (after + * every delivery/perturbation/replicate) and idempotent LOG convergence at the end; it + * returns 0 on zero violations and < 0 (the negated first failing invariant) on any + * violation. + * + * The library is correct (bug-1095/1096 fixed — the duplicate-vote double-count and the + * stale-term vote-count are gone), so a faithful drop/dup/reorder MUST complete with + * zero violations: a return < 0 here is a REAL divergence the scenario surfaced (the + * step already dumped the replayable trace to stderr) — NOT something to mask by + * hand-picking a seed. Several (n, seed) combinations are exercised so the property is + * not a single-seed accident. + * + * Also asserts the duplicated-AE-is-a-no-op property DIRECTLY (independent of the + * scenario): deliver the SAME AppendEntries twice and confirm the second delivery does + * NOT grow or corrupt the follower's log (the library's already-have branch). + */ +static void test_sim_5_3_drop_dup_reorder(void) +{ + /* (n, seed) pairs across 3- and 5-node clusters. The scenario inits the caller's + * cluster internally from (n, seed). */ + struct { int n; uint64_t seed; } cases[] = { + { 3, 13ULL }, { 3, 0x4A33ULL }, { 3, 4ULL }, + { 5, 17ULL }, { 5, 0xBEEFULL }, + }; + + for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) { + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + int rc = raft_sim_scenario_drop_dup_reorder(&c, cases[k].n, cases[k].seed); + + /* Zero invariant violations throughout the drop/dup/reorder + retransmit. */ + TEST_ASSERT_TRUE_MESSAGE(rc == 0, + "drop/dup/reorder scenario surfaced an invariant violation on the correct library"); + + /* Post-convergence: the cluster still satisfies all four invariants and has a + * single leader. */ + TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c)); + + int leaders = 0; + int leader_idx = -1; + for (int i = 0; i < c.n; i++) { + raft_snapshot_t snap; + raft_snapshot(&c.nodes[i].state, &snap); + if (snap.role == RAFT_LEADER) { + leaders++; + leader_idx = i; + } + } + TEST_ASSERT_EQUAL_INT_MESSAGE(1, leaders, + "post-drop/dup/reorder cluster does not have exactly one leader"); + + /* Convergence is observable AND non-vacuous: the leader holds a non-trivial log, + * and despite dropped/duplicated/reordered delivery every other node's log caught + * up to the leader's tip (length AND every entry's term match over the FULL log + * via Log Matching). A node stuck behind a dropped entry (shorter log) or one + * corrupted by a duplicate/reorder (divergent term) would FAIL here. */ + TEST_ASSERT_TRUE_MESSAGE(c.nodes[leader_idx].log_count > 0, + "leader never replicated any entry across the scenario"); + + for (int i = 0; i < c.n; i++) { + if (i == leader_idx) + continue; + TEST_ASSERT_EQUAL_INT_MESSAGE( + c.nodes[leader_idx].log_count, c.nodes[i].log_count, + "follower log did not catch up to the leader's tip after drop/dup/reorder"); + for (int e = 1; e <= c.nodes[leader_idx].log_count; e++) { + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + c.nodes[leader_idx].log[e - 1].term, + c.nodes[i].log[e - 1].term, + "follower diverges from leader log after drop/dup/reorder convergence"); + } + } + } + + /* ---- Duplicated-AE-is-a-no-op demonstrated DIRECTLY via the conformance bag ---- + * Elect a leader, replicate one entry to a follower, then DUPLICATE the next + * AppendEntries and deliver it twice. The second (duplicate) delivery hits the + * library's already-have branch and must NOT grow or corrupt the follower's log. */ + { + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 3, 2000, 99ULL)); + + /* Elect node 0 and drain the vote traffic so it becomes LEADER. */ + (void)raft_start_election(&c.nodes[0].state); + for (int g = 0; g < RAFT_SIM_MAX_MSGS * 4 && c.bag_count > 0; g++) { + int idx = raft_sim_bag_pick(&c); + if (idx < 0) + break; + (void)raft_sim_bag_deliver(&c, idx); + } + TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_LEADER, c.nodes[0].state.role, + "node 0 did not become leader for the duplicate-AE no-op probe"); + + /* Append one entry and send an AppendEntries for it to node 1. */ + uint64_t value = 0xABCDULL, out_index = 0; + (void)raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index); + (void)raft_send_append_entries(&c.nodes[0].state, 1, + c.nodes[0].state.commit_index); + + /* Find the AE_REQ targeting node 1 in the bag, DUPLICATE it (multiplicity += 1). */ + int ae_idx = -1; + for (int i = 0; i < c.bag_count; i++) { + if (c.bag[i].kind == RAFT_SIM_MSG_AE_REQ && c.bag[i].dst == 1) { + ae_idx = i; + break; + } + } + TEST_ASSERT_TRUE_MESSAGE(ae_idx >= 0, + "no AppendEntries to node 1 found in the bag for the duplicate probe"); + TEST_ASSERT_EQUAL_INT(0, raft_sim_bag_duplicate(&c, ae_idx)); + + /* Deliver the FIRST copy: node 1 appends the entry. Record its log_count. */ + int first = -1; + for (int i = 0; i < c.bag_count; i++) { + if (c.bag[i].kind == RAFT_SIM_MSG_AE_REQ && c.bag[i].dst == 1) { + first = i; + break; + } + } + TEST_ASSERT_TRUE(first >= 0); + (void)raft_sim_bag_deliver(&c, first); + int log_count_after_first = c.nodes[1].log_count; + uint64_t term_after_first = (log_count_after_first > 0) + ? c.nodes[1].log[log_count_after_first - 1].term : 0; + TEST_ASSERT_TRUE_MESSAGE(log_count_after_first >= 1, + "node 1 did not append the entry on the first AE delivery"); + + /* Deliver the SECOND (duplicate) copy: already-have branch — log unchanged. */ + int second = -1; + for (int i = 0; i < c.bag_count; i++) { + if (c.bag[i].kind == RAFT_SIM_MSG_AE_REQ && c.bag[i].dst == 1) { + second = i; + break; + } + } + TEST_ASSERT_TRUE_MESSAGE(second >= 0, + "duplicate AE to node 1 was not present for the second delivery"); + (void)raft_sim_bag_deliver(&c, second); + + /* Idempotent: the duplicate delivery did NOT grow or corrupt the log. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(log_count_after_first, c.nodes[1].log_count, + "duplicated AppendEntries grew the follower log (not idempotent)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_after_first, + c.nodes[1].log[c.nodes[1].log_count - 1].term, + "duplicated AppendEntries corrupted the follower log tail term"); + + TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c)); + } +} #endif /* XCE_TESTING */ /* ---------- test_raft entry point ---------- */ @@ -8459,6 +8619,7 @@ void test_raft(void) RUN_TEST(test_sim_4_4_run_deterministic_replay); RUN_TEST(test_sim_5_1_partition_heal); RUN_TEST(test_sim_5_2_restart_churn); + RUN_TEST(test_sim_5_3_drop_dup_reorder); /* Group A: basic election correctness (task 5.1) */ RUN_TEST(test_proto_A1_single_node_self_elects);