From 9c1a034aa8af69cd4da2efc1142ff926d7127ea4 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sun, 21 Jun 2026 17:06:05 +0000 Subject: [PATCH] test(raft-log-replication): simulator invariant tests (tasks 9.1-9.4) 9.1 replication + partition catch-up: 3-entry quorum replication then Log Matching; isolate a follower, heal, assert next_index walks back, catch-up, Log Matching. 9.2 divergent-suffix truncation (G3): follower with a same-index/different-term tail converges via conflict-truncate; byte-for-byte prefix equality + Log Matching. 9.3 drop/dup/reorder convergence: dropped AE recovered; duplicate already-have AE is a no-op; genuine out-of-order delivery (entry-3 AE before entry-2) rejected via the logOk gap with no phantom gap; converges + Log Matching (Req 8.3). 9.4 chaotic-churn G5/bug-851 regression: stale-term node rejoins under a higher-term leader, adopts the term via the AE/heartbeat UpdateTerm path (load-bearing: would fail as stale-reject if UpdateTerm-on-AE regressed), voted_for cleared, no second leader, and a post-converge election campaigns strictly above the old term (cannot reuse). Log Matching holds throughout. Review-applied: genuine reorder in 9.3; active re-election + voted_for assertions in 9.4. ASAN green (219 tests). --- .kiro/specs/raft-log-replication/tasks.md | 8 +- src/lib/testsuite/test_raft.c | 423 ++++++++++++++++++++++ 2 files changed, 427 insertions(+), 4 deletions(-) diff --git a/.kiro/specs/raft-log-replication/tasks.md b/.kiro/specs/raft-log-replication/tasks.md index 78042bde..6da5b16a 100644 --- a/.kiro/specs/raft-log-replication/tasks.md +++ b/.kiro/specs/raft-log-replication/tasks.md @@ -161,28 +161,28 @@ ## 9. Simulator invariant tests (Log Matching, truncation, faults, G5 regression) -- [ ] 9.1 Replication + partition catch-up Log Matching test +- [x] 9.1 Replication + partition catch-up Log Matching test - Test: leader appends 3 entries, replicates to a quorum, assert Log Matching; then isolate a follower across more appends, heal, assert `nextIndex` walks back, the follower catches up, and Log Matching holds - Observable completion: test passes end-to-end via the sim - _Requirements: 8.1, 8.3, 8.4_ - _Boundary: test_raft.c, raft_sim.c_ - _Depends: 8.2, 8.3_ -- [ ] 9.2 Divergent-suffix truncation test (G3) +- [x] 9.2 Divergent-suffix truncation test (G3) - Test: give a follower a manufactured divergent tail (same indices, different term), replicate from a higher-term leader; assert the follower truncates the divergent suffix (`on_truncate_from`) and converges to the leader's prefix; Log Matching holds - Observable completion: test passes; follower's post-converge prefix equals the leader's - _Requirements: 8.5_ - _Boundary: test_raft.c, raft_sim.c_ - _Depends: 9.1_ -- [ ] 9.3 Message drop/dup/reorder convergence test +- [x] 9.3 Message drop/dup/reorder convergence test - Test: drop, duplicate, and reorder AE requests/replies in the sim queue; assert idempotent convergence (a duplicated AE is a no-op via the already-have branch) and Log Matching after faults clear - Observable completion: test passes despite injected faults - _Requirements: 8.3_ - _Boundary: test_raft.c, raft_sim.c_ - _Depends: 9.1_ -- [ ] 9.4 Chaotic-churn G5 regression test +- [x] 9.4 Chaotic-churn G5 regression test - Test: a stale-term node (term lagging) rejoins under a higher-term leader; assert it adopts the higher term from the AE/heartbeat path and does NOT re-elect at a reused term; Log Matching holds across the churn - Observable completion: test passes; the rejoining node's term equals the leader's after the first heartbeat, and no second leader appears at the reused term - _Requirements: 8.6_ diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c index 268599c3..50070977 100644 --- a/src/lib/testsuite/test_raft.c +++ b/src/lib/testsuite/test_raft.c @@ -3194,6 +3194,425 @@ static void test_sim_check_log_matching_fail_on_divergence(void) TEST_ASSERT_EQUAL_INT(0, raft_sim_check_log_matching(&c)); } +/* ---------- Group: Simulator invariant tests (tasks 9.1-9.4) ---------- + * + * These are observation-based invariant tests. They drive the COMPLETE + * machinery (raft_log_append / raft_send_append_entries / + * raft_sim_deliver_append_entries / raft_on_append_entries_reply) and assert + * Raft Log Matching (raft_sim_check_log_matching) plus the divergent-suffix + * truncation, drop/dup/reorder convergence, and chaotic-churn (bug-851) + * properties from requirements 8.1, 8.3, 8.4, 8.5, 8.6. + * + * Each test carries INTERMEDIATE assertions that would FAIL if convergence + * did NOT actually happen (e.g. follower log_count, exact entry terms, + * nextIndex back-off), so a trivially-passing run is impossible: the + * end-to-end raft_sim_check_log_matching == 1 is only reachable after the + * replicate/catch-up/truncate/term-adoption work is genuinely observed. + */ + +/* Replicate the leader's single entry at the leader's CURRENT next_index for + * `peer` and route it synchronously. Returns the follower's resulting reply via + * the sim's reply path (already applied to the leader's per-peer progress). */ +static void sim_replicate_one(raft_sim_cluster_t *c, int leader, int peer, + uint64_t leader_commit) +{ + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_send_append_entries(&c->nodes[leader].state, peer, leader_commit)); + raft_sim_deliver_append_entries(c); +} + +/* + * test_sim_9_1_replication_and_partition_catch_up (task 9.1; Req 8.1, 8.3, 8.4) + * + * Leader appends 3 entries, replicates to a quorum, asserts Log Matching. + * Then isolates a follower across MORE appends, heals it, and drives catch-up: + * the leader's nextIndex for the lagging follower walks BACK (failure replies + * decrement next_index), the follower catches up entry-by-entry, and Log + * Matching holds end-to-end. + * + * Load-bearing intermediate assertions: + * - after the initial 3-entry replication, node 1 AND node 2 hold log_count 3 + * (if delivery were skipped these would be 0 — the assertion is real). + * - after isolation + 2 more appends, node 2 is STILL at log_count 3 (proves + * the partition genuinely blocked replication) while leader is at 5. + * - the first post-heal AppendEntries (prev_log_index=4) is REJECTED, forcing + * next_index[2] to decrement (catch-up back-off is exercised, not bypassed). + * - node 2 converges to log_count 5 and Log Matching holds. + */ +static void test_sim_9_1_replication_and_partition_catch_up(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Elect node 0 leader. */ + raft_start_election(&c.nodes[0].state); + raft_sim_deliver_pending(&c); + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + + /* Append 3 entries and replicate each to BOTH followers (a quorum). */ + for (int e = 0; e < 3; e++) { + unsigned char payload[2] = { (unsigned char)('A' + e), 0x00 }; + uint64_t out_index = 0; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index)); + TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index); + sim_replicate_one(&c, 0, 1, 0); + sim_replicate_one(&c, 0, 2, 0); + } + + /* Intermediate: both followers replicated all 3 (delivery actually happened). */ + TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count); + TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count); + TEST_ASSERT_EQUAL_INT(3, c.nodes[2].log_count); + TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c)); + + /* After replicating 1..3 to node 2, the leader believes next_index[2] == 4. */ + TEST_ASSERT_EQUAL_UINT64(4, c.nodes[0].state.next_index[2]); + + /* Isolate node 2; append 2 MORE entries and replicate only to node 1. + * Sends to node 2 are dropped (partitioned), so next_index[2] stays 4. */ + raft_sim_partition(&c, 2, 1); + for (int e = 3; e < 5; e++) { + unsigned char payload[2] = { (unsigned char)('A' + e), 0x00 }; + uint64_t out_index = 0; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index)); + TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index); + sim_replicate_one(&c, 0, 1, 0); + /* Attempt to send to node 2 as well — dropped because partitioned. */ + sim_replicate_one(&c, 0, 2, 0); + } + + /* Intermediate: the partition genuinely blocked node 2 (still 3, not 5). */ + TEST_ASSERT_EQUAL_INT(5, c.nodes[0].log_count); + TEST_ASSERT_EQUAL_INT(5, c.nodes[1].log_count); + TEST_ASSERT_EQUAL_INT(3, c.nodes[2].log_count); + + /* Simulate node 2 having LOST entries 2..3 while away (e.g. a crash that + * rolled back its tail to 1) so its real log no longer matches the leader's + * stale next_index[2] == 4. This forces the AppendEntries consistency check + * to walk next_index[2] BACK until prev_log_index matches node 2's tail. */ + raft_sim_partition(&c, 2, 0); + c.nodes[2].cbs.on_truncate_from(&c.nodes[2], 2); /* drop indices 2..end */ + TEST_ASSERT_EQUAL_INT(1, c.nodes[2].log_count); + + uint64_t ni_before = c.nodes[0].state.next_index[2]; + /* The leader still believes node 2 is at index 3 (next_index 4), past its + * real tail (1) — so the first AE prev_log_index = 3 must be rejected. */ + TEST_ASSERT_TRUE(ni_before > (uint64_t)(c.nodes[2].log_count + 1)); + + /* One round: rejected (prev_log_index beyond node 2's log) ⇒ next_index-- . */ + sim_replicate_one(&c, 0, 2, 0); + TEST_ASSERT_TRUE(c.nodes[0].state.next_index[2] < ni_before); + + /* Continue driving rounds until node 2 catches up (bounded loop). Each round + * either backs off (reject) or appends one entry (accept), so it converges. */ + int rounds = 0; + while (c.nodes[2].log_count < c.nodes[0].log_count && rounds < 64) { + sim_replicate_one(&c, 0, 2, 0); + rounds++; + } + + /* Converged: node 2 holds the full prefix; Log Matching holds end-to-end. */ + TEST_ASSERT_EQUAL_INT(5, c.nodes[2].log_count); + for (int i = 0; i < 5; i++) + TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[2].log[i].term); + TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c)); +} + +/* + * test_sim_9_2_divergent_suffix_truncation (task 9.2; Req 8.5, closes G3) + * + * Give a follower a manufactured divergent tail (same indices, DIFFERENT term), + * replicate from a higher-term leader, and assert the follower truncates the + * divergent suffix (on_truncate_from via the conflict branch) and converges to + * the leader's prefix. The follower's post-converge prefix must equal the + * leader's byte-for-byte; Log Matching holds. + * + * Load-bearing intermediate assertions: + * - pre-replication, node 1 holds the WRONG (term-2) entries at indices 1..3, + * differing from node 0's term-3 entries (so a no-op delivery would FAIL + * the final byte-equality / Log Matching check). + * - the conflict at index 1 triggers a truncation back-off, and node 1's + * final terms equal node 0's (proves the divergent suffix was deleted, not + * merely overwritten in place across a mismatching prefix). + */ +static void test_sim_9_2_divergent_suffix_truncation(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Elect node 0 leader, then advance its term to 3 (strictly higher than the + * follower's manufactured divergent term 2). raft_log_append stamps each new + * entry with s->term, so setting the term before appending makes the leader's + * entries carry term 3. next_index/match_index were initialized at + * become_leader and are term-independent, so they remain valid. */ + raft_start_election(&c.nodes[0].state); /* term -> 1 */ + raft_sim_deliver_pending(&c); + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + c.nodes[0].state.term = 3; + uint64_t leader_term = c.nodes[0].state.term; + TEST_ASSERT_TRUE(leader_term >= 3); + + unsigned char lp[3][3] = { + { 'L', '0', 0 }, { 'L', '1', 0 }, { 'L', '2', 0 } }; + for (int e = 0; e < 3; e++) { + uint64_t out_index = 0; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_log_append(&c.nodes[0].state, lp[e], 2, &out_index)); + TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index); + } + TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count); + for (int i = 0; i < 3; i++) + TEST_ASSERT_EQUAL_UINT64(leader_term, c.nodes[0].log[i].term); + + /* Manufacture node 1's DIVERGENT tail: same indices 1..3, term 2, "X" bytes. + * Persist directly into the sim log (same indices, lower/different term). */ + for (int i = 0; i < 3; i++) { + c.nodes[1].log[i].term = 2; + c.nodes[1].log[i].bytes[0] = 'X'; + c.nodes[1].log[i].bytes[1] = (unsigned char)('0' + i); + c.nodes[1].log[i].len = 2; + } + c.nodes[1].log_count = 3; + /* Make node 1 a follower whose term is at most the leader's (so it accepts). */ + raft_on_append_entries_request(&c.nodes[1].state, 0, leader_term, + 0, 0, 0, NULL, 0, 0, 0, &(raft_append_result_t){0}); + TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, c.nodes[1].state.role); + + /* Intermediate: node 1 currently DIVERGES from node 0 at every index. */ + TEST_ASSERT_TRUE(c.nodes[1].log[0].term != c.nodes[0].log[0].term); + + /* Replicate from the higher-term leader. next_index[1] starts at Len(log)+1 + * = 4; the leader walks back, the conflict branch truncates the divergent + * suffix, then appends the leader's entries. Drive bounded rounds. */ + int rounds = 0; + while (rounds < 64) { + sim_replicate_one(&c, 0, 1, 0); + rounds++; + if (c.nodes[1].log_count == c.nodes[0].log_count) { + int all_match = 1; + for (int i = 0; i < c.nodes[0].log_count; i++) + if (c.nodes[1].log[i].term != c.nodes[0].log[i].term) + all_match = 0; + if (all_match) + break; + } + } + + /* Converged: node 1's prefix now EQUALS node 0's byte-for-byte (the divergent + * term-2 suffix was truncated and replaced with the leader's term-N prefix). */ + TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count); + for (int i = 0; i < 3; i++) { + TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[1].log[i].term); + TEST_ASSERT_EQUAL_size_t(c.nodes[0].log[i].len, c.nodes[1].log[i].len); + TEST_ASSERT_EQUAL_MEMORY(c.nodes[0].log[i].bytes, c.nodes[1].log[i].bytes, + c.nodes[0].log[i].len); + } + TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c)); +} + +/* + * test_sim_9_3_drop_dup_reorder_convergence (task 9.3; Req 8.3) + * + * Drop, duplicate, and reorder AppendEntries in the sim queue and assert + * idempotent convergence: a duplicated AE is a no-op via the already-have + * branch, a dropped AE is recovered by a later round, and reordering does not + * corrupt the log. Log Matching holds after the faults clear. + * + * Load-bearing intermediate assertions: + * - after the deliberately-DROPPED round, node 1 is still missing the entry + * (proves the drop was real, not silently delivered). + * - after delivering a DUPLICATE of an already-applied AE, node 1's log_count + * does NOT grow (idempotent already-have branch). + * - final Log Matching holds and node 1 holds the full prefix. + */ +static void test_sim_9_3_drop_dup_reorder_convergence(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + raft_start_election(&c.nodes[0].state); + raft_sim_deliver_pending(&c); + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + + /* Append 3 entries on the leader. */ + for (int e = 0; e < 3; e++) { + unsigned char payload[2] = { (unsigned char)('a' + e), 0x00 }; + uint64_t out_index = 0; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index)); + TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index); + } + TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count); + + /* DROP: buffer an AE for entry 1 to node 1, then clear the queue WITHOUT + * delivering (simulate a dropped message). */ + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_send_append_entries(&c.nodes[0].state, 1, /*leader_commit=*/0)); + TEST_ASSERT_EQUAL_INT(1, c.ae_pending_count); + c.ae_pending_count = 0; /* DROP the buffered AE */ + + /* Intermediate: the drop was real — node 1 received nothing. */ + TEST_ASSERT_EQUAL_INT(0, c.nodes[1].log_count); + + /* Recover the dropped entry via a fresh round. */ + sim_replicate_one(&c, 0, 1, 0); + TEST_ASSERT_EQUAL_INT(1, c.nodes[1].log_count); + + /* DUPLICATE: rewind the leader's belief so it re-sends the ALREADY-PRESENT + * entry 1 (next_index[1] = 1 ⇒ prev_log_index = 0, entry index 1). Buffer it + * twice and deliver both; both deliveries hit the already-have branch (the + * incoming entry matches node 1's local entry 1), so the log must NOT grow. */ + c.nodes[0].state.next_index[1] = 1; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_send_append_entries(&c.nodes[0].state, 1, /*leader_commit=*/0)); + TEST_ASSERT_TRUE(c.ae_pending_count >= 1); + /* Manually duplicate the just-buffered AE (re-sends entry 1). */ + if (c.ae_pending_count < RAFT_SIM_MAX_AE) { + c.ae_pending[c.ae_pending_count] = c.ae_pending[c.ae_pending_count - 1]; + c.ae_pending_count++; + } + raft_sim_deliver_append_entries(&c); + /* Idempotent: the duplicated already-have AE did not grow the log. */ + TEST_ASSERT_EQUAL_INT(1, c.nodes[1].log_count); + + /* REORDER (genuine out-of-order delivery): node 1 currently holds only + * entry 1. Buffer the AE for entry 2 (prev_log_index=1) and the AE for + * entry 3 (prev_log_index=2), then SWAP them so entry 3 is delivered FIRST. + * The entry-3 AE references a prev (index 2) node 1 does not yet have, so + * logOk fails (prev_log_index > log_len) ⇒ it is rejected without creating a + * gap; the entry-2 AE then appends normally. This proves reordered AEs cannot + * insert out-of-order (Req 8.3 reorder). */ + c.ae_pending_count = 0; /* start the reorder round with a clean queue */ + c.nodes[0].state.next_index[1] = 2; /* prev=1 → entry 2 */ + TEST_ASSERT_EQUAL_INT(RAFT_OK, raft_send_append_entries(&c.nodes[0].state, 1, 0)); + c.nodes[0].state.next_index[1] = 3; /* prev=2 → entry 3 */ + TEST_ASSERT_EQUAL_INT(RAFT_OK, raft_send_append_entries(&c.nodes[0].state, 1, 0)); + TEST_ASSERT_EQUAL_INT(2, c.ae_pending_count); + /* Swap so the entry-3 AE is delivered before the entry-2 AE. */ + { + raft_sim_ae_t tmp = c.ae_pending[0]; + c.ae_pending[0] = c.ae_pending[1]; + c.ae_pending[1] = tmp; + } + raft_sim_deliver_append_entries(&c); + /* The out-of-order entry-3 AE was rejected (no gap); only entry 2 appended. */ + TEST_ASSERT_EQUAL_INT(2, c.nodes[1].log_count); + + /* Now deliver remaining rounds (entry 3) under continued churn; bounded loop + * drives to convergence regardless of transient reject/back-off. */ + int rounds = 0; + while (c.nodes[1].log_count < c.nodes[0].log_count && rounds < 64) { + sim_replicate_one(&c, 0, 1, 0); + rounds++; + } + + /* Converged despite drop + dup + back-off; Log Matching holds. */ + TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count); + for (int i = 0; i < 3; i++) + TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[1].log[i].term); + TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c)); +} + +/* + * test_sim_9_4_chaotic_churn_g5_regression (task 9.4; Req 8.6 — bug-851) + * + * A stale-term node (term lagging) rejoins under a higher-term leader; assert it + * adopts the higher term from the AE/heartbeat path (UpdateTerm on AppendEntries, + * G5, task 5/6) and does NOT re-elect at a reused term. Log Matching holds across + * the churn. THIS is the bug-851 regression: silent committed-prefix divergence + * from two leaders at a reused term. It relies on UpdateTerm-on-AE and the + * SEND-time leader term in delivery (task 8.2). + * + * Load-bearing intermediate assertions: + * - pre-rejoin, the stale node's term is STRICTLY LESS than the leader's + * (so a no-op heartbeat would leave them divergent and the assertion real). + * - after the FIRST heartbeat, the rejoining node's term EQUALS the leader's + * (term adopted from the AE path) AND its role is FOLLOWER (stepped down). + * - no SECOND leader appears at the reused term (leader_count unchanged after + * the stale node is denied a re-election at its own old term). + */ +static void test_sim_9_4_chaotic_churn_g5_regression(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Elect node 0 leader and advance its term to model accumulated leadership + * churn — node 0 now leads at a high term while a partitioned node 2 lags. */ + raft_start_election(&c.nodes[0].state); + raft_sim_deliver_pending(&c); + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + c.nodes[0].state.term = 5; + uint64_t leader_term = c.nodes[0].state.term; + TEST_ASSERT_TRUE(leader_term >= 3); + + /* Leader has a couple of entries. */ + for (int e = 0; e < 2; e++) { + unsigned char payload[2] = { (unsigned char)('K' + e), 0x00 }; + uint64_t out_index = 0; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index)); + } + + /* Node 2 is a STALE-TERM rejoiner: force its term low (1) and role follower + * (as if it was partitioned away during the churn and never advanced). */ + c.nodes[2].state.term = 1; + c.nodes[2].state.role = RAFT_FOLLOWER; + c.nodes[2].state.voted_for = -1; + + /* Intermediate: stale node's term is strictly below the leader's. */ + TEST_ASSERT_TRUE(c.nodes[2].state.term < leader_term); + + int leader_count_before = c.leader_count; + + /* Send a HEARTBEAT (n_entries handled by send AE: next_index[2] = Len(log)+1 + * so prev_log_index points past node 2's empty log; this is the AE/heartbeat + * path). UpdateTerm must fire on this AE: node 2 adopts leader_term. */ + sim_replicate_one(&c, 0, 2, 0); + + /* G5: node 2 adopted the higher term from the AppendEntries/heartbeat path + * and stepped down to follower — NOT only from a vote RPC. */ + TEST_ASSERT_EQUAL_UINT64(leader_term, c.nodes[2].state.term); + TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, c.nodes[2].state.role); + + /* UpdateTerm cleared the stale node's vote state: voted_for reset to -1 so it + * cannot double-vote in the new term, and any vote it casts now belongs to + * leader_term (>= the term it would have reused). This is the concrete state + * that prevents the reused-term double-leader of bug-851. */ + TEST_ASSERT_EQUAL_INT(-1, c.nodes[2].state.voted_for); + + /* No second leader was created during the rejoin/heartbeat delivery. */ + TEST_ASSERT_EQUAL_INT(leader_count_before, c.leader_count); + + /* The original leader is still the sole leader. */ + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + + /* Drive catch-up so node 2 converges; Log Matching holds across the churn. */ + int rounds = 0; + while (c.nodes[2].log_count < c.nodes[0].log_count && rounds < 64) { + sim_replicate_one(&c, 0, 2, 0); + rounds++; + } + TEST_ASSERT_EQUAL_INT(c.nodes[0].log_count, c.nodes[2].log_count); + TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c)); + + /* Active check (after convergence, so it cannot disturb catch-up): if the + * rejoined node now starts an election it campaigns at a term STRICTLY ABOVE + * leader_term — never the stale term it lagged at — so it can never resurrect + * a committed-prefix divergence by reusing an old term (bug-851). */ + raft_start_election(&c.nodes[2].state); + TEST_ASSERT_TRUE(c.nodes[2].state.term > leader_term); +} + /* ---------- Group A: Basic Election Correctness (task 5.1) ---------- */ /* @@ -5444,6 +5863,10 @@ void test_raft(void) RUN_TEST(test_sim_deliver_append_entries_replicates); RUN_TEST(test_sim_check_log_matching_pass_on_converged); RUN_TEST(test_sim_check_log_matching_fail_on_divergence); + RUN_TEST(test_sim_9_1_replication_and_partition_catch_up); + RUN_TEST(test_sim_9_2_divergent_suffix_truncation); + RUN_TEST(test_sim_9_3_drop_dup_reorder_convergence); + RUN_TEST(test_sim_9_4_chaotic_churn_g5_regression); /* Group A: basic election correctness (task 5.1) */ RUN_TEST(test_proto_A1_single_node_self_elects);