test(raft-log-replication): simulator invariant tests (tasks 9.1-9.4)
9.1 replication + partition catch-up: 3-entry quorum replication then Log Matching; isolate a follower, heal, assert next_index walks back, catch-up, Log Matching. 9.2 divergent-suffix truncation (G3): follower with a same-index/different-term tail converges via conflict-truncate; byte-for-byte prefix equality + Log Matching. 9.3 drop/dup/reorder convergence: dropped AE recovered; duplicate already-have AE is a no-op; genuine out-of-order delivery (entry-3 AE before entry-2) rejected via the logOk gap with no phantom gap; converges + Log Matching (Req 8.3). 9.4 chaotic-churn G5/bug-851 regression: stale-term node rejoins under a higher-term leader, adopts the term via the AE/heartbeat UpdateTerm path (load-bearing: would fail as stale-reject if UpdateTerm-on-AE regressed), voted_for cleared, no second leader, and a post-converge election campaigns strictly above the old term (cannot reuse). Log Matching holds throughout. Review-applied: genuine reorder in 9.3; active re-election + voted_for assertions in 9.4. ASAN green (219 tests).
This commit is contained in:
parent
6ff07e5843
commit
9c1a034aa8
2 changed files with 427 additions and 4 deletions
|
|
@ -161,28 +161,28 @@
|
|||
|
||||
## 9. Simulator invariant tests (Log Matching, truncation, faults, G5 regression)
|
||||
|
||||
- [ ] 9.1 Replication + partition catch-up Log Matching test
|
||||
- [x] 9.1 Replication + partition catch-up Log Matching test
|
||||
- Test: leader appends 3 entries, replicates to a quorum, assert Log Matching; then isolate a follower across more appends, heal, assert `nextIndex` walks back, the follower catches up, and Log Matching holds
|
||||
- Observable completion: test passes end-to-end via the sim
|
||||
- _Requirements: 8.1, 8.3, 8.4_
|
||||
- _Boundary: test_raft.c, raft_sim.c_
|
||||
- _Depends: 8.2, 8.3_
|
||||
|
||||
- [ ] 9.2 Divergent-suffix truncation test (G3)
|
||||
- [x] 9.2 Divergent-suffix truncation test (G3)
|
||||
- Test: give a follower a manufactured divergent tail (same indices, different term), replicate from a higher-term leader; assert the follower truncates the divergent suffix (`on_truncate_from`) and converges to the leader's prefix; Log Matching holds
|
||||
- Observable completion: test passes; follower's post-converge prefix equals the leader's
|
||||
- _Requirements: 8.5_
|
||||
- _Boundary: test_raft.c, raft_sim.c_
|
||||
- _Depends: 9.1_
|
||||
|
||||
- [ ] 9.3 Message drop/dup/reorder convergence test
|
||||
- [x] 9.3 Message drop/dup/reorder convergence test
|
||||
- Test: drop, duplicate, and reorder AE requests/replies in the sim queue; assert idempotent convergence (a duplicated AE is a no-op via the already-have branch) and Log Matching after faults clear
|
||||
- Observable completion: test passes despite injected faults
|
||||
- _Requirements: 8.3_
|
||||
- _Boundary: test_raft.c, raft_sim.c_
|
||||
- _Depends: 9.1_
|
||||
|
||||
- [ ] 9.4 Chaotic-churn G5 regression test
|
||||
- [x] 9.4 Chaotic-churn G5 regression test
|
||||
- Test: a stale-term node (term lagging) rejoins under a higher-term leader; assert it adopts the higher term from the AE/heartbeat path and does NOT re-elect at a reused term; Log Matching holds across the churn
|
||||
- Observable completion: test passes; the rejoining node's term equals the leader's after the first heartbeat, and no second leader appears at the reused term
|
||||
- _Requirements: 8.6_
|
||||
|
|
|
|||
|
|
@ -3194,6 +3194,425 @@ static void test_sim_check_log_matching_fail_on_divergence(void)
|
|||
TEST_ASSERT_EQUAL_INT(0, raft_sim_check_log_matching(&c));
|
||||
}
|
||||
|
||||
/* ---------- Group: Simulator invariant tests (tasks 9.1-9.4) ----------
|
||||
*
|
||||
* These are observation-based invariant tests. They drive the COMPLETE
|
||||
* machinery (raft_log_append / raft_send_append_entries /
|
||||
* raft_sim_deliver_append_entries / raft_on_append_entries_reply) and assert
|
||||
* Raft Log Matching (raft_sim_check_log_matching) plus the divergent-suffix
|
||||
* truncation, drop/dup/reorder convergence, and chaotic-churn (bug-851)
|
||||
* properties from requirements 8.1, 8.3, 8.4, 8.5, 8.6.
|
||||
*
|
||||
* Each test carries INTERMEDIATE assertions that would FAIL if convergence
|
||||
* did NOT actually happen (e.g. follower log_count, exact entry terms,
|
||||
* nextIndex back-off), so a trivially-passing run is impossible: the
|
||||
* end-to-end raft_sim_check_log_matching == 1 is only reachable after the
|
||||
* replicate/catch-up/truncate/term-adoption work is genuinely observed.
|
||||
*/
|
||||
|
||||
/* Replicate the leader's single entry at the leader's CURRENT next_index for
|
||||
* `peer` and route it synchronously. Returns the follower's resulting reply via
|
||||
* the sim's reply path (already applied to the leader's per-peer progress). */
|
||||
static void sim_replicate_one(raft_sim_cluster_t *c, int leader, int peer,
|
||||
uint64_t leader_commit)
|
||||
{
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_send_append_entries(&c->nodes[leader].state, peer, leader_commit));
|
||||
raft_sim_deliver_append_entries(c);
|
||||
}
|
||||
|
||||
/*
|
||||
* test_sim_9_1_replication_and_partition_catch_up (task 9.1; Req 8.1, 8.3, 8.4)
|
||||
*
|
||||
* Leader appends 3 entries, replicates to a quorum, asserts Log Matching.
|
||||
* Then isolates a follower across MORE appends, heals it, and drives catch-up:
|
||||
* the leader's nextIndex for the lagging follower walks BACK (failure replies
|
||||
* decrement next_index), the follower catches up entry-by-entry, and Log
|
||||
* Matching holds end-to-end.
|
||||
*
|
||||
* Load-bearing intermediate assertions:
|
||||
* - after the initial 3-entry replication, node 1 AND node 2 hold log_count 3
|
||||
* (if delivery were skipped these would be 0 — the assertion is real).
|
||||
* - after isolation + 2 more appends, node 2 is STILL at log_count 3 (proves
|
||||
* the partition genuinely blocked replication) while leader is at 5.
|
||||
* - the first post-heal AppendEntries (prev_log_index=4) is REJECTED, forcing
|
||||
* next_index[2] to decrement (catch-up back-off is exercised, not bypassed).
|
||||
* - node 2 converges to log_count 5 and Log Matching holds.
|
||||
*/
|
||||
static void test_sim_9_1_replication_and_partition_catch_up(void)
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
/* Elect node 0 leader. */
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
raft_sim_deliver_pending(&c);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
/* Append 3 entries and replicate each to BOTH followers (a quorum). */
|
||||
for (int e = 0; e < 3; e++) {
|
||||
unsigned char payload[2] = { (unsigned char)('A' + e), 0x00 };
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index));
|
||||
TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index);
|
||||
sim_replicate_one(&c, 0, 1, 0);
|
||||
sim_replicate_one(&c, 0, 2, 0);
|
||||
}
|
||||
|
||||
/* Intermediate: both followers replicated all 3 (delivery actually happened). */
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count);
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count);
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[2].log_count);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c));
|
||||
|
||||
/* After replicating 1..3 to node 2, the leader believes next_index[2] == 4. */
|
||||
TEST_ASSERT_EQUAL_UINT64(4, c.nodes[0].state.next_index[2]);
|
||||
|
||||
/* Isolate node 2; append 2 MORE entries and replicate only to node 1.
|
||||
* Sends to node 2 are dropped (partitioned), so next_index[2] stays 4. */
|
||||
raft_sim_partition(&c, 2, 1);
|
||||
for (int e = 3; e < 5; e++) {
|
||||
unsigned char payload[2] = { (unsigned char)('A' + e), 0x00 };
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index));
|
||||
TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index);
|
||||
sim_replicate_one(&c, 0, 1, 0);
|
||||
/* Attempt to send to node 2 as well — dropped because partitioned. */
|
||||
sim_replicate_one(&c, 0, 2, 0);
|
||||
}
|
||||
|
||||
/* Intermediate: the partition genuinely blocked node 2 (still 3, not 5). */
|
||||
TEST_ASSERT_EQUAL_INT(5, c.nodes[0].log_count);
|
||||
TEST_ASSERT_EQUAL_INT(5, c.nodes[1].log_count);
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[2].log_count);
|
||||
|
||||
/* Simulate node 2 having LOST entries 2..3 while away (e.g. a crash that
|
||||
* rolled back its tail to 1) so its real log no longer matches the leader's
|
||||
* stale next_index[2] == 4. This forces the AppendEntries consistency check
|
||||
* to walk next_index[2] BACK until prev_log_index matches node 2's tail. */
|
||||
raft_sim_partition(&c, 2, 0);
|
||||
c.nodes[2].cbs.on_truncate_from(&c.nodes[2], 2); /* drop indices 2..end */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[2].log_count);
|
||||
|
||||
uint64_t ni_before = c.nodes[0].state.next_index[2];
|
||||
/* The leader still believes node 2 is at index 3 (next_index 4), past its
|
||||
* real tail (1) — so the first AE prev_log_index = 3 must be rejected. */
|
||||
TEST_ASSERT_TRUE(ni_before > (uint64_t)(c.nodes[2].log_count + 1));
|
||||
|
||||
/* One round: rejected (prev_log_index beyond node 2's log) ⇒ next_index-- . */
|
||||
sim_replicate_one(&c, 0, 2, 0);
|
||||
TEST_ASSERT_TRUE(c.nodes[0].state.next_index[2] < ni_before);
|
||||
|
||||
/* Continue driving rounds until node 2 catches up (bounded loop). Each round
|
||||
* either backs off (reject) or appends one entry (accept), so it converges. */
|
||||
int rounds = 0;
|
||||
while (c.nodes[2].log_count < c.nodes[0].log_count && rounds < 64) {
|
||||
sim_replicate_one(&c, 0, 2, 0);
|
||||
rounds++;
|
||||
}
|
||||
|
||||
/* Converged: node 2 holds the full prefix; Log Matching holds end-to-end. */
|
||||
TEST_ASSERT_EQUAL_INT(5, c.nodes[2].log_count);
|
||||
for (int i = 0; i < 5; i++)
|
||||
TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[2].log[i].term);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c));
|
||||
}
|
||||
|
||||
/*
|
||||
* test_sim_9_2_divergent_suffix_truncation (task 9.2; Req 8.5, closes G3)
|
||||
*
|
||||
* Give a follower a manufactured divergent tail (same indices, DIFFERENT term),
|
||||
* replicate from a higher-term leader, and assert the follower truncates the
|
||||
* divergent suffix (on_truncate_from via the conflict branch) and converges to
|
||||
* the leader's prefix. The follower's post-converge prefix must equal the
|
||||
* leader's byte-for-byte; Log Matching holds.
|
||||
*
|
||||
* Load-bearing intermediate assertions:
|
||||
* - pre-replication, node 1 holds the WRONG (term-2) entries at indices 1..3,
|
||||
* differing from node 0's term-3 entries (so a no-op delivery would FAIL
|
||||
* the final byte-equality / Log Matching check).
|
||||
* - the conflict at index 1 triggers a truncation back-off, and node 1's
|
||||
* final terms equal node 0's (proves the divergent suffix was deleted, not
|
||||
* merely overwritten in place across a mismatching prefix).
|
||||
*/
|
||||
static void test_sim_9_2_divergent_suffix_truncation(void)
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
/* Elect node 0 leader, then advance its term to 3 (strictly higher than the
|
||||
* follower's manufactured divergent term 2). raft_log_append stamps each new
|
||||
* entry with s->term, so setting the term before appending makes the leader's
|
||||
* entries carry term 3. next_index/match_index were initialized at
|
||||
* become_leader and are term-independent, so they remain valid. */
|
||||
raft_start_election(&c.nodes[0].state); /* term -> 1 */
|
||||
raft_sim_deliver_pending(&c);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
c.nodes[0].state.term = 3;
|
||||
uint64_t leader_term = c.nodes[0].state.term;
|
||||
TEST_ASSERT_TRUE(leader_term >= 3);
|
||||
|
||||
unsigned char lp[3][3] = {
|
||||
{ 'L', '0', 0 }, { 'L', '1', 0 }, { 'L', '2', 0 } };
|
||||
for (int e = 0; e < 3; e++) {
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, lp[e], 2, &out_index));
|
||||
TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index);
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count);
|
||||
for (int i = 0; i < 3; i++)
|
||||
TEST_ASSERT_EQUAL_UINT64(leader_term, c.nodes[0].log[i].term);
|
||||
|
||||
/* Manufacture node 1's DIVERGENT tail: same indices 1..3, term 2, "X" bytes.
|
||||
* Persist directly into the sim log (same indices, lower/different term). */
|
||||
for (int i = 0; i < 3; i++) {
|
||||
c.nodes[1].log[i].term = 2;
|
||||
c.nodes[1].log[i].bytes[0] = 'X';
|
||||
c.nodes[1].log[i].bytes[1] = (unsigned char)('0' + i);
|
||||
c.nodes[1].log[i].len = 2;
|
||||
}
|
||||
c.nodes[1].log_count = 3;
|
||||
/* Make node 1 a follower whose term is at most the leader's (so it accepts). */
|
||||
raft_on_append_entries_request(&c.nodes[1].state, 0, leader_term,
|
||||
0, 0, 0, NULL, 0, 0, 0, &(raft_append_result_t){0});
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, c.nodes[1].state.role);
|
||||
|
||||
/* Intermediate: node 1 currently DIVERGES from node 0 at every index. */
|
||||
TEST_ASSERT_TRUE(c.nodes[1].log[0].term != c.nodes[0].log[0].term);
|
||||
|
||||
/* Replicate from the higher-term leader. next_index[1] starts at Len(log)+1
|
||||
* = 4; the leader walks back, the conflict branch truncates the divergent
|
||||
* suffix, then appends the leader's entries. Drive bounded rounds. */
|
||||
int rounds = 0;
|
||||
while (rounds < 64) {
|
||||
sim_replicate_one(&c, 0, 1, 0);
|
||||
rounds++;
|
||||
if (c.nodes[1].log_count == c.nodes[0].log_count) {
|
||||
int all_match = 1;
|
||||
for (int i = 0; i < c.nodes[0].log_count; i++)
|
||||
if (c.nodes[1].log[i].term != c.nodes[0].log[i].term)
|
||||
all_match = 0;
|
||||
if (all_match)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Converged: node 1's prefix now EQUALS node 0's byte-for-byte (the divergent
|
||||
* term-2 suffix was truncated and replaced with the leader's term-N prefix). */
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[1].log[i].term);
|
||||
TEST_ASSERT_EQUAL_size_t(c.nodes[0].log[i].len, c.nodes[1].log[i].len);
|
||||
TEST_ASSERT_EQUAL_MEMORY(c.nodes[0].log[i].bytes, c.nodes[1].log[i].bytes,
|
||||
c.nodes[0].log[i].len);
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c));
|
||||
}
|
||||
|
||||
/*
|
||||
* test_sim_9_3_drop_dup_reorder_convergence (task 9.3; Req 8.3)
|
||||
*
|
||||
* Drop, duplicate, and reorder AppendEntries in the sim queue and assert
|
||||
* idempotent convergence: a duplicated AE is a no-op via the already-have
|
||||
* branch, a dropped AE is recovered by a later round, and reordering does not
|
||||
* corrupt the log. Log Matching holds after the faults clear.
|
||||
*
|
||||
* Load-bearing intermediate assertions:
|
||||
* - after the deliberately-DROPPED round, node 1 is still missing the entry
|
||||
* (proves the drop was real, not silently delivered).
|
||||
* - after delivering a DUPLICATE of an already-applied AE, node 1's log_count
|
||||
* does NOT grow (idempotent already-have branch).
|
||||
* - final Log Matching holds and node 1 holds the full prefix.
|
||||
*/
|
||||
static void test_sim_9_3_drop_dup_reorder_convergence(void)
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
raft_sim_deliver_pending(&c);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
/* Append 3 entries on the leader. */
|
||||
for (int e = 0; e < 3; e++) {
|
||||
unsigned char payload[2] = { (unsigned char)('a' + e), 0x00 };
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index));
|
||||
TEST_ASSERT_EQUAL_UINT64((uint64_t)(e + 1), out_index);
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count);
|
||||
|
||||
/* DROP: buffer an AE for entry 1 to node 1, then clear the queue WITHOUT
|
||||
* delivering (simulate a dropped message). */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_send_append_entries(&c.nodes[0].state, 1, /*leader_commit=*/0));
|
||||
TEST_ASSERT_EQUAL_INT(1, c.ae_pending_count);
|
||||
c.ae_pending_count = 0; /* DROP the buffered AE */
|
||||
|
||||
/* Intermediate: the drop was real — node 1 received nothing. */
|
||||
TEST_ASSERT_EQUAL_INT(0, c.nodes[1].log_count);
|
||||
|
||||
/* Recover the dropped entry via a fresh round. */
|
||||
sim_replicate_one(&c, 0, 1, 0);
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[1].log_count);
|
||||
|
||||
/* DUPLICATE: rewind the leader's belief so it re-sends the ALREADY-PRESENT
|
||||
* entry 1 (next_index[1] = 1 ⇒ prev_log_index = 0, entry index 1). Buffer it
|
||||
* twice and deliver both; both deliveries hit the already-have branch (the
|
||||
* incoming entry matches node 1's local entry 1), so the log must NOT grow. */
|
||||
c.nodes[0].state.next_index[1] = 1;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_send_append_entries(&c.nodes[0].state, 1, /*leader_commit=*/0));
|
||||
TEST_ASSERT_TRUE(c.ae_pending_count >= 1);
|
||||
/* Manually duplicate the just-buffered AE (re-sends entry 1). */
|
||||
if (c.ae_pending_count < RAFT_SIM_MAX_AE) {
|
||||
c.ae_pending[c.ae_pending_count] = c.ae_pending[c.ae_pending_count - 1];
|
||||
c.ae_pending_count++;
|
||||
}
|
||||
raft_sim_deliver_append_entries(&c);
|
||||
/* Idempotent: the duplicated already-have AE did not grow the log. */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[1].log_count);
|
||||
|
||||
/* REORDER (genuine out-of-order delivery): node 1 currently holds only
|
||||
* entry 1. Buffer the AE for entry 2 (prev_log_index=1) and the AE for
|
||||
* entry 3 (prev_log_index=2), then SWAP them so entry 3 is delivered FIRST.
|
||||
* The entry-3 AE references a prev (index 2) node 1 does not yet have, so
|
||||
* logOk fails (prev_log_index > log_len) ⇒ it is rejected without creating a
|
||||
* gap; the entry-2 AE then appends normally. This proves reordered AEs cannot
|
||||
* insert out-of-order (Req 8.3 reorder). */
|
||||
c.ae_pending_count = 0; /* start the reorder round with a clean queue */
|
||||
c.nodes[0].state.next_index[1] = 2; /* prev=1 → entry 2 */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK, raft_send_append_entries(&c.nodes[0].state, 1, 0));
|
||||
c.nodes[0].state.next_index[1] = 3; /* prev=2 → entry 3 */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK, raft_send_append_entries(&c.nodes[0].state, 1, 0));
|
||||
TEST_ASSERT_EQUAL_INT(2, c.ae_pending_count);
|
||||
/* Swap so the entry-3 AE is delivered before the entry-2 AE. */
|
||||
{
|
||||
raft_sim_ae_t tmp = c.ae_pending[0];
|
||||
c.ae_pending[0] = c.ae_pending[1];
|
||||
c.ae_pending[1] = tmp;
|
||||
}
|
||||
raft_sim_deliver_append_entries(&c);
|
||||
/* The out-of-order entry-3 AE was rejected (no gap); only entry 2 appended. */
|
||||
TEST_ASSERT_EQUAL_INT(2, c.nodes[1].log_count);
|
||||
|
||||
/* Now deliver remaining rounds (entry 3) under continued churn; bounded loop
|
||||
* drives to convergence regardless of transient reject/back-off. */
|
||||
int rounds = 0;
|
||||
while (c.nodes[1].log_count < c.nodes[0].log_count && rounds < 64) {
|
||||
sim_replicate_one(&c, 0, 1, 0);
|
||||
rounds++;
|
||||
}
|
||||
|
||||
/* Converged despite drop + dup + back-off; Log Matching holds. */
|
||||
TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count);
|
||||
for (int i = 0; i < 3; i++)
|
||||
TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[1].log[i].term);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c));
|
||||
}
|
||||
|
||||
/*
|
||||
* test_sim_9_4_chaotic_churn_g5_regression (task 9.4; Req 8.6 — bug-851)
|
||||
*
|
||||
* A stale-term node (term lagging) rejoins under a higher-term leader; assert it
|
||||
* adopts the higher term from the AE/heartbeat path (UpdateTerm on AppendEntries,
|
||||
* G5, task 5/6) and does NOT re-elect at a reused term. Log Matching holds across
|
||||
* the churn. THIS is the bug-851 regression: silent committed-prefix divergence
|
||||
* from two leaders at a reused term. It relies on UpdateTerm-on-AE and the
|
||||
* SEND-time leader term in delivery (task 8.2).
|
||||
*
|
||||
* Load-bearing intermediate assertions:
|
||||
* - pre-rejoin, the stale node's term is STRICTLY LESS than the leader's
|
||||
* (so a no-op heartbeat would leave them divergent and the assertion real).
|
||||
* - after the FIRST heartbeat, the rejoining node's term EQUALS the leader's
|
||||
* (term adopted from the AE path) AND its role is FOLLOWER (stepped down).
|
||||
* - no SECOND leader appears at the reused term (leader_count unchanged after
|
||||
* the stale node is denied a re-election at its own old term).
|
||||
*/
|
||||
static void test_sim_9_4_chaotic_churn_g5_regression(void)
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
/* Elect node 0 leader and advance its term to model accumulated leadership
|
||||
* churn — node 0 now leads at a high term while a partitioned node 2 lags. */
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
raft_sim_deliver_pending(&c);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
c.nodes[0].state.term = 5;
|
||||
uint64_t leader_term = c.nodes[0].state.term;
|
||||
TEST_ASSERT_TRUE(leader_term >= 3);
|
||||
|
||||
/* Leader has a couple of entries. */
|
||||
for (int e = 0; e < 2; e++) {
|
||||
unsigned char payload[2] = { (unsigned char)('K' + e), 0x00 };
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index));
|
||||
}
|
||||
|
||||
/* Node 2 is a STALE-TERM rejoiner: force its term low (1) and role follower
|
||||
* (as if it was partitioned away during the churn and never advanced). */
|
||||
c.nodes[2].state.term = 1;
|
||||
c.nodes[2].state.role = RAFT_FOLLOWER;
|
||||
c.nodes[2].state.voted_for = -1;
|
||||
|
||||
/* Intermediate: stale node's term is strictly below the leader's. */
|
||||
TEST_ASSERT_TRUE(c.nodes[2].state.term < leader_term);
|
||||
|
||||
int leader_count_before = c.leader_count;
|
||||
|
||||
/* Send a HEARTBEAT (n_entries handled by send AE: next_index[2] = Len(log)+1
|
||||
* so prev_log_index points past node 2's empty log; this is the AE/heartbeat
|
||||
* path). UpdateTerm must fire on this AE: node 2 adopts leader_term. */
|
||||
sim_replicate_one(&c, 0, 2, 0);
|
||||
|
||||
/* G5: node 2 adopted the higher term from the AppendEntries/heartbeat path
|
||||
* and stepped down to follower — NOT only from a vote RPC. */
|
||||
TEST_ASSERT_EQUAL_UINT64(leader_term, c.nodes[2].state.term);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, c.nodes[2].state.role);
|
||||
|
||||
/* UpdateTerm cleared the stale node's vote state: voted_for reset to -1 so it
|
||||
* cannot double-vote in the new term, and any vote it casts now belongs to
|
||||
* leader_term (>= the term it would have reused). This is the concrete state
|
||||
* that prevents the reused-term double-leader of bug-851. */
|
||||
TEST_ASSERT_EQUAL_INT(-1, c.nodes[2].state.voted_for);
|
||||
|
||||
/* No second leader was created during the rejoin/heartbeat delivery. */
|
||||
TEST_ASSERT_EQUAL_INT(leader_count_before, c.leader_count);
|
||||
|
||||
/* The original leader is still the sole leader. */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
/* Drive catch-up so node 2 converges; Log Matching holds across the churn. */
|
||||
int rounds = 0;
|
||||
while (c.nodes[2].log_count < c.nodes[0].log_count && rounds < 64) {
|
||||
sim_replicate_one(&c, 0, 2, 0);
|
||||
rounds++;
|
||||
}
|
||||
TEST_ASSERT_EQUAL_INT(c.nodes[0].log_count, c.nodes[2].log_count);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c));
|
||||
|
||||
/* Active check (after convergence, so it cannot disturb catch-up): if the
|
||||
* rejoined node now starts an election it campaigns at a term STRICTLY ABOVE
|
||||
* leader_term — never the stale term it lagged at — so it can never resurrect
|
||||
* a committed-prefix divergence by reusing an old term (bug-851). */
|
||||
raft_start_election(&c.nodes[2].state);
|
||||
TEST_ASSERT_TRUE(c.nodes[2].state.term > leader_term);
|
||||
}
|
||||
|
||||
/* ---------- Group A: Basic Election Correctness (task 5.1) ---------- */
|
||||
|
||||
/*
|
||||
|
|
@ -5444,6 +5863,10 @@ void test_raft(void)
|
|||
RUN_TEST(test_sim_deliver_append_entries_replicates);
|
||||
RUN_TEST(test_sim_check_log_matching_pass_on_converged);
|
||||
RUN_TEST(test_sim_check_log_matching_fail_on_divergence);
|
||||
RUN_TEST(test_sim_9_1_replication_and_partition_catch_up);
|
||||
RUN_TEST(test_sim_9_2_divergent_suffix_truncation);
|
||||
RUN_TEST(test_sim_9_3_drop_dup_reorder_convergence);
|
||||
RUN_TEST(test_sim_9_4_chaotic_churn_g5_regression);
|
||||
|
||||
/* Group A: basic election correctness (task 5.1) */
|
||||
RUN_TEST(test_proto_A1_single_node_self_elects);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue