diff --git a/.kiro/specs/raft-library/tasks.md b/.kiro/specs/raft-library/tasks.md index 102c643d..623402a2 100644 --- a/.kiro/specs/raft-library/tasks.md +++ b/.kiro/specs/raft-library/tasks.md @@ -128,7 +128,7 @@ - All 7 Unity test cases pass; trace buffer confirms `on_cancel_timeout` fires in E1 and E2 - _Requirements: 4.1, 4.2, 4.3, 4.4, 6.1, 6.2, 6.3, 8.2, 8.3_ -- [ ] 5.4 Test Groups F and G: RDHUP failover and split-brain prevention (6 cases) +- [x] 5.4 Test Groups F and G: RDHUP failover and split-brain prevention (6 cases) - F1: nodes 1 and 2 call `raft_on_leader_lost`; mock `on_start_election_after_delay` receives `1×timeout` for node 1 and `2×timeout` for node 2 (priority-based jitter) - F2: during node 1's delay, node 1 receives and grants a vote request from another node → `voted_for != self_idx`; `raft_check_and_start_election` returns `RAFT_SKIP_ELECTION` - F3: `raft_on_leader_lost` followed by `raft_check_and_start_election` with no intervening vote grant → returns `RAFT_ELECTION_STARTED`; `state.role==RAFT_CANDIDATE` diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c index dcfea37d..ab6bf4f9 100644 --- a/src/lib/testsuite/test_raft.c +++ b/src/lib/testsuite/test_raft.c @@ -2584,6 +2584,256 @@ static void test_proto_E3_new_election_after_timeout(void) TEST_ASSERT_EQUAL_INT(2, g_mock.schedule_timeout_count); } +/* ---------- Group F: RDHUP Failover (task 5.4) ---------- */ + +/* + * test_proto_F1_leader_lost_priority_delay + * + * Verify that on_start_election_after_delay fires with delay = my_priority * election_timeout_ms. + * Node 1 (priority=1, timeout=1000) → delay=1000ms. + * Node 2 (priority=2, timeout=1000) → delay=2000ms. + */ +static void test_proto_F1_leader_lost_priority_delay(void) +{ + /* --- Node 1: priority=1, election_timeout_ms=1000 → expected delay=1000 --- */ + { + raft_state_t s; + raft_callbacks_t cbs = make_mock_cbs(); + raft_config_t cfg; + + reset_mock(); + memset(&cfg, 0, sizeof(cfg)); + cfg.n_peers = 3; + cfg.self_idx = 1; + cfg.my_priority = 1; + cfg.election_timeout_ms = 1000; + cfg.initial_voted_for = -1; + cfg.cbs = &cbs; + TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg)); + + raft_on_leader_lost(&s); + + TEST_ASSERT_EQUAL_INT(1, g_mock.start_election_after_delay_count); + TEST_ASSERT_EQUAL_UINT32(1000, g_mock.last_start_election_delay); + } + + /* --- Node 2: priority=2, election_timeout_ms=1000 → expected delay=2000 --- */ + { + raft_state_t s; + raft_callbacks_t cbs = make_mock_cbs(); + raft_config_t cfg; + + reset_mock(); + memset(&cfg, 0, sizeof(cfg)); + cfg.n_peers = 3; + cfg.self_idx = 2; + cfg.my_priority = 2; + cfg.election_timeout_ms = 1000; + cfg.initial_voted_for = -1; + cfg.cbs = &cbs; + TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg)); + + raft_on_leader_lost(&s); + + TEST_ASSERT_EQUAL_INT(1, g_mock.start_election_after_delay_count); + TEST_ASSERT_EQUAL_UINT32(2000, g_mock.last_start_election_delay); + } +} + +/* + * test_proto_F2_voted_during_delay_skip + * + * Node calls raft_on_leader_lost (voted_for=-1). + * During the jitter delay, a vote request arrives from node 0 at term=1. + * Node grants the vote (voted_for=0). + * Then raft_check_and_start_election is called. + * Since voted_for==0 != self_idx==1, must return RAFT_SKIP_ELECTION. + */ +static void test_proto_F2_voted_during_delay_skip(void) +{ + raft_state_t s; + raft_callbacks_t cbs = make_mock_cbs(); + raft_config_t cfg; + uint64_t reply_term = 0; + + reset_mock(); + memset(&cfg, 0, sizeof(cfg)); + cfg.n_peers = 3; + cfg.self_idx = 1; + cfg.my_priority = 1; + cfg.election_timeout_ms = 1000; + cfg.initial_voted_for = -1; + cfg.cbs = &cbs; + TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg)); + + /* Leader lost: clears voted_for to -1, schedules delayed election */ + raft_on_leader_lost(&s); + TEST_ASSERT_EQUAL_INT(-1, s.voted_for); + + /* During the delay, a vote request arrives from node 0 at term=1 */ + int rc = raft_on_vote_request(&s, 0, 1, &reply_term); + TEST_ASSERT_EQUAL_INT(RAFT_GRANT, rc); + TEST_ASSERT_EQUAL_INT(0, s.voted_for); + + /* Timer fires: check-and-start sees voted_for=0 != self_idx=1 → skip */ + int check_rc = raft_check_and_start_election(&s); + TEST_ASSERT_EQUAL_INT(RAFT_SKIP_ELECTION, check_rc); + + /* Role must stay FOLLOWER — no election started */ + TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, s.role); +} + +/* + * test_proto_F3_no_vote_during_delay_starts_election + * + * Node 0 (priority=0, self_idx=0) calls raft_on_leader_lost. + * No vote arrives during the delay (voted_for stays -1). + * raft_check_and_start_election must return RAFT_ELECTION_STARTED and role==CANDIDATE. + */ +static void test_proto_F3_no_vote_during_delay_starts_election(void) +{ + raft_state_t s; + raft_callbacks_t cbs = make_mock_cbs(); + raft_config_t cfg; + + reset_mock(); + memset(&cfg, 0, sizeof(cfg)); + cfg.n_peers = 3; + cfg.self_idx = 0; + cfg.my_priority = 0; + cfg.election_timeout_ms = 1000; + cfg.initial_voted_for = -1; + cfg.cbs = &cbs; + TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg)); + + /* Leader lost: voted_for cleared to -1 */ + raft_on_leader_lost(&s); + TEST_ASSERT_EQUAL_INT(-1, s.voted_for); + + /* No vote arrived during the delay */ + int rc = raft_check_and_start_election(&s); + + TEST_ASSERT_EQUAL_INT(RAFT_ELECTION_STARTED, rc); + TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, s.role); +} + +/* + * test_proto_F4_simultaneous_rdhup_one_leader + * + * Nodes 1 and 2 both receive RDHUP (leader/node 0 lost). + * Both call raft_on_leader_lost → voted_for=-1 for both. + * Node 1 (lower idx, lower priority) calls check_and_start first → ELECTION_STARTED. + * Node 1 sends vote requests; deliver them to node 2. + * Node 2 grants node 1 (voted_for=1). + * Node 2 then calls check_and_start → RAFT_SKIP_ELECTION (voted_for=1 != self_idx=2). + * Result: leader_count==1. + */ +static void test_proto_F4_simultaneous_rdhup_one_leader(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Simulate: node 0 was the leader, now lost. Nodes 1 and 2 see RDHUP. */ + raft_on_leader_lost(&c.nodes[1].state); + raft_on_leader_lost(&c.nodes[2].state); + + /* Both have voted_for=-1 now */ + TEST_ASSERT_EQUAL_INT(-1, c.nodes[1].state.voted_for); + TEST_ASSERT_EQUAL_INT(-1, c.nodes[2].state.voted_for); + + /* Node 1 fires its delayed timer first (lower priority = smaller delay) */ + int rc1 = raft_check_and_start_election(&c.nodes[1].state); + TEST_ASSERT_EQUAL_INT(RAFT_ELECTION_STARTED, rc1); + TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, c.nodes[1].state.role); + + /* Deliver node 1's vote requests to the cluster (node 2 gets one, node 0 is gone) */ + /* Only deliver the RPC intended for node 2; skip node 0 (it's gone/isolated) */ + raft_sim_partition(&c, 0, 1); /* node 0 is isolated (lost leader) */ + raft_sim_deliver_pending(&c); + + /* Node 2 received node 1's vote request and granted it */ + TEST_ASSERT_EQUAL_INT(1, c.nodes[2].state.voted_for); + + /* Node 2's timer fires: check_and_start sees voted_for=1 != self_idx=2 → skip */ + int rc2 = raft_check_and_start_election(&c.nodes[2].state); + TEST_ASSERT_EQUAL_INT(RAFT_SKIP_ELECTION, rc2); + + /* Exactly one leader (node 1) elected */ + TEST_ASSERT_EQUAL_INT(1, c.leader_count); +} + +/* ---------- Group G: Split-Brain Prevention (task 5.4) ---------- */ + +/* + * test_proto_G1_no_split_brain_100_rounds + * + * Run 100 independent election rounds on a fresh 3-node cluster each time. + * Node 0 always starts first. After delivering pending RPCs, leader_count must + * equal exactly 1 (no split brain, no starvation). + */ +static void test_proto_G1_no_split_brain_100_rounds(void) +{ + int total_leaders = 0; + + for (int round = 0; round < 100; round++) { + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Node 0 always starts first (highest priority, idx=0) */ + raft_start_election(&c.nodes[0].state); + raft_sim_deliver_pending(&c); + + /* Each round must produce exactly one leader — no split brain */ + TEST_ASSERT_EQUAL_INT(1, c.leader_count); + total_leaders += c.leader_count; + } + + /* Across 100 rounds, exactly 100 leaders must have been elected */ + TEST_ASSERT_EQUAL_INT(100, total_leaders); +} + +/* + * test_proto_G2_partition_minority_no_quorum + * + * 3-node cluster. Node 2 is isolated (partition of 1). + * Nodes 0+1 form the majority partition (size=2 = quorum). + * Node 0 starts election in the majority partition → wins (leader_count=1). + * Node 2 tries to start election in isolation → RPCs dropped → cannot win. + * leader_count remains 1. + */ +static void test_proto_G2_partition_minority_no_quorum(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Isolate node 2 (minority partition of 1) */ + raft_sim_partition(&c, 2, 1); + + /* Node 0 starts election in the majority partition {0, 1} */ + raft_start_election(&c.nodes[0].state); + raft_sim_deliver_pending(&c); + + /* Nodes 0+1 have quorum=2; node 0 must be LEADER */ + TEST_ASSERT_EQUAL_INT(1, c.leader_count); + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + + /* Node 2 attempts to start an election in isolation */ + raft_start_election(&c.nodes[2].state); + raft_sim_deliver_pending(&c); /* RPCs to/from node 2 are dropped */ + + /* Node 2 cannot win — still CANDIDATE (no grants received) */ + TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, c.nodes[2].state.role); + + /* leader_count still 1 — no split brain */ + TEST_ASSERT_EQUAL_INT(1, c.leader_count); +} + #endif /* XCE_TESTING */ /* ---------- test_raft entry point ---------- */ @@ -2689,5 +2939,15 @@ void test_raft(void) RUN_TEST(test_proto_E1_timeout_before_quorum); RUN_TEST(test_proto_E2_timeout_after_quorum_noop); RUN_TEST(test_proto_E3_new_election_after_timeout); + + /* Group F: RDHUP Failover (task 5.4) */ + RUN_TEST(test_proto_F1_leader_lost_priority_delay); + RUN_TEST(test_proto_F2_voted_during_delay_skip); + RUN_TEST(test_proto_F3_no_vote_during_delay_starts_election); + RUN_TEST(test_proto_F4_simultaneous_rdhup_one_leader); + + /* Group G: Split-Brain Prevention (task 5.4) */ + RUN_TEST(test_proto_G1_no_split_brain_100_rounds); + RUN_TEST(test_proto_G2_partition_minority_no_quorum); #endif }