feat(raft-library): task 5.4 — Groups F+G, 6 test cases (RDHUP failover + split-brain)
This commit is contained in:
parent
5c677bc52d
commit
6bdf7edacf
2 changed files with 261 additions and 1 deletions
|
|
@ -128,7 +128,7 @@
|
|||
- All 7 Unity test cases pass; trace buffer confirms `on_cancel_timeout` fires in E1 and E2
|
||||
- _Requirements: 4.1, 4.2, 4.3, 4.4, 6.1, 6.2, 6.3, 8.2, 8.3_
|
||||
|
||||
- [ ] 5.4 Test Groups F and G: RDHUP failover and split-brain prevention (6 cases)
|
||||
- [x] 5.4 Test Groups F and G: RDHUP failover and split-brain prevention (6 cases)
|
||||
- F1: nodes 1 and 2 call `raft_on_leader_lost`; mock `on_start_election_after_delay` receives `1×timeout` for node 1 and `2×timeout` for node 2 (priority-based jitter)
|
||||
- F2: during node 1's delay, node 1 receives and grants a vote request from another node → `voted_for != self_idx`; `raft_check_and_start_election` returns `RAFT_SKIP_ELECTION`
|
||||
- F3: `raft_on_leader_lost` followed by `raft_check_and_start_election` with no intervening vote grant → returns `RAFT_ELECTION_STARTED`; `state.role==RAFT_CANDIDATE`
|
||||
|
|
|
|||
|
|
@ -2584,6 +2584,256 @@ static void test_proto_E3_new_election_after_timeout(void)
|
|||
TEST_ASSERT_EQUAL_INT(2, g_mock.schedule_timeout_count);
|
||||
}
|
||||
|
||||
/* ---------- Group F: RDHUP Failover (task 5.4) ---------- */
|
||||
|
||||
/*
|
||||
* test_proto_F1_leader_lost_priority_delay
|
||||
*
|
||||
* Verify that on_start_election_after_delay fires with delay = my_priority * election_timeout_ms.
|
||||
* Node 1 (priority=1, timeout=1000) → delay=1000ms.
|
||||
* Node 2 (priority=2, timeout=1000) → delay=2000ms.
|
||||
*/
|
||||
static void test_proto_F1_leader_lost_priority_delay(void)
|
||||
{
|
||||
/* --- Node 1: priority=1, election_timeout_ms=1000 → expected delay=1000 --- */
|
||||
{
|
||||
raft_state_t s;
|
||||
raft_callbacks_t cbs = make_mock_cbs();
|
||||
raft_config_t cfg;
|
||||
|
||||
reset_mock();
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.n_peers = 3;
|
||||
cfg.self_idx = 1;
|
||||
cfg.my_priority = 1;
|
||||
cfg.election_timeout_ms = 1000;
|
||||
cfg.initial_voted_for = -1;
|
||||
cfg.cbs = &cbs;
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg));
|
||||
|
||||
raft_on_leader_lost(&s);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(1, g_mock.start_election_after_delay_count);
|
||||
TEST_ASSERT_EQUAL_UINT32(1000, g_mock.last_start_election_delay);
|
||||
}
|
||||
|
||||
/* --- Node 2: priority=2, election_timeout_ms=1000 → expected delay=2000 --- */
|
||||
{
|
||||
raft_state_t s;
|
||||
raft_callbacks_t cbs = make_mock_cbs();
|
||||
raft_config_t cfg;
|
||||
|
||||
reset_mock();
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.n_peers = 3;
|
||||
cfg.self_idx = 2;
|
||||
cfg.my_priority = 2;
|
||||
cfg.election_timeout_ms = 1000;
|
||||
cfg.initial_voted_for = -1;
|
||||
cfg.cbs = &cbs;
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg));
|
||||
|
||||
raft_on_leader_lost(&s);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(1, g_mock.start_election_after_delay_count);
|
||||
TEST_ASSERT_EQUAL_UINT32(2000, g_mock.last_start_election_delay);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* test_proto_F2_voted_during_delay_skip
|
||||
*
|
||||
* Node calls raft_on_leader_lost (voted_for=-1).
|
||||
* During the jitter delay, a vote request arrives from node 0 at term=1.
|
||||
* Node grants the vote (voted_for=0).
|
||||
* Then raft_check_and_start_election is called.
|
||||
* Since voted_for==0 != self_idx==1, must return RAFT_SKIP_ELECTION.
|
||||
*/
|
||||
static void test_proto_F2_voted_during_delay_skip(void)
|
||||
{
|
||||
raft_state_t s;
|
||||
raft_callbacks_t cbs = make_mock_cbs();
|
||||
raft_config_t cfg;
|
||||
uint64_t reply_term = 0;
|
||||
|
||||
reset_mock();
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.n_peers = 3;
|
||||
cfg.self_idx = 1;
|
||||
cfg.my_priority = 1;
|
||||
cfg.election_timeout_ms = 1000;
|
||||
cfg.initial_voted_for = -1;
|
||||
cfg.cbs = &cbs;
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg));
|
||||
|
||||
/* Leader lost: clears voted_for to -1, schedules delayed election */
|
||||
raft_on_leader_lost(&s);
|
||||
TEST_ASSERT_EQUAL_INT(-1, s.voted_for);
|
||||
|
||||
/* During the delay, a vote request arrives from node 0 at term=1 */
|
||||
int rc = raft_on_vote_request(&s, 0, 1, &reply_term);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_GRANT, rc);
|
||||
TEST_ASSERT_EQUAL_INT(0, s.voted_for);
|
||||
|
||||
/* Timer fires: check-and-start sees voted_for=0 != self_idx=1 → skip */
|
||||
int check_rc = raft_check_and_start_election(&s);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_SKIP_ELECTION, check_rc);
|
||||
|
||||
/* Role must stay FOLLOWER — no election started */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, s.role);
|
||||
}
|
||||
|
||||
/*
|
||||
* test_proto_F3_no_vote_during_delay_starts_election
|
||||
*
|
||||
* Node 0 (priority=0, self_idx=0) calls raft_on_leader_lost.
|
||||
* No vote arrives during the delay (voted_for stays -1).
|
||||
* raft_check_and_start_election must return RAFT_ELECTION_STARTED and role==CANDIDATE.
|
||||
*/
|
||||
static void test_proto_F3_no_vote_during_delay_starts_election(void)
|
||||
{
|
||||
raft_state_t s;
|
||||
raft_callbacks_t cbs = make_mock_cbs();
|
||||
raft_config_t cfg;
|
||||
|
||||
reset_mock();
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.n_peers = 3;
|
||||
cfg.self_idx = 0;
|
||||
cfg.my_priority = 0;
|
||||
cfg.election_timeout_ms = 1000;
|
||||
cfg.initial_voted_for = -1;
|
||||
cfg.cbs = &cbs;
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_init(&s, &cfg));
|
||||
|
||||
/* Leader lost: voted_for cleared to -1 */
|
||||
raft_on_leader_lost(&s);
|
||||
TEST_ASSERT_EQUAL_INT(-1, s.voted_for);
|
||||
|
||||
/* No vote arrived during the delay */
|
||||
int rc = raft_check_and_start_election(&s);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_ELECTION_STARTED, rc);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, s.role);
|
||||
}
|
||||
|
||||
/*
|
||||
* test_proto_F4_simultaneous_rdhup_one_leader
|
||||
*
|
||||
* Nodes 1 and 2 both receive RDHUP (leader/node 0 lost).
|
||||
* Both call raft_on_leader_lost → voted_for=-1 for both.
|
||||
* Node 1 (lower idx, lower priority) calls check_and_start first → ELECTION_STARTED.
|
||||
* Node 1 sends vote requests; deliver them to node 2.
|
||||
* Node 2 grants node 1 (voted_for=1).
|
||||
* Node 2 then calls check_and_start → RAFT_SKIP_ELECTION (voted_for=1 != self_idx=2).
|
||||
* Result: leader_count==1.
|
||||
*/
|
||||
static void test_proto_F4_simultaneous_rdhup_one_leader(void)
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
/* Simulate: node 0 was the leader, now lost. Nodes 1 and 2 see RDHUP. */
|
||||
raft_on_leader_lost(&c.nodes[1].state);
|
||||
raft_on_leader_lost(&c.nodes[2].state);
|
||||
|
||||
/* Both have voted_for=-1 now */
|
||||
TEST_ASSERT_EQUAL_INT(-1, c.nodes[1].state.voted_for);
|
||||
TEST_ASSERT_EQUAL_INT(-1, c.nodes[2].state.voted_for);
|
||||
|
||||
/* Node 1 fires its delayed timer first (lower priority = smaller delay) */
|
||||
int rc1 = raft_check_and_start_election(&c.nodes[1].state);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_ELECTION_STARTED, rc1);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, c.nodes[1].state.role);
|
||||
|
||||
/* Deliver node 1's vote requests to the cluster (node 2 gets one, node 0 is gone) */
|
||||
/* Only deliver the RPC intended for node 2; skip node 0 (it's gone/isolated) */
|
||||
raft_sim_partition(&c, 0, 1); /* node 0 is isolated (lost leader) */
|
||||
raft_sim_deliver_pending(&c);
|
||||
|
||||
/* Node 2 received node 1's vote request and granted it */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[2].state.voted_for);
|
||||
|
||||
/* Node 2's timer fires: check_and_start sees voted_for=1 != self_idx=2 → skip */
|
||||
int rc2 = raft_check_and_start_election(&c.nodes[2].state);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_SKIP_ELECTION, rc2);
|
||||
|
||||
/* Exactly one leader (node 1) elected */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.leader_count);
|
||||
}
|
||||
|
||||
/* ---------- Group G: Split-Brain Prevention (task 5.4) ---------- */
|
||||
|
||||
/*
|
||||
* test_proto_G1_no_split_brain_100_rounds
|
||||
*
|
||||
* Run 100 independent election rounds on a fresh 3-node cluster each time.
|
||||
* Node 0 always starts first. After delivering pending RPCs, leader_count must
|
||||
* equal exactly 1 (no split brain, no starvation).
|
||||
*/
|
||||
static void test_proto_G1_no_split_brain_100_rounds(void)
|
||||
{
|
||||
int total_leaders = 0;
|
||||
|
||||
for (int round = 0; round < 100; round++) {
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
/* Node 0 always starts first (highest priority, idx=0) */
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
raft_sim_deliver_pending(&c);
|
||||
|
||||
/* Each round must produce exactly one leader — no split brain */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.leader_count);
|
||||
total_leaders += c.leader_count;
|
||||
}
|
||||
|
||||
/* Across 100 rounds, exactly 100 leaders must have been elected */
|
||||
TEST_ASSERT_EQUAL_INT(100, total_leaders);
|
||||
}
|
||||
|
||||
/*
|
||||
* test_proto_G2_partition_minority_no_quorum
|
||||
*
|
||||
* 3-node cluster. Node 2 is isolated (partition of 1).
|
||||
* Nodes 0+1 form the majority partition (size=2 = quorum).
|
||||
* Node 0 starts election in the majority partition → wins (leader_count=1).
|
||||
* Node 2 tries to start election in isolation → RPCs dropped → cannot win.
|
||||
* leader_count remains 1.
|
||||
*/
|
||||
static void test_proto_G2_partition_minority_no_quorum(void)
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000));
|
||||
|
||||
/* Isolate node 2 (minority partition of 1) */
|
||||
raft_sim_partition(&c, 2, 1);
|
||||
|
||||
/* Node 0 starts election in the majority partition {0, 1} */
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
raft_sim_deliver_pending(&c);
|
||||
|
||||
/* Nodes 0+1 have quorum=2; node 0 must be LEADER */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.leader_count);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
/* Node 2 attempts to start an election in isolation */
|
||||
raft_start_election(&c.nodes[2].state);
|
||||
raft_sim_deliver_pending(&c); /* RPCs to/from node 2 are dropped */
|
||||
|
||||
/* Node 2 cannot win — still CANDIDATE (no grants received) */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, c.nodes[2].state.role);
|
||||
|
||||
/* leader_count still 1 — no split brain */
|
||||
TEST_ASSERT_EQUAL_INT(1, c.leader_count);
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ---------- test_raft entry point ---------- */
|
||||
|
|
@ -2689,5 +2939,15 @@ void test_raft(void)
|
|||
RUN_TEST(test_proto_E1_timeout_before_quorum);
|
||||
RUN_TEST(test_proto_E2_timeout_after_quorum_noop);
|
||||
RUN_TEST(test_proto_E3_new_election_after_timeout);
|
||||
|
||||
/* Group F: RDHUP Failover (task 5.4) */
|
||||
RUN_TEST(test_proto_F1_leader_lost_priority_delay);
|
||||
RUN_TEST(test_proto_F2_voted_during_delay_skip);
|
||||
RUN_TEST(test_proto_F3_no_vote_during_delay_starts_election);
|
||||
RUN_TEST(test_proto_F4_simultaneous_rdhup_one_leader);
|
||||
|
||||
/* Group G: Split-Brain Prevention (task 5.4) */
|
||||
RUN_TEST(test_proto_G1_no_split_brain_100_rounds);
|
||||
RUN_TEST(test_proto_G2_partition_minority_no_quorum);
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue