diff --git a/.kiro/specs/raft-log-replication/tasks.md b/.kiro/specs/raft-log-replication/tasks.md index 30e622d..e7dd260 100644 --- a/.kiro/specs/raft-log-replication/tasks.md +++ b/.kiro/specs/raft-log-replication/tasks.md @@ -201,5 +201,5 @@ ## Implementation Notes - **Validation (2026-06-21): GO.** Clean ASAN build green from scratch (`xce-build.sh lib`, Fail 0); 8/8 requirement sections covered (40/41 acceptance criteria); boundary ALIGNED (no controller files; no commit advance / vote freshness / transport I/O / entry-bytes in library state); out-of-scope items confirmed absent. -- **Known minor gap — Req 8.3 "node restart while replication proceeds" (PARTIAL):** drop/dup/reorder are fully covered by 9.3, and the `initial_log_len` restore primitive is tested at init; 9.4's stale-rejoin covers the equivalent observable state. But no dedicated simulator test chains "restart a sim node mid-replication (re-init from restored log_len) then resume AE delivery" — the sim has no in-place single-node restart helper. Low risk; a follow-up could add a `raft_sim_restart_node` + test to close it strictly. +- **Req 8.3 "node restart while replication proceeds" — CLOSED (2026-06-21):** added `raft_sim_restart_node` (durable log[]/(term,voted_for) survive; volatile consensus state cleared via raft_init → FOLLOWER) + `test_sim_9_5_node_restart_resumes_replication` (restart a follower mid-replication with a partial log; assert log_count preserved — load-bearing: would fail if the log were wiped — role/term restored, replication resumes, Log Matching holds). drop/dup/reorder remain covered by 9.3. All 8/8 sections now fully covered. - **Adapter invariant (from validation LOW note):** the AE accept already-have branch treats `on_get_entry_term(index) == entry_term` as a match, which is also true when both are 0. A correct leader never sends `entry_term == 0` for a real entry (`raft_log_append` stamps `s->term >= 1`), so this is safe in-library — but the controller adapter (raft-commit-safety) MUST guarantee it never delivers `entry_term == 0` with `entries_len == 1`. diff --git a/src/lib/raft/raft.h b/src/lib/raft/raft.h index d61bcd2..8f7033e 100644 --- a/src/lib/raft/raft.h +++ b/src/lib/raft/raft.h @@ -513,6 +513,18 @@ void raft_sim_partition(raft_sim_cluster_t *c, int node_idx, int isolated); */ int raft_sim_check_log_matching(const raft_sim_cluster_t *c); +/* + * raft_sim_restart_node: model a process restart of node_idx (Req 8.3). + * + * Durable state survives a restart: the in-memory log[] (which models the WAL on + * disk) and the persisted (term, voted_for) are retained. Volatile consensus + * state (role, votes, election bookkeeping, leader progress) is cleared by + * re-running raft_init with the restored (term, voted_for, log_count), returning + * the node to FOLLOWER. The four callbacks, ctx, idx, and partition flag are + * preserved. Returns raft_init's status (0 on success). + */ +int raft_sim_restart_node(raft_sim_cluster_t *c, int node_idx); + #endif /* XCE_TESTING */ #endif /* RAFT_H */ diff --git a/src/lib/raft/raft_sim.c b/src/lib/raft/raft_sim.c index 545e74f..1d8ef39 100644 --- a/src/lib/raft/raft_sim.c +++ b/src/lib/raft/raft_sim.c @@ -413,4 +413,37 @@ int raft_sim_check_log_matching(const raft_sim_cluster_t *c) return 1; /* holds */ } +int raft_sim_restart_node(raft_sim_cluster_t *c, int node_idx) +{ + if (node_idx < 0 || node_idx >= c->n) + return -1; + + raft_sim_node_t *node = &c->nodes[node_idx]; + + /* Snapshot the DURABLE state that survives a process restart: the persisted + * (term, voted_for) and the log length (the in-memory log[] models the WAL on + * disk and is left untouched). Volatile consensus state is discarded by the + * re-init below. */ + uint64_t restored_term = node->state.term; + int restored_voted_for = node->state.voted_for; + uint64_t restored_log_len = (uint64_t)node->log_count; + + raft_config_t cfg; + memset(&cfg, 0, sizeof(cfg)); + cfg.n_peers = node->state.n_peers; + cfg.self_idx = node->idx; + cfg.my_priority = node->idx; + cfg.election_timeout_ms = node->state.election_timeout_ms; + cfg.initial_term = restored_term; + cfg.initial_voted_for = restored_voted_for; + cfg.initial_log_len = restored_log_len; + cfg.cbs = &node->cbs; + cfg.ctx = node; /* same ctx convention as raft_sim_init */ + + /* raft_init returns the node to FOLLOWER and clears votes/election/leader + * progress while restoring (term, voted_for, log_len). The log[] array and + * the partition flag are preserved. */ + return raft_init(&node->state, &cfg); +} + #endif /* XCE_TESTING */ diff --git a/src/lib/testsuite/test_raft.c b/src/lib/testsuite/test_raft.c index 5007097..08a4f78 100644 --- a/src/lib/testsuite/test_raft.c +++ b/src/lib/testsuite/test_raft.c @@ -3613,6 +3613,75 @@ static void test_sim_9_4_chaotic_churn_g5_regression(void) TEST_ASSERT_TRUE(c.nodes[2].state.term > leader_term); } +/* + * test_sim_9_5_node_restart_resumes_replication (task 9.x; Req 8.3 — node restart + * while replication proceeds) + * + * A follower restarts MID-replication: its volatile consensus state is lost but + * its durable log (the in-memory log[], modelling the WAL) and persisted + * (term, voted_for) survive. Replication then resumes and converges with Log + * Matching intact. This closes the "node restart" sub-clause of Req 8.3 that the + * drop/dup/reorder test (9.3) did not chain. + * + * Load-bearing assertions: + * - immediately after restart the follower STILL holds its partial log + * (log_count preserved, NOT wiped to 0) and is back to FOLLOWER at the same + * durable term — proving the restart cleared volatile state while keeping the + * durable log; + * - replication resumes from the partial prefix and the follower converges to + * the leader's full log with Log Matching holding. + */ +static void test_sim_9_5_node_restart_resumes_replication(void) +{ + raft_sim_cluster_t c; + memset(&c, 0, sizeof(c)); + + TEST_ASSERT_EQUAL_INT(0, raft_sim_init(&c, 3, 2000)); + + /* Elect node 0 leader. */ + raft_start_election(&c.nodes[0].state); + raft_sim_deliver_pending(&c); + TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role); + + /* Leader appends 3 entries. */ + for (int e = 0; e < 3; e++) { + unsigned char payload[2] = { (unsigned char)('p' + e), 0x00 }; + uint64_t out_index = 0; + TEST_ASSERT_EQUAL_INT(RAFT_OK, + raft_log_append(&c.nodes[0].state, payload, sizeof(payload), &out_index)); + } + TEST_ASSERT_EQUAL_INT(3, c.nodes[0].log_count); + + /* Replicate PARTIALLY to node 1 (entries 1 and 2), leaving entry 3 pending. */ + sim_replicate_one(&c, 0, 1, 0); + sim_replicate_one(&c, 0, 1, 0); + TEST_ASSERT_EQUAL_INT(2, c.nodes[1].log_count); + uint64_t durable_term = c.nodes[1].state.term; + + /* RESTART node 1 mid-replication. */ + TEST_ASSERT_EQUAL_INT(0, raft_sim_restart_node(&c, 1)); + + /* Durable log survived the restart (NOT wiped); volatile state reset to a + * fresh FOLLOWER at the restored term and log length. */ + TEST_ASSERT_EQUAL_INT(2, c.nodes[1].log_count); + TEST_ASSERT_EQUAL_INT(RAFT_FOLLOWER, c.nodes[1].state.role); + TEST_ASSERT_EQUAL_UINT64(durable_term, c.nodes[1].state.term); + TEST_ASSERT_EQUAL_UINT64(2, c.nodes[1].state.log_len); /* restored from log_count */ + + /* Resume replication; bounded loop drives the rebooted follower to converge. */ + int rounds = 0; + while (c.nodes[1].log_count < c.nodes[0].log_count && rounds < 64) { + sim_replicate_one(&c, 0, 1, 0); + rounds++; + } + + /* Converged across the restart; Log Matching holds. */ + TEST_ASSERT_EQUAL_INT(3, c.nodes[1].log_count); + for (int i = 0; i < 3; i++) + TEST_ASSERT_EQUAL_UINT64(c.nodes[0].log[i].term, c.nodes[1].log[i].term); + TEST_ASSERT_EQUAL_INT(1, raft_sim_check_log_matching(&c)); +} + /* ---------- Group A: Basic Election Correctness (task 5.1) ---------- */ /* @@ -5867,6 +5936,7 @@ void test_raft(void) RUN_TEST(test_sim_9_2_divergent_suffix_truncation); RUN_TEST(test_sim_9_3_drop_dup_reorder_convergence); RUN_TEST(test_sim_9_4_chaotic_churn_g5_regression); + RUN_TEST(test_sim_9_5_node_restart_resumes_replication); /* Group A: basic election correctness (task 5.1) */ RUN_TEST(test_proto_A1_single_node_self_elects);