feat(raft-sim-conformance): restart-churn adversarial scenario (task 5.2)
This commit is contained in:
parent
7d0964dd16
commit
6976392d77
4 changed files with 304 additions and 1 deletions
|
|
@ -992,6 +992,27 @@ int raft_sim_run(raft_sim_cluster_t *c, int max_steps);
|
|||
*/
|
||||
int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed);
|
||||
|
||||
/*
|
||||
* raft_sim_scenario_restart_churn: process-restart scenario (raft-sim-conformance
|
||||
* task 5.2; Req 4.2). Initializes the caller's cluster from (n, election_timeout=2000,
|
||||
* seed), then:
|
||||
* 1. drives it to a single leader + a committed prefix,
|
||||
* 2. MID-replication, restarts one or more nodes (raft_sim_restart_node) — a follower
|
||||
* holding a partial log AND the leader (forcing a re-election). The persisted
|
||||
* (term, votedFor, log) survives each restart (task 4.3 guarantees this); the
|
||||
* restart resets only volatile state to a fresh FOLLOWER,
|
||||
* 3. re-elects if the leader was restarted, replicates more, and PROPAGATES COMMIT
|
||||
* (a follow-up AppendEntries round carrying the higher leader_commit) so the
|
||||
* restarted nodes catch up and their committed/applied prefix advances.
|
||||
* raft_sim_check_all is run after EVERY action it applies — DURING and after the churn —
|
||||
* so a violation is caught the moment it appears. Returns 0 if the scenario completes
|
||||
* with ZERO invariant violations; < 0 (the negated first failing invariant, matching
|
||||
* raft_sim_step) on any violation, after dumping the replayable trace to stderr. Returns
|
||||
* -1 on an invalid node count (n < 3). Drives only via published library entry points +
|
||||
* harness primitives — no protocol re-implementation.
|
||||
*/
|
||||
int raft_sim_scenario_restart_churn(raft_sim_cluster_t *c, int n, uint64_t seed);
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
#endif /* RAFT_H */
|
||||
|
|
|
|||
|
|
@ -1518,4 +1518,181 @@ int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed
|
|||
return sim_scen_check(c);
|
||||
}
|
||||
|
||||
/* ---------- restart-churn scenario (raft-sim-conformance task 5.2) ---------- */
|
||||
|
||||
/*
|
||||
* sim_scen_elect: campaign one node at a time (raft_start_election), draining the bag
|
||||
* after each, until a single LEADER emerges or the bounded attempts are exhausted.
|
||||
* Returns the leader index (>= 0), -1 if none emerged, or < 0 propagated from
|
||||
* sim_scen_check on an invariant violation (callers distinguish: a violation value is
|
||||
* always <= -(int)RAFT_SIM_INV_NONE - 1 < -1, but to keep this simple the caller treats
|
||||
* any returned violation by re-checking; here we surface the violation via *viol). On a
|
||||
* violation *viol holds the negated invariant and the return is -1.
|
||||
*/
|
||||
static int sim_scen_elect(raft_sim_cluster_t *c, int *viol)
|
||||
{
|
||||
*viol = 0;
|
||||
int leader = sim_find_leader(c);
|
||||
for (int attempt = 0; attempt < 2 * c->n && leader < 0; attempt++) {
|
||||
(void)raft_start_election(&c->nodes[attempt % c->n].state);
|
||||
int chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0) {
|
||||
*viol = chk;
|
||||
return -1;
|
||||
}
|
||||
leader = sim_find_leader(c);
|
||||
}
|
||||
return leader;
|
||||
}
|
||||
|
||||
int raft_sim_scenario_restart_churn(raft_sim_cluster_t *c, int n, uint64_t seed)
|
||||
{
|
||||
if (n < 3 || n > RAFT_SIM_MAX_NODES)
|
||||
return -1;
|
||||
|
||||
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
|
||||
return -1;
|
||||
|
||||
int chk, viol;
|
||||
|
||||
/* ---- 1. Elect a leader and replicate a committed prefix. ---- */
|
||||
int leader = sim_scen_elect(c, &viol);
|
||||
if (viol < 0)
|
||||
return viol;
|
||||
if (leader < 0)
|
||||
return 0; /* no leader emerged — observable, not a violation; test asserts it */
|
||||
|
||||
for (int r = 0; r < 3; r++) {
|
||||
chk = sim_scen_replicate_round(c, leader, 0x500ULL + (uint64_t)r);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
if (leader < 0) {
|
||||
leader = sim_scen_elect(c, &viol);
|
||||
if (viol < 0)
|
||||
return viol;
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- 2. Restart a FOLLOWER mid-replication (it holds a partial log). ---- */
|
||||
int follower = -1;
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (i != leader) {
|
||||
follower = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
/* Append one more entry on the leader and send it out, but DO NOT fully drain —
|
||||
* leave replication in flight, then restart the follower so its restart lands
|
||||
* mid-replication (it may hold only a partial prefix). */
|
||||
{
|
||||
uint64_t value = 0x5A0ULL;
|
||||
uint64_t out_index = 0;
|
||||
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
for (int p = 0; p < n; p++) {
|
||||
if (p == leader)
|
||||
continue;
|
||||
(void)raft_send_append_entries(&c->nodes[leader].state, p,
|
||||
c->nodes[leader].state.commit_index);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
}
|
||||
}
|
||||
|
||||
(void)raft_sim_restart_node(c, follower); /* preserves persisted (term, votedFor, log) */
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* Drain in-flight traffic; the restarted follower (now a fresh FOLLOWER) re-syncs
|
||||
* via AppendEntries. */
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* Re-establish a leader if the in-flight churn stepped the current one down. */
|
||||
leader = sim_find_leader(c);
|
||||
if (leader < 0) {
|
||||
leader = sim_scen_elect(c, &viol);
|
||||
if (viol < 0)
|
||||
return viol;
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ---- 3. Restart the LEADER mid-replication, forcing a re-election. ---- */
|
||||
{
|
||||
uint64_t value = 0x5B0ULL;
|
||||
uint64_t out_index = 0;
|
||||
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
for (int p = 0; p < n; p++) {
|
||||
if (p == leader)
|
||||
continue;
|
||||
(void)raft_send_append_entries(&c->nodes[leader].state, p,
|
||||
c->nodes[leader].state.commit_index);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
}
|
||||
}
|
||||
|
||||
(void)raft_sim_restart_node(c, leader); /* persisted state survives; volatile reset */
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* The old leader is now a FOLLOWER; a survivor must win a fresh election. */
|
||||
leader = sim_scen_elect(c, &viol);
|
||||
if (viol < 0)
|
||||
return viol;
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
|
||||
/* ---- 4. Catch-up rounds: replicate + propagate commit so EVERY node (including the
|
||||
* two formerly-restarted ones) converges to the leader's log and advances its
|
||||
* committed/applied prefix. Re-establish the leader inside the loop if a late
|
||||
* vote-request steps it down. Bounded. ---- */
|
||||
for (int r = 0; r < n + 4; r++) {
|
||||
chk = sim_scen_replicate_round(c, leader, 0x600ULL + (uint64_t)r);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
if (leader < 0) {
|
||||
leader = sim_scen_elect(c, &viol);
|
||||
if (viol < 0)
|
||||
return viol;
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* A final heartbeat round so any follower lagging on leader_commit observes the
|
||||
* latest committed position and advances its applied prefix (commit propagation). */
|
||||
for (int p = 0; p < n; p++) {
|
||||
if (p == leader)
|
||||
continue;
|
||||
(void)raft_send_append_entries(&c->nodes[leader].state, p,
|
||||
c->nodes[leader].state.commit_index);
|
||||
}
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* ---- 5. Final convergence check (also covered after every step above). ---- */
|
||||
return sim_scen_check(c);
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue