feat(raft-sim-conformance): partition/heal adversarial scenario (task 5.1)
This commit is contained in:
parent
b844e75f3b
commit
7d0964dd16
4 changed files with 364 additions and 1 deletions
|
|
@ -971,6 +971,27 @@ int raft_sim_step(raft_sim_cluster_t *c);
|
|||
*/
|
||||
int raft_sim_run(raft_sim_cluster_t *c, int max_steps);
|
||||
|
||||
/*
|
||||
* raft_sim_scenario_partition_heal: network-partition scenario (raft-sim-conformance
|
||||
* task 5.1; Req 4.1). Initializes the caller's cluster from (n, election_timeout=2000,
|
||||
* seed), then:
|
||||
* 1. drives it to a single leader + a committed prefix,
|
||||
* 2. isolates a MINORITY of nodes (floor((n-1)/2), the highest indices) via
|
||||
* raft_sim_partition(..., 1) during a phase of client appends/replication while
|
||||
* the MAJORITY keeps making progress (the minority cannot win a new-term
|
||||
* election — the library enforces it),
|
||||
* 3. HEALS the partition (raft_sim_partition(..., 0)) and drains the bag (bounded)
|
||||
* so the formerly-isolated nodes catch up.
|
||||
* raft_sim_check_all is run after EVERY action it applies — DURING the partitioned
|
||||
* phase and after heal/convergence — so a violation is caught the moment it appears.
|
||||
* Returns 0 if the scenario completes with ZERO invariant violations; < 0 (the negated
|
||||
* first failing invariant, matching raft_sim_step) on any violation, after dumping the
|
||||
* replayable trace to stderr. Returns -1 on an invalid node count (n < 3 cannot host a
|
||||
* minority partition with a surviving majority). Drives only via published library
|
||||
* entry points + harness primitives — no protocol re-implementation.
|
||||
*/
|
||||
int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed);
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
#endif /* RAFT_H */
|
||||
|
|
|
|||
|
|
@ -1268,4 +1268,254 @@ int raft_sim_run(raft_sim_cluster_t *c, int max_steps)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* ---------- partition/heal scenario (raft-sim-conformance task 5.1) ---------- */
|
||||
|
||||
/*
|
||||
* sim_scen_check: run all four invariants over `c`. On a violation, dump the
|
||||
* replayable trace to stderr and return the negated failing invariant (matching the
|
||||
* raft_sim_step / raft_sim_run contract); 0 when all hold. The scenario calls this
|
||||
* after EVERY action it applies so a violation is surfaced the moment it appears
|
||||
* (Req 4.1 — invariants asserted THROUGHOUT, not only at the end).
|
||||
*/
|
||||
static int sim_scen_check(raft_sim_cluster_t *c)
|
||||
{
|
||||
raft_sim_invariant_t failing = raft_sim_check_all(c);
|
||||
if (failing != RAFT_SIM_INV_NONE) {
|
||||
raft_sim_dump_trace(c, stderr, failing);
|
||||
return -(int)failing;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* sim_scen_drain_bag: deliver every in-flight bag message (PRNG-picked order, so a
|
||||
* partition leaves partitioned-endpoint messages parked rather than delivered),
|
||||
* running the invariant check after EACH delivery. `cap` bounds the work so a
|
||||
* duplicate/heal storm can never spin unbounded. Stops early and propagates < 0 on
|
||||
* the first violation. Returns 0 when the bag drains (or only undeliverable,
|
||||
* partitioned messages remain) with no violation.
|
||||
*/
|
||||
static int sim_scen_drain_bag(raft_sim_cluster_t *c, int cap)
|
||||
{
|
||||
for (int g = 0; g < cap && c->bag_count > 0; g++) {
|
||||
int i = raft_sim_bag_pick(c);
|
||||
if (i < 0)
|
||||
break;
|
||||
|
||||
int rc = raft_sim_bag_deliver(c, i);
|
||||
/* rc == -2 means the picked message targets a partitioned endpoint and was
|
||||
* left in the bag (a heal can deliver it later). To avoid re-picking the same
|
||||
* parked message forever, sweep for ANY deliverable message; if none remain,
|
||||
* the bag holds only partitioned traffic and we are done draining for now. */
|
||||
if (rc == -2) {
|
||||
int progressed = 0;
|
||||
for (int j = 0; j < c->bag_count; j++) {
|
||||
int s = c->bag[j].src, d = c->bag[j].dst;
|
||||
if (!c->nodes[s].partition && !c->nodes[d].partition) {
|
||||
if (raft_sim_bag_deliver(c, j) >= 0)
|
||||
progressed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!progressed)
|
||||
break; /* only partitioned traffic left */
|
||||
}
|
||||
|
||||
int chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* sim_scen_replicate_round: on the current leader, append one client value, send
|
||||
* AppendEntries to every other node, drain the bag, then recompute commit — checking
|
||||
* invariants after every action. Non-leaders and partitioned peers are handled by the
|
||||
* library / bag delivery (a partitioned peer's AE simply never delivers). Returns 0 on
|
||||
* a clean round, < 0 on the first invariant violation.
|
||||
*/
|
||||
static int sim_scen_replicate_round(raft_sim_cluster_t *c, int leader, uint64_t value)
|
||||
{
|
||||
uint64_t out_index = 0;
|
||||
(void)raft_log_append(&c->nodes[leader].state, &value, sizeof(value), &out_index);
|
||||
int chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
for (int p = 0; p < c->n; p++) {
|
||||
if (p == leader)
|
||||
continue;
|
||||
(void)raft_send_append_entries(&c->nodes[leader].state, p,
|
||||
c->nodes[leader].state.commit_index);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
}
|
||||
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
(void)raft_advance_commit(&c->nodes[leader].state);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* Commit advance can release a higher leader_commit; replicate it out so followers
|
||||
* apply the newly-committed entries, then re-check. */
|
||||
for (int p = 0; p < c->n; p++) {
|
||||
if (p == leader)
|
||||
continue;
|
||||
(void)raft_send_append_entries(&c->nodes[leader].state, p,
|
||||
c->nodes[leader].state.commit_index);
|
||||
}
|
||||
return sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
|
||||
}
|
||||
|
||||
int raft_sim_scenario_partition_heal(raft_sim_cluster_t *c, int n, uint64_t seed)
|
||||
{
|
||||
/* A minority partition with a surviving majority needs n >= 3 (floor((n-1)/2) >= 1
|
||||
* isolated, with a strict majority still connected). */
|
||||
if (n < 3 || n > RAFT_SIM_MAX_NODES)
|
||||
return -1;
|
||||
|
||||
if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
|
||||
return -1;
|
||||
|
||||
int chk;
|
||||
|
||||
/* ---- 1. Elect a leader: node 0 campaigns; drain the vote traffic. ---- */
|
||||
(void)raft_start_election(&c->nodes[0].state);
|
||||
chk = sim_scen_check(c);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
int leader = sim_find_leader(c);
|
||||
if (leader < 0) {
|
||||
/* No leader emerged — retry a couple of campaigns (still all via the library).
|
||||
* If the cluster genuinely cannot elect, that is observable, not a violation;
|
||||
* the post-scenario single-leader assertion in the test will catch it. */
|
||||
for (int retry = 0; retry < 3 && leader < 0; retry++) {
|
||||
(void)raft_start_election(&c->nodes[retry % n].state);
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
}
|
||||
if (leader < 0)
|
||||
return 0; /* no violation occurred; nothing more to drive */
|
||||
}
|
||||
|
||||
/* ---- 2. Replicate + commit a few entries with the cluster fully connected. ---- */
|
||||
for (int r = 0; r < 3; r++) {
|
||||
chk = sim_scen_replicate_round(c, leader, 0x100ULL + (uint64_t)r);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ---- 3. Isolate a MINORITY (highest indices); majority keeps progressing. ---- */
|
||||
int minority = (n - 1) / 2; /* < n/2, so the remaining nodes are a strict majority */
|
||||
for (int m = 0; m < minority; m++)
|
||||
raft_sim_partition(c, n - 1 - m, 1);
|
||||
|
||||
/* If the isolated set happened to include the leader, re-elect within the majority
|
||||
* so the partitioned phase still makes progress on the connected side. */
|
||||
{
|
||||
int leader_isolated = c->nodes[leader].partition;
|
||||
if (leader_isolated) {
|
||||
leader = -1;
|
||||
for (int cand = 0; cand < n - minority && leader < 0; cand++) {
|
||||
if (c->nodes[cand].partition)
|
||||
continue;
|
||||
(void)raft_start_election(&c->nodes[cand].state);
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
/* Only accept a CONNECTED leader (a stale isolated leader does not count). */
|
||||
if (leader >= 0 && c->nodes[leader].partition)
|
||||
leader = -1;
|
||||
}
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* Drive a phase of client appends/replication while partitioned. The majority must
|
||||
* keep committing; AE to isolated peers simply never delivers. A minority node also
|
||||
* times out and campaigns — it must NOT be able to win a higher-term election (the
|
||||
* library enforces quorum; the harness just drives + checks). */
|
||||
for (int r = 0; r < 4; r++) {
|
||||
chk = sim_scen_replicate_round(c, leader, 0x200ULL + (uint64_t)r);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* A minority node campaigns mid-partition (cannot reach a majority of votes). */
|
||||
if (minority > 0) {
|
||||
(void)raft_start_election(&c->nodes[n - 1].state);
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
}
|
||||
|
||||
leader = sim_find_leader(c);
|
||||
if (leader < 0 || c->nodes[leader].partition)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ---- 4. HEAL the partition and drain everything so the minority catches up. ---- */
|
||||
for (int m = 0; m < minority; m++)
|
||||
raft_sim_partition(c, n - 1 - m, 0);
|
||||
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
|
||||
/* Draining the parked minority traffic (those nodes campaigned at higher terms while
|
||||
* isolated) can step the connected leader down via UpdateTerm — correct library
|
||||
* behavior. Re-establish a single leader over the now-fully-connected cluster before
|
||||
* catch-up: campaign one node at a time, draining each round, until exactly one
|
||||
* leader holds. This is the natural Raft recovery, driven only via library entry
|
||||
* points. */
|
||||
leader = sim_find_leader(c);
|
||||
for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) {
|
||||
(void)raft_start_election(&c->nodes[attempt % n].state);
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
}
|
||||
if (leader < 0)
|
||||
return 0; /* no leader emerged — observable, not a violation; test asserts it */
|
||||
|
||||
/* Catch-up rounds: re-replicate to ALL nodes (now reachable) until the formerly
|
||||
* isolated followers converge to the leader's committed prefix. Re-establish the
|
||||
* leader inside the loop if a late vote-request steps it down again. Bounded. */
|
||||
for (int r = 0; r < n + 4; r++) {
|
||||
chk = sim_scen_replicate_round(c, leader, 0x300ULL + (uint64_t)r);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
for (int attempt = 0; attempt < 2 * n && leader < 0; attempt++) {
|
||||
(void)raft_start_election(&c->nodes[attempt % n].state);
|
||||
chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
|
||||
if (chk < 0)
|
||||
return chk;
|
||||
leader = sim_find_leader(c);
|
||||
}
|
||||
if (leader < 0)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ---- 5. Final convergence check (also covered after every step above). ---- */
|
||||
return sim_scen_check(c);
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue