feat(raft-apply-decoupling): task 7.1 apply-ordering conformance assertions
Extend raft_sim.c's deterministic TLA+-conformance harness with two fault-injectable checks: no reaction before commit, and idempotent re-apply of an already-committed entry. Wired as dedicated tests only, not into raft_sim_check_all's fixed four-invariant contract (owned by a separate spec). RED/GREEN independently verified by adversarial review; zero regression to existing scenario functions.
This commit is contained in:
parent
2de410c877
commit
5f98d1b0ef
4 changed files with 347 additions and 5 deletions
|
|
@ -8832,6 +8832,184 @@ static void test_sim_7_2_negative_both_directions(void)
|
|||
* (broken_rc < 0, Log Matching) AND the correct build passes (correct_rc == 0). */
|
||||
}
|
||||
|
||||
/*
|
||||
* test_sim_apply_decoupling_apply_after_commit_ordering
|
||||
* (raft-apply-decoupling task 7.1; Req 10.1)
|
||||
*
|
||||
* New conformance-harness assertion, layered on the existing apply-after-
|
||||
* on_commit_advanced loop: no entry's state mutation/reaction (the harness's apply
|
||||
* model, node->applied[]) may occur before that entry commits (node->commit_index
|
||||
* reaching its index). Proven in BOTH directions within one test, mirroring the
|
||||
* task 7.1/7.2 fault-injection idiom used above for the four raft.tla invariants:
|
||||
*
|
||||
* (1) CORRECT (fault OFF, the shipped sim_conf_on_commit_advanced adapter):
|
||||
* a persisted-but-not-yet-committed entry is NOT applied; once replicated
|
||||
* to a quorum and committed, it IS applied — raft_sim_check_apply_after_commit
|
||||
* holds throughout.
|
||||
* (2) BROKEN (RAFT_SIM_BREAK_APPLY_BEFORE_COMMIT set): the SAME persist step
|
||||
* eagerly marks the entry applied while commit_index is still 0 — the SAME
|
||||
* check now FAILS, proving the assertion has real teeth, not a vacuous pass.
|
||||
*/
|
||||
static void test_sim_apply_decoupling_apply_after_commit_ordering(void)
|
||||
{
|
||||
/* ---- (1) CORRECT: fault OFF — apply never precedes commit ---- */
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 3, 2000, 0xA0715ULL));
|
||||
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
uint64_t value = 0x99AAULL;
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index));
|
||||
TEST_ASSERT_EQUAL_UINT64(1, out_index);
|
||||
|
||||
/* Persisted but not yet replicated/committed: nothing applied yet, and the
|
||||
* ordering check holds (vacuously — nothing to violate it with). */
|
||||
TEST_ASSERT_EQUAL_UINT64(0, c.nodes[0].applied[0]);
|
||||
TEST_ASSERT_EQUAL_UINT64(0, c.nodes[0].commit_index);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_apply_after_commit(&c));
|
||||
|
||||
/* Replicate to one follower (2 of 3 = quorum) and commit. */
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_send_append_entries(&c.nodes[0].state, 1, 0));
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
raft_sim_advance_commit(&c, 0);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, c.nodes[0].commit_index);
|
||||
|
||||
/* Now applied, and ordering still holds. */
|
||||
TEST_ASSERT_EQUAL_UINT64(value, c.nodes[0].applied[0]);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_apply_after_commit(&c));
|
||||
}
|
||||
|
||||
/* ---- (2) BROKEN: RAFT_SIM_BREAK_APPLY_BEFORE_COMMIT — persisted-but-uncommitted
|
||||
* entry is nonetheless marked applied; the SAME check now fails. ---- */
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 3, 2000, 0xA0715ULL));
|
||||
raft_sim_set_fault_inject(&c, RAFT_SIM_BREAK_APPLY_BEFORE_COMMIT);
|
||||
TEST_ASSERT_EQUAL_UINT(RAFT_SIM_BREAK_APPLY_BEFORE_COMMIT, c.fault_inject);
|
||||
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
uint64_t value = 0x99AAULL;
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index));
|
||||
TEST_ASSERT_EQUAL_UINT64(1, out_index);
|
||||
|
||||
/* Never replicated, never committed — commit_index is still 0 ... */
|
||||
TEST_ASSERT_EQUAL_UINT64(0, c.nodes[0].commit_index);
|
||||
/* ... yet the corrupted adapter already marked it applied. */
|
||||
TEST_ASSERT_EQUAL_UINT64(value, c.nodes[0].applied[0]);
|
||||
|
||||
/* The oracle has teeth: this deliberately-broken build FAILS the SAME check
|
||||
* that passed throughout the correct-code block above. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, raft_sim_check_apply_after_commit(&c),
|
||||
"BROKEN config (apply-before-commit) did not trip the ordering check — "
|
||||
"the assertion lacks teeth");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* test_sim_apply_decoupling_idempotent_reapply
|
||||
* (raft-apply-decoupling task 7.1; Req 10.3)
|
||||
*
|
||||
* New conformance-harness assertion: re-applying an already-committed entry
|
||||
* (whether a genuine duplicate on_commit_advanced fire or a forced
|
||||
* raft_sim_replay_commit retry) is idempotent — its apply-time reaction fires
|
||||
* exactly once per index, never twice. Proven in BOTH directions:
|
||||
*
|
||||
* (1) CORRECT (fault OFF): commit one entry, then force a REPLAY of the SAME
|
||||
* commit event via raft_sim_replay_commit — applied[]/commit_index stay
|
||||
* byte-identical and raft_sim_check_idempotent_reapply holds.
|
||||
* (2) BROKEN (RAFT_SIM_BREAK_REAPPLY_GUARD set): the SAME replay re-executes
|
||||
* the apply-time reaction a second time — the SAME check now FAILS.
|
||||
*/
|
||||
static void test_sim_apply_decoupling_idempotent_reapply(void)
|
||||
{
|
||||
/* ---- (1) CORRECT: fault OFF — replay of an already-applied commit is a no-op ---- */
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 3, 2000, 0xB0715ULL));
|
||||
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
uint64_t value = 0x77BBULL;
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index));
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_send_append_entries(&c.nodes[0].state, 1, 0));
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
raft_sim_advance_commit(&c, 0);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, c.nodes[0].commit_index);
|
||||
TEST_ASSERT_EQUAL_UINT64(value, c.nodes[0].applied[0]);
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[0].apply_reaction_count[0]);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_idempotent_reapply(&c));
|
||||
|
||||
/* Force a REPLAY of the already-processed commit (simulated retry/duplicate
|
||||
* notification), bypassing the library's own fire-once gate. */
|
||||
raft_sim_replay_commit(&c, 0);
|
||||
|
||||
/* No residual mutation: applied[]/commit_index unchanged, reaction still 1. */
|
||||
TEST_ASSERT_EQUAL_UINT64(1, c.nodes[0].commit_index);
|
||||
TEST_ASSERT_EQUAL_UINT64(value, c.nodes[0].applied[0]);
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[0].apply_reaction_count[0]);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_idempotent_reapply(&c));
|
||||
}
|
||||
|
||||
/* ---- (2) BROKEN: RAFT_SIM_BREAK_REAPPLY_GUARD — replay re-fires the reaction ---- */
|
||||
{
|
||||
raft_sim_cluster_t c;
|
||||
memset(&c, 0, sizeof(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 3, 2000, 0xB0715ULL));
|
||||
|
||||
raft_start_election(&c.nodes[0].state);
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
|
||||
|
||||
uint64_t value = 0x77BBULL;
|
||||
uint64_t out_index = 0;
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index));
|
||||
TEST_ASSERT_EQUAL_INT(RAFT_OK,
|
||||
raft_send_append_entries(&c.nodes[0].state, 1, 0));
|
||||
for (int g = 0; g < 64 && c.bag_count > 0; g++)
|
||||
raft_sim_bag_deliver(&c, 0);
|
||||
raft_sim_advance_commit(&c, 0);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, c.nodes[0].commit_index);
|
||||
TEST_ASSERT_EQUAL_INT(1, c.nodes[0].apply_reaction_count[0]);
|
||||
TEST_ASSERT_EQUAL_INT(1, raft_sim_check_idempotent_reapply(&c));
|
||||
|
||||
/* NOW arm the fault and replay — the corrupted adapter re-fires the reaction. */
|
||||
raft_sim_set_fault_inject(&c, RAFT_SIM_BREAK_REAPPLY_GUARD);
|
||||
TEST_ASSERT_EQUAL_UINT(RAFT_SIM_BREAK_REAPPLY_GUARD, c.fault_inject);
|
||||
raft_sim_replay_commit(&c, 0);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(2, c.nodes[0].apply_reaction_count[0]);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, raft_sim_check_idempotent_reapply(&c),
|
||||
"BROKEN config (reapply guard disabled) did not trip the idempotency "
|
||||
"check — the assertion lacks teeth");
|
||||
}
|
||||
}
|
||||
|
||||
#define RAFT_SIM_SWEEP_M 32 /* configured minimum number of randomized seeds (R6.2) */
|
||||
#define RAFT_SIM_SWEEP_STEPS 400 /* per-run step cap — bounded for the suite budget (R6.4) */
|
||||
|
||||
|
|
@ -9129,6 +9307,8 @@ void test_raft(void)
|
|||
RUN_TEST(test_sim_5_5_stale_rejoin);
|
||||
RUN_TEST(test_sim_7_1_fault_switch_mechanism);
|
||||
RUN_TEST(test_sim_7_2_negative_both_directions);
|
||||
RUN_TEST(test_sim_apply_decoupling_apply_after_commit_ordering);
|
||||
RUN_TEST(test_sim_apply_decoupling_idempotent_reapply);
|
||||
RUN_TEST(test_sim_6_1_seed_sweep);
|
||||
|
||||
/* Group A: basic election correctness (task 5.1) */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue