wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(raft-sim-conformance): runtime fault-injection switch (G1 term-stamp, G3 truncate) (task 7.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-24 16:15:13 +00:00
parent 7087dc19d9
commit 62523892d9
4 changed files with 156 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.kiro/specs/raft-sim-conformance/tasks.md
View file

@ -185,7 +185,7 @@
## 7. Deliberately-broken-build negative test (the oracle's teeth)
- [ ] 7.1 RED+GREEN: compile-time fault switch emulates the bug-851 defects in the adapter
- [x] 7.1 RED+GREEN: compile-time fault switch emulates the bug-851 defects in the adapter
- Implement `RAFT_SIM_BREAK_TERM_STAMP` (adapter `on_persist_entry` writes term 0) and `RAFT_SIM_BREAK_TRUNCATE` (adapter `on_truncate_from` no-op), confined to the test/harness build configuration; default off
- Observable completion: with a macro off, scenarios are clean; the macro is not set in the default CI run and does not touch `raft.c`
- _Requirements: 7.1, 7.3_

31
src/lib/raft/raft.h
View file

@ -455,6 +455,22 @@ void raft_snapshot(const raft_state_t *s, raft_snapshot_t *snap);
* failure traces. One raft_sim_action_t is recorded per step. */
#define RAFT_SIM_MAX_STEPS 1024
/* ---- deliberately-broken-build fault switch (raft-sim-conformance task 7.1) ----
* Runtime-togglable fault-injection flags, confined to this XCE_TESTING build and
* DEFAULT OFF (cluster.fault_inject is zero-initialized by the init routines, so
* neither legacy nor conformance runs are perturbed, and the default CI conformance
* sweep is unaffected — Req 7.3). The corruption lives ONLY in the harness ADAPTER
* callbacks (raft_sim.c); raft.c is never touched. Setting a flag reproduces a
* bug-851-class invariant divergence so the negative test (task 7.2) can assert that
* a broken adapter config FAILS an invariant while the correct config PASSES.
* - RAFT_SIM_BREAK_TERM_STAMP: on_persist_entry writes term 0 instead of the
* supplied term (emulates G1: entries carry term 0 so on_get_entry_term cannot
* distinguish same-index/different-term entries).
* - RAFT_SIM_BREAK_TRUNCATE: on_truncate_from is a no-op (emulates G3: a divergent
* suffix is never deleted). */
#define RAFT_SIM_BREAK_TERM_STAMP 0x1u
#define RAFT_SIM_BREAK_TRUNCATE 0x2u
/* Buffered RPC entry: one pending vote-request to route.
* last_log_term/last_log_index capture the CANDIDATE's advertised last-log tuple
* at SEND time (mirroring raft_sim_ae_t.leader_term) so the (lastLogTerm,
@ -633,6 +649,11 @@ typedef struct {
/* Replayable per-step action trace for failure dumps. */
raft_sim_action_t trace[RAFT_SIM_MAX_STEPS];
int trace_len;
/* Deliberately-broken-build fault switch (task 7.1). Bitmask of
* RAFT_SIM_BREAK_* flags consulted by the harness adapter callbacks via
* node->ctx. Zero (DEFAULT OFF) on init — legacy and conformance runs are
* unaffected unless a test explicitly sets it. */
unsigned fault_inject;
} raft_sim_cluster_t;
/*
@ -1094,6 +1115,16 @@ int raft_sim_scenario_figure8(raft_sim_cluster_t *c, int n, uint64_t seed);
*/
int raft_sim_scenario_stale_rejoin(raft_sim_cluster_t *c, int n, uint64_t seed);
/*
* raft_sim_set_fault_inject: set the cluster's deliberately-broken-build fault mask
* (raft-sim-conformance task 7.1). `flags` is a bitmask of RAFT_SIM_BREAK_* values;
* pass 0 to clear (restore correct adapter behavior). The flags are consulted only by
* the harness ADAPTER callbacks (raft_sim.c); raft.c is never modified. DEFAULT OFF —
* a cluster that never calls this behaves exactly as before. Provided so the negative
* test (task 7.2) can toggle the fault within a single binary.
*/
void raft_sim_set_fault_inject(raft_sim_cluster_t *c, unsigned flags);
#endif /* XCE_TESTING */
#endif /* RAFT_H */

30
src/lib/raft/raft_sim.c
View file

@ -106,6 +106,15 @@ static int sim_persist_entry(void *ctx, uint64_t index, uint64_t term,
if (index < 1 || index > RAFT_SIM_MAX_LOG)
return -1; /* out of fixed sim bounds */
/* Deliberately-broken-build fault switch (task 7.1): when RAFT_SIM_BREAK_TERM_STAMP
* is set on the cluster, stamp the entry with term 0 instead of `term`. This
* emulates bug-851 G1 (entries carry term 0 so on_get_entry_term can't distinguish
* same-index/different-term entries). Purely additive — DEFAULT OFF, so legacy and
* conformance runs that never set the flag are byte-identical to before. */
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
if (c && (c->fault_inject & RAFT_SIM_BREAK_TERM_STAMP))
term = 0;
raft_sim_entry_t *slot = &node->log[index - 1];
slot->term = term;
@ -147,6 +156,15 @@ static int sim_truncate_from(void *ctx, uint64_t index)
if (index < 1)
return -1;
/* Deliberately-broken-build fault switch (task 7.1): when RAFT_SIM_BREAK_TRUNCATE
* is set on the cluster, make truncation a NO-OP so the divergent suffix is never
* deleted (emulates bug-851 G3). Returns 0 (success) without touching the log, so
* the library believes the truncate succeeded while the stale suffix lingers.
* Purely additive — DEFAULT OFF; runs that never set the flag are unaffected. */
raft_sim_cluster_t *c = (raft_sim_cluster_t *)node->ctx;
if (c && (c->fault_inject & RAFT_SIM_BREAK_TRUNCATE))
return 0;
for (uint64_t i = index; i <= (uint64_t)node->log_count; i++) {
if (i <= RAFT_SIM_MAX_LOG) {
raft_sim_entry_t *slot = &node->log[i - 1];
@ -2343,4 +2361,16 @@ int raft_sim_scenario_stale_rejoin(raft_sim_cluster_t *c, int n, uint64_t seed)
return sim_scen_check(c);
}
/*
* raft_sim_set_fault_inject: set the cluster's deliberately-broken-build fault mask
* (task 7.1). Consulted only by the harness adapter callbacks (sim_persist_entry /
* sim_truncate_from); raft.c is never touched. DEFAULT OFF — only a test that calls
* this enables the corruption, and passing 0 clears it again within the same binary.
*/
void raft_sim_set_fault_inject(raft_sim_cluster_t *c, unsigned flags)
{
if (c)
c->fault_inject = flags;
}
#endif /* XCE_TESTING */

94
src/lib/testsuite/test_raft.c
View file

@ -8547,6 +8547,99 @@ static void test_sim_5_5_stale_rejoin(void)
* cluster sizes, each capped at RAFT_SIM_SWEEP_STEPS steps; the harness is fixed-array
* (no allocation) so the whole batch stays within the lib test-suite time budget.
*/
/*
* test_sim_7_1_fault_switch_mechanism (raft-sim-conformance task 7.1; Req 7.1, 7.3):
* the deliberately-broken-build fault switch is RUNTIME-togglable, DEFAULT OFF, and
* observably corrupts the harness adapter (term->0 / truncate no-op) only when set.
* This is the MECHANISM check; the full both-directions invariant-violation demo is
* task 7.2. Single-node conformance cluster so raft_start_election forces LEADER
* without depending on bag delivery, mirroring test_sim_conformance_init_append_readback.
*/
static void test_sim_7_1_fault_switch_mechanism(void)
{
/* ---- DEFAULT OFF: the switch does not perturb a correct run ---- */
{
raft_sim_cluster_t c;
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 1, 2000, 0x7A17ULL));
/* fault_inject is zero-initialized -> OFF by default. */
TEST_ASSERT_EQUAL_UINT(0u, c.fault_inject);
raft_start_election(&c.nodes[0].state);
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
uint64_t value = 0xABCDEF0011223344ULL;
uint64_t out_index = 0;
TEST_ASSERT_EQUAL_INT(RAFT_OK,
raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index));
TEST_ASSERT_EQUAL_UINT64(1, out_index);
/* With the switch OFF the persisted term equals the leader's term (not 0). */
raft_sim_node_t *L = &c.nodes[0];
TEST_ASSERT_EQUAL_UINT64(L->state.term, L->log[0].term);
TEST_ASSERT_NOT_EQUAL(0, L->log[0].term);
}
/* ---- RAFT_SIM_BREAK_TERM_STAMP: on_persist_entry writes term 0 (emulates G1) ---- */
{
raft_sim_cluster_t c;
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 1, 2000, 0x7A17ULL));
raft_sim_set_fault_inject(&c, RAFT_SIM_BREAK_TERM_STAMP);
TEST_ASSERT_EQUAL_UINT(RAFT_SIM_BREAK_TERM_STAMP, c.fault_inject);
raft_start_election(&c.nodes[0].state);
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, c.nodes[0].state.role);
uint64_t value = 0xABCDEF0011223344ULL;
uint64_t out_index = 0;
TEST_ASSERT_EQUAL_INT(RAFT_OK,
raft_log_append(&c.nodes[0].state, &value, sizeof(value), &out_index));
TEST_ASSERT_EQUAL_UINT64(1, out_index);
raft_sim_node_t *L = &c.nodes[0];
/* The leader's term is non-zero, yet the adapter stamped the entry term 0. */
TEST_ASSERT_NOT_EQUAL(0, L->state.term);
TEST_ASSERT_EQUAL_UINT64(0, L->log[0].term);
/* on_get_entry_term reads back the corrupted term (0 => indistinguishable). */
TEST_ASSERT_EQUAL_UINT64(0, L->cbs.on_get_entry_term(L, 1));
/* Clearing the flag restores correct stamping within the SAME binary. */
raft_sim_set_fault_inject(&c, 0);
TEST_ASSERT_EQUAL_UINT(0u, c.fault_inject);
uint64_t value2 = 0x55667788AABBCCDDULL;
TEST_ASSERT_EQUAL_INT(RAFT_OK,
raft_log_append(&c.nodes[0].state, &value2, sizeof(value2), &out_index));
TEST_ASSERT_EQUAL_UINT64(2, out_index);
TEST_ASSERT_EQUAL_UINT64(L->state.term, L->log[1].term); /* term again correct */
TEST_ASSERT_NOT_EQUAL(0, L->log[1].term);
}
/* ---- RAFT_SIM_BREAK_TRUNCATE: on_truncate_from is a no-op (emulates G3) ---- */
{
raft_sim_cluster_t c;
memset(&c, 0, sizeof(c));
TEST_ASSERT_EQUAL_INT(0, raft_sim_conformance_init(&c, 1, 2000, 0x7A17ULL));
raft_sim_node_t *L = &c.nodes[0];
/* Seed two entries directly via the persist adapter (term != 0). */
uint64_t e1 = 0x1111111111111111ULL, e2 = 0x2222222222222222ULL;
TEST_ASSERT_EQUAL_INT(0, L->cbs.on_persist_entry(L, 1, 7, &e1, sizeof(e1)));
TEST_ASSERT_EQUAL_INT(0, L->cbs.on_persist_entry(L, 2, 7, &e2, sizeof(e2)));
TEST_ASSERT_EQUAL_INT(2, L->log_count);
/* With the switch OFF, truncate-from index 2 drops the suffix. */
TEST_ASSERT_EQUAL_INT(0, L->cbs.on_truncate_from(L, 2));
TEST_ASSERT_EQUAL_INT(1, L->log_count);
/* Re-seed, then with the switch ON the same truncate is a no-op: suffix stays. */
TEST_ASSERT_EQUAL_INT(0, L->cbs.on_persist_entry(L, 2, 7, &e2, sizeof(e2)));
TEST_ASSERT_EQUAL_INT(2, L->log_count);
raft_sim_set_fault_inject(&c, RAFT_SIM_BREAK_TRUNCATE);
TEST_ASSERT_EQUAL_INT(0, L->cbs.on_truncate_from(L, 2)); /* reports success */
TEST_ASSERT_EQUAL_INT(2, L->log_count); /* but suffix lingers */
TEST_ASSERT_EQUAL_UINT64(7, L->log[1].term); /* entry 2 still present */
}
}
#define RAFT_SIM_SWEEP_M 32 /* configured minimum number of randomized seeds (R6.2) */
#define RAFT_SIM_SWEEP_STEPS 400 /* per-run step cap — bounded for the suite budget (R6.4) */
@ -8841,6 +8934,7 @@ void test_raft(void)
RUN_TEST(test_sim_5_3_drop_dup_reorder);
RUN_TEST(test_sim_5_4_figure8);
RUN_TEST(test_sim_5_5_stale_rejoin);
RUN_TEST(test_sim_7_1_fault_switch_mechanism);
RUN_TEST(test_sim_6_1_seed_sweep);
/* Group A: basic election correctness (task 5.1) */