feat(raft-sim-conformance): stale-rejoin scenario, bug-851 G5 term adoption via AE (task 5.5)

.kiro/specs/raft-sim-conformance/tasks.md

@@ -159,7 +159,7 @@
   - _Boundary: raft_sim.c, test_raft.c_
   - _Depends: 4.2_
 
-- [ ] 5.5 Stale-rejoin scenario (bug-851 G5)
+- [x] 5.5 Stale-rejoin scenario (bug-851 G5)
   - Implement `raft_sim_scenario_stale_rejoin`: a term-lagging node rejoins under a higher-term leader; assert it adopts the higher term from the AE/heartbeat path and does not re-elect at a reused term (Election Safety preserved)
   - Observable completion: the rejoining node's term advances via AE; no duplicate-term leader appears
   - _Requirements: 4.5_

src/lib/raft/raft.h

@@ -1067,6 +1067,33 @@ int raft_sim_scenario_drop_dup_reorder(raft_sim_cluster_t *c, int n, uint64_t se
  */
 int raft_sim_scenario_figure8(raft_sim_cluster_t *c, int n, uint64_t seed);
 
+/*
+ * raft_sim_scenario_stale_rejoin: bug-851 G5 stale-rejoin scenario (raft-sim-
+ * conformance task 5.5; Req 4.5). Initializes the caller's cluster from
+ * (n, election_timeout=2000, seed), then:
+ *   1. elects a leader and replicates a committed prefix,
+ *   2. PARTITIONS one node N_stale away and, while it is isolated, advances the
+ *      connected majority's TERM via a fresh election so the surviving (connected)
+ *      leader is at a STRICTLY-HIGHER term than N_stale (which keeps its old/stale
+ *      term; it may also campaign while isolated but can never win — no quorum),
+ *   3. HEALS the partition and drives one AppendEntries/heartbeat from the higher-term
+ *      leader to N_stale, DELIVERED through the bag,
+ *   4. ASSERTS (R4.5): N_stale ADOPTS the higher term via the AE path — after delivery
+ *      N_stale's term == the leader's term (the library's raft_on_append_entries_request
+ *      UpdateTerm stepped it down to FOLLOWER at the leader's term) and N_stale is a
+ *      FOLLOWER, having NOT re-elected at a reused term. The adoption is NON-VACUOUS:
+ *      N_stale's term was confirmed STRICTLY LOWER than the leader's BEFORE the rejoin
+ *      AE. Election Safety holds THROUGHOUT (raft_sim_check_all after every action).
+ * The term adoption comes from the LIBRARY (driven via bag deliver) — the harness never
+ * sets N_stale's term directly. Returns 0 if the scenario completes with ZERO invariant
+ * violations AND the adoption property holds; < 0 (the negated first failing invariant,
+ * matching raft_sim_step) on any violation or if the property is broken, after dumping
+ * the replayable trace to stderr. Returns -1 on an invalid node count (n < 3). Drives
+ * only via published library entry points + harness primitives — no protocol
+ * re-implementation.
+ */
+int raft_sim_scenario_stale_rejoin(raft_sim_cluster_t *c, int n, uint64_t seed);
+
 #endif /* XCE_TESTING */
 
 #endif /* RAFT_H */

src/lib/raft/raft_sim.c

@@ -2113,4 +2113,234 @@ int raft_sim_scenario_drop_dup_reorder(raft_sim_cluster_t *c, int n, uint64_t se
     return sim_scen_check(c);
 }
 
+/* ---------- stale-rejoin scenario (raft-sim-conformance task 5.5; bug-851 G5) ---------- */
+
+/*
+ * sim_scen_elect_connected: campaign one CONNECTED node at a time (raft_start_election),
+ * draining the bag after each, until a CONNECTED node holds role==RAFT_LEADER or the
+ * bounded attempts are exhausted. Differs from sim_scen_elect in two ways the stale-
+ * rejoin needs: (1) it skips partitioned candidates, and (2) it returns the index of a
+ * CONNECTED leader (a stale, partitioned-away node that still reports LEADER at an old
+ * term is NOT accepted — sim_find_leader would return that lowest-index stale leader).
+ * On an invariant violation *viol holds the negated invariant and the return is -1;
+ * otherwise *viol is 0 and the return is the connected-leader index (>= 0) or -1.
+ */
+static int sim_scen_elect_connected(raft_sim_cluster_t *c, int *viol)
+{
+    *viol = 0;
+    for (int attempt = 0; attempt < 4 * c->n; attempt++) {
+        int cand = attempt % c->n;
+        if (c->nodes[cand].partition)
+            continue; /* never campaign an isolated node */
+
+        (void)raft_start_election(&c->nodes[cand].state);
+        int chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
+        if (chk < 0) {
+            *viol = chk;
+            return -1;
+        }
+
+        /* Accept only a CONNECTED leader — check candidate roles directly rather than
+         * sim_find_leader (which returns the lowest index, possibly a stale isolated
+         * leader still at an old term). */
+        for (int p = 0; p < c->n; p++) {
+            if (c->nodes[p].partition)
+                continue;
+            raft_snapshot_t snap;
+            raft_snapshot(&c->nodes[p].state, &snap);
+            if (snap.role == RAFT_LEADER)
+                return p;
+        }
+    }
+    return -1; /* no connected leader emerged */
+}
+
+int raft_sim_scenario_stale_rejoin(raft_sim_cluster_t *c, int n, uint64_t seed)
+{
+    /* A surviving connected majority while one node is isolated needs n >= 3. */
+    if (n < 3 || n > RAFT_SIM_MAX_NODES)
+        return -1;
+
+    if (raft_sim_conformance_init(c, n, 2000, seed) != 0)
+        return -1;
+
+    int chk, viol;
+
+    /* ---- 1. Elect a leader and replicate a committed prefix (fully connected). ---- */
+    int leader = sim_scen_elect(c, &viol);
+    if (viol < 0)
+        return viol;
+    if (leader < 0)
+        return 0; /* no leader emerged — observable, not a violation; test asserts it */
+
+    for (int r = 0; r < 3; r++) {
+        chk = sim_scen_replicate_round(c, leader, 0xA00ULL + (uint64_t)r);
+        if (chk < 0)
+            return chk;
+        leader = sim_find_leader(c);
+        if (leader < 0) {
+            leader = sim_scen_elect(c, &viol);
+            if (viol < 0)
+                return viol;
+            if (leader < 0)
+                return 0;
+        }
+    }
+
+    /* ---- 2. Pick N_stale = a node != leader; ISOLATE it. While isolated it keeps its
+     * (now stale) term. The connected majority will advance its term below. ---- */
+    int n_stale = (leader == 0) ? 1 : 0; /* any node other than the current leader */
+    raft_sim_partition(c, n_stale, 1);
+    chk = sim_scen_check(c);
+    if (chk < 0)
+        return chk;
+
+    /* The isolated node times out and campaigns — it can bump its OWN term but can NEVER
+     * win (no quorum reachable). Drive a couple of its campaigns so the run exercises the
+     * realistic "stale node spins elections in isolation" path; its messages park in the
+     * bag (partitioned endpoint) and never grant it leadership. */
+    for (int e = 0; e < 2; e++) {
+        (void)raft_start_election(&c->nodes[n_stale].state);
+        chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 4);
+        if (chk < 0)
+            return chk;
+    }
+
+    /* ---- 3. Advance the CONNECTED majority's TERM via fresh elections until a connected
+     * leader sits at a STRICTLY-HIGHER term than N_stale. The connected leader from phase
+     * 1 may have stepped down (or N_stale's isolated campaigns left stale parked traffic);
+     * re-establish a connected leader, then keep campaigning connected nodes until the
+     * connected leader's term exceeds N_stale's term. Bounded. ---- */
+    int conn_leader = sim_scen_elect_connected(c, &viol);
+    if (viol < 0)
+        return viol;
+    if (conn_leader < 0)
+        return 0; /* connected majority could not elect — observable, not a violation */
+
+    for (int bump = 0; bump < 4 * n &&
+                       c->nodes[conn_leader].state.term <= c->nodes[n_stale].state.term;
+         bump++) {
+        /* Force another term bump on the connected side: campaign a different connected
+         * node (a higher term steps the current connected leader down, and the survivors
+         * re-elect at a strictly-higher term). */
+        conn_leader = sim_scen_elect_connected(c, &viol);
+        if (viol < 0)
+            return viol;
+        if (conn_leader < 0)
+            return 0;
+
+        if (c->nodes[conn_leader].state.term > c->nodes[n_stale].state.term)
+            break;
+
+        /* Not yet higher — bump the connected leader's term explicitly by campaigning a
+         * connected follower (its prevote/term bump raises the connected epoch). */
+        for (int p = 0; p < n; p++) {
+            if (p == conn_leader || c->nodes[p].partition)
+                continue;
+            (void)raft_start_election(&c->nodes[p].state);
+            chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
+            if (chk < 0)
+                return chk;
+            break;
+        }
+        conn_leader = sim_scen_elect_connected(c, &viol);
+        if (viol < 0)
+            return viol;
+        if (conn_leader < 0)
+            return 0;
+    }
+
+    /* The connected leader MUST now outrank N_stale; otherwise the construction failed to
+     * create the term gap (observable, not a violation). */
+    if (c->nodes[conn_leader].state.term <= c->nodes[n_stale].state.term)
+        return 0;
+
+    /* Replicate a current-term entry on the connected leader so its log tip carries the
+     * higher term, then drain — this gives the rejoin AE real content to carry. */
+    chk = sim_scen_replicate_round(c, conn_leader, 0xA50ULL);
+    if (chk < 0)
+        return chk;
+    /* sim_scen_replicate_round can step the leader down via a late vote; re-establish a
+     * connected leader still outranking N_stale. */
+    {
+        raft_snapshot_t lsnap;
+        raft_snapshot(&c->nodes[conn_leader].state, &lsnap);
+        if (lsnap.role != RAFT_LEADER) {
+            conn_leader = sim_scen_elect_connected(c, &viol);
+            if (viol < 0)
+                return viol;
+            if (conn_leader < 0)
+                return 0;
+        }
+    }
+    if (c->nodes[conn_leader].state.term <= c->nodes[n_stale].state.term)
+        return 0;
+
+    /* ---- 4. NON-VACUITY: capture the leader's term and N_stale's term BEFORE the rejoin
+     * AE. The adoption assertion is real only if N_stale is STRICTLY behind here. ---- */
+    uint64_t leader_term_pre = c->nodes[conn_leader].state.term;
+    uint64_t stale_term_pre  = c->nodes[n_stale].state.term;
+    if (stale_term_pre >= leader_term_pre)
+        return 0; /* no term gap — construction failed; not a safety violation */
+
+    /* ---- 5. HEAL the partition and drive ONE AppendEntries from the higher-term leader
+     * to N_stale, then DELIVER it through the bag. The library's
+     * raft_on_append_entries_request UpdateTerm path adopts the leader's higher term and
+     * steps N_stale down to FOLLOWER — the term is NOT set by the harness. ---- */
+    raft_sim_partition(c, n_stale, 0);
+    chk = sim_scen_check(c);
+    if (chk < 0)
+        return chk;
+
+    (void)raft_send_append_entries(&c->nodes[conn_leader].state, n_stale,
+                                   c->nodes[conn_leader].state.commit_index);
+    chk = sim_scen_check(c);
+    if (chk < 0)
+        return chk;
+
+    /* Deliver the just-sent AE (and its reply) to N_stale via the bag. Each delivery is
+     * invariant-checked so an Election-Safety violation surfaces the moment it appears. */
+    chk = sim_scen_drain_bag(c, RAFT_SIM_MAX_MSGS * 8);
+    if (chk < 0)
+        return chk;
+
+    /* ---- 6. ADOPTION ASSERTIONS (R4.5), all observable via raft_snapshot:
+     *   (a) N_stale adopted the leader's higher term via the AE path,
+     *   (b) N_stale is a FOLLOWER (it did NOT re-elect at a reused term). ---- */
+    raft_snapshot_t stale_snap;
+    raft_snapshot(&c->nodes[n_stale].state, &stale_snap);
+
+    /* (a) Term adopted from the AE path: N_stale's term advanced to AT LEAST the leader's
+     * pre-rejoin term (UpdateTerm adopts the higher term). It was strictly lower before
+     * (asserted at phase 4), so this is a genuine library-driven adoption, not a no-op. */
+    if (stale_snap.term < leader_term_pre)
+        return -(int)RAFT_SIM_INV_ELECTION_SAFETY;
+
+    /* (b) N_stale stepped to FOLLOWER on adopting the higher term — it did not become a
+     * leader at a reused/duplicate term. */
+    if (stale_snap.role == RAFT_LEADER)
+        return -(int)RAFT_SIM_INV_ELECTION_SAFETY;
+
+    /* ---- 7. Let the cluster converge to a single leader and re-replicate so N_stale
+     * catches up the committed prefix. Bounded; invariants checked after every action via
+     * the helpers (Election Safety asserted THROUGHOUT). ---- */
+    leader = sim_find_leader(c);
+    for (int r = 0; r < n + 4; r++) {
+        if (leader < 0 || c->nodes[leader].partition) {
+            leader = sim_scen_elect(c, &viol);
+            if (viol < 0)
+                return viol;
+            if (leader < 0)
+                return 0;
+        }
+        chk = sim_scen_replicate_round(c, leader, 0xA80ULL + (uint64_t)r);
+        if (chk < 0)
+            return chk;
+        leader = sim_find_leader(c);
+    }
+
+    /* ---- 8. Final invariant check (also covered after every step above). ---- */
+    return sim_scen_check(c);
+}
+
 #endif /* XCE_TESTING */

src/lib/testsuite/test_raft.c

@@ -8468,6 +8468,64 @@ static void test_sim_5_4_figure8(void)
             "figure-8 leader did not commit the prior-term + current-term entries");
     }
 }
+
+/*
+ * test_sim_5_5_stale_rejoin (raft-sim-conformance task 5.5; Req 4.5)
+ *
+ * The bug-851 G5 stale-rejoin: a term-lagging node is isolated while the connected
+ * majority advances its TERM via a fresh election, so the surviving leader is at a
+ * STRICTLY-HIGHER term than the isolated node. On HEAL, an AppendEntries/heartbeat from
+ * the higher-term leader is driven to the stale node and DELIVERED through the bag — the
+ * library's raft_on_append_entries_request UpdateTerm path adopts the leader's higher
+ * term and steps the node down to FOLLOWER. The node does NOT re-elect at a reused term;
+ * Election Safety holds throughout (no two leaders at one term).
+ *
+ * raft_sim_scenario_stale_rejoin returns 0 ONLY when (a) the stale node's term was
+ * STRICTLY LOWER than the leader's BEFORE the rejoin AE (non-vacuous adoption),
+ * (b) after delivering the leader's AE the stale node's term == leader's term and its
+ * role is FOLLOWER (adopted via the LIBRARY AE path, not forged), and (c) no invariant
+ * (notably Election Safety) ever fired. A return < 0 is a REAL safety violation the
+ * scenario surfaced (the step already dumped the replayable trace to stderr) — NOT
+ * something to mask. Several (n, seed) pairs are exercised so the property is not a
+ * single-seed accident.
+ */
+static void test_sim_5_5_stale_rejoin(void)
+{
+    /* (n, seed) pairs across 3- and 5-node clusters. The scenario inits the caller's
+     * cluster internally from (n, seed). */
+    struct { int n; uint64_t seed; } cases[] = {
+        { 3, 11ULL }, { 3, 0x4A33ULL }, { 3, 7ULL },
+        { 5, 23ULL }, { 5, 0xBEEFULL },
+    };
+
+    for (size_t k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
+        raft_sim_cluster_t c;
+        memset(&c, 0, sizeof(c));
+
+        int rc = raft_sim_scenario_stale_rejoin(&c, cases[k].n, cases[k].seed);
+
+        /* Zero invariant violations AND the stale-rejoin adoption property held
+         * throughout (stale term strictly lower pre-rejoin; adopted == leader term and
+         * FOLLOWER post-AE; Election Safety never fired). */
+        TEST_ASSERT_TRUE_MESSAGE(rc == 0,
+            "stale-rejoin scenario surfaced an invariant/adoption violation on the correct library");
+
+        /* Post-scenario: the cluster still satisfies all four invariants. */
+        TEST_ASSERT_EQUAL_INT(RAFT_SIM_INV_NONE, raft_sim_check_all(&c));
+
+        /* Election Safety holds in the final converged state (no two leaders at one
+         * term). Count leaders observably; exactly one must hold. */
+        int leaders = 0;
+        for (int i = 0; i < c.n; i++) {
+            raft_snapshot_t snap;
+            raft_snapshot(&c.nodes[i].state, &snap);
+            if (snap.role == RAFT_LEADER)
+                leaders++;
+        }
+        TEST_ASSERT_TRUE_MESSAGE(leaders <= 1,
+            "post-stale-rejoin cluster reports more than one leader (Election Safety)");
+    }
+}
 #endif /* XCE_TESTING */
 
 /* ---------- test_raft entry point ---------- */
@@ -8682,6 +8740,7 @@ void test_raft(void)
     RUN_TEST(test_sim_5_2_restart_churn);
     RUN_TEST(test_sim_5_3_drop_dup_reorder);
     RUN_TEST(test_sim_5_4_figure8);
+    RUN_TEST(test_sim_5_5_stale_rejoin);
 
     /* Group A: basic election correctness (task 5.1) */
     RUN_TEST(test_proto_A1_single_node_self_elects);