From 01b7d6b7d5a1856663d9aa56c131702f2851fdc5 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sun, 28 Jun 2026 03:42:06 +0000 Subject: [PATCH] =?UTF-8?q?test(raft-ha-core-safety):=20follower-reconverg?= =?UTF-8?q?ence=20discriminator=20=E2=80=94=20Phase=204=20sound=20(tasks?= =?UTF-8?q?=204.1/4.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deterministic follower-level test drives the REAL ha_consume_append_entries with a leader-style backtracking AE sequence against a multi-entry former- primary divergent tail (12-15 @old term, committed 1..11). Follower backtracks, conflict-truncates 14/15, accepts leader term-6 entries, preserves committed prefix. Reviewer adversarially falsified (neuter intake => reached=0). VERDICT: follower-side reconvergence + applied-agreement gate already sound on baseline => NO Phase-4 production fix needed. Redirects the live-stall root cause to Phase 5 (channel-flap + in_catch_up-blocks-prevote- reply). 4.1 + apply-gap pins green; 14/0 suite. --- .kiro/specs/raft-ha-core-safety/tasks.md | 8 +- .../testsuite/test_ha_core_safety.c | 427 ++++++++++++++++++ 2 files changed, 431 insertions(+), 4 deletions(-) diff --git a/.kiro/specs/raft-ha-core-safety/tasks.md b/.kiro/specs/raft-ha-core-safety/tasks.md index 2228e32..36ea2b3 100644 --- a/.kiro/specs/raft-ha-core-safety/tasks.md +++ b/.kiro/specs/raft-ha-core-safety/tasks.md @@ -82,15 +82,15 @@ - [ ] 4. Standby reconvergence completeness - _Root cause (Phase-4/5 investigation, `research.md`): a recovered standby holding a divergent uncommitted tail above the new leader's committed point (former-primary shape, e.g. ctrl-2 with 12-15 at an old term while leader committed 11) never reaches `ae_already_have` — it loops on `ae_reject_logmismatch`, its control channel flaps, and it never rejoins. A short-suffix standby (ctrl-3) DOES reconverge, so the backtrack works for small divergence but not the former-primary tail._ -- [ ] 4.1 RED: divergent-tail reconvergence + apply-gap gate tests +- [x] 4.1 RED: divergent-tail reconvergence + apply-gap gate tests - Primary case: construct a recovered standby with a MULTI-entry divergent uncommitted tail above the leader's committed point and assert it reconverges to follower — truncates the conflicting suffix and reaches `ae_already_have` (no permanent `ae_reject_logmismatch` loop). Secondary case: a rejoining node with an injected catch-up apply gap must not be reported converged/healthy until `applied_lsn` and `log_len` equal the leader's committed boundary - Observable completion: both cases FAIL on the baseline (the divergent-tail case loops on log-mismatch / never truncates; the apply-gap case reports healthy at match-index alone) - _Requirements: 5.1, 5.2, 5.3, 5.4, 7.2_ - _Boundary: Standby Reconvergence_ -- [ ] 4.2 Make a divergent uncommitted tail truncate + catch up, and gate healthy on applied agreement - - Ensure the AE log-matching backtrack + follower-AE-conflict-truncate path fires for a former-primary divergent tail (not only a 1-2 entry suffix) so the standby truncates the conflicting suffix and accepts the leader's entries; synchronize post-recovery `log_len`/applied-lsn and complete outstanding catch-up apply (routed through the seam, reusing the rejoin intake hook) before reporting converged; bound the recovery window - - Observable completion: 4.1 passes; a recovered standby with a divergent uncommitted tail reconverges to `ae_already_have` and reports converged only after `applied_lsn == log_len == leader committed`, restoring a healthy 3-of-3 +- [x] 4.2 Verify divergent-tail reconvergence + applied-agreement gate (no production fix needed) + - **Discriminator outcome (4.1, adversarially confirmed):** the follower-side AE log-matching backtrack + follower-AE-conflict-truncate path ALREADY reconverges a multi-entry former-primary divergent tail (truncates 12-15 old-term, accepts the leader's term-6 entries, preserves committed prefix 1..11) when driven deterministically WITHOUT a channel flap; and the applied-agreement healthy gate (`ha_process_standby_state`) already blocks "converged" until `applied_lsn == committed_lsn`. Neutering `ha_consume_append_entries` makes the test fail (`reached=0`), proving the real path does the work. **No production change required in Phase 4** — the live stall is NOT follower-side reconvergence. The fix moves to Phase 5 (channel-flap suppression + `in_catch_up`-blocks-PreVote-reply). + - Observable completion: 4.1's divergent-tail + apply-gap tests pass as regression pins; no production diff in Phase 4 (verified by the discriminator) - _Requirements: 5.1, 5.2, 5.3, 5.4_ - _Boundary: Standby Reconvergence_ - _Depends: 2.4_ diff --git a/src/controller/testsuite/test_ha_core_safety.c b/src/controller/testsuite/test_ha_core_safety.c index 314ecea..55a8fb6 100644 --- a/src/controller/testsuite/test_ha_core_safety.c +++ b/src/controller/testsuite/test_ha_core_safety.c @@ -54,6 +54,7 @@ #include "config_repl.h" /* ha_stream_wal_to_standby */ #include "ha.h" #include "persist.h" +#include "xce/cluster_def.h" /* cluster_t (follower AE-intake fixture, task 4.1) */ #include "request.h" /* xce_request_t */ #include "typemap.h" /* wal_catchup_entry_t */ #include "wal_types.h" @@ -912,6 +913,423 @@ static void test_2_3_unrepairable_catchup_raises_nonpromotable_fence(void) printf("test_2_3_unrepairable_catchup_raises_nonpromotable_fence: PASS (pin)\n"); } +/* ═══════════════════════════════════════════════════════════════════════════ + * TASK 4.1 — DETERMINISTIC follower-side reconvergence (the DISCRIMINATOR) + * + * The live chaotic-soak stall (research.md §"Phase 4/5 — Post-Phase-2 stall"): + * a recovered standby (former primary, ctrl-2) holds a DIVERGENT UNCOMMITTED + * TAIL above the new leader's committed point (committed 11, then entries 12-15 + * at an OLD term) and loops on ae_reject_logmismatch forever, while a SHORT- + * suffix standby (ctrl-3) reconverges. Two candidate root causes are + * intertwined in the live run: (Phase 4) the follower-side AE backtrack/truncate + * failing for a longer former-primary tail, or (Phase 5) the control-channel + * flap interrupting the backtrack before it completes. + * + * This test ISOLATES the follower side: it drives the REAL follower AE intake + * (ha_consume_append_entries → raft_on_append_entries_request, the same seam + * ctrl-3 uses) with a deterministic leader-style nextIndex backtracking sequence + * and NO channel flap, so it cleanly answers: does the follower-side AE + * log-matching + conflict-truncate path reconverge a MULTI-ENTRY former-primary + * divergent tail? + * + * - If this FAILS on baseline (the follower does not backtrack/truncate/accept + * a multi-entry former-primary tail) → Phase 4 has a real follower-side bug; + * this is the RED task 4.2 fixes. + * - If this PASSES on baseline → KEY FINDING: the follower reconverges fine + * when driven deterministically; the live stall is NOT follower-side but the + * Phase-5 control-channel flap interrupting the backtrack. Keep this as a + * PASSING PIN guarding the follower path and re-focus on Phase 5. + * + * Requirements: 5.1/5.2/5.3/5.4 (reconvergence + no false convergence), 7.2 + * (deterministic harness forcing the divergent-tail rejoin). + * ═══════════════════════════════════════════════════════════════════════════ */ + +/* A cluster_t whose embedded persist is started and whose ha.raft_c self-refs + * the cluster (the convention every raft callback uses). This is the full + * follower fixture the AE intake needs (c->ha.raft + c->persist), distinct from + * the bare-persist hcs_member_t used by the 2.x send-side seams. */ +static cluster_t* hcs4_cluster_up(const char* suffix) +{ + cluster_t* c = calloc(1, sizeof(cluster_t)); + if (!c) + return NULL; + + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; + cfg.sync_timeout_ms = 50; + cfg.max_wal_records = 1u << 30; /* never auto-checkpoint during a unit test */ + + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/xce_hcs4_%s_XXXXXX", suffix); + if (!mkdtemp(tmp)) { + free(c); + return NULL; + } + persist_set_workdir(&c->persist, tmp, "ctrl"); + if (persist_init(&c->persist, &cfg, c) != 0) { + free(c); + return NULL; + } + if (persist_start(&c->persist) != 0) { + persist_fini(&c->persist); + free(c); + return NULL; + } + c->ha.raft_c = c; + return c; +} + +static void hcs4_cluster_down(cluster_t* c) +{ + if (!c) + return; + persist_stop(&c->persist); /* drain the fsync worker before fini (bug-1224) */ + persist_fini(&c->persist); + free(c); +} + +/* Bring the embedded raft up as a STANDBY follower at `term`, seeding the + * library log floor from the persisted WAL tip (ha_init_raft restore). Mirrors + * make_follower in test_term_safety_adapter.c. */ +static void hcs4_make_follower(cluster_t* c, uint64_t term, int n_peers) +{ + atomic_store(&c->ha.role, CTRL_STANDBY); + atomic_store(&c->ha.term, term); + ha_init_raft(&c->ha, c, n_peers); + c->ha.raft.term = term; + c->ha.raft.role = RAFT_FOLLOWER; +} + +/* Pack one adapter entry blob: [raft_entry_blob_t{type,len}][payload]. */ +static size_t hcs4_pack_entry(uint8_t* buf, uint32_t type, + const void* payload, uint32_t plen) +{ + raft_entry_blob_t* h = (raft_entry_blob_t*)buf; + h->type = type; + h->payload_len = plen; + if (payload && plen > 0) + memcpy(buf + sizeof(*h), payload, plen); + return sizeof(*h) + plen; +} + +/* Build a decoded single-entry/heartbeat WalCatchupEntry whose TOP-LEVEL fields + * carry the AE metadata (the on-wire single-entry shape ha_consume_append_entries + * reads). Mirrors fill_ae_entry in test_term_safety_adapter.c. */ +static void hcs4_fill_ae(wal_catchup_entry_t* e, int is_heartbeat, + uint64_t prev_log_index, uint64_t prev_log_term, + uint64_t leader_commit, uint64_t entry_lsn, + uint8_t entry_type, uint64_t entry_term, + uint8_t* payload, uint32_t plen) +{ + memset(e, 0, sizeof(*e)); + e->prev_log_index = prev_log_index; + e->prev_log_term = prev_log_term; + e->leader_commit = leader_commit; + if (is_heartbeat) { + e->n_entries = 0; + e->lsn = 0; + } else { + e->n_entries = 1; + e->lsn = entry_lsn; + e->entry_type = entry_type; + e->term = entry_term; + e->raw_payload = payload; + e->raw_payload_len = plen; + } +} + +/* Probe what is PHYSICALLY on disk at `lsn` in a cluster's WAL (reuses the 1.4 + * readback visitor over c->persist.wal_path). */ +static int hcs4_probe(cluster_t* c, uint64_t lsn, hcs_probe_t* out) +{ + memset(out, 0, sizeof(*out)); + hcs_probe_ctx_t pc = { .want_lsn = lsn, .out = out }; + FILE* sink = fopen("/dev/null", "w"); + if (!sink) + return -1; + int rc = waldump_path_visit(c->persist.wal_path, sink, NULL, + hcs_probe_visit, &pc); + fclose(sink); + return rc; +} + +/* The follower's logOk decision for (prev_log_index, prev_log_term) read from + * its REAL observable state — this is exactly the accept/reject signal a real + * leader reads back from the follower's AppendEntries reply and uses to drive + * nextIndex backtracking (raft.c logOk). */ +static int hcs4_follower_logok(cluster_t* c, uint64_t prev_log_index, + uint64_t prev_log_term) +{ + if (prev_log_index == 0) + return 1; + uint64_t f_term = persist_entry_term(&c->persist, prev_log_index); + uint64_t f_log_len = c->ha.raft.log_len; + return (prev_log_index <= f_log_len) && (f_term == prev_log_term); +} + +/* HCS_MAX_TAIL: the leader-model log capacity for the backtrack driver. */ +#define HCS4_MAX_LOG 32u + +/* Drive a REAL leader-style nextIndex backtracking sequence against the + * follower's REAL AE intake. The leader holds an authoritative log of + * `leader_log_len` entries (per-LSN term in leader_term[1..], payload in + * leader_pay/leader_paylen). Starting at nextIndex = leader_log_len + 1, each + * step sends prev=(nextIndex-1, leader_term[nextIndex-1]) plus, when one exists, + * the entry at nextIndex; accept advances nextIndex, reject decrements it — the + * decision taken from the follower's real reply (hcs4_follower_logok). Bounded: + * a permanent reject loop is observable as reached==0 within the step budget. + * + * *out_rejects → number of ae_reject_logmismatch backtrack steps observed + * *out_reached → 1 iff the follower reconverged (nextIndex passed the tip) + * Returns the step count. */ +static int hcs4_drive_backtrack(cluster_t* c, const uint64_t leader_term[], + uint8_t* const leader_pay[], + const uint32_t leader_paylen[], + uint64_t leader_log_len, uint64_t leader_commit, + int leader_idx, int* out_rejects, + int* out_reached) +{ + uint64_t next_index = leader_log_len + 1; + int rejects = 0; + int reached = 0; + int max_steps = (int)leader_log_len + 8; /* bounded — no infinite loop */ + int steps = 0; + + for (; steps < max_steps; steps++) { + uint64_t prev = next_index - 1; + uint64_t prev_term = (prev == 0) ? 0 : leader_term[prev]; + int has_entry = (next_index <= leader_log_len) ? 1 : 0; + + wal_catchup_entry_t ae; + if (has_entry) { + hcs4_fill_ae(&ae, /*heartbeat*/0, prev, prev_term, leader_commit, + /*entry_lsn*/next_index, WAL_JOB_SUBMIT, + /*entry_term*/leader_term[next_index], + leader_pay[next_index], leader_paylen[next_index]); + } else { + /* nextIndex past the leader tip → a 0-entry heartbeat probe carrying + * the leader's live term (its current term == its tip's term). */ + hcs4_fill_ae(&ae, /*heartbeat*/1, prev, prev_term, leader_commit, + 0, 0, 0, NULL, 0); + ae.term = leader_term[leader_log_len]; + } + + /* Decide accept/reject from the follower's state BEFORE this RPC mutates + * its log (the RPC only touches index = prev+1 and above, never `prev`), + * so this is the logOk the follower will answer. */ + int accept = hcs4_follower_logok(c, prev, prev_term); + + ha_consume_append_entries(c, &ae, leader_idx); + + if (accept) { + next_index++; + if (next_index > leader_log_len) { + reached = 1; + steps++; + break; + } + } else { + rejects++; + if (next_index <= 1) + break; /* safety: cannot backtrack below the base */ + next_index--; + } + } + + if (out_rejects) + *out_rejects = rejects; + if (out_reached) + *out_reached = reached; + return steps; +} + +/* THE DISCRIMINATOR — a former-primary multi-entry divergent uncommitted tail is + * reconverged by the follower-side AE backtrack + conflict-truncate, driven + * deterministically with NO channel flap. */ +static void test_4_1_follower_reconverges_multientry_divergent_tail(void) +{ + cluster_t* c = hcs4_cluster_up("recv"); + TEST_ASSERT_NOT_NULL(c); + + /* The recovered former primary believes itself at the OLD term 4. */ + enum { OLD_TERM = 4, NEW_TERM = 6, COMMITTED = 11, FOLLOWER_TIP = 15, + LEADER_TIP = 13 }; + hcs4_make_follower(c, /*term*/OLD_TERM, /*n_peers*/3); + + uint8_t full[HCS_FULL_LEN]; + hcs_fill_body(full); + uint8_t blob[sizeof(raft_entry_blob_t) + HCS_FULL_LEN]; + + /* ── Seed the ctrl-2 shape ──────────────────────────────────────────────── + * Committed prefix [1..11] @ OLD_TERM (the agreed prefix the leader shares), + * then a DIVERGENT UNCOMMITTED tail [12..15] @ OLD_TERM — the former-primary + * entries that never committed. Built through the WIRED standby persist + * callback so each LSN carries its real per-LSN term in the WAL. */ + for (uint64_t lsn = 1; lsn <= FOLLOWER_TIP; lsn++) { + size_t n = hcs4_pack_entry(blob, WAL_JOB_SUBMIT, full, HCS_FULL_LEN); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, + ha_cb_persist_entry_for_test(&c->ha, lsn, OLD_TERM, blob, n), + "seed write must persist (real fs required for inline fdatasync — bug-924)"); + } + c->ha.raft.log_len = FOLLOWER_TIP; + atomic_store_explicit(&c->persist.committed_lsn, (uint64_t)COMMITTED, + memory_order_release); + + /* Sanity: the divergent tail is on disk at OLD_TERM with a full body. */ + for (uint64_t lsn = COMMITTED + 1; lsn <= FOLLOWER_TIP; lsn++) + TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)OLD_TERM, + persist_entry_term(&c->persist, lsn), + "precondition: divergent tail stored at the OLD term"); + + /* ── Leader model ───────────────────────────────────────────────────────── + * [1..11] @ OLD_TERM (the shared agreed prefix) + [12,13] @ NEW_TERM (the new + * leader's own authoritative entries written after winning the election). */ + uint64_t leader_term[HCS4_MAX_LOG]; + uint8_t* leader_pay[HCS4_MAX_LOG]; + uint32_t leader_paylen[HCS4_MAX_LOG]; + memset(leader_term, 0, sizeof(leader_term)); + memset(leader_pay, 0, sizeof(leader_pay)); + memset(leader_paylen, 0, sizeof(leader_paylen)); + for (uint64_t lsn = 1; lsn <= COMMITTED; lsn++) { + leader_term[lsn] = OLD_TERM; + leader_pay[lsn] = full; + leader_paylen[lsn] = HCS_FULL_LEN; + } + for (uint64_t lsn = COMMITTED + 1; lsn <= LEADER_TIP; lsn++) { + leader_term[lsn] = NEW_TERM; + leader_pay[lsn] = full; + leader_paylen[lsn] = HCS_FULL_LEN; + } + + /* ── Drive the deterministic backtrack (no channel flap) ──────────────────*/ + int rejects = -1, reached = -1; + int steps = hcs4_drive_backtrack(c, leader_term, leader_pay, leader_paylen, + (uint64_t)LEADER_TIP, /*leader_commit*/ + (uint64_t)COMMITTED, /*leader_idx*/1, + &rejects, &reached); + printf("test_4_1 backtrack: steps=%d rejects=%d reached=%d " + "follower_log_len=%" PRIu64 " follower_term=%" PRIu64 "\n", + steps, rejects, reached, c->ha.raft.log_len, c->ha.raft.term); + + /* (1) The backtrack actually exercised the ae_reject_logmismatch path — the + * exact reject ctrl-2 loops on (former-primary tail above the match point). */ + TEST_ASSERT_TRUE_MESSAGE(rejects >= 1, + "the divergent former-primary tail must force at least one " + "ae_reject_logmismatch backtrack step (else the RED is not exercising " + "the ctrl-2 reject path)"); + + /* (2) THE DISCRIMINATOR: the follower must RECONVERGE — backtrack to the + * match point, conflict-truncate the multi-entry divergent suffix, and accept + * the leader's authoritative entries — with NO permanent reject loop. If + * this FAILS on baseline, Phase 4 has a real follower-side reconvergence bug + * (task 4.2's RED). If it PASSES, the follower path is sound and the live + * stall is the Phase-5 channel flap. */ + TEST_ASSERT_TRUE_MESSAGE(reached, + "DISCRIMINATOR: the follower must reconverge a MULTI-ENTRY former-primary " + "divergent tail (backtrack → conflict-truncate → accept) within the " + "bounded step budget — a non-reached outcome is a permanent reject loop " + "(Phase-4 follower-side bug, the RED task 4.2 fixes)"); + + /* (3) The agreed committed prefix [1..11] @ OLD_TERM is untouched. */ + for (uint64_t lsn = 1; lsn <= COMMITTED; lsn++) + TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)OLD_TERM, + persist_entry_term(&c->persist, lsn), + "the agreed committed prefix must NOT be truncated or re-termed"); + + /* (4) The leader's authoritative entries [12,13] now carry NEW_TERM. */ + for (uint64_t lsn = COMMITTED + 1; lsn <= LEADER_TIP; lsn++) + TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)NEW_TERM, + persist_entry_term(&c->persist, lsn), + "the leader's authoritative tail must be accepted at the NEW term"); + + /* (5) The divergent OLD-term suffix above the leader tip [14,15] is GONE — + * the conflict-truncate removed the whole divergent suffix in one shot. */ + for (uint64_t lsn = LEADER_TIP + 1; lsn <= FOLLOWER_TIP; lsn++) + TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, + persist_entry_term(&c->persist, lsn), + "the divergent suffix above the leader tip must be truncated (absent)"); + + /* (6) The follower's library log matches the leader's tip + term adopted. */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)LEADER_TIP, c->ha.raft.log_len, + "the follower's log_len must equal the leader tip after reconvergence"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)NEW_TERM, c->ha.raft.term, + "the follower must have adopted the leader's NEW term (UpdateTerm/G5)"); + + /* (7) Payload integrity (the live divergence was payload LOSS): the + * reconverged entries hold the FULL leader body via the (term,pcrc,len) + * readback probe — not an empty/partial body. */ + for (uint64_t lsn = COMMITTED + 1; lsn <= LEADER_TIP; lsn++) { + hcs_probe_t pr; + TEST_ASSERT_EQUAL_INT(0, hcs4_probe(c, lsn, &pr)); + TEST_ASSERT_TRUE_MESSAGE(pr.present, + "the reconverged entry must be present on disk"); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(HCS_FULL_LEN, pr.len, + "the reconverged entry must hold the FULL leader payload, not empty"); + TEST_ASSERT_NOT_EQUAL_MESSAGE(0u, pr.pcrc, + "the reconverged entry must carry a real payload CRC, not 0"); + } + + hcs4_cluster_down(c); + printf("test_4_1_follower_reconverges_multientry_divergent_tail: done\n"); +} + +/* SECONDARY 4.1 case — a rejoining node with a catch-up APPLY GAP must NOT be + * reported converged until applied_lsn catches the committed boundary. Uses the + * 1.4 apply-gap seam to park committed > applied and the REAL primary-side + * convergence predicate (ha_process_standby_state: a standby's reported_lsn is + * its applied_lsn; converged iff reported_lsn >= committed_lsn). Baseline- + * passing PIN — the predicate already exists; this guards "WAL on disk is not + * convergence; the state machine must have applied it". */ +static void test_4_1_apply_gap_blocks_converged_until_applied(void) +{ + hcs_member_t m; + TEST_ASSERT_EQUAL_INT(0, hcs_member_up(&m, "agap")); + ha_state_t ha; + ha_init(&ha, &g_hcs_cfg, 0); + + enum { COMMITTED = 11, T = 1 }; + + /* Inject a catch-up apply gap: the WAL is committed to 11 but only applied + * to 8 (the rejoining node has the bytes on disk but not yet in its state + * machine). */ + hcs_inject_apply_gap(&ha, m.p, /*applied=*/8, /*committed=*/COMMITTED); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, hcs_apply_gap(&ha, m.p), + "an injected catch-up apply gap must be observable (committed 11 - applied 8)"); + + /* The primary's convergence predicate sees the standby's reported applied_lsn + * (8) below committed (11) → NOT converged. */ + ha_process_standby_state(&ha, /*peer_idx*/0, /*reported_tag*/T, + atomic_load_explicit(&ha.applied_lsn, memory_order_acquire), + /*committed_tag*/T, + atomic_load_explicit(&m.p->committed_lsn, memory_order_acquire)); + TEST_ASSERT_FALSE_MESSAGE(ha.standby_converged[0], + "a node with a catch-up apply gap (applied 8 < committed 11) must NOT be " + "reported converged — bytes on disk are not an applied state machine"); + + /* Partial catch-up (applied 10, still < committed 11) — still not converged. */ + hcs_inject_apply_gap(&ha, m.p, /*applied=*/10, /*committed=*/COMMITTED); + ha_process_standby_state(&ha, 0, T, + atomic_load_explicit(&ha.applied_lsn, memory_order_acquire), T, + atomic_load_explicit(&m.p->committed_lsn, memory_order_acquire)); + TEST_ASSERT_FALSE_MESSAGE(ha.standby_converged[0], + "a node still one entry short (applied 10 < committed 11) must remain " + "not-converged (no premature convergence)"); + + /* Fully applied: applied == committed == log boundary → gap 0 → converged. */ + hcs_inject_apply_gap(&ha, m.p, /*applied=*/COMMITTED, /*committed=*/COMMITTED); + TEST_ASSERT_EQUAL_UINT64(0, hcs_apply_gap(&ha, m.p)); + ha_process_standby_state(&ha, 0, T, + atomic_load_explicit(&ha.applied_lsn, memory_order_acquire), T, + atomic_load_explicit(&m.p->committed_lsn, memory_order_acquire)); + TEST_ASSERT_TRUE_MESSAGE(ha.standby_converged[0], + "once applied_lsn catches the committed boundary the node is reported " + "converged (applied 11 == committed 11)"); + + ha_fini(&ha); + hcs_member_down(&m); + printf("test_4_1_apply_gap_blocks_converged_until_applied: PASS (pin)\n"); +} + /* ═══════════════════════════════════════════════════════════════════════════ * SCAFFOLDING (PASS) — prove each shared seam is usable by later phases. * ═══════════════════════════════════════════════════════════════════════════ */ @@ -1048,6 +1466,15 @@ void test_ha_core_safety(void) RUN_TEST(test_2_4_streamer_short_read_retry_recovers); RUN_TEST(test_2_3_termblind_route_payload_guarded); RUN_TEST(test_2_3_unrepairable_catchup_raises_nonpromotable_fence); + /* Task 4.1 follower-side reconvergence (the DISCRIMINATOR): + * DISCRIMINATOR — a MULTI-ENTRY former-primary divergent tail is + * reconverged by the follower AE backtrack + conflict- + * truncate (no channel flap). PASS ⇒ root cause is the + * Phase-5 channel flap; FAIL ⇒ Phase-4 follower bug (4.2). + * PIN — a catch-up apply gap blocks "converged" until applied + * catches the committed boundary. */ + RUN_TEST(test_4_1_follower_reconverges_multientry_divergent_tail); + RUN_TEST(test_4_1_apply_gap_blocks_converged_until_applied); /* Scaffolding: each shared seam is usable by 2.3 / 4.1 / 5.1. */ RUN_TEST(test_seam_wal_construct_and_probe); RUN_TEST(test_seam_divergence_is_observable);