lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ha-term-safety): remove task-2.4 commit min-ceiling (Option A) + fix bug-991 catch-up off-by-one

Browse source 
Two coupled fixes for a live commit-stall (committed_lsn pinned at the own-term
NOOP while a 3-node cluster was durable to LSN 5 — jobs acked but never committed):

bug-991: the WAL catch-up applied_lsn was advanced to lsn+1 (next-to-apply) in
the three live/catch-up apply paths, but recovery/checkpoint/init and the leader's
catch-up streamer treat it as last-held. The over-advertise made the streamer skip
an LSN -> library logOk wedge -> no WalAck. Fixed to last-held (= lsn).

Option A (design reconsideration of REVIEW ISSUE 3): removed the
committed = min(durable-majority, raft_commit_index) ceiling. Under single-writer
the durable-majority watermark is already (term,lsn)-current-term-safe (term-carrying
replication + (term,lsn) reconcile/truncate-before-durable + own-term NOOP gate +
post-election truncate-to-safe), so the library commit_index was a redundant,
starvation-prone gate that pinned committed below the safe durable majority whenever
the library lagged. The durable-majority barrier + own-term writable gate are now
authoritative; raft_commit_index is inert-advisory.

Safety: reviewer-verified the durable-majority watermark is (term,lsn)-safe and
Figure-8 holds via the own-term gate (new RED->GREEN test); bug-851 cannot recur.
Live: commit trace 4->5 + walcompare committed-prefix identical across 3 nodes.
177 unit tests pass.
This commit is contained in:
Rongsong Shen 2026-06-23 00:21:06 +00:00
parent df4bc4f86a
commit 83fbed2370
5 changed files with 396 additions and 172 deletions
Download patch file Download diff file Expand all files Collapse all files

53
src/controller/testsuite/test_config_repl.c
View file

@ -2020,8 +2020,11 @@ static void test_wal_catchup_entry_advances_applied_lsn(void)
/* GREEN will implement ha_apply_wal_catchup_entry() */
int rc = ha_apply_wal_catchup_entry(&ha, p, &entry, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to lsn+1 = 43 (mirrors persist->current_lsn) */
TEST_ASSERT_EQUAL_UINT64(43,
/* bug-991: applied_lsn is the LAST CONTIGUOUSLY-HELD LSN, so a single applied
* entry at LSN 42 advances applied_lsn to 42 (NOT 43). The +1 over-advertised
* the standby's watermark and made the leader's catch-up streamer skip the
* real next-needed LSN. */
TEST_ASSERT_EQUAL_UINT64(42,
(uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire));
ha_fini(&ha);
@ -2128,8 +2131,9 @@ static void test_wal_catchup_batch_advances_applied_lsn(void)
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to last_entry->lsn + 1 = 13 */
TEST_ASSERT_EQUAL_UINT64(13,
/* bug-991: applied_lsn = last contiguously-held LSN = last_entry->lsn = 12
* (NOT 13). */
TEST_ASSERT_EQUAL_UINT64(12,
(uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire));
ha_fini(&ha);
@ -2175,8 +2179,9 @@ static void test_wal_catchup_partial_batch(void)
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to 21+1=22 */
TEST_ASSERT_EQUAL_UINT64(22,
/* bug-991: applied_lsn = last contiguously-held LSN = 21 (NOT 22). Monotonic:
* advances from the pre-set 5 to 21. */
TEST_ASSERT_EQUAL_UINT64(21,
(uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire));
ha_fini(&ha);
@ -2225,10 +2230,12 @@ static void test_wal_catchup_batch_error_entry_skipped(void)
/* Overall rc == 0: outer batch call succeeds even with inner skips */
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must reach at least 31 (e0 applied); ideally 33 (e2 applied) */
/* bug-991: applied_lsn = last contiguously-held LSN. e0 (LSN 30) applies,
* the NULL sub-entry is skipped, e2 (LSN 32) applies → last-held is 32 (or at
* least 30 if e2 were dropped). Must reach at least 30 (e0 held). */
uint64_t final_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn,
memory_order_acquire);
TEST_ASSERT_TRUE(final_lsn >= 31);
TEST_ASSERT_TRUE(final_lsn >= 30);
ha_fini(&ha);
destroy_test_persist_catchup(p);
@ -2263,8 +2270,9 @@ static void test_wal_catchup_batch_zero_entries_fallback(void)
int rc = ha_apply_wal_catchup_entry(&ha, p, &entry, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Single-entry path: applied_lsn = 50+1 = 51 */
TEST_ASSERT_EQUAL_UINT64(51,
/* bug-991: single-entry path advances applied_lsn to the last-held LSN = 50
* (NOT 51). */
TEST_ASSERT_EQUAL_UINT64(50,
(uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire));
ha_fini(&ha);
@ -2321,8 +2329,9 @@ static void test_wal_catchup_batch_256_entries(void)
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must advance to last_entry->lsn + 1 = 1000+255+1 = 1256 */
TEST_ASSERT_EQUAL_UINT64(1256,
/* bug-991: applied_lsn = last contiguously-held LSN = last_entry->lsn =
* 1000+255 = 1255 (NOT 1256). */
TEST_ASSERT_EQUAL_UINT64(1255,
(uint64_t)atomic_load_explicit(&ha.applied_lsn, memory_order_acquire));
ha_fini(&ha);
@ -2340,9 +2349,10 @@ static void test_wal_catchup_batch_256_entries(void)
*
* Strategy: start with a pre-set applied_lsn, apply a batch of 4 entries,
* verify that applied_lsn does NOT advance to an intermediate value and IS
* exactly last_entry+1 at the end. We confirm the "only-after-full-batch"
* invariant by checking the exact final value equals last_lsn+1 (not some
* intermediate lsn+1 from a mid-batch advance).
* exactly the last contiguously-held LSN at the end (bug-991: last_lsn, not
* last_lsn+1). We confirm the "only-after-full-batch" invariant by checking the
* exact final value equals last_lsn (not some intermediate held LSN from a
* mid-batch advance).
* Requirements 5.3, 5.5. */
static void test_wal_catchup_batch_applied_lsn_only_after_full_batch(void)
{
@ -2382,17 +2392,16 @@ static void test_wal_catchup_batch_applied_lsn_only_after_full_batch(void)
int rc = ha_apply_wal_catchup_entry(&ha, p, &outer, NULL, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
/* applied_lsn must be exactly 204 (203+1) — the post-batch value.
* If the implementation advanced after each sub-entry, it would still
* reach 204, but the test for correctness is that the final value
* equals the LAST entry lsn+1, not any intermediate lsn+1.
* Specifically, it must NOT remain at 201 or 202 (partial advance). */
/* bug-991: applied_lsn must be exactly 203 (the last contiguously-held LSN) —
* the post-batch value. The test for correctness is that the final value
* equals the LAST entry's lsn, not any intermediate held LSN.
* Specifically, it must NOT remain at 200 or 201 (partial advance). */
uint64_t final_lsn = (uint64_t)atomic_load_explicit(&ha.applied_lsn,
memory_order_acquire);
/* Must have advanced past starting value (100) */
TEST_ASSERT_GREATER_THAN_UINT64(100, final_lsn);
/* Must equal the last-entry+1 (204), not any intermediate value */
TEST_ASSERT_EQUAL_UINT64(204, final_lsn);
/* Must equal the last-held LSN (203), not any intermediate value */
TEST_ASSERT_EQUAL_UINT64(203, final_lsn);
ha_fini(&ha);
destroy_test_persist_catchup(p);

375
src/controller/testsuite/test_term_safety_adapter.c
View file

@ -688,115 +688,177 @@ static uint64_t t24_committed(void)
return atomic_load(&g_t24_p.committed_lsn);
}
/* Test: the library commit index is the commit SOURCE once fed. A durable
* majority covers LSN 3, but the library has only admitted (current-term commit)
* up to LSN 1, so the barrier caps committed at 1 — NOT at the durable-majority
* floor of 3. Raising the library ceiling to 3 then releases the barrier to 3.
* This proves the library current-term commit index gates the commit, replacing
* the term-blind safe-commit (R4.1, R4.5, R5.4).
/* commit-ceiling-optA — REFRAMED from test_commit_advanced_is_commit_source.
*
* (raft_commit_index == 0 is the INERT pre-wire state — the library source is
* not yet active and the barrier keeps autonomous durable-majority behavior;
* that flag-OFF fallback is covered by the existing test_commit_engine suite,
* whose autonomous semantics this task must not regress.) */
static void test_commit_advanced_is_commit_source(void)
* The durable-majority barrier is AUTHORITATIVE for commit-advance; the recorded
* library commit_index is inert-advisory and does NOT cap committed_lsn. A
* durable majority covering LSN 3 commits to 3 EVEN WHILE the recorded library
* commit_index lags at 1 — this is the starvation the old ceiling caused (commit
* pinned at 1 below the safe durable majority) and the Option A fix removes.
*
* Meaningful both ways: it FAILS if commit were wrongly gated by the recorded
* index (committed would stick at 1), AND it FAILS if commit over-advanced past
* the durable majority (it asserts == 3, the exact durable watermark). */
static void test_commit_advances_to_durable_majority_index_inert(void)
{
/* 3 peers = self + 2 standbys; majority needs self + 1 standby. */
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/3);
/* The library is the commit source from the first commit: establish the
* ceiling at LSN 1 BEFORE any ACK drives evaluation (production order — the
* ACK path drives raft_advance_commit, which fires on_commit_advanced, then
* the barrier re-evaluates). */
/* Record an advisory library commit_index that LAGS at 1 (e.g. match_index[]
* not yet advanced via catch-up). Under the old design this pinned committed
* at 1; under Option A it is inert. */
commit_set_raft_commit_index(&g_t24_ce, 1);
/* A standby ACKs LSN 3; self is durable to LSN 3 → durable majority = 3.
* The barrier must cap committed at the library ceiling (1), NOT at the
* durable majority (3). */
* committed advances to the durable majority (3), NOT pinned at the lagging
* advisory index (1). */
t24_standby_ack(0, 3);
TEST_ASSERT_EQUAL_UINT64(1, t24_committed());
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
/* Library admits up to LSN 3 → barrier releases to the durable majority (3). */
/* Advancing the advisory record changes nothing — already at the durable
* majority; record is inert. */
commit_set_raft_commit_index(&g_t24_ce, 3);
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
t24_engine_teardown();
printf("test_commit_advanced_is_commit_source: PASS\n");
printf("test_commit_advances_to_durable_majority_index_inert: PASS\n");
}
/* Test: the library commit index is a CEILING — committed never advances past
* it even when a durable majority covers a higher LSN (current-term gate). The
* barrier's durable cap is the OTHER ceiling: committed = min(durable-majority,
* raft_commit_index). No gap: raising either ceiling alone cannot overshoot the
* other (R4.1, R4.5). */
static void test_commit_advanced_ceiling_caps_durable_majority(void)
/* commit-ceiling-optA — REFRAMED from test_commit_advanced_ceiling_caps_durable_majority.
*
* The recorded library commit_index is NOT a ceiling: committed advances to the
* full durable majority even when the advisory index sits below it. Pins the
* removal of the redundant ceiling.
*
* Meaningful both ways: FAILS if the advisory index still capped committed at 3
* (the old, now-rejected behavior), AND FAILS if committed over-advanced past
* the durable majority of 5 (asserts == 5 exactly). */
static void test_commit_index_record_is_not_a_ceiling(void)
{
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/5);
/* Establish the library ceiling at 3 BEFORE any ACK (production order). The
* library has only committed up to 3 (higher entries prev-term or not yet
* current-term-agreed). */
/* Advisory record lags at 3 while the durable majority will reach 5. */
commit_set_raft_commit_index(&g_t24_ce, 3);
/* Durable majority covers up to LSN 5 (self durable 5 + standby acked 5).
* committed must cap at the library ceiling (3), not the durable floor (5). */
* committed advances to 5 — the lagging advisory index (3) does NOT cap it. */
t24_standby_ack(0, 5);
TEST_ASSERT_EQUAL_UINT64(3, t24_committed());
TEST_ASSERT_EQUAL_UINT64(5, t24_committed());
/* Library advances the ceiling to 5 → committed catches up to the durable
* majority floor (5). Monotonic, no regression. */
/* Catching the advisory record up to 5 is a no-op (already there). */
commit_set_raft_commit_index(&g_t24_ce, 5);
TEST_ASSERT_EQUAL_UINT64(5, t24_committed());
t24_engine_teardown();
printf("test_commit_advanced_ceiling_caps_durable_majority: PASS\n");
printf("test_commit_index_record_is_not_a_ceiling: PASS\n");
}
/* Test: the library commit index NEVER lets committed advance past what is
* durable locally. With raft_commit_index=4 but self only durable to LSN 2 and
* the standby acked LSN 2, committed must stay at the durable-majority floor (2)
* — the library ceiling does not bypass persist-before-ACK / durable cap
* (R5.4 — barrier durability mechanism unchanged). */
/* commit-ceiling-optA — RETAINED (unchanged contract): committed never advances
* past what is durable locally. The durable self-cap (persist_durable_lsn) is
* still authoritative regardless of any advisory library index. With the
* advisory index well ahead (4) but self durable only to LSN 2 and the standby
* acked LSN 2, committed stays at the durable-majority floor (2). This invariant
* is unchanged by Option A — the durable cap was always orthogonal to the
* removed ceiling — but is re-pinned here to guard Commit Durability (R5.4). */
static void test_commit_advanced_does_not_bypass_durable_cap(void)
{
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/4);
atomic_store(&g_t24_p.durable_lsn, 2); /* self durable only to 2 */
t24_standby_ack(0, 2); /* standby agrees to 2 */
commit_set_raft_commit_index(&g_t24_ce, 4); /* library ceiling well ahead */
commit_set_raft_commit_index(&g_t24_ce, 4); /* advisory index well ahead */
commit_on_ack(&g_t24_ce, 0, 2);
/* Durable-majority floor is 2; the library ceiling (4) cannot raise it. */
/* Durable-majority floor is 2; nothing (advisory index 4 included) raises it. */
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
t24_engine_teardown();
printf("test_commit_advanced_does_not_bypass_durable_cap: PASS\n");
}
/* commit-ceiling-optA — NEW load-bearing safety test: the own-term writable gate
* (Figure-8). With Option A removing the library-commit-index ceiling, the
* own-term NOOP gate (commit_set_writable_lsn / wait_done_locked) is now the
* load-bearing realization of "no acked-committed prior-term entry until the
* current-term NOOP is durable on a majority".
*
* Scenario: self holds a PRIOR-term entry at LSN 1 and the current-term NOOP at
* LSN 2 (the promotion NOOP). The own-term gate is armed at LSN 2. A durable
* majority reaches LSN 1 FIRST (the NOOP not yet majority-durable):
* - committed_lsn MAY advance to 1 (the barrier is contiguous-prefix), but a
* WAIT for LSN 1 must NOT complete — the gate blocks acknowledgement until
* committed_lsn >= writable_lsn (2). This is the safety property: the
* prior-term entry at LSN 1 is NOT acknowledged-committed yet.
* Then the NOOP becomes majority-durable (committed reaches 2):
* - the gate opens and the WAIT completes.
*
* RED if the writable gate were bypassed: a WAIT(lsn=1) would complete as soon
* as committed_lsn >= 1, acknowledging a prior-term entry before the own-term
* NOOP is majority-durable — the Figure-8 violation. GREEN: it blocks (times
* out) until the NOOP commits, then completes. */
static void test_own_term_gate_blocks_priorterm_ack_until_noop_durable(void)
{
/* 3 peers; majority needs self + 1 standby. Appended tip = LSN 2 (prior-term
* entry at 1 + current-term NOOP at 2). */
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2);
/* Arm the own-term gate at the NOOP's LSN (2): no acknowledgement of any
* entry until committed_lsn >= 2 (the current-term NOOP is majority-durable). */
commit_set_writable_lsn(&g_t24_ce, 2);
/* Durable majority reaches LSN 1 only: self durable to 2, but the standby has
* acked just LSN 1, so the median (self + 1 of 2 peers) covers LSN 1. */
atomic_store(&g_t24_p.durable_lsn, 2);
t24_standby_ack(0, 1);
/* The contiguous-prefix barrier MAY advance committed_lsn to 1 ... */
TEST_ASSERT_EQUAL_UINT64(1, t24_committed());
/* ... but a WAIT for LSN 1 must NOT complete: the own-term gate (writable_lsn
* == 2) holds it. Pre-NOOP-durable, this is the Figure-8 safety block — the
* prior-term entry at LSN 1 is committed-but-not-acknowledged. A short
* timeout proves the gate blocks (returns -ETIMEDOUT, not 0). */
TEST_ASSERT_EQUAL_INT(-ETIMEDOUT,
commit_submit(&g_t24_ce, /*lsn*/1, COMMIT_MODE_WAIT, /*timeout_ms*/150));
/* The current-term NOOP at LSN 2 now becomes majority-durable (standby acks
* LSN 2). committed_lsn advances to 2 and the gate opens. */
t24_standby_ack(0, 2);
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
/* The WAIT for LSN 1 now completes: committed_lsn (2) >= lsn (1) AND
* >= writable_lsn (2). The prior-term entry is acknowledged only AFTER the
* own-term NOOP is majority-durable. */
TEST_ASSERT_EQUAL_INT(0,
commit_submit(&g_t24_ce, /*lsn*/1, COMMIT_MODE_WAIT, /*timeout_ms*/500));
t24_engine_teardown();
printf("test_own_term_gate_blocks_priorterm_ack_until_noop_durable: PASS\n");
}
/* Test (pins the own-term-gate DECISION — RETAINED, complementary, no deadlock):
* the own-term writable_lsn gate is a WAIT-completion predicate, NOT a cap on
* committed_lsn. It is retained alongside the library commit source. With the
* gate armed at LSN 2 and the library commit index + durable majority both at 2,
* committed advances to 2 AND a WAIT(lsn=2) completes — the two gates cannot
* deadlock (the own-term gate never holds committed_lsn back; the library
* ceiling never holds the WAIT predicate back beyond committed_lsn). */
* committed_lsn. With the gate armed at LSN 2 and the durable majority at 2,
* committed advances to 2 AND a WAIT(lsn=2) completes — the gate never holds
* committed_lsn back, and once committed reaches the gate the WAIT completes (no
* deadlock). The advisory library index is irrelevant here (inert under optA). */
static void test_own_term_gate_retained_no_deadlock(void)
{
t24_engine_setup(/*n_peers*/3, /*n_standbys*/2, /*appended*/2);
/* Arm the own-term gate at LSN 2 (retained complementary durability gate). */
/* Arm the own-term gate at LSN 2 (the load-bearing complementary gate). */
commit_set_writable_lsn(&g_t24_ce, 2);
/* Library commit source + durable majority both reach LSN 2. */
/* Durable majority reaches LSN 2. */
t24_standby_ack(0, 2);
commit_set_raft_commit_index(&g_t24_ce, 2);
commit_set_raft_commit_index(&g_t24_ce, 2); /* advisory; inert */
commit_on_ack(&g_t24_ce, 0, 2);
/* committed_lsn advanced to 2 (own-term gate never caps committed_lsn). */
TEST_ASSERT_EQUAL_UINT64(2, t24_committed());
/* A WAIT for LSN 2 completes immediately: committed_lsn (2) >= lsn (2) AND
* >= writable_lsn (2). No deadlock between the gate and the library source. */
* >= writable_lsn (2). No deadlock between the gate and the barrier. */
TEST_ASSERT_EQUAL_INT(0,
commit_submit(&g_t24_ce, /*lsn*/2, COMMIT_MODE_WAIT, /*timeout_ms*/500));
@ -804,14 +866,20 @@ static void test_own_term_gate_retained_no_deadlock(void)
printf("test_own_term_gate_retained_no_deadlock: PASS\n");
}
/* Test: the WIRED on_commit_advanced callback routes commit_index into the
* cluster's commit engine end-to-end (ha_cb_commit_advanced_for_test) as the
* commit SOURCE. A durable majority covers LSN 2, but the wired callback feeds
* commit_index=1 first → the barrier caps committed at 1 (library current-term
* gate), NOT at the durable majority of 2. A second callback at commit_index=2
* releases the barrier to 2. Proves the stub→real transition: the no-op stub
* never fed the source, so committed would have followed the durable majority. */
static void test_on_commit_advanced_wired_feeds_barrier(void)
/* commit-ceiling-optA — REFRAMED from test_on_commit_advanced_wired_feeds_barrier.
*
* The WIRED on_commit_advanced callback (ha_cb_commit_advanced_for_test) routes
* the library commit_index into the cluster's commit engine as an INERT-ADVISORY
* record — it records ce->raft_commit_index and wakes the barrier, but does NOT
* gate committed_lsn. The durable-majority barrier is authoritative: with a
* durable majority covering LSN 2, committed advances to 2 even though the wired
* callback last fed a LAGGING commit_index of 1.
*
* Meaningful both ways: FAILS if the wired callback still capped committed at 1
* (the old, now-rejected ceiling — the starvation), AND FAILS if committed
* over-advanced past the durable majority of 2 (asserts == 2). Also pins that
* the wired callback still RECORDS the advisory index (sink intact). */
static void test_on_commit_advanced_wired_records_advisory_index(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_commitadv_XXXXXX");
@ -842,25 +910,26 @@ static void test_on_commit_advanced_wired_feeds_barrier(void)
TEST_ASSERT_EQUAL_INT(0,
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
/* Fire the WIRED callback FIRST to establish the library ceiling at LSN 1
* (production order: the ACK path drives raft_advance_commit, which fires
* on_commit_advanced before/with the barrier re-evaluation). */
/* Fire the WIRED callback with a LAGGING commit_index of 1 (e.g. match_index[]
* not yet caught up). Under optA this only RECORDS the advisory index. */
ha_cb_commit_advanced_for_test(&c->ha, 1);
TEST_ASSERT_EQUAL_UINT64(1, c->commit.raft_commit_index); /* sink intact */
/* Standby ACKs LSN 2 → durable majority covers LSN 2. The wired source caps
* committed at 1, BELOW the durable majority (2): the library current-term
* gate governs (the term-blind safe-commit is retired as the source). */
/* Standby ACKs LSN 2 → durable majority covers LSN 2. committed advances to
* the durable majority (2), NOT pinned at the lagging advisory index (1):
* the advisory record does not gate commit. */
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
commit_on_ack(&c->commit, 0, 2);
TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->persist.committed_lsn));
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
/* Library admits LSN 2 → barrier releases to the durable majority (2). */
/* A later callback advancing the advisory record to 2 is a no-op for commit. */
ha_cb_commit_advanced_for_test(&c->ha, 2);
TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index);
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
commit_engine_fini(&c->commit);
teardown_cluster(c);
printf("test_on_commit_advanced_wired_feeds_barrier: PASS\n");
printf("test_on_commit_advanced_wired_records_advisory_index: PASS\n");
}
/* ================================================================== */
@ -1355,6 +1424,111 @@ static void test_live_broadcast_receive_persists_leader_term(void)
printf("test_live_broadcast_receive_persists_leader_term: PASS\n");
}
/* ================================================================== */
/* bug-991 — catch-up applied_lsn must be "last contiguously-held LSN" */
/* (NOT next-to-apply / lsn+1) so the leader's catch-up */
/* streamer never SKIPS the standby's real next-needed LSN. */
/* ================================================================== */
/* The standby's applied_lsn is the watermark it advertises to the leader
* (ha_nop_reply_state_report → sr.applied_lsn → standby_reported_lsn). The
* leader's catch-up streamer (ha_stream_wal_to_standby) and the snapshot/stream
* decision (ha_evaluate_repl_action) BOTH treat that value as the
* "last contiguously-held LSN" and stream the half-open range
* (standby_lsn, committed_lsn]. The recovery/checkpoint/init setters already use
* last-held semantics (persist_durable_floor / checkpoint_lsn / 0). The three
* live/catch-up apply paths historically advanced to lsn+1 (next-to-apply) —
* an off-by-one that made the standby advertise one LSN too high, so after it
* applied catch-up entry N the leader streamed (N+1, committed] and SKIPPED the
* real next-needed entry N+1 forever (logOk wedge → no WalAck → commit stall).
*
* RED (pre-fix): after applying a catch-up entry at LSN N the standby's
* applied_lsn reads N+1, and ha_evaluate_repl_action(standby_lsn=applied_lsn,
* committed=N) reports CONVERGED one entry early (skip).
* GREEN (post-fix): applied_lsn reads N (last-held); a standby that has applied
* up to N with committed > N still STREAMS (no skip), and the streamer's
* (standby_lsn, committed] range begins at exactly N+1. */
static void test_catchup_applied_lsn_is_last_held_no_skip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ts_bug991_lasthelp_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
make_follower(c, /*term*/7, /*n_peers*/3);
/* applied_lsn starts at 0 (init = empty/last-held-nothing). */
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&c->ha.applied_lsn));
/* Catch-up stream of LSN 1 then LSN 2 at the leader's term 7, contiguous
* (prev tuples chain 0/0 → 1/7), routed through the production receive entry
* point exactly as ha_stream_wal_to_standby → xce_decode would deliver them. */
uint8_t p1[1] = { 'a' };
wal_catchup_entry_t sub1;
memset(&sub1, 0, sizeof(sub1));
sub1.lsn = 1; sub1.entry_type = WAL_JOB_SUBMIT; sub1.term = 7;
sub1.prev_log_index = 0; sub1.prev_log_term = 0;
sub1.raw_payload = p1; sub1.raw_payload_len = sizeof(p1);
wal_catchup_entry_t* arr1[1] = { &sub1 };
wal_catchup_entry_t batch1;
memset(&batch1, 0, sizeof(batch1));
batch1.n_entries = 1; batch1.entries = arr1;
batch1.term = 7; batch1.prev_log_index = 0; batch1.prev_log_term = 0;
TEST_ASSERT_EQUAL_INT(0,
ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch1, /*ep*/NULL, c));
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 1));
/* THE bug-991 load-bearing assertion: after applying LSN 1 the standby
* advertises applied_lsn == 1 (last-held), NOT 2 (next-to-apply). Pre-fix
* this read 2 and the leader's next stream skipped LSN 2. */
TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->ha.applied_lsn));
/* No-skip property at the leader's decision seam: a standby holding up to
* LSN 1 with committed_lsn == 2 must STREAM (its next-needed LSN 2 is missing),
* NOT report CONVERGED. With the off-by-one advertised value (2) the leader
* would have seen standby_lsn(2) >= committed(2) → CONVERGED (the skip). */
{
uint64_t standby_lsn = atomic_load(&c->ha.applied_lsn);
ha_repl_action_t act = ha_evaluate_repl_action(
/*standby_tag*/1, standby_lsn,
/*committed_tag*/1, /*committed_lsn*/2, /*oldest_wal_lsn*/1);
TEST_ASSERT_EQUAL_INT(HA_REPL_STREAM_WAL, act);
/* The streamer's start LSN is standby_lsn + 1 == 2 — exactly the missing
* entry, no skip. */
TEST_ASSERT_EQUAL_UINT64(2, standby_lsn + 1);
}
/* Apply the next contiguous entry (LSN 2) — applied_lsn becomes 2 (last-held),
* not 3 — and the standby is now CONVERGED at committed_lsn == 2. */
uint8_t p2[1] = { 'b' };
wal_catchup_entry_t sub2;
memset(&sub2, 0, sizeof(sub2));
sub2.lsn = 2; sub2.entry_type = WAL_JOB_SUBMIT; sub2.term = 7;
sub2.prev_log_index = 1; sub2.prev_log_term = 7;
sub2.raw_payload = p2; sub2.raw_payload_len = sizeof(p2);
wal_catchup_entry_t* arr2[1] = { &sub2 };
wal_catchup_entry_t batch2;
memset(&batch2, 0, sizeof(batch2));
batch2.n_entries = 1; batch2.entries = arr2;
batch2.term = 7; batch2.prev_log_index = 1; batch2.prev_log_term = 7;
TEST_ASSERT_EQUAL_INT(0,
ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch2, /*ep*/NULL, c));
TEST_ASSERT_EQUAL_UINT64(7, persist_entry_term(&c->persist, 2));
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->ha.applied_lsn));
{
ha_repl_action_t act = ha_evaluate_repl_action(
/*standby_tag*/1, atomic_load(&c->ha.applied_lsn),
/*committed_tag*/1, /*committed_lsn*/2, /*oldest_wal_lsn*/1);
TEST_ASSERT_EQUAL_INT(HA_REPL_CONVERGED, act);
}
teardown_cluster(c);
printf("test_catchup_applied_lsn_is_last_held_no_skip: PASS\n");
}
/* ================================================================== */
/* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */
/* on the replication ACK path */
@ -1401,11 +1575,13 @@ static void leader_seed_entries(cluster_t* c, uint64_t term, int count)
/* (a) A current-term quorum ACK drives raft_on_append_entries_reply →
* raft_advance_commit → on_commit_advanced fires with the agreed commit_index,
* which raises the barrier ceiling. 3-peer cluster (self + 2 standbys, quorum=2):
* self holds LSN 1,2 at the current term; one standby ACKs LSN 2 → {self, peer1}
* is a quorum for LSN 2, the agreed entry is current-term, so commit advances to
* 2 and the library ceiling (commit_set_raft_commit_index effect) caps committed
* at the library point under a durable majority. */
* which is recorded as the advisory library index AND wakes the barrier. 3-peer
* cluster (self + 2 standbys, quorum=2): self holds LSN 1,2 at the current term;
* one standby ACKs LSN 2 → {self, peer1} is a quorum for LSN 2, the agreed entry
* is current-term, so the library commit_index advances to 2; the durable
* majority (also at 2) is what advances committed_lsn (optA: the durable-majority
* barrier is authoritative; the recorded index is inert-advisory). This leg
* still verifies the full ACK→library→callback→barrier wiring is intact. */
static void test_repl_ack_currentterm_quorum_advances_commit(void)
{
char tmp[256];
@ -1429,13 +1605,16 @@ static void test_repl_ack_currentterm_quorum_advances_commit(void)
TEST_ASSERT_EQUAL_INT(0,
repl_add_standby(&c->persist.repl, &g_ae_dummy_ep, "sb0", 7000));
/* Durable majority: the standby's durable position covers LSN 2 so the
* barrier ceiling is not held below the library commit point. */
/* Durable majority: the standby's durable position covers LSN 2. Under optA
* the durable-majority barrier is authoritative, so this ACK already advances
* committed_lsn to 2 (asserted at the end); the library wiring below is still
* exercised to confirm the ACK→library→callback path is intact. */
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
/* Nothing committed yet: the library has not advanced commit_index (no ACK
* routed through the library) so the commit source is still inert at 0. */
/* The library has not advanced commit_index yet (no ACK routed through the
* library) so the advisory record is still 0 — but committed_lsn is NOT
* gated by it (optA). */
TEST_ASSERT_EQUAL_UINT64(0, c->ha.raft.commit_index);
/* The standby ACKs LSN 2 at the leader's current term. The caller has
@ -1452,9 +1631,12 @@ static void test_repl_ack_currentterm_quorum_advances_commit(void)
* ({self, peer1} quorum, entry at LSN 2 is term T). */
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
/* (3) on_commit_advanced fired → the barrier ceiling rose and committed_lsn
* advanced to the library point under the durable majority (R4.1). */
/* (3) committed_lsn advanced to the durable majority (2). on_commit_advanced
* fired and recorded the advisory index (2), but the durable-majority barrier
* is what governs committed_lsn under optA (R4.1 realized via the authoritative
* durable barrier + own-term gate). */
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); /* advisory recorded */
commit_engine_fini(&c->commit);
teardown_cluster(c);
@ -1787,27 +1969,30 @@ static void test_every_callback_and_marker_path_end_to_end(void)
TEST_ASSERT_EQUAL_INT(-ENOENT,
config_committed_marker(c, &got_tag, got_sha));
/* on_commit_advanced is LOAD-BEARING (production order: the ACK path fires
* on_commit_advanced, which records the library ceiling, BEFORE the barrier
* re-evaluates the durable majority). Fire the WIRED callback at
* commit_index=1 FIRST to set the ceiling at 1, THEN supply a durable
* majority covering LSN 2. committed_lsn caps at the library ceiling (1),
* BELOW the durable majority (2). This can only hold if on_commit_advanced
* fed the library current-term commit point as the barrier ceiling (the
* term-blind safe-commit is no longer the source); a no-op callback would
* let the barrier release committed_lsn to the durable majority (2). */
/* on_commit_advanced records the advisory library index and wakes the
* barrier (optA: it does NOT gate committed_lsn). Production order: the ACK
* path fires on_commit_advanced with a (possibly lagging) commit_index, then
* the durable-majority barrier — which is authoritative — advances
* committed_lsn. Fire the WIRED callback with a LAGGING commit_index=1, then
* supply a durable majority covering LSN 2: committed_lsn advances to the
* durable majority (2), NOT pinned at the lagging advisory index (1). Under
* the OLD (rejected) ceiling design this would have stuck committed at 1 —
* the starvation. The advisory index is still RECORDED (sink intact). */
ha_cb_commit_advanced_for_test(&c->ha, 1);
TEST_ASSERT_EQUAL_UINT64(1, c->commit.raft_commit_index); /* advisory recorded */
atomic_store(&c->persist.repl.standbys[0].acked_lsn, 2);
commit_on_ack(&c->commit, /*peer_idx*/0, /*acked_lsn*/2);
TEST_ASSERT_EQUAL_UINT64(1, atomic_load(&c->persist.committed_lsn));
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
/* Now route the real current-term quorum ACK through the library
/* Route the real current-term quorum ACK through the library
* (raft_on_append_entries_reply → raft_advance_commit → on_commit_advanced).
* The agreed entries are current-term (T) so the library commit advances to
* 2 and the callback raises the ceiling, releasing committed_lsn to 2. */
* The agreed entries are current-term (T) so the library commit advances to 2
* and the callback records the advisory index 2. committed_lsn is already 2
* (durable barrier). This still exercises the full ACK→library→callback wiring. */
TEST_ASSERT_EQUAL_INT(0,
ha_raft_on_repl_ack(c, /*abs_peer_idx*/1, /*acked_lsn*/2, /*reply_term*/T));
TEST_ASSERT_EQUAL_UINT64(2, c->ha.raft.commit_index);
TEST_ASSERT_EQUAL_UINT64(2, c->commit.raft_commit_index); /* advisory recorded */
TEST_ASSERT_EQUAL_UINT64(2, atomic_load(&c->persist.committed_lsn));
/* The marker (LSN 2) is now in the committed prefix → the DERIVED committed
@ -1864,11 +2049,12 @@ void test_term_safety_adapter(void)
RUN_TEST(test_on_truncate_from_removes_all);
RUN_TEST(test_send_append_entries_one_entry_from_wal);
RUN_TEST(test_send_append_entries_heartbeat_zero_entries);
RUN_TEST(test_commit_advanced_is_commit_source);
RUN_TEST(test_commit_advanced_ceiling_caps_durable_majority);
RUN_TEST(test_commit_advances_to_durable_majority_index_inert);
RUN_TEST(test_commit_index_record_is_not_a_ceiling);
RUN_TEST(test_commit_advanced_does_not_bypass_durable_cap);
RUN_TEST(test_own_term_gate_blocks_priorterm_ack_until_noop_durable);
RUN_TEST(test_own_term_gate_retained_no_deadlock);
RUN_TEST(test_on_commit_advanced_wired_feeds_barrier);
RUN_TEST(test_on_commit_advanced_wired_records_advisory_index);
RUN_TEST(test_init_restores_recovered_floor);
RUN_TEST(test_init_empty_wal_zero_floor);
RUN_TEST(test_primary_log_append_stamps_term_and_assigns_lsn);
@ -1878,6 +2064,7 @@ void test_term_safety_adapter(void)
RUN_TEST(test_intake_divergent_tail_truncated_and_reconverged);
RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
RUN_TEST(test_live_broadcast_receive_persists_leader_term);
RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);
RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit);
RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit);
RUN_TEST(test_repl_ack_staleterm_reply_dropped);