diff --git a/src/controller/ha.c b/src/controller/ha.c index ce272694..92175d9e 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -5184,12 +5184,29 @@ static void ha_enter_real_election(ha_state_t *ha, cluster_t *c) return; } - /* Truncate uncommitted WAL entries above committed_lsn (mirrors the - * existing failover sequence in ha_on_primary_ctrl_fd_rdhup). */ - uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn, - memory_order_acquire); - if (committed > 0) - persist_truncate_wal_above(&c->persist, committed); + /* bug-828 final slice (Leader Completeness): do NOT pre-truncate the WAL + * here. The earlier code truncated above ha->last_known_committed_lsn + * BEFORE the vote — but that watermark is the LOCALLY-LEARNED commit point, + * which LAGS the true cluster commit point (the primary commits an entry on + * a durable majority, then dies before a heartbeat carries that committed + * LSN to this standby). persist_truncate_wal_above clamps durable_lsn down + * to the cutoff, so a standby that had DURABLY persisted committed entries + * [committed+1 .. tip] would discard them and then advertise the stale-low + * cutoff as its durable floor. Every voter doing the same made + * safe-committed-from-votes collapse to the stale watermark, and the + * AUTHORITATIVE post-election truncate (raft_cb_become_leader, + * persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale + * prefix — losing committed jobs on a clean failover (bug-828/795). + * + * Raft model: a candidate does NOT truncate its log before campaigning. It + * keeps its full durable log and advertises its real durable tip + * (ha_candidate_advertise_lsn == persist_durable_floor), so voters holding + * more committed entries deny a colder candidate and the winner has every + * committed entry (Leader Completeness). Divergent UNCOMMITTED tails are + * removed ONLY by the authoritative post-election safe-commit truncate + * (above the majority-held point) and by the library's per-entry (term,lsn) + * reconcile (on_truncate_from) during subsequent AppendEntries — never by a + * stale-watermark pre-truncate that can erase committed state. */ /* Candidate freshness for the RequestVote payload (ha-commit-durability * task 4.2). The TAG axis uses the same config-epoch fold the voter side diff --git a/src/controller/testsuite/test_repl_apply.c b/src/controller/testsuite/test_repl_apply.c index b0e9722d..03668d8d 100644 --- a/src/controller/testsuite/test_repl_apply.c +++ b/src/controller/testsuite/test_repl_apply.c @@ -464,6 +464,71 @@ static void test_repl_write_rejects_checkpoint(void) printf("test_repl_write_rejects_checkpoint: PASS\n"); } +/* ================================================================== */ +/* bug-828 final slice: a catch-up'd standby durable at LSN 5 must */ +/* advertise its REAL durable tip (5) through a failover — the */ +/* pre-election truncate must NOT discard durably-committed entries */ +/* based on a STALE-LOW locally-known committed watermark. */ +/* ================================================================== */ + +/* A standby streams catch-up entries up to LSN 5 (all fdatasync'd durable), + * so persist_durable_floor == 5. This is the LSN the candidate advertises + * (ha_candidate_advertise_lsn == persist_durable_floor) and the voter floor + * (ha_vote_self_freshness over persist_durable_floor) at the failover vote, + * which feeds safe-commit-from-votes. + * + * The clean-failover job-loss bug was that ha_enter_real_election truncated + * the WAL above the standby's STALE locally-known last_known_committed_lsn + * (e.g. 1 — the primary committed 2..5 but died before a heartbeat carried + * committed=5 to this standby) BEFORE the vote. persist_truncate_wal_above + * clamps durable_lsn down to the cutoff, so the standby then advertised 1, + * safe-commit computed 1, and the AUTHORITATIVE post-election truncate kept + * only LSN 1 — the committed jobs 2..5 (durable on this very node) were lost. + * + * This test pins the fix at the persist seam: after catch-up to 5 the durable + * floor is 5, and a truncate at the genuine safe-commit point (5, derived from + * the un-clamped voter floors) PRESERVES all five entries. A truncate at the + * stale-low watermark (1) — the removed pre-election step — would instead + * destroy LSN 2..5; this test asserts the durable committed prefix survives. */ +static void test_repl_catchup_durable_floor_survives_failover(void) +{ + char tmpdir[256]; + persist_t* p = make_test_persist_in(tmpdir, sizeof(tmpdir)); + + /* Stream the committed prefix [1..5] as a standby does on catch-up — each + * persist_wal_write_replicated fdatasyncs inline (persist-before-ACK). */ + for (uint64_t lsn = 1; lsn <= 5; lsn++) { + int rc = persist_wal_write_replicated(p, lsn, WAL_JOB_UPDATE, + "committed-job", + (uint32_t)strlen("committed-job")); + TEST_ASSERT_EQUAL_INT(0, rc); + } + + /* All five are fdatasync'd durable, so the advertised floor is 5 — the + * value the failover vote must carry (RED before the partial fix: 1). */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, persist_durable_floor(p), + "catch-up'd standby must advertise its real fsync'd durable tip (5)"); + + /* The genuine safe-commit point (from the un-clamped voter floors) is 5; + * truncating above it is a no-op and preserves the committed prefix. This + * is the ONLY truncate the fixed failover path runs (post-election). */ + TEST_ASSERT_EQUAL_INT(0, persist_truncate_wal_above(p, 5)); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, persist_durable_floor(p), + "safe-commit truncate at the real tip must preserve all committed entries"); + TEST_ASSERT_EQUAL_UINT64(6, p->current_lsn); + + /* Recovery over the same WAL replays the full committed prefix [1..5]. */ + destroy_test_persist(p); + persist_t* p2 = reopen_test_persist(tmpdir); + TEST_ASSERT_TRUE(persist_recover(p2) >= 0); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(6, p2->current_lsn, + "the durable committed prefix [1..5] must survive a restart"); + TEST_ASSERT_EQUAL_UINT64(5ULL, persist_durable_floor(p2)); + destroy_test_persist(p2); + + printf("test_repl_catchup_durable_floor_survives_failover: PASS\n"); +} + /* ------------------------------------------------------------------ */ /* Suite entry point (registered in controller_test.c) */ /* ------------------------------------------------------------------ */ @@ -482,4 +547,7 @@ void test_repl_apply(void) RUN_TEST(test_repl_write_dedup_idempotent); RUN_TEST(test_repl_write_rejects_forward_gap); RUN_TEST(test_repl_write_rejects_checkpoint); + + /* bug-828 final: durable committed prefix survives the failover vote */ + RUN_TEST(test_repl_catchup_durable_floor_survives_failover); }