lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ha): close permanent catch-up wedge from duplicated replication-target formula (bug-1478)

Browse source 
Two independent call sites duplicated target = max(committed_lsn,
last_written): ha.c's heartbeat/stateReport dispatch (repl_ack_decode_and_
dispatch) and config_repl.c's catchup_derive_target (used by the ACK-driven
catch-up continuation path). Once config-upgrade-snapshot's marker-triggered
checkpoint advances last_written one LSN past an already-committed
committed_lsn (the checkpoint's own consumed, never-replicated LSN slot),
the unconditional max() permanently wedges every standby's catch-up target
at an unreachable value -- the primary re-streams an empty range every
heartbeat forever.

Extracted the shared, pure ha_compute_repl_target_lsn(committed_lsn,
last_written) (ha.h/ha.c), which only falls back to last_written on genuine
committed_lsn==0 bootstrap, and pointed both call sites at it so they can
never disagree again.

Fixing the second call site exposed three pre-existing tests in
test_convergence_verification.c that hardcoded the old buggy formula's
output as their expected target; updated their fixture values (committed_lsn
4->9, matching appended_lsn) to decouple them from the now-fixed edge case,
which already has dedicated pure-function coverage in test_ha_role.c.

GREEN: full controller_unit_test suite = 1040 tests, 0 failures, clean
across multiple runs. Live-verified via run-ha-soak.sh
--config-upgrade-wal-trim (18 jobs, zero job loss, ctrl-1 reconverged 3-of-3
using the preserved floor segment) -- unblocks config-upgrade-snapshot
task 8.9.
This commit is contained in:
Rongsong Shen 2026-07-28 09:53:20 +00:00
parent 71223ec712
commit 102e263cce
9 changed files with 850 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

27
src/controller/testsuite/test_convergence_verification.c
View file

@ -276,17 +276,22 @@ static int cv_standby_in_catch_up(repl_ep_t ep)
/* ── Tests ──────────────────────────────────────────────────────────────── */
/* 5.1: a walAck with acked_lsn < target = max(committed_lsn, last_written)
* schedules EXACTLY ONE continuation that streams [acked_lsn+1, target] without
* any intervening heartbeat/stateReport dispatch.
/* 5.1: a walAck with acked_lsn < target = ha_compute_repl_target_lsn(
* committed_lsn, last_written) schedules EXACTLY ONE continuation that
* streams [acked_lsn+1, target] without any intervening heartbeat/
* stateReport dispatch.
* RED: the walAck continuation scaffold schedules nothing → no task, the stream
* hook is never called. */
static void test_walack_behind_target_schedules_one_continuation(void)
{
/* appended_lsn=9 → last_written=9; committed_lsn=4 → target=max(4,9)=9. */
/* appended_lsn=9 → last_written=9; committed_lsn=9 → target=9. Kept equal
* to last_written (rather than a lower value like the historical 4) so
* this test exercises catch-up continuation scheduling only — the
* committed_lsn-vs-last_written edge case itself (bug-1478) has its own
* dedicated pure-function coverage in test_ha_role.c. */
cv_setup("behind", /*n_peers=*/3, /*n_standbys=*/1,
/*appended_lsn=*/9, /*n_vcpus=*/8);
atomic_store(&g_cv.persist.committed_lsn, 4);
atomic_store(&g_cv.persist.committed_lsn, 9);
const int peer_idx = 0;
const uint64_t acked_lsn = 3; /* behind target 9 */
@ -325,8 +330,8 @@ static void test_walack_behind_target_schedules_one_continuation(void)
"[acked_lsn+1, target]) (Req 5.1)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(target,
atomic_load(&g_stream_last_target_lsn),
"the continuation must stream up to target = max(committed_lsn, "
"last_written) (Req 5.1)");
"the continuation must stream up to target = "
"ha_compute_repl_target_lsn(committed_lsn, last_written) (Req 5.1)");
cv_teardown();
printf("test_walack_behind_target_schedules_one_continuation: checked\n");
@ -338,9 +343,11 @@ static void test_walack_behind_target_schedules_one_continuation(void)
* "exactly one continuation across two ACKs" assertion fails (0 != 1). */
static void test_walack_single_flight_no_duplicate_continuation(void)
{
/* committed_lsn == appended_lsn (target=9 either way) — see the "behind"
* test's own note on why these don't use a lower committed_lsn. */
cv_setup("single", /*n_peers=*/3, /*n_standbys=*/1,
/*appended_lsn=*/9, /*n_vcpus=*/8);
atomic_store(&g_cv.persist.committed_lsn, 4);
atomic_store(&g_cv.persist.committed_lsn, 9);
const int peer_idx = 0;
@ -382,9 +389,11 @@ static void test_walack_single_flight_no_duplicate_continuation(void)
* RED: the scaffold never schedules/clears, so neither continuation appears. */
static void test_walack_reschedules_after_continuation_completes(void)
{
/* committed_lsn == appended_lsn (target=9 either way) — see the "behind"
* test's own note on why these don't use a lower committed_lsn. */
cv_setup("respawn", /*n_peers=*/3, /*n_standbys=*/1,
/*appended_lsn=*/9, /*n_vcpus=*/8);
atomic_store(&g_cv.persist.committed_lsn, 4);
atomic_store(&g_cv.persist.committed_lsn, 9);
const int peer_idx = 0;

63
src/controller/testsuite/test_ha_role.c
View file

@ -660,6 +660,64 @@ static void test_evaluate_repl_action_stream_wal_fresh_system(void)
printf("test_evaluate_repl_action_stream_wal_fresh_system: PASS\n");
}
/* ── bug-1478: ha_compute_repl_target_lsn ─────────────────────────────────
*
* repl_ack_decode_and_dispatch (ha.c) computes the replication/convergence
* target passed into ha_dispatch_repl_action_for_standby as
* max(committed_lsn, last_written), where last_written is the raw locally-
* appended WAL tip (persist.current_lsn - 1). A live 3-node HA soak
* (config-upgrade-snapshot task 8.9, --config-upgrade-wal-trim) caught this
* unconditional max() permanently wedging catch-up: once a marker-triggered
* checkpoint (a local-only, NEVER-replicated WAL_CHECKPOINT entry —
* persist_write_checkpoint_marker bumps current_lsn directly, bypassing
* raft_log_append/the commit engine, bug-1466) trails an already-advancing
* committed_lsn, last_written sits permanently one LSN past it — a position
* no standby's applied_lsn can EVER legitimately reach (the checkpoint's own
* slot is, by design, never streamed or applied). ha_evaluate_repl_action's
* standby_lsn < committed_lsn check then never reaches HA_REPL_CONVERGED,
* and the primary re-streams the same empty range every heartbeat, forever.
* Ground-truthed via `xce_test waldump` on the live soak's preserved WAL
* (ctrl-2: lsn 21 = the WAL_CONFIG_MARKER, lsn 22 = WAL_CHECKPOINT, 0 bytes)
* and via ctrl-2's own commit_advance_locked log (the REAL, majority-gated
* committed_lsn never advanced past 21 — only the target_lsn fallback
* leaked 22 into the heartbeat decision).
*/
static void test_compute_repl_target_lsn_bootstrap_fallback(void)
{
/* committed_lsn genuinely stuck at 0 (no standby has ever acked) — the
* documented bootstrap exception: fall back to last_written so a
* first-ever connecting standby can catch up to real data. */
TEST_ASSERT_EQUAL_UINT64(42, ha_compute_repl_target_lsn(0, 42));
printf("test_compute_repl_target_lsn_bootstrap_fallback: PASS\n");
}
static void test_compute_repl_target_lsn_zero_zero(void)
{
/* Genuinely fresh cluster, nothing appended yet. */
TEST_ASSERT_EQUAL_UINT64(0, ha_compute_repl_target_lsn(0, 0));
printf("test_compute_repl_target_lsn_zero_zero: PASS\n");
}
static void test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn(void)
{
/* bug-1478's exact reproduction shape: committed_lsn=21 is already
* advancing (non-zero, real, majority-gated), but last_written=22
* reflects a local-only WAL_CHECKPOINT that trails it. The bootstrap
* fallback must NOT fire here — the target must stay at the real
* committed_lsn (21), never the unstreamable checkpoint slot (22). */
TEST_ASSERT_EQUAL_UINT64(21, ha_compute_repl_target_lsn(21, 22));
printf("test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn: PASS\n");
}
static void test_compute_repl_target_lsn_committed_already_ahead(void)
{
/* Normal steady-state case: committed_lsn is already the higher/current
* value; last_written (a possibly-stale, lock-free read) trails it.
* The target must simply be committed_lsn. */
TEST_ASSERT_EQUAL_UINT64(100, ha_compute_repl_target_lsn(100, 50));
printf("test_compute_repl_target_lsn_committed_already_ahead: PASS\n");
}
/* The transitional-leader timeout default test was removed: the config knob
* and its default macro are deleted by ha-single-leader-role (the wall-clock
* force-promotion watchdog it configured is gone — Req 1.5). */
@ -892,6 +950,11 @@ void test_ha_role(void)
RUN_TEST(test_evaluate_repl_action_stream_wal);
/* Task 6.1: (tag=T, lsn=0) with oldest_wal_lsn=0 → STREAM_WAL */
RUN_TEST(test_evaluate_repl_action_stream_wal_fresh_system);
/* bug-1478: ha_compute_repl_target_lsn */
RUN_TEST(test_compute_repl_target_lsn_bootstrap_fallback);
RUN_TEST(test_compute_repl_target_lsn_zero_zero);
RUN_TEST(test_compute_repl_target_lsn_ignores_trailing_checkpoint_lsn);
RUN_TEST(test_compute_repl_target_lsn_committed_already_ahead);
/* Task 3.3: ha_dispatch_repl_action_for_standby PUSH_CONFIG wiring */
RUN_TEST(test_dispatch_repl_action_push_config_when_tag_behind);
RUN_TEST(test_dispatch_repl_action_no_push_when_converged);