From 0649e17f62d32f2b191334c49b26a0bc9e732cd8 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 23 Jul 2026 04:23:50 +0000 Subject: [PATCH] fix(controller): clamp committed_config_tag to own loaded config on promotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bug-1462: raft_cb_become_leader baselined committed_config_tag to the majority-safe floor across voters, correct for WAL LSN safety but wrong for config tags — a leader's own actively-loaded config isn't speculative like an unacknowledged WAL entry. On a fresh deploy every peer reports tag=0 except the new primary, so the floor stuck at 0 forever and late-joining standbys never received a config push. ha_promotion_config_tag_floor() clamps to max(safe_tag, self_tag), mirroring bug-1060's WAL-truncation clamp pattern. Validated with 20-round sequential (63 jobs, zero loss) and 20-round chaotic overlapping-kill (zero committed divergence, zero committed job loss) HA failover soaks; unit suite 977/977 green. --- src/controller/ha.c | 19 +++++- src/controller/ha.h | 22 +++++++ src/controller/testsuite/test_ha_election.c | 67 +++++++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/controller/ha.c b/src/controller/ha.c index e56d8fb..eb7f9fe 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -814,6 +814,13 @@ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip) return safe_lsn > durable_tip ? safe_lsn : durable_tip; } +/* ── bug-1462: never floor committed_config_tag below the leader's own + * loaded config — see ha.h for full rationale. */ +uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag) +{ + return safe_tag > self_tag ? safe_tag : self_tag; +} + /* ── raft-drive-async task 1.1: peer-scaled liveness reservation ───────────── * * Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is @@ -1275,8 +1282,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term) * Req 2.3 (ha-repl-hardening). */ atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn, memory_order_release); - /* Update committed config tag baseline for the post-election decision loop */ - atomic_store_explicit(&ha->committed_config_tag, safe_tag, + /* Update committed config tag baseline for the post-election decision + * loop. bug-1462: clamp to the leader's OWN loaded config tag — + * unlike WAL LSN, a node's actively-loaded config is never + * speculative, so the bare majority floor (correct for LSN) must + * not be allowed to strand a real config behind an all-zero floor + * reported by voters that never received a live config push. + * See ha_promotion_config_tag_floor's doc comment (ha.h). */ + uint64_t committed_tag_floor = + ha_promotion_config_tag_floor(safe_tag, ectx->self_tag); + atomic_store_explicit(&ha->committed_config_tag, committed_tag_floor, memory_order_release); } } diff --git a/src/controller/ha.h b/src/controller/ha.h index b7324b1..c6dc080 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -817,6 +817,28 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx, * producing divergent WALs and a permanent prevote deadlock. */ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip); +/* ── bug-1462: config_tag floor for post-election committed_config_tag ─────── + * + * ha_promotion_config_tag_floor: clamp the post-election committed_config_tag + * baseline so it never drops below the config the new leader already has + * loaded. + * + * Returns max(safe_tag, self_tag). safe_tag is the majority-threshold config + * tag from the vote replies (ha_compute_safe_committed_lsn); self_tag is the + * candidate's own actively-loaded config tag at election time. Unlike WAL LSN + * (where an unacknowledged entry is genuinely speculative), a node's own + * loaded config is never speculative — so the majority floor, correct for + * LSN, is wrong for config_tag. + * + * Root cause it fixes: on a cluster where no live admin config push had ever + * happened, every peer voter reported config_tag=0 while the new leader's own + * self_tag was a real, non-zero, loaded tag. The majority floor picked 0, + * permanently stalling the heartbeat-driven HA_REPL_PUSH_CONFIG decision for + * any standby not connected at the exact instant of promotion (a late + * joiner) — WAL/job replication was unaffected since it correctly floors at + * genuinely-speculative LSN 0. */ +uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag); + /* ── raft-drive-async task 1.1: peer-scaled liveness reservation ───────────── * * drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED — diff --git a/src/controller/testsuite/test_ha_election.c b/src/controller/testsuite/test_ha_election.c index d9d1b92..bd4ae6c 100644 --- a/src/controller/testsuite/test_ha_election.c +++ b/src/controller/testsuite/test_ha_election.c @@ -1216,6 +1216,69 @@ static void test_promotion_never_truncates_own_committed_entries(void) printf("test_promotion_never_truncates_own_committed_entries: PASS\n"); } +/* ── bug-1462: a new leader must never floor its OWN config below what it + * already has loaded ─────────────────────────────────────────────────────── + * + * ha_compute_safe_committed_lsn's majority-floor math is correct for WAL LSN + * (an unacknowledged entry is genuinely speculative) but wrong for config_tag: + * the new primary's own actively-loaded config is not speculative the way an + * unacknowledged WAL entry is. + * + * Production root cause (bug-1462): on a cluster where no live admin config + * push has ever happened (deploy-ha.sh seeds cluster.bin on disk pre-boot), + * every peer voter reports config_tag=0 while the new primary's own self_tag + * is a real, non-zero, currently-loaded tag. The majority floor picks 0, so + * raft_cb_become_leader baselines ha->committed_config_tag at 0. The one-shot + * promotion push still gets already-connected standbys the real config (it + * reads myconf directly), but any LATE-joining standby depends entirely on + * the heartbeat-driven `standby_tag < committed_tag` decision loop — which + * can never fire again for the rest of the term because committed_tag is + * stuck at 0. Config replication silently stops while WAL/job replication + * (which floors correctly, since LSN 0 IS legitimately unacknowledged) + * continues to work — exactly the observed symptom. + * + * Fix (mirrors bug-1060's ha_promotion_truncate_floor precedent): clamp the + * post-election config_tag baseline to max(majority_safe_tag, self_tag). + * + * RED before the fix: ha_promotion_config_tag_floor does not exist (undefined + * reference). + */ +static void test_promotion_config_tag_never_drops_below_own_loaded_config(void) +{ + /* The bug-1462 case: majority floor (0) is BELOW the leader's own loaded + * tag (0x860cfd4f59ef9381) — no peer has ever received a live config push, + * but the leader itself demonstrably has one. The floor must clamp UP to + * self_tag so the heartbeat decision loop can still push it to late + * joiners for the rest of the term. */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + 0x860cfd4f59ef9381ULL, + ha_promotion_config_tag_floor(/*safe_tag=*/0ULL, + /*self_tag=*/0x860cfd4f59ef9381ULL), + "config_tag floor must never fall below the leader's own loaded config " + "(a node's actively-loaded config is not speculative like an " + "unacknowledged WAL entry)"); + + /* Equal: floor stays at self_tag (no-op). */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + 100ULL, ha_promotion_config_tag_floor(100ULL, 100ULL), + "equal safe_tag and self_tag -> floor at that tag"); + + /* safe_tag ABOVE self_tag: a majority genuinely holds a newer config than + * this node loaded (e.g. this node was behind before winning) — follow + * the majority-safe tag, same as the LSN case follows safe_lsn above the + * durable tip. */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + 200ULL, ha_promotion_config_tag_floor(200ULL, 100ULL), + "when safe_tag exceeds self_tag, the floor follows the majority-safe tag"); + + /* Fresh boot / never configured: both zero -> floor 0. */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + 0ULL, ha_promotion_config_tag_floor(0ULL, 0ULL), + "no config anywhere yet: floor 0"); + + printf("test_promotion_config_tag_never_drops_below_own_loaded_config: PASS\n"); +} + /* ── Test suite entry point ───────────────────────────────────────────────── */ void test_ha_election(void) @@ -1252,6 +1315,10 @@ void test_ha_election(void) /* Req 2.8 — zero config_tag candidate must not win when peer has config */ RUN_TEST(test_zero_config_tag_candidate_does_not_win_when_peer_has_config); + /* bug-1462 — post-election config_tag floor must never drop below the + * leader's own loaded config */ + RUN_TEST(test_promotion_config_tag_never_drops_below_own_loaded_config); + /* Req 2.9 — single-node self-election */ RUN_TEST(test_single_node_self_elects_as_primary);