fix(controller): clamp committed_config_tag to own loaded config on promotion
bug-1462: raft_cb_become_leader baselined committed_config_tag to the majority-safe floor across voters, correct for WAL LSN safety but wrong for config tags — a leader's own actively-loaded config isn't speculative like an unacknowledged WAL entry. On a fresh deploy every peer reports tag=0 except the new primary, so the floor stuck at 0 forever and late-joining standbys never received a config push. ha_promotion_config_tag_floor() clamps to max(safe_tag, self_tag), mirroring bug-1060's WAL-truncation clamp pattern. Validated with 20-round sequential (63 jobs, zero loss) and 20-round chaotic overlapping-kill (zero committed divergence, zero committed job loss) HA failover soaks; unit suite 977/977 green.
This commit is contained in:
parent
ee8473bd99
commit
0649e17f62
3 changed files with 106 additions and 2 deletions
|
|
@ -814,6 +814,13 @@ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
|
|||
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
|
||||
}
|
||||
|
||||
/* ── bug-1462: never floor committed_config_tag below the leader's own
|
||||
* loaded config — see ha.h for full rationale. */
|
||||
uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag)
|
||||
{
|
||||
return safe_tag > self_tag ? safe_tag : self_tag;
|
||||
}
|
||||
|
||||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||||
*
|
||||
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
|
||||
|
|
@ -1275,8 +1282,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
|||
* Req 2.3 (ha-repl-hardening). */
|
||||
atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn,
|
||||
memory_order_release);
|
||||
/* Update committed config tag baseline for the post-election decision loop */
|
||||
atomic_store_explicit(&ha->committed_config_tag, safe_tag,
|
||||
/* Update committed config tag baseline for the post-election decision
|
||||
* loop. bug-1462: clamp to the leader's OWN loaded config tag —
|
||||
* unlike WAL LSN, a node's actively-loaded config is never
|
||||
* speculative, so the bare majority floor (correct for LSN) must
|
||||
* not be allowed to strand a real config behind an all-zero floor
|
||||
* reported by voters that never received a live config push.
|
||||
* See ha_promotion_config_tag_floor's doc comment (ha.h). */
|
||||
uint64_t committed_tag_floor =
|
||||
ha_promotion_config_tag_floor(safe_tag, ectx->self_tag);
|
||||
atomic_store_explicit(&ha->committed_config_tag, committed_tag_floor,
|
||||
memory_order_release);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -817,6 +817,28 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
|||
* producing divergent WALs and a permanent prevote deadlock. */
|
||||
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip);
|
||||
|
||||
/* ── bug-1462: config_tag floor for post-election committed_config_tag ───────
|
||||
*
|
||||
* ha_promotion_config_tag_floor: clamp the post-election committed_config_tag
|
||||
* baseline so it never drops below the config the new leader already has
|
||||
* loaded.
|
||||
*
|
||||
* Returns max(safe_tag, self_tag). safe_tag is the majority-threshold config
|
||||
* tag from the vote replies (ha_compute_safe_committed_lsn); self_tag is the
|
||||
* candidate's own actively-loaded config tag at election time. Unlike WAL LSN
|
||||
* (where an unacknowledged entry is genuinely speculative), a node's own
|
||||
* loaded config is never speculative — so the majority floor, correct for
|
||||
* LSN, is wrong for config_tag.
|
||||
*
|
||||
* Root cause it fixes: on a cluster where no live admin config push had ever
|
||||
* happened, every peer voter reported config_tag=0 while the new leader's own
|
||||
* self_tag was a real, non-zero, loaded tag. The majority floor picked 0,
|
||||
* permanently stalling the heartbeat-driven HA_REPL_PUSH_CONFIG decision for
|
||||
* any standby not connected at the exact instant of promotion (a late
|
||||
* joiner) — WAL/job replication was unaffected since it correctly floors at
|
||||
* genuinely-speculative LSN 0. */
|
||||
uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag);
|
||||
|
||||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||||
*
|
||||
* drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED —
|
||||
|
|
|
|||
|
|
@ -1216,6 +1216,69 @@ static void test_promotion_never_truncates_own_committed_entries(void)
|
|||
printf("test_promotion_never_truncates_own_committed_entries: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1462: a new leader must never floor its OWN config below what it
|
||||
* already has loaded ───────────────────────────────────────────────────────
|
||||
*
|
||||
* ha_compute_safe_committed_lsn's majority-floor math is correct for WAL LSN
|
||||
* (an unacknowledged entry is genuinely speculative) but wrong for config_tag:
|
||||
* the new primary's own actively-loaded config is not speculative the way an
|
||||
* unacknowledged WAL entry is.
|
||||
*
|
||||
* Production root cause (bug-1462): on a cluster where no live admin config
|
||||
* push has ever happened (deploy-ha.sh seeds cluster.bin on disk pre-boot),
|
||||
* every peer voter reports config_tag=0 while the new primary's own self_tag
|
||||
* is a real, non-zero, currently-loaded tag. The majority floor picks 0, so
|
||||
* raft_cb_become_leader baselines ha->committed_config_tag at 0. The one-shot
|
||||
* promotion push still gets already-connected standbys the real config (it
|
||||
* reads myconf directly), but any LATE-joining standby depends entirely on
|
||||
* the heartbeat-driven `standby_tag < committed_tag` decision loop — which
|
||||
* can never fire again for the rest of the term because committed_tag is
|
||||
* stuck at 0. Config replication silently stops while WAL/job replication
|
||||
* (which floors correctly, since LSN 0 IS legitimately unacknowledged)
|
||||
* continues to work — exactly the observed symptom.
|
||||
*
|
||||
* Fix (mirrors bug-1060's ha_promotion_truncate_floor precedent): clamp the
|
||||
* post-election config_tag baseline to max(majority_safe_tag, self_tag).
|
||||
*
|
||||
* RED before the fix: ha_promotion_config_tag_floor does not exist (undefined
|
||||
* reference).
|
||||
*/
|
||||
static void test_promotion_config_tag_never_drops_below_own_loaded_config(void)
|
||||
{
|
||||
/* The bug-1462 case: majority floor (0) is BELOW the leader's own loaded
|
||||
* tag (0x860cfd4f59ef9381) — no peer has ever received a live config push,
|
||||
* but the leader itself demonstrably has one. The floor must clamp UP to
|
||||
* self_tag so the heartbeat decision loop can still push it to late
|
||||
* joiners for the rest of the term. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
0x860cfd4f59ef9381ULL,
|
||||
ha_promotion_config_tag_floor(/*safe_tag=*/0ULL,
|
||||
/*self_tag=*/0x860cfd4f59ef9381ULL),
|
||||
"config_tag floor must never fall below the leader's own loaded config "
|
||||
"(a node's actively-loaded config is not speculative like an "
|
||||
"unacknowledged WAL entry)");
|
||||
|
||||
/* Equal: floor stays at self_tag (no-op). */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
100ULL, ha_promotion_config_tag_floor(100ULL, 100ULL),
|
||||
"equal safe_tag and self_tag -> floor at that tag");
|
||||
|
||||
/* safe_tag ABOVE self_tag: a majority genuinely holds a newer config than
|
||||
* this node loaded (e.g. this node was behind before winning) — follow
|
||||
* the majority-safe tag, same as the LSN case follows safe_lsn above the
|
||||
* durable tip. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
200ULL, ha_promotion_config_tag_floor(200ULL, 100ULL),
|
||||
"when safe_tag exceeds self_tag, the floor follows the majority-safe tag");
|
||||
|
||||
/* Fresh boot / never configured: both zero -> floor 0. */
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||
0ULL, ha_promotion_config_tag_floor(0ULL, 0ULL),
|
||||
"no config anywhere yet: floor 0");
|
||||
|
||||
printf("test_promotion_config_tag_never_drops_below_own_loaded_config: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_election(void)
|
||||
|
|
@ -1252,6 +1315,10 @@ void test_ha_election(void)
|
|||
/* Req 2.8 — zero config_tag candidate must not win when peer has config */
|
||||
RUN_TEST(test_zero_config_tag_candidate_does_not_win_when_peer_has_config);
|
||||
|
||||
/* bug-1462 — post-election config_tag floor must never drop below the
|
||||
* leader's own loaded config */
|
||||
RUN_TEST(test_promotion_config_tag_never_drops_below_own_loaded_config);
|
||||
|
||||
/* Req 2.9 — single-node self-election */
|
||||
RUN_TEST(test_single_node_self_elects_as_primary);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue