fix(controller): clamp committed_config_tag to own loaded config on promotion
bug-1462: raft_cb_become_leader baselined committed_config_tag to the majority-safe floor across voters, correct for WAL LSN safety but wrong for config tags — a leader's own actively-loaded config isn't speculative like an unacknowledged WAL entry. On a fresh deploy every peer reports tag=0 except the new primary, so the floor stuck at 0 forever and late-joining standbys never received a config push. ha_promotion_config_tag_floor() clamps to max(safe_tag, self_tag), mirroring bug-1060's WAL-truncation clamp pattern. Validated with 20-round sequential (63 jobs, zero loss) and 20-round chaotic overlapping-kill (zero committed divergence, zero committed job loss) HA failover soaks; unit suite 977/977 green.
This commit is contained in:
parent
ee8473bd99
commit
0649e17f62
3 changed files with 106 additions and 2 deletions
|
|
@ -814,6 +814,13 @@ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
|
||||||
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
|
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── bug-1462: never floor committed_config_tag below the leader's own
|
||||||
|
* loaded config — see ha.h for full rationale. */
|
||||||
|
uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag)
|
||||||
|
{
|
||||||
|
return safe_tag > self_tag ? safe_tag : self_tag;
|
||||||
|
}
|
||||||
|
|
||||||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||||||
*
|
*
|
||||||
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
|
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
|
||||||
|
|
@ -1275,8 +1282,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
||||||
* Req 2.3 (ha-repl-hardening). */
|
* Req 2.3 (ha-repl-hardening). */
|
||||||
atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn,
|
atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn,
|
||||||
memory_order_release);
|
memory_order_release);
|
||||||
/* Update committed config tag baseline for the post-election decision loop */
|
/* Update committed config tag baseline for the post-election decision
|
||||||
atomic_store_explicit(&ha->committed_config_tag, safe_tag,
|
* loop. bug-1462: clamp to the leader's OWN loaded config tag —
|
||||||
|
* unlike WAL LSN, a node's actively-loaded config is never
|
||||||
|
* speculative, so the bare majority floor (correct for LSN) must
|
||||||
|
* not be allowed to strand a real config behind an all-zero floor
|
||||||
|
* reported by voters that never received a live config push.
|
||||||
|
* See ha_promotion_config_tag_floor's doc comment (ha.h). */
|
||||||
|
uint64_t committed_tag_floor =
|
||||||
|
ha_promotion_config_tag_floor(safe_tag, ectx->self_tag);
|
||||||
|
atomic_store_explicit(&ha->committed_config_tag, committed_tag_floor,
|
||||||
memory_order_release);
|
memory_order_release);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -817,6 +817,28 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
||||||
* producing divergent WALs and a permanent prevote deadlock. */
|
* producing divergent WALs and a permanent prevote deadlock. */
|
||||||
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip);
|
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip);
|
||||||
|
|
||||||
|
/* ── bug-1462: config_tag floor for post-election committed_config_tag ───────
|
||||||
|
*
|
||||||
|
* ha_promotion_config_tag_floor: clamp the post-election committed_config_tag
|
||||||
|
* baseline so it never drops below the config the new leader already has
|
||||||
|
* loaded.
|
||||||
|
*
|
||||||
|
* Returns max(safe_tag, self_tag). safe_tag is the majority-threshold config
|
||||||
|
* tag from the vote replies (ha_compute_safe_committed_lsn); self_tag is the
|
||||||
|
* candidate's own actively-loaded config tag at election time. Unlike WAL LSN
|
||||||
|
* (where an unacknowledged entry is genuinely speculative), a node's own
|
||||||
|
* loaded config is never speculative — so the majority floor, correct for
|
||||||
|
* LSN, is wrong for config_tag.
|
||||||
|
*
|
||||||
|
* Root cause it fixes: on a cluster where no live admin config push had ever
|
||||||
|
* happened, every peer voter reported config_tag=0 while the new leader's own
|
||||||
|
* self_tag was a real, non-zero, loaded tag. The majority floor picked 0,
|
||||||
|
* permanently stalling the heartbeat-driven HA_REPL_PUSH_CONFIG decision for
|
||||||
|
* any standby not connected at the exact instant of promotion (a late
|
||||||
|
* joiner) — WAL/job replication was unaffected since it correctly floors at
|
||||||
|
* genuinely-speculative LSN 0. */
|
||||||
|
uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag);
|
||||||
|
|
||||||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||||||
*
|
*
|
||||||
* drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED —
|
* drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED —
|
||||||
|
|
|
||||||
|
|
@ -1216,6 +1216,69 @@ static void test_promotion_never_truncates_own_committed_entries(void)
|
||||||
printf("test_promotion_never_truncates_own_committed_entries: PASS\n");
|
printf("test_promotion_never_truncates_own_committed_entries: PASS\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── bug-1462: a new leader must never floor its OWN config below what it
|
||||||
|
* already has loaded ───────────────────────────────────────────────────────
|
||||||
|
*
|
||||||
|
* ha_compute_safe_committed_lsn's majority-floor math is correct for WAL LSN
|
||||||
|
* (an unacknowledged entry is genuinely speculative) but wrong for config_tag:
|
||||||
|
* the new primary's own actively-loaded config is not speculative the way an
|
||||||
|
* unacknowledged WAL entry is.
|
||||||
|
*
|
||||||
|
* Production root cause (bug-1462): on a cluster where no live admin config
|
||||||
|
* push has ever happened (deploy-ha.sh seeds cluster.bin on disk pre-boot),
|
||||||
|
* every peer voter reports config_tag=0 while the new primary's own self_tag
|
||||||
|
* is a real, non-zero, currently-loaded tag. The majority floor picks 0, so
|
||||||
|
* raft_cb_become_leader baselines ha->committed_config_tag at 0. The one-shot
|
||||||
|
* promotion push still gets already-connected standbys the real config (it
|
||||||
|
* reads myconf directly), but any LATE-joining standby depends entirely on
|
||||||
|
* the heartbeat-driven `standby_tag < committed_tag` decision loop — which
|
||||||
|
* can never fire again for the rest of the term because committed_tag is
|
||||||
|
* stuck at 0. Config replication silently stops while WAL/job replication
|
||||||
|
* (which floors correctly, since LSN 0 IS legitimately unacknowledged)
|
||||||
|
* continues to work — exactly the observed symptom.
|
||||||
|
*
|
||||||
|
* Fix (mirrors bug-1060's ha_promotion_truncate_floor precedent): clamp the
|
||||||
|
* post-election config_tag baseline to max(majority_safe_tag, self_tag).
|
||||||
|
*
|
||||||
|
* RED before the fix: ha_promotion_config_tag_floor does not exist (undefined
|
||||||
|
* reference).
|
||||||
|
*/
|
||||||
|
static void test_promotion_config_tag_never_drops_below_own_loaded_config(void)
|
||||||
|
{
|
||||||
|
/* The bug-1462 case: majority floor (0) is BELOW the leader's own loaded
|
||||||
|
* tag (0x860cfd4f59ef9381) — no peer has ever received a live config push,
|
||||||
|
* but the leader itself demonstrably has one. The floor must clamp UP to
|
||||||
|
* self_tag so the heartbeat decision loop can still push it to late
|
||||||
|
* joiners for the rest of the term. */
|
||||||
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||||
|
0x860cfd4f59ef9381ULL,
|
||||||
|
ha_promotion_config_tag_floor(/*safe_tag=*/0ULL,
|
||||||
|
/*self_tag=*/0x860cfd4f59ef9381ULL),
|
||||||
|
"config_tag floor must never fall below the leader's own loaded config "
|
||||||
|
"(a node's actively-loaded config is not speculative like an "
|
||||||
|
"unacknowledged WAL entry)");
|
||||||
|
|
||||||
|
/* Equal: floor stays at self_tag (no-op). */
|
||||||
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||||
|
100ULL, ha_promotion_config_tag_floor(100ULL, 100ULL),
|
||||||
|
"equal safe_tag and self_tag -> floor at that tag");
|
||||||
|
|
||||||
|
/* safe_tag ABOVE self_tag: a majority genuinely holds a newer config than
|
||||||
|
* this node loaded (e.g. this node was behind before winning) — follow
|
||||||
|
* the majority-safe tag, same as the LSN case follows safe_lsn above the
|
||||||
|
* durable tip. */
|
||||||
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||||
|
200ULL, ha_promotion_config_tag_floor(200ULL, 100ULL),
|
||||||
|
"when safe_tag exceeds self_tag, the floor follows the majority-safe tag");
|
||||||
|
|
||||||
|
/* Fresh boot / never configured: both zero -> floor 0. */
|
||||||
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
|
||||||
|
0ULL, ha_promotion_config_tag_floor(0ULL, 0ULL),
|
||||||
|
"no config anywhere yet: floor 0");
|
||||||
|
|
||||||
|
printf("test_promotion_config_tag_never_drops_below_own_loaded_config: PASS\n");
|
||||||
|
}
|
||||||
|
|
||||||
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
/* ── Test suite entry point ───────────────────────────────────────────────── */
|
||||||
|
|
||||||
void test_ha_election(void)
|
void test_ha_election(void)
|
||||||
|
|
@ -1252,6 +1315,10 @@ void test_ha_election(void)
|
||||||
/* Req 2.8 — zero config_tag candidate must not win when peer has config */
|
/* Req 2.8 — zero config_tag candidate must not win when peer has config */
|
||||||
RUN_TEST(test_zero_config_tag_candidate_does_not_win_when_peer_has_config);
|
RUN_TEST(test_zero_config_tag_candidate_does_not_win_when_peer_has_config);
|
||||||
|
|
||||||
|
/* bug-1462 — post-election config_tag floor must never drop below the
|
||||||
|
* leader's own loaded config */
|
||||||
|
RUN_TEST(test_promotion_config_tag_never_drops_below_own_loaded_config);
|
||||||
|
|
||||||
/* Req 2.9 — single-node self-election */
|
/* Req 2.9 — single-node self-election */
|
||||||
RUN_TEST(test_single_node_self_elects_as_primary);
|
RUN_TEST(test_single_node_self_elects_as_primary);
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue