lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): pre-promotion eligibility gate + catch-up fence/release (task 4.3)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 09:40:32 +00:00
parent 3049b620f3
commit e9e2a6d703
4 changed files with 409 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

155
src/controller/ha.c
View file

@ -538,6 +538,91 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
" (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
*
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
* Policy Decision Table inputs from the promoting member's recovery/durability
* state and delegate to persist_recover_decision so the 3.2/3.3 table is the
* single source of truth (DRY). "Behind the HWM" is durable_floor < safe_hwm;
* a truncated log is treated as behind regardless of its volatile position
* (design.md note). A hollow node (truncated OR behind) can never be ELIGIBLE
* until it catches up — independent of quorum size (Req 2.5).
*/
recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
uint64_t durable_floor,
uint64_t safe_hwm,
bool peer_reachable,
bool catch_up_succeeded)
{
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < safe_hwm),
.healthy_peer_reachable = peer_reachable,
.catch_up_succeeded = catch_up_succeeded,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
return persist_recover_decision(&in);
}
/* ha_evaluate_promotion_fence: live gate over the existing in_catch_up fence
* (Req 2.2, 2.6). Reuses ha->in_catch_up — does NOT invent a parallel fence.
* - ELIGIBLE → clear the fence, return ELIGIBLE (a member that caught up past
* the committed boundary regains eligibility, Req 2.6).
* - FENCE → set the fence, return FENCE (refuse promotion; catch up from a
* healthy peer).
* - FATAL → run the 3.3 loud-terminate path (logs the truncation boundary,
* then terminates — or fires persist_fatal_hook in tests).
*/
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
bool recovery_truncated,
uint64_t durable_floor,
uint64_t safe_hwm,
bool peer_reachable,
bool catch_up_succeeded)
{
recover_decision_t d = ha_promotion_gate_decision(
recovery_truncated, durable_floor, safe_hwm, peer_reachable,
catch_up_succeeded);
switch (d) {
case RECOVER_DECISION_ELIGIBLE:
/* Release the fence: catch-up brought the member to a
* committed-consistent state at/above the cluster boundary (Req 2.6). */
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
break;
case RECOVER_DECISION_FENCE:
/* Refuse promotion: extend the catch-up fence so the member catches up
* from a healthy peer before it can lead (Req 2.2). */
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
INFO("HA: promotion FENCED — recovery_truncated=%d durable_floor=%"
PRIu64 " < cluster HWM=%" PRIu64 " (catch up before leading; "
"ha-commit-durability Req 2.2)",
recovery_truncated, durable_floor, safe_hwm);
break;
case RECOVER_DECISION_FATAL: {
/* Bounded-unrecoverable: route through the 3.3 loud-terminate action,
* which logs the truncation boundary at ERROR then terminates (or fires
* persist_fatal_hook under test). Req 2.4. */
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < safe_hwm),
.healthy_peer_reachable = peer_reachable,
.catch_up_succeeded = catch_up_succeeded,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
persist_recover_fatal_if_unrecoverable(&in);
break;
}
}
return d;
}
/*
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
*
@ -559,6 +644,13 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
/* Sync ha->term from raft term (raft owns term increments) */
atomic_store_explicit(&ha->term, term, memory_order_release);
/* The cluster committed high-water mark for this promotion — derived from
* the voter replies (NOT a stale ctrlhello). The task-4.3 pre-promotion
* eligibility gate (below) compares the local CLAMPED durable position
* against it. Lifted to function scope so the gate can read it after the
* truncate-clamp. */
uint64_t promotion_safe_lsn = 0;
/* Task 4.1 (ha-repl-hardening): compute safe_committed_lsn from vote replies.
* Uses voter LSNs collected during election (ha->elect_ctx->votes[]) to find
* the highest LSN held by a majority without an extra round-trip.
@ -570,6 +662,7 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
uint64_t safe_lsn = 0;
uint64_t safe_tag = 0;
ha_compute_safe_committed_lsn(ectx, &safe_lsn, &safe_tag);
promotion_safe_lsn = safe_lsn;
/* Store results for use in WAL truncation and post-election commit.
* safe_lsn is the highest LSN on a majority; entries above it
* may not have been committed on all nodes and must be truncated. */
@ -593,6 +686,68 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
}
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────
* Refuse promotion BEFORE the node becomes authoritative CTRL_PRIMARY if it
* carries the recovery-truncated flag (task 3.1 — a structurally corrupt log
* must never lead) OR its CLAMPED durable position is below the cluster
* committed high-water mark (promotion_safe_lsn, from the voter replies).
* The durable position is read AFTER persist_truncate_wal_above above, so it
* reflects the 2.2 truncate-clamp (a node whose durable tip was clamped down
* is correctly seen as behind). A hollow node (truncated OR behind) is
* refused regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote (Req 2.5). On refusal we set
* the in_catch_up fence and return WITHOUT becoming primary; the member
* catches up from a healthy peer and a later promotion attempt passes the
* gate (Req 2.2/2.6). Only the production path (real persist) is gated; the
* c==NULL unit shim has no persist and skips it. */
if (c != NULL) {
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
/* A healthy peer is reachable for catch-up iff some voter granted with a
* live endpoint survives this election (these become the standby
* channels promoted below). */
bool peer_reachable = false;
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
for (int i = 0; i < ectx->n_peers; i++) {
if (ectx->votes[i].vote_granted && ectx->votes[i].ep != NULL) {
peer_reachable = true;
break;
}
}
}
}
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
* the live bounded-window catch-up loop is wired by task 5.1; here the
* gate refuses (FENCE) or fails loud (FATAL when no peer) immediately,
* never silently proceeding while hollow. */
recover_decision_t gate = ha_evaluate_promotion_fence(
ha, truncated, durable_floor, promotion_safe_lsn,
peer_reachable, /*catch_up_succeeded=*/false);
if (gate != RECOVER_DECISION_ELIGIBLE) {
/* FENCE: refuse promotion — do NOT become CTRL_PRIMARY. The
* in_catch_up fence is set; the member catches up from a healthy
* peer and re-attempts promotion later (Req 2.2, 2.6).
* FATAL already terminated in production (or fired the test hook);
* still bail out here so a test-hooked FATAL does not fall through
* to authoritative leadership. */
WARN("HA: promotion REFUSED (decision=%d) — recovery_truncated=%d "
"durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64
" peer_reachable=%d; remaining non-primary until caught up "
"(ha-commit-durability Req 2.2/2.4)",
(int)gate, truncated, durable_floor, promotion_safe_lsn,
peer_reachable);
return;
}
/* ELIGIBLE: clear of the fence (release on a caught-up member, Req 2.6)
* — proceed to authoritative promotion below. */
}
int is_first_boot = !ha->phase2_done;
/* First boot: call cluster_init_phase2 */