lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): bounded-window catch-up before FATAL — fence no longer FATALs synchronously (task 4.4)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 10:35:13 +00:00
parent e14bd62115
commit addf53e3b4
4 changed files with 562 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

282
src/controller/ha.c
View file

@ -623,6 +623,226 @@ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
return d;
}
/* ── ha-commit-durability task 4.4: bounded-window catch-up before FATAL ─────
*
* The 4.3 gate above can DECIDE FATAL synchronously, but a member that won an
* election with no peer reachable AT THE PROMOTION INSTANT may simply be seeing
* a transient no-peer snapshot during aggressive failover (task-5.2 soak: a
* recovery-truncated winner self-FATAL'd immediately and repeatedly until
* quorum was lost — an availability collapse, not durable loss). Per the 3.2
* decision table the FATAL terminal must be reached ONLY after a BOUNDED
* failure: the member is fenced (in_catch_up), given DEFAULT_RECOVERY_CATCHUP_-
* WINDOW_MS to find a peer / make catch-up progress, and FATAL'd from a one-shot
* timer only if it is STILL hollow with no peer and no progress when the window
* elapses.
*
* The window is a settable seam so the timer test runs fast. Default =
* DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */
static _Atomic uint32_t s_promotion_fence_window_ms =
DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
uint32_t ha_promotion_fence_window_ms(void)
{
uint32_t w = atomic_load_explicit(&s_promotion_fence_window_ms,
memory_order_acquire);
return w ? w : DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
}
#ifdef XCE_TESTING
void ha_set_promotion_fence_window_ms(uint32_t ms)
{
atomic_store_explicit(&s_promotion_fence_window_ms, ms,
memory_order_release);
}
#endif
/* promotion_fence_ctx_t: delaymgr context for the bounded catch-up window.
* Captures the cluster committed high-water mark (target_hwm) and the durable
* floor AT SCHEDULE TIME (baseline_durable_floor) so the timer can detect
* catch-up progress (durable_floor advanced toward the HWM). `cancelled` is set
* by ha_cancel_promotion_fence_timer (recovery / shutdown) so a fired-but-
* cancelled callback is a safe no-op. */
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
bool was_truncated; /* recovery-truncated at schedule time */
uint64_t target_hwm; /* cluster committed high-water mark */
uint64_t baseline_durable_floor; /* durable floor at schedule time */
} promotion_fence_ctx_t;
/* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer.
* Idempotent; safe to call when no timer is armed. Used when the member catches
* up (fence released) or when the controller is shutting down — so neither a
* caught-up nor a stopped member ever reaches the deferred FATAL. */
void ha_cancel_promotion_fence_timer(ha_state_t *ha)
{
if (!ha) return;
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)atomic_exchange_explicit(
&ha->promotion_fence_ctx, NULL, memory_order_acq_rel);
if (t)
atomic_store_explicit(&t->cancelled, 1, memory_order_release);
}
/* promotion_fence_timeout_fn: the bounded-window timer callback. Owns its ctx —
* frees it on exit. Re-evaluates the LIVE recovery/durability state and reaches
* the 3.3 FATAL ONLY on a real bounded failure (Req 2.4):
* - still recovery-truncated/behind AND no healthy peer reachable AND no
* catch-up progress (durable_floor has not advanced past the baseline)
* → persist_recover_fatal_if_unrecoverable (fail loud).
* - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
* in_catch_up fence; do NOT FATAL (Req 2.6 — eligible again).
* - a healthy peer is reachable now, OR catch-up is making progress but is not
* yet complete → leave the member fenced under the existing catch-up
* machinery and do NOT FATAL while progress is possible. The single bounded
* window has elapsed, so we do not re-arm: a member with a live peer is no
* longer in the "no peer + no progress" bounded-failure state the FATAL
* guards, and the existing catch-up / re-election path will release the
* fence (ha_evaluate_promotion_fence → ELIGIBLE) when it completes. */
static int promotion_fence_timeout_fn(void *arg)
{
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg;
ha_state_t *ha = t->ha;
cluster_t *c = t->c;
/* Clear the stored handle before checking cancelled so a concurrent
* ha_cancel_promotion_fence_timer is a safe no-op (matches the
* raft_election_timeout_fn convention). */
promotion_fence_ctx_t *cur = (promotion_fence_ctx_t *)atomic_load_explicit(
&ha->promotion_fence_ctx, memory_order_acquire);
if (cur == t)
atomic_store_explicit(&ha->promotion_fence_ctx, NULL,
memory_order_release);
if (atomic_load_explicit(&t->cancelled, memory_order_acquire)) {
free(t);
return 0;
}
if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */
/* Re-read the LIVE state after the bounded window. */
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool below_hwm = durable_floor < t->target_hwm;
bool made_progress = durable_floor > t->baseline_durable_floor;
bool peer_reachable = (ha->repl != NULL)
? (repl_active_standby_count(ha->repl) > 0)
: false;
/* Caught up: not truncated AND durable position reached the boundary →
* release the fence; the member is eligible on its next election (Req 2.6). */
if (!truncated && !below_hwm) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)",
durable_floor, t->target_hwm);
free(t);
return 0;
}
/* Still hollow, but a peer is reachable or catch-up advanced the durable
* floor: progress is possible — do NOT FATAL. Leave the member fenced
* (in_catch_up stays set) under the existing catch-up machinery; the bounded
* window has elapsed so we do not re-arm. A member with a live peer is no
* longer in the no-peer bounded-failure state the FATAL guards. */
if (peer_reachable || made_progress) {
INFO("HA: promotion fence catch-up window elapsed but progress is "
"possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, NOT FATAL "
"(ha-commit-durability Req 2.2)",
peer_reachable, durable_floor, t->baseline_durable_floor,
t->target_hwm);
free(t);
return 0;
}
/* Bounded failure: STILL truncated/behind AND no healthy peer AND no
* catch-up progress after the window → fail loud (Req 2.4). This is the
* ONLY path that reaches the 3.3 FATAL — never synchronously at promotion. */
{
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = below_hwm,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = t->target_hwm,
};
ERROR("HA: promotion catch-up window (%u ms) elapsed with no peer and "
"no progress — recovery_truncated=%d durable_floor=%" PRIu64
" < cluster HWM=%" PRIu64 "; failing loud rather than serving "
"divergent state (ha-commit-durability Req 2.4)",
ha_promotion_fence_window_ms(), truncated, durable_floor,
t->target_hwm);
persist_recover_fatal_if_unrecoverable(&in);
}
free(t);
return 0;
}
/* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule
* the one-shot bounded-window FATAL timer. Cancels any previously-armed timer
* first so a re-evaluation never leaks or double-fires. Returns 0 on success.
* On a failure to schedule the timer it FATALs immediately via the 3.3 action so
* the bounded-failure terminal is never silently lost (a member that could not
* arm its own catch-up deadline must not run on indefinitely). */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm)
{
/* Fence first (Req 2.2): refuse promotion while hollow. */
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
/* Cancel a stale timer (e.g. a prior promotion attempt). */
ha_cancel_promotion_fence_timer(ha);
promotion_fence_ctx_t *t = calloc(1, sizeof(*t));
if (!t)
goto fail_loud;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->was_truncated = truncated;
t->target_hwm = target_hwm;
t->baseline_durable_floor = durable_floor;
uint32_t window_ms = ha_promotion_fence_window_ms();
int rc = delay_task_ms(&c->delaymgr, (int)window_ms, 1,
promotion_fence_timeout_fn, t);
if (rc < 0) {
WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc);
free(t);
goto fail_loud;
}
atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release);
INFO("HA: promotion FENCED + bounded catch-up window armed (%u ms) — "
"recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64
"; FATAL deferred to the timer (ha-commit-durability Req 2.2/2.4)",
window_ms, truncated, durable_floor, target_hwm);
return;
fail_loud:
{
/* Could not arm the bounded deadline — reach the FATAL terminal now
* rather than fence indefinitely. */
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < target_hwm),
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
persist_recover_fatal_if_unrecoverable(&in);
}
}
/*
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
*
@ -722,30 +942,41 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
}
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
* the live bounded-window catch-up loop is wired by task 5.1; here the
* gate refuses (FENCE) or fails loud (FATAL when no peer) immediately,
* never silently proceeding while hollow. */
recover_decision_t gate = ha_evaluate_promotion_fence(
ha, truncated, durable_floor, promotion_safe_lsn,
* a hollow member never silently proceeds to leadership. We evaluate
* the PURE 3.2/3.3 decision here and DEFER its terminal — task 4.4 does
* NOT reach the 3.3 FATAL synchronously at the promotion instant. A
* FENCE or a would-be-FATAL (truncated/behind + no peer right now) both
* fence the member (in_catch_up) and arm the bounded catch-up window;
* the FATAL is reached ONLY from that timer after a real bounded failure
* (still hollow + no peer + no progress). This avoids the task-5.2 soak
* collapse where a transient no-peer snapshot during aggressive failover
* self-FATAL'd a recovering winner repeatedly until quorum was lost. */
recover_decision_t gate = ha_promotion_gate_decision(
truncated, durable_floor, promotion_safe_lsn,
peer_reachable, /*catch_up_succeeded=*/false);
if (gate != RECOVER_DECISION_ELIGIBLE) {
/* FENCE: refuse promotion — do NOT become CTRL_PRIMARY. The
* in_catch_up fence is set; the member catches up from a healthy
* peer and re-attempts promotion later (Req 2.2, 2.6).
* FATAL already terminated in production (or fired the test hook);
* still bail out here so a test-hooked FATAL does not fall through
* to authoritative leadership. */
if (gate == RECOVER_DECISION_ELIGIBLE) {
/* ELIGIBLE: release the fence (a caught-up member regains
* eligibility, Req 2.6) and cancel any pending bounded-window FATAL
* timer — a member that is now promotable must never be FATAL'd by a
* stale timer. Proceed to authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE or would-be-FATAL: refuse promotion — do NOT become
* CTRL_PRIMARY. Fence the member and arm the bounded catch-up
* window; the 3.3 FATAL is deferred to the timer (Req 2.2/2.4).
* Never FATAL synchronously, never fence indefinitely. */
WARN("HA: promotion REFUSED (decision=%d) — recovery_truncated=%d "
"durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64
" peer_reachable=%d; remaining non-primary until caught up "
" peer_reachable=%d; fencing + arming bounded catch-up window "
"(ha-commit-durability Req 2.2/2.4)",
(int)gate, truncated, durable_floor, promotion_safe_lsn,
peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;
}
/* ELIGIBLE: clear of the fence (release on a caught-up member, Req 2.6)
* — proceed to authoritative promotion below. */
}
int is_first_boot = !ha->phase2_done;
@ -1230,6 +1461,13 @@ void ha_fini(ha_state_t *ha)
{
if (!ha)
return;
/* ha-commit-durability task 4.4: cancel any pending bounded-window FATAL
* timer so a stopped member never reaches the deferred FATAL. The delaymgr
* task ctx is freed by promotion_fence_timeout_fn when it fires; cancelling
* makes that firing a no-op (it observes cancelled=1 and frees). */
ha_cancel_promotion_fence_timer(ha);
pthread_mutex_destroy(&ha->lock);
sem_destroy(&ha->heartbeat_seen);
@ -3600,6 +3838,20 @@ void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term)
raft_cb_become_leader((void *)ha, term);
}
#ifdef XCE_TESTING
/* ha_arm_promotion_fence_timer_for_test (ha-commit-durability task 4.4):
* expose the file-static bounded-window arm path so the durability suite can
* drive the exact deferral wired into raft_cb_become_leader without staging a
* full election. Production never references this symbol (XCE_TESTING only). */
void ha_arm_promotion_fence_timer_for_test(cluster_t *c, bool truncated,
uint64_t durable_floor,
uint64_t target_hwm)
{
ha_arm_promotion_fence_timer(&c->ha, c, truncated, durable_floor,
target_hwm);
}
#endif
/*
* raft_cb_become_follower_for_test: drive the follower path without a live
* cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production