lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(job-array-single-wal-record): gate promotion on the live apply fence

Browse source 
Threads !persist_apply_resource_fenced(&c->persist) as a required
conjunct into both existing promotion-eligibility call sites in ha.c:
the periodic fence-release check (promotion_fence_timeout_fn) and the
election-time check (raft_cb_become_leader). A node with the fence
engaged is refused promotion at both points; once it self-clears
(task 4.2), the node is eligible again with no other action.

Task 4.3 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
Rongsong Shen 2026-07-15 04:21:35 +00:00
parent b47cc660ba
commit 8cb8c25d4d
4 changed files with 288 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

73
src/controller/ha.c
View file

@ -1062,11 +1062,25 @@ static int promotion_fence_timeout_fn(void *arg)
bool self_restart_pending =
atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
/* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
* 9.4/9.5): same unconditional, never-live-cleared-by-structure refusal
* as apply_divergent/self_restart_pending above — a node stuck on a
* resource-exhaustion apply failure (task 4.1/4.2's Live Apply Resource
* Fence) can have a perfectly healthy WAL (not truncated, durable_floor
* >= HWM) by construction, since apply success and WAL durability are
* orthogonal. This purely structural re-check must not release the fence
* on WAL health alone — only the apply pump's own successful retry of the
* stuck LSN clears it (persist_clear_apply_resource_fence, called from
* inside persist_apply_pump_run itself). */
bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
/* Caught up: not truncated AND durable position reached the boundary AND
* the current apply verdict is not DIVERGENT AND no self-restart is
* pending → release the fence + clear the sticky flag; the member is
* eligible on its next election (Req 2.6). */
if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending) {
* pending AND no live apply resource fence is engaged → release the
* fence + clear the sticky flag; the member is eligible on its next
* election (Req 2.6, Req 9.5). */
if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending
&& !apply_resource_fenced) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
@ -1312,6 +1326,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
bool self_restart_pending =
atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
/* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
* 9.4/9.5): same unconditional-refuse treatment as apply_divergent/
* self_restart_pending above — a node stuck on a resource-exhaustion
* apply failure (task 4.1/4.2's Live Apply Resource Fence) must never
* become primary while the fence is engaged, regardless of how
* structurally ELIGIBLE its WAL looks (truncated/durable_floor are
* orthogonal to apply success). Only the apply pump's own successful
* retry of the stuck LSN clears it — no election/HWM condition can. */
bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
/* A healthy peer is reachable for catch-up iff some voter granted with a
* live endpoint survives this election (these become the standby
* channels promoted below). */
@ -1342,34 +1366,41 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
peer_reachable, /*catch_up_succeeded=*/false);
if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent
&& !self_restart_pending) {
/* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, AND no
* pending self-restart: release the fence (a caught-up member
* regains eligibility, Req 2.6), clear the sticky flag if set, and
* cancel any pending re-alert timer. Proceed to authoritative
* promotion below. */
&& !self_restart_pending && !apply_resource_fenced) {
/* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, no
* pending self-restart, AND no live apply resource fence engaged:
* release the fence (a caught-up member regains eligibility, Req
* 2.6), clear the sticky flag if set, and cancel any pending
* re-alert timer. Proceed to authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent, or
* self-restart-pending: refuse promotion — do NOT become
* CTRL_PRIMARY. Fence the member and arm the catch-up re-alert
* timer. The member STAYS ALIVE, non-promotable, retrying catch-up
* (Req 2.2/2.4). NEVER self-exit, never fence silently. An
* apply-divergent or self-restart-pending member is refused
* unconditionally (Req 8.4; ha-convergence-recovery Req 1.4) — no
* peer/HWM condition can clear either trigger; only a fresh boot
* (clean restart replay, or a successful re-exec) can. */
/* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent,
* self-restart-pending, or apply-resource-fenced: refuse
* promotion — do NOT become CTRL_PRIMARY. Fence the member and
* arm the catch-up re-alert timer. The member STAYS ALIVE,
* non-promotable, retrying catch-up (Req 2.2/2.4). NEVER
* self-exit, never fence silently. An apply-divergent,
* self-restart-pending, or apply-resource-fenced member is
* refused unconditionally (Req 8.4; ha-convergence-recovery Req
* 1.4; job-array-single-wal-record Req 9.4/9.5) — no peer/HWM
* condition can clear any of these triggers; apply-resource-
* fenced clears only from inside the apply pump itself on a
* successful retry (task 4.2), the others only on a fresh boot
* (clean restart replay, or a successful re-exec). */
WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d "
"apply_divergent=%d self_restart_pending=%d) — "
"apply_divergent=%d self_restart_pending=%d "
"apply_resource_fenced=%d) — "
"recovery_truncated=%d durable_floor=%" PRIu64
" cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + "
"arming catch-up re-alert window, staying ALIVE "
"(ha-commit-durability Req 2.2/2.4; raft-apply-decoupling "
"Req 8.4; ha-convergence-recovery Req 1.4)",
"Req 8.4; ha-convergence-recovery Req 1.4; "
"job-array-single-wal-record Req 9.4/9.5)",
(int)gate, unrecoverable, apply_divergent, self_restart_pending,
truncated, durable_floor, promotion_safe_lsn, peer_reachable);
apply_resource_fenced, truncated, durable_floor,
promotion_safe_lsn, peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;