lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ha-commit-durability)!: controller never self-terminates — loud non-promotable fence replaces recovery FATAL/abort

Browse source 
Per design invariant (a controller exits only on crash or admin kill), replace the
self-abort recovery terminal (tasks 3.3/4.3/4.4) with a loud, permanent, non-promotable
fence that stays alive and keeps retrying catch-up (self-heals on peer return; no manual
admin recovery). Records the never-self-terminate invariant as project-wide guidance in
src/controller/CLAUDE.md + .kiro/steering/ha-async-io.md.
This commit is contained in:
Rongsong Shen 2026-06-18 11:10:19 +00:00
parent addf53e3b4
commit 59d9df246c
10 changed files with 607 additions and 406 deletions
Download patch file Download diff file Expand all files Collapse all files

202
src/controller/ha.c
View file

@ -572,8 +572,10 @@ recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
* the committed boundary regains eligibility, Req 2.6).
* - FENCE → set the fence, return FENCE (refuse promotion; catch up from a
* healthy peer).
* - FATAL → run the 3.3 loud-terminate path (logs the truncation boundary,
* then terminates — or fires persist_fatal_hook in tests).
* - FENCE_UNRECOVERABLE → run the 3.3 loud-fence action (logs LOUDLY + sets
* the sticky non-promotable flag) and ALSO set the in_catch_up
* fence. The member STAYS ALIVE, non-promotable, retrying
* catch-up — it NEVER self-exits (Req 2.4).
*/
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
bool recovery_truncated,
@ -603,10 +605,11 @@ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
"ha-commit-durability Req 2.2)",
recovery_truncated, durable_floor, safe_hwm);
break;
case RECOVER_DECISION_FATAL: {
/* Bounded-unrecoverable: route through the 3.3 loud-terminate action,
* which logs the truncation boundary at ERROR then terminates (or fires
* persist_fatal_hook under test). Req 2.4. */
case RECOVER_DECISION_FENCE_UNRECOVERABLE: {
/* Unrecoverable: route through the 3.3 loud-fence action — logs LOUDLY
* and sets the sticky non-promotable flag. Also hold the in_catch_up
* fence so the member refuses promotion. The process STAYS ALIVE and
* keeps retrying catch-up; it NEVER self-exits (Req 2.4). */
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
@ -616,25 +619,36 @@ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
persist_recover_fatal_if_unrecoverable(&in);
persist_t *p = (ha != NULL && ha->raft_c != NULL) ? &ha->raft_c->persist
: NULL;
persist_recover_loud_fence_if_unrecoverable(p, &in);
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
break;
}
}
return d;
}
/* ── ha-commit-durability task 4.4: bounded-window catch-up before FATAL ─────
/* ── ha-commit-durability task 4.4: bounded-window catch-up re-alert ─────────
*
* The 4.3 gate above can DECIDE FATAL synchronously, but a member that won an
* election with no peer reachable AT THE PROMOTION INSTANT may simply be seeing
* a transient no-peer snapshot during aggressive failover (task-5.2 soak: a
* recovery-truncated winner self-FATAL'd immediately and repeatedly until
* quorum was lost — an availability collapse, not durable loss). Per the 3.2
* decision table the FATAL terminal must be reached ONLY after a BOUNDED
* failure: the member is fenced (in_catch_up), given DEFAULT_RECOVERY_CATCHUP_-
* WINDOW_MS to find a peer / make catch-up progress, and FATAL'd from a one-shot
* timer only if it is STILL hollow with no peer and no progress when the window
* elapses.
* (never-self-exit rework 2026-06-18.) A member that wins an election while
* recovery-truncated / behind the HWM with no peer reachable AT THE PROMOTION
* INSTANT is FENCED (in_catch_up set, non-promotable) and STAYS ALIVE, retrying
* catch-up indefinitely. This one-shot timer exists ONLY for operator
* VISIBILITY: when it fires, if the member is STILL unrecoverable (still
* truncated/behind AND no peer AND no progress) it RE-EMITS the loud,
* operator-visible alert via persist_recover_loud_fence_if_unrecoverable (which
* also keeps the sticky non-promotable flag set) — it NEVER terminates the
* process. If the member has caught up (not truncated AND durable_floor >=
* HWM) the timer RELEASES the fence (Req 2.6). If progress is possible (peer
* back, or durable floor advanced) it leaves the member fenced under the
* existing catch-up machinery.
*
* Earlier this timer DEFERRED-then-ABORTED. The abort is GONE: a controller
* MUST NEVER self-terminate as a designed response to an operational condition
* (user invariant, 2026-06-18). The task-5.2 soak quorum collapse was caused
* by that self-abort; staying alive + retrying self-heals when a peer returns.
*
* The window is a settable seam so the timer test runs fast. Default =
* DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */
@ -684,21 +698,26 @@ void ha_cancel_promotion_fence_timer(ha_state_t *ha)
atomic_store_explicit(&t->cancelled, 1, memory_order_release);
}
/* promotion_fence_timeout_fn: the bounded-window timer callback. Owns its ctx —
* frees it on exit. Re-evaluates the LIVE recovery/durability state and reaches
* the 3.3 FATAL ONLY on a real bounded failure (Req 2.4):
* - still recovery-truncated/behind AND no healthy peer reachable AND no
* catch-up progress (durable_floor has not advanced past the baseline)
* → persist_recover_fatal_if_unrecoverable (fail loud).
/* Forward decl: the timeout callback re-arms itself for periodic re-alert. */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm);
/* promotion_fence_timeout_fn: the bounded-window re-alert timer callback. Owns
* its ctx — frees it on exit. Re-evaluates the LIVE recovery/durability state.
* It NEVER terminates the process (never-self-exit invariant, 2026-06-18):
* - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
* in_catch_up fence; do NOT FATAL (Req 2.6 — eligible again).
* - a healthy peer is reachable now, OR catch-up is making progress but is not
* yet complete → leave the member fenced under the existing catch-up
* machinery and do NOT FATAL while progress is possible. The single bounded
* window has elapsed, so we do not re-arm: a member with a live peer is no
* longer in the "no peer + no progress" bounded-failure state the FATAL
* guards, and the existing catch-up / re-election path will release the
* fence (ha_evaluate_promotion_fence → ELIGIBLE) when it completes. */
* in_catch_up fence + clear the sticky unrecoverable flag; eligible again
* (Req 2.6).
* - a healthy peer is reachable now, OR catch-up is making progress → leave
* the member fenced under the existing catch-up machinery (it will release
* via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not
* re-arm the re-alert while progress is happening.
* - STILL unrecoverable (truncated/behind AND no peer AND no progress) →
* RE-EMIT the loud operator alert via persist_recover_loud_fence_if_-
* unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the
* timer so the alert keeps repeating until a peer returns. The process
* STAYS ALIVE and keeps retrying — it NEVER aborts. */
static int promotion_fence_timeout_fn(void *arg)
{
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg;
@ -721,7 +740,7 @@ static int promotion_fence_timeout_fn(void *arg)
if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */
/* Re-read the LIVE state after the bounded window. */
/* Re-read the LIVE state after the window. */
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool below_hwm = durable_floor < t->target_hwm;
@ -731,9 +750,11 @@ static int promotion_fence_timeout_fn(void *arg)
: false;
/* Caught up: not truncated AND durable position reached the boundary →
* release the fence; the member is eligible on its next election (Req 2.6). */
* release the fence + clear the sticky flag; the member is eligible on its
* next election (Req 2.6). */
if (!truncated && !below_hwm) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)",
durable_floor, t->target_hwm);
@ -742,14 +763,13 @@ static int promotion_fence_timeout_fn(void *arg)
}
/* Still hollow, but a peer is reachable or catch-up advanced the durable
* floor: progress is possible — do NOT FATAL. Leave the member fenced
* (in_catch_up stays set) under the existing catch-up machinery; the bounded
* window has elapsed so we do not re-arm. A member with a live peer is no
* longer in the no-peer bounded-failure state the FATAL guards. */
* floor: progress is possible. Leave the member fenced (in_catch_up stays
* set) under the existing catch-up machinery; do not re-arm the re-alert
* while progress is happening. */
if (peer_reachable || made_progress) {
INFO("HA: promotion fence catch-up window elapsed but progress is "
"possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, NOT FATAL "
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, catching up "
"(ha-commit-durability Req 2.2)",
peer_reachable, durable_floor, t->baseline_durable_floor,
t->target_hwm);
@ -757,9 +777,11 @@ static int promotion_fence_timeout_fn(void *arg)
return 0;
}
/* Bounded failure: STILL truncated/behind AND no healthy peer AND no
* catch-up progress after the window → fail loud (Req 2.4). This is the
* ONLY path that reaches the 3.3 FATAL — never synchronously at promotion. */
/* STILL unrecoverable: truncated/behind AND no healthy peer AND no catch-up
* progress. RE-EMIT the loud operator alert and keep the member fenced +
* non-promotable + ALIVE (Req 2.4) — NEVER terminate. Re-arm the re-alert
* timer so the operator keeps hearing about it until a peer returns. */
uint64_t target_hwm = t->target_hwm; /* capture before free (UAF guard) */
{
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
@ -768,27 +790,27 @@ static int promotion_fence_timeout_fn(void *arg)
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = t->target_hwm,
.cluster_hwm_lsn = target_hwm,
};
ERROR("HA: promotion catch-up window (%u ms) elapsed with no peer and "
"no progress — recovery_truncated=%d durable_floor=%" PRIu64
" < cluster HWM=%" PRIu64 "; failing loud rather than serving "
"divergent state (ha-commit-durability Req 2.4)",
ha_promotion_fence_window_ms(), truncated, durable_floor,
t->target_hwm);
persist_recover_fatal_if_unrecoverable(&in);
/* Loud, operator-visible alert + sticky non-promotable flag. STAYS
* ALIVE: no abort/exit anywhere on this path. */
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
}
free(t);
/* Re-arm so the loud alert repeats periodically while the member remains
* unrecoverable. The member keeps retrying catch-up; the next firing
* re-evaluates and releases the fence once a peer returns and it catches up. */
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, target_hwm);
return 0;
}
/* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule
* the one-shot bounded-window FATAL timer. Cancels any previously-armed timer
* first so a re-evaluation never leaks or double-fires. Returns 0 on success.
* On a failure to schedule the timer it FATALs immediately via the 3.3 action so
* the bounded-failure terminal is never silently lost (a member that could not
* arm its own catch-up deadline must not run on indefinitely). */
* the one-shot bounded-window RE-ALERT timer. Cancels any previously-armed
* timer first so a re-evaluation never leaks or double-fires. On a failure to
* allocate/schedule the timer it emits the loud unrecoverable fence ONCE (stays
* alive, non-promotable) — it NEVER aborts (never-self-exit invariant). */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm)
@ -801,7 +823,7 @@ static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
promotion_fence_ctx_t *t = calloc(1, sizeof(*t));
if (!t)
goto fail_loud;
goto loud_fence_no_timer;
atomic_init(&t->cancelled, 0);
t->ha = ha;
@ -816,20 +838,22 @@ static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
if (rc < 0) {
WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc);
free(t);
goto fail_loud;
goto loud_fence_no_timer;
}
atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release);
INFO("HA: promotion FENCED + bounded catch-up window armed (%u ms) — "
INFO("HA: promotion FENCED + catch-up re-alert window armed (%u ms) — "
"recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64
"; FATAL deferred to the timer (ha-commit-durability Req 2.2/2.4)",
"; staying ALIVE + non-promotable, retrying catch-up "
"(ha-commit-durability Req 2.2/2.4)",
window_ms, truncated, durable_floor, target_hwm);
return;
fail_loud:
loud_fence_no_timer:
{
/* Could not arm the bounded deadline — reach the FATAL terminal now
* rather than fence indefinitely. */
/* Could not arm the re-alert timer — still emit the loud unrecoverable
* fence once so the operator sees it; the member stays fenced
* (in_catch_up set above) + non-promotable + ALIVE. NO abort. */
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
@ -839,7 +863,7 @@ fail_loud:
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
persist_recover_fatal_if_unrecoverable(&in);
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
}
}
@ -919,10 +943,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
* the in_catch_up fence and return WITHOUT becoming primary; the member
* catches up from a healthy peer and a later promotion attempt passes the
* gate (Req 2.2/2.6). Only the production path (real persist) is gated; the
* c==NULL unit shim has no persist and skips it. */
* c==NULL unit shim has no persist and skips it.
*
* Belt-and-suspenders (never-self-exit rework 2026-06-18): a member that has
* been marked sticky-unrecoverable (persist_recovery_unrecoverable) must
* NEVER become primary while that flag is set — it stays alive, fenced, and
* keeps retrying catch-up. The flag clears automatically on catch-up. */
if (c != NULL) {
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool unrecoverable = persist_recovery_unrecoverable(&c->persist);
/* A healthy peer is reachable for catch-up iff some voter granted with a
* live endpoint survives this election (these become the standby
@ -943,36 +973,36 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
* a hollow member never silently proceeds to leadership. We evaluate
* the PURE 3.2/3.3 decision here and DEFER its terminal — task 4.4 does
* NOT reach the 3.3 FATAL synchronously at the promotion instant. A
* FENCE or a would-be-FATAL (truncated/behind + no peer right now) both
* fence the member (in_catch_up) and arm the bounded catch-up window;
* the FATAL is reached ONLY from that timer after a real bounded failure
* (still hollow + no peer + no progress). This avoids the task-5.2 soak
* collapse where a transient no-peer snapshot during aggressive failover
* self-FATAL'd a recovering winner repeatedly until quorum was lost. */
* the PURE 3.2/3.3 decision here. A FENCE or an UNRECOVERABLE
* (truncated/behind + no peer right now) both fence the member
* (in_catch_up) and arm the catch-up re-alert timer; the member STAYS
* ALIVE and keeps retrying — it NEVER self-exits (never-self-exit
* invariant, 2026-06-18; this is what fixes the task-5.2 soak collapse
* where a self-aborting recovering winner repeatedly dropped quorum). */
recover_decision_t gate = ha_promotion_gate_decision(
truncated, durable_floor, promotion_safe_lsn,
peer_reachable, /*catch_up_succeeded=*/false);
if (gate == RECOVER_DECISION_ELIGIBLE) {
/* ELIGIBLE: release the fence (a caught-up member regains
* eligibility, Req 2.6) and cancel any pending bounded-window FATAL
* timer — a member that is now promotable must never be FATAL'd by a
* stale timer. Proceed to authoritative promotion below. */
if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable) {
/* ELIGIBLE and not sticky-unrecoverable: release the fence (a
* caught-up member regains eligibility, Req 2.6), clear the sticky
* flag if set, and cancel any pending re-alert timer. Proceed to
* authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE or would-be-FATAL: refuse promotion — do NOT become
* CTRL_PRIMARY. Fence the member and arm the bounded catch-up
* window; the 3.3 FATAL is deferred to the timer (Req 2.2/2.4).
* Never FATAL synchronously, never fence indefinitely. */
WARN("HA: promotion REFUSED (decision=%d) — recovery_truncated=%d "
"durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64
" peer_reachable=%d; fencing + arming bounded catch-up window "
/* FENCE, UNRECOVERABLE, or sticky-unrecoverable: refuse promotion —
* do NOT become CTRL_PRIMARY. Fence the member and arm the catch-up
* re-alert timer. The member STAYS ALIVE, non-promotable, retrying
* catch-up (Req 2.2/2.4). NEVER self-exit, never fence silently. */
WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d) — "
"recovery_truncated=%d durable_floor=%" PRIu64
" cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + arming "
"catch-up re-alert window, staying ALIVE "
"(ha-commit-durability Req 2.2/2.4)",
(int)gate, truncated, durable_floor, promotion_safe_lsn,
peer_reachable);
(int)gate, unrecoverable, truncated, durable_floor,
promotion_safe_lsn, peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;