fix(ha-commit-durability)!: controller never self-terminates — loud non-promotable fence replaces recovery FATAL/abort
Per design invariant (a controller exits only on crash or admin kill), replace the self-abort recovery terminal (tasks 3.3/4.3/4.4) with a loud, permanent, non-promotable fence that stays alive and keeps retrying catch-up (self-heals on peer return; no manual admin recovery). Records the never-self-terminate invariant as project-wide guidance in src/controller/CLAUDE.md + .kiro/steering/ha-async-io.md.
This commit is contained in:
parent
addf53e3b4
commit
59d9df246c
10 changed files with 607 additions and 406 deletions
|
|
@ -46,19 +46,29 @@ typedef enum {
|
|||
* ELIGIBLE — may promote (no fence). Rows 1, 6.
|
||||
* FENCE — ineligible to promote; catch up from a healthy peer, regain
|
||||
* eligibility on success (Req 2.6). Rows 2, 3, 5. A reachable
|
||||
* healthy peer NEVER yields FATAL.
|
||||
* FATAL — fail loud: cannot recover locally AND cannot catch up within
|
||||
* the bounded window. Terminate with the truncation/boundary
|
||||
* logged rather than serve divergent state (Req 2.4). Rows 4, 4b.
|
||||
* healthy peer NEVER yields FENCE_UNRECOVERABLE.
|
||||
* FENCE_UNRECOVERABLE — loud, PERMANENT, NON-terminating fence: the member
|
||||
* cannot recover locally AND cannot catch up (no healthy peer).
|
||||
* It logs LOUDLY, stays PERMANENTLY non-promotable, and keeps
|
||||
* retrying catch-up indefinitely — but the process STAYS ALIVE.
|
||||
* Rows 4, 4b. This is NOT process exit: a controller MUST NEVER
|
||||
* self-terminate as a designed response to an operational
|
||||
* condition (user invariant, 2026-06-18). A self-exited controller
|
||||
* needs MANUAL admin recovery and leaves quorum degraded
|
||||
* meanwhile; a controller that stays alive and keeps retrying
|
||||
* SELF-HEALS when a peer returns. The fence RELEASES (→ ELIGIBLE,
|
||||
* Req 2.6) automatically when catch-up reaches a
|
||||
* committed-consistent state.
|
||||
*
|
||||
* AUTHORITATIVE INVARIANT (design.md): the fence is never a silent indefinite
|
||||
* wait — a truncated/behind member resolves to exactly one of FENCE→ELIGIBLE
|
||||
* or a BOUNDED FATAL. There is no "fenced forever, silently divergent" state.
|
||||
* AUTHORITATIVE INVARIANT (design.md): a truncated/behind member resolves to
|
||||
* either FENCE→ELIGIBLE (caught up) or a loud PERMANENT FENCE_UNRECOVERABLE
|
||||
* (stays alive, non-promotable, keeps retrying). It is NEVER silently divergent
|
||||
* (it refuses to lead) and the process NEVER self-exits.
|
||||
* ============================================================ */
|
||||
typedef enum {
|
||||
RECOVER_DECISION_ELIGIBLE = 0,
|
||||
RECOVER_DECISION_FENCE = 1,
|
||||
RECOVER_DECISION_FATAL = 2,
|
||||
RECOVER_DECISION_ELIGIBLE = 0,
|
||||
RECOVER_DECISION_FENCE = 1,
|
||||
RECOVER_DECISION_FENCE_UNRECOVERABLE = 2,
|
||||
} recover_decision_t;
|
||||
|
||||
/* Policy inputs evaluated at boot / promotion-eligibility (design.md table).
|
||||
|
|
@ -177,6 +187,22 @@ struct persist_s {
|
|||
* before the node participates; read by the promotion gate thereafter. */
|
||||
recover_verdict_t recovery_verdict;
|
||||
|
||||
/* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4,
|
||||
* NEVER-SELF-EXIT rework 2026-06-18). Set by
|
||||
* persist_recover_loud_fence_if_unrecoverable when a member is
|
||||
* recovery-truncated AND/OR durable below the cluster committed high-water
|
||||
* mark with NO successful catch-up: the member stays ALIVE but is held
|
||||
* PERMANENTLY non-promotable and keeps retrying catch-up. A controller MUST
|
||||
* NEVER self-exit as a designed response to this operational condition; it
|
||||
* self-heals automatically when a healthy peer returns and catch-up reaches
|
||||
* a committed-consistent state — at which point this flag is CLEARED via
|
||||
* persist_clear_recovery_unrecoverable (wired into the fence-release /
|
||||
* catch-up path). Read by the promotion gate as belt-and-suspenders
|
||||
* (a member must never lead while this is set). Atomic: set on the recovery
|
||||
* path, cleared on the catch-up path, read by the promotion/heartbeat
|
||||
* threads. Defaults to false before recovery runs. */
|
||||
_Atomic bool recovery_unrecoverable;
|
||||
|
||||
/* Paths (derived from workdir + name via persist_set_workdir) */
|
||||
char wal_path[4096];
|
||||
char snap_path[4096];
|
||||
|
|
@ -347,53 +373,64 @@ uint64_t persist_durable_floor(persist_t* p);
|
|||
recover_verdict_t persist_recovery_verdict(persist_t* p);
|
||||
bool persist_recovery_truncated(persist_t* p);
|
||||
|
||||
/* Bounded catch-up window (ha-commit-durability task 3.3, Req 2.4).
|
||||
* The FINITE deadline a recovery-truncated / behind member is given to reach a
|
||||
* committed-consistent state from a healthy peer before the boot policy FATALs.
|
||||
* The fence must NEVER be an indefinite wait (design.md authoritative
|
||||
* invariant), so the window is always finite. Default 30s — generous enough for
|
||||
* a healthy-peer catch-up stream yet bounded; the verification mode overrides it
|
||||
* via the inputs/timeout the caller (task 4.3 boot path) supplies. */
|
||||
/* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4;
|
||||
* never-self-exit rework 2026-06-18).
|
||||
* The interval after which a still-unrecoverable, loudly-fenced member RE-EMITS
|
||||
* its operator alert (it does NOT terminate — see RECOVER_DECISION_FENCE_-
|
||||
* UNRECOVERABLE). The member stays alive and keeps retrying catch-up
|
||||
* indefinitely; this window only controls how often the loud operator-visible
|
||||
* alert is repeated. Default 30s. */
|
||||
#define DEFAULT_RECOVERY_CATCHUP_WINDOW_MS 30000u
|
||||
|
||||
/* persist_recover_decision (ha-commit-durability task 3.3, Req 2.4).
|
||||
*
|
||||
* Pure decision over the Boot Policy Decision Table: maps the policy inputs to
|
||||
* RECOVER_DECISION_{ELIGIBLE,FENCE,FATAL}. No side effects, no termination — it
|
||||
* only classifies, so it is unit-testable in isolation. The FATAL terminal is
|
||||
* acted on by persist_recover_fatal_if_unrecoverable (which logs + terminates).
|
||||
* RECOVER_DECISION_{ELIGIBLE,FENCE,FENCE_UNRECOVERABLE}. No side effects, no
|
||||
* termination — it only classifies, so it is unit-testable in isolation. The
|
||||
* FENCE_UNRECOVERABLE terminal is acted on by
|
||||
* persist_recover_loud_fence_if_unrecoverable (which logs LOUDLY + sets the
|
||||
* sticky non-promotable flag — it NEVER terminates the process).
|
||||
*
|
||||
* Contract (design.md rows): a reachable healthy peer NEVER returns FATAL
|
||||
* (rows 3, 5); FATAL is returned ONLY when the member is truncated or behind
|
||||
* the HWM, has NO healthy peer, and bounded catch-up did not reach a
|
||||
* committed-consistent state (rows 4, 4b). */
|
||||
* Contract (design.md rows): a reachable healthy peer NEVER returns
|
||||
* FENCE_UNRECOVERABLE (rows 3, 5); it is returned ONLY when the member is
|
||||
* truncated or behind the HWM, has NO healthy peer, and catch-up did not reach
|
||||
* a committed-consistent state (rows 4, 4b). */
|
||||
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in);
|
||||
|
||||
/* persist_recover_fatal_if_unrecoverable (ha-commit-durability task 3.3,
|
||||
* Req 2.4).
|
||||
/* persist_recover_loud_fence_if_unrecoverable (ha-commit-durability task 3.3,
|
||||
* Req 2.4; never-self-exit rework 2026-06-18).
|
||||
*
|
||||
* Evaluate persist_recover_decision over `in`; if the outcome is FATAL, log at
|
||||
* ERROR LOUDLY — INCLUDING the truncation boundary (durable_floor_lsn) and the
|
||||
* cluster high-water mark — and then TERMINATE (fail loud rather than serve
|
||||
* divergent state). Returns the decision so a non-FATAL caller can fence/proceed
|
||||
* (FENCE/ELIGIBLE are acted on by task 4.3).
|
||||
* Evaluate persist_recover_decision over `in`; if the outcome is
|
||||
* FENCE_UNRECOVERABLE, log LOUDLY at ERROR — INCLUDING the truncation boundary
|
||||
* (durable_floor_lsn) and the cluster high-water mark — stating the member is
|
||||
* staying fenced + non-promotable and will keep retrying catch-up, then SET the
|
||||
* sticky `recovery_unrecoverable` flag on `p` (so the promotion gate refuses
|
||||
* leadership as belt-and-suspenders). Returns the decision.
|
||||
*
|
||||
* Termination routes through persist_fatal_hook: in production the hook is NULL
|
||||
* and the function flushes logs and abort()s (NEVER returns on FATAL); a test
|
||||
* build may install a hook to capture the FATAL decision + boundary log without
|
||||
* the process dying (mirrors the ha_fatal_hook seam in ha_convergence.c). */
|
||||
* CRITICAL — the process STAYS ALIVE. There is NO abort()/exit()/tlog_exit()
|
||||
* here: a controller MUST NEVER self-terminate as a designed response to an
|
||||
* operational condition (user invariant, 2026-06-18). The member self-heals —
|
||||
* the fence releases (→ ELIGIBLE, Req 2.6) and the flag clears via
|
||||
* persist_clear_recovery_unrecoverable — automatically when a healthy peer
|
||||
* returns and catch-up reaches a committed-consistent state.
|
||||
*
|
||||
* `p` may be NULL (pure-decision callers that only want the classification +
|
||||
* loud log without touching a persist_t); the flag set is then skipped. */
|
||||
recover_decision_t
|
||||
persist_recover_fatal_if_unrecoverable(const recover_policy_inputs_t* in);
|
||||
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
|
||||
const recover_policy_inputs_t* in);
|
||||
|
||||
/* persist_fatal_hook: test-only seam for the FATAL (abort) contract of
|
||||
* persist_recover_fatal_if_unrecoverable (ha-commit-durability task 3.3,
|
||||
* Req 2.4). Mirrors ha_fatal_hook (ha_convergence.c).
|
||||
/* persist_recovery_unrecoverable / persist_clear_recovery_unrecoverable
|
||||
* (ha-commit-durability Req 2.4/2.6, never-self-exit rework 2026-06-18).
|
||||
*
|
||||
* When non-NULL, the FATAL path invokes this INSTEAD OF abort(), so a test
|
||||
* binary can assert the FATAL decision was taken and the truncation boundary
|
||||
* was logged without the process dying. Production leaves it NULL and actually
|
||||
* terminates. Signature: void hook(const char *msg). */
|
||||
extern void (*persist_fatal_hook)(const char* msg);
|
||||
* Accessor + clear for the sticky "unrecoverable, loudly fenced" flag. The flag
|
||||
* is SET by persist_recover_loud_fence_if_unrecoverable and CLEARED on the
|
||||
* catch-up / fence-release path once the member reaches a committed-consistent
|
||||
* state (not truncated AND durable_floor >= HWM). While set, the member is held
|
||||
* permanently non-promotable but stays alive. Safe on a NULL p (returns false /
|
||||
* no-op). */
|
||||
bool persist_recovery_unrecoverable(persist_t* p);
|
||||
void persist_clear_recovery_unrecoverable(persist_t* p);
|
||||
|
||||
/* Register a durable-advance wake callback (ha-commit-durability task 2.2).
|
||||
* The group-commit flusher calls fn(ctx) after it advances durable_lsn and
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue