feat(ha-commit-durability): loud bounded FATAL on unrecoverable recovery (task 3.3)

src/controller/persist.c

@@ -1642,6 +1642,98 @@ bool persist_recovery_truncated(persist_t* p)
     return p && p->recovery_verdict == RECOVER_TRUNCATED;
 }
 
+/* ------------------------------------------------------------------ *
+ * Boot-policy FATAL decision (ha-commit-durability task 3.3, Req 2.4).
+ *
+ * Implements the FATAL terminal of the Boot Policy Decision Table
+ * (design.md). The decision is a PURE function of its inputs; the FATAL
+ * action (loud log + terminate) is separated into
+ * persist_recover_fatal_if_unrecoverable so the decision can be unit-tested
+ * without aborting. A reachable healthy peer NEVER FATALs (rows 3, 5);
+ * FATAL is reserved for a member that cannot recover locally AND cannot catch
+ * up within the bounded window (rows 4, 4b) — never a silent indefinite fence.
+ * ------------------------------------------------------------------ */
+
+/* FATAL seam: NULL in production (real abort); a test may install a hook to
+ * capture the FATAL path without dying. Mirrors ha_fatal_hook. */
+void (*persist_fatal_hook)(const char* msg) = NULL;
+
+recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in)
+{
+    if (in == NULL)
+        return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */
+
+    bool truncated = (in->verdict == RECOVER_TRUNCATED);
+
+    /* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster
+     * high-water mark is a normal promotable member — no fence, no FATAL. */
+    if (!truncated && !in->durable_below_hwm)
+        return RECOVER_DECISION_ELIGIBLE;
+
+    /* From here the member is truncated OR behind the HWM — it must NOT lead
+     * as-is. A truncated log is by definition not provably at/above the HWM
+     * (design.md note), so it is treated as behind for fence purposes. */
+
+    /* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain
+     * eligibility (task 4.3). A reachable peer NEVER yields FATAL — even for a
+     * truncated log or a legitimate snapshot-sentinel. If catch-up has already
+     * succeeded the member is eligible; otherwise it stays fenced (4.3 drives
+     * the catch-up + release; this task only refuses FATAL while a peer
+     * exists). */
+    if (in->healthy_peer_reachable) {
+        if (in->catch_up_succeeded)
+            return RECOVER_DECISION_ELIGIBLE;
+        return RECOVER_DECISION_FENCE;
+    }
+
+    /* No healthy peer. The bounded catch-up window is the member's only path to
+     * a committed-consistent state; if it did not reach one, the member can
+     * neither recover locally nor catch up — rows 4 & 4b: FATAL (fail loud),
+     * never a silent indefinite fence (Req 2.4). */
+    if (!in->catch_up_succeeded)
+        return RECOVER_DECISION_FATAL;
+
+    /* No peer but catch-up is reported succeeded (e.g. already at/above after a
+     * local catch-up): eligible. Defensive — the boot path reaches the
+     * no-peer branch with catch_up_succeeded=false in the bounded-failure case
+     * this task owns. */
+    return RECOVER_DECISION_ELIGIBLE;
+}
+
+recover_decision_t
+persist_recover_fatal_if_unrecoverable(const recover_policy_inputs_t* in)
+{
+    recover_decision_t decision = persist_recover_decision(in);
+
+    if (decision != RECOVER_DECISION_FATAL)
+        return decision;
+
+    /* Fail loud: log the truncation boundary (durable_floor_lsn) and the
+     * cluster high-water mark so an operator sees exactly where the local log
+     * stops short of the cluster's committed state (Req 2.4). */
+    const char* why = (in->verdict == RECOVER_TRUNCATED)
+                          ? "recovery-truncated log (structural corruption)"
+                          : "log behind the cluster committed high-water mark";
+    ERROR("persist_recover: FATAL — %s and no healthy peer reachable to catch "
+          "up within the bounded window (%u ms); truncation boundary "
+          "durable_floor_lsn=%" PRIu64 " < cluster committed high-water "
+          "hwm=%" PRIu64 " — terminating rather than serving divergent state "
+          "(ha-commit-durability Req 2.4)",
+          why, (unsigned)DEFAULT_RECOVERY_CATCHUP_WINDOW_MS,
+          in->durable_floor_lsn, in->cluster_hwm_lsn);
+
+    if (persist_fatal_hook != NULL) {
+        /* Test seam: capture the FATAL decision + boundary log without dying.
+         * Production leaves the hook NULL and aborts below. */
+        persist_fatal_hook("persist_recover: bounded-unrecoverable (FATAL)");
+        return RECOVER_DECISION_FATAL;
+    }
+
+    tlog_exit(); /* flush the async log buffer before dying */
+    abort();     /* NEVER returns in production */
+    return RECOVER_DECISION_FATAL; /* unreachable */
+}
+
 /* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
  * entry is already durable on local stable storage (ha-commit-durability task
  * 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the