myxce/src/controller/include/commit.h

#pragma once
/*
 * commit.h — Commit engine: single authority for per-entry commit evaluation,
 * committed_lsn advancement, and waiter completion.
 *
 * Service interface is umbrella-verbatim from
 * .kiro/specs/ha-consistency-invariants/design.md (sliced by ha-commit-engine).
 *
 * All functions thread-safe; commit_on_ack callable from iomgr (non-blocking).
 */
#include <pthread.h>
#include <stdint.h>
#include <time.h> /* struct timespec — per-continuation timeout deadline (task 1.3) */

#include "persist.h" /* persist_t (owns committed_lsn atomic + replicator_t registry) */
#include "vtaskmgr.h" /* vtaskmgr_t — LivenessAdmission blocking-admission seam */

/* Headroom kept free of blocking commit waits; cap = max(1, n_vcpus - reserved).
 * Used ONLY by the local (no-pool) admission fallback. When a vtaskmgr pool is
 * wired (commit_set_pool, raft-drive-async task 1.6), the pool owns the single
 * reservation (cap = max(1, n_vcpus - VTASK_LIVENESS_RESERVED)) and this
 * constant is not consulted. */
#define COMMIT_WAIT_RESERVED 2

typedef enum { COMMIT_MODE_WAIT, COMMIT_MODE_RETAIN } commit_mode_t;

/* raft-apply-decoupling task 1.1: event-driven resolve outcome, delivered to
 * a registered continuation in place of a COMMIT_MODE_WAIT caller's blocking
 * return value. COMMITTED mirrors commit_submit's rc==0; FAILED mirrors any
 * of commit_submit's negative outcomes (timeout, demotion, truncation), with
 * the specific cause carried in the continuation's `rc` argument. */
typedef enum { COMMIT_OUTCOME_COMMITTED, COMMIT_OUTCOME_FAILED } commit_outcome_t;

/* Invoked exactly once per registered lsn when its outcome becomes known.
 * Delivered via commit_resolve (wired by a follow-up task onto
 * commit_advance_locked's majority path and the existing timeout/demotion/
 * truncation failure handling), dispatched off the calling thread via
 * vtask_enq_classified -- never inline on the replication-ACK or iomgr
 * path. */
typedef void (*commit_continuation_fn)(void* ctx, uint64_t lsn,
                                        commit_outcome_t outcome, int rc);

/* raft-apply-decoupling task 1.1: one registered COMMIT_MODE_WAIT-equivalent
 * continuation. lsn == 0 marks a free slot -- LSN 0 is the reserved
 * empty/none sentinel (see persist.h reserve-lsn-0), so it is never a real
 * registration and is safe as the "unused" marker. */
typedef struct commit_continuation {
    uint64_t               lsn;
    commit_continuation_fn fn;
    void*                  ctx;
    /* raft-apply-decoupling task 1.3: absolute CLOCK_REALTIME deadline this
     * registration must resolve by if majority is never reached -- computed
     * at registration time via the SAME wait_deadline() helper and the SAME
     * DEFAULT_SYNC_TIMEOUT_MS constant commit_submit's blocking WAIT path
     * uses, so an async registration observes identical failure timing to
     * today's blocking path (Req 11.1). Only meaningful while lsn != 0;
     * overwritten on every fresh registration of a reused slot. Swept by the
     * commit engine's dedicated timeout-sweep thread (see commit.c). */
    struct timespec         deadline;
} commit_continuation_t;

/*
 * Commit engine state — in-memory only (commit state volatile, Raft-conformant).
 * Per-standby view delegated to standby_conn_t.acked_lsn / in_catch_up via
 * p->repl (no duplication).
 */
typedef struct commit_engine {
    pthread_mutex_t lock;
    pthread_cond_t  cond; /* broadcast on committed_lsn advance / membership change */
    persist_t*      p;    /* back-reference: committed_lsn store + repl registry */
    int             n_peers;
    int             n_vcpus;
    /* raft-drive-async task 1.6: when non-NULL, commit-wait admission is folded
     * onto the vtaskmgr pool's single LivenessAdmission reservation (the pool
     * owns the cap). When NULL (commit-only unit fixtures), the legacy local
     * cap max(1, n_vcpus - COMMIT_WAIT_RESERVED) is used. inflight_waits is
     * maintained in BOTH modes as the observable in-flight commit-wait count. */
    vtaskmgr_t*     pool;
    int             inflight_waits;        /* in-flight blocking commit waits */
    uint64_t        writable_lsn;          /* own-term gate; 0 = no gate */
    /* ha-term-safety commit-ceiling-optA: the raft library's commit_index,
     * recorded for OBSERVABILITY / TRACE / conformance-oracle ONLY. It does NOT
     * gate commit-advance: majority_candidate_locked no longer folds it into the
     * commit candidate. Commit-advance is authoritative via the durable-majority
     * barrier + the own-term writable_lsn gate. Kept (and still fed by
     * on_commit_advanced) so the library half keeps a sink and tooling can
     * compare the library current-term commit point vs committed_lsn. Monotonic
     * non-decreasing, set under ce->lock via commit_set_raft_commit_index. */
    uint64_t        raft_commit_index;
    uint64_t        scratch[MAX_STANDBYS]; /* sorted matchIndex scratch array */
    /* raft-apply-decoupling task 1.1: per-LSN continuation registry for
     * commit_submit_async (event-driven resolve, replacing the
     * COMMIT_MODE_WAIT blocking-park model). Sized to n_vcpus at init: the
     * SAME admission-cap discipline (wait_admit_locked/wait_cap, the shared
     * inflight_waits budget) bounds concurrent registrations to
     * wait_cap(ce) <= n_vcpus, so n_vcpus slots always suffice regardless of
     * the blocking/async call mix. Resolution (commit_resolve, invoked from
     * commit_advance_locked and the timeout/demotion/truncation failure
     * paths) is wired by a follow-up task; until then a registered slot is
     * inert -- nothing reads it back. */
    commit_continuation_t* continuations;
    int n_continuations; /* capacity, == n_vcpus */
    /* raft-apply-decoupling task 1.3: dedicated timeout-sweep thread, private
     * to this commit_engine_t (NOT the shared controller delaymgr/iomgr --
     * see commit.c file header for why). Sleeps on `cond` to the earliest
     * pending continuation deadline (or indefinitely when none are pending)
     * and resolves any still-registered continuation FAILED/-ETIMEDOUT once
     * its deadline passes. Started in commit_engine_init, stopped+joined in
     * commit_engine_fini before `lock`/`cond`/`continuations` are torn down.
     * sweep_stop is read/written only under `lock`, alongside the broadcast
     * that wakes the thread to observe it. */
    pthread_t sweep_thread;
    int       sweep_stop;
} commit_engine_t;

/* Lifecycle. init validates n_peers >= 0, n_vcpus >= 1, non-NULL persist_t;
 * returns -EINVAL otherwise. Call after n_peers and n_vcpus are final. */
int  commit_engine_init(commit_engine_t* ce, persist_t* p, int n_peers, int n_vcpus);
void commit_engine_fini(commit_engine_t* ce);

/* raft-drive-async task 1.6: fold commit-wait admission onto the vtaskmgr
 * pool's single LivenessAdmission reservation. After this, commit_submit(WAIT)
 * acquires a pool blocking-admission slot (cap = n_vcpus - VTASK_LIVENESS_RESERVED)
 * instead of using the local COMMIT_WAIT_RESERVED cap. pool == NULL restores
 * the local fallback. Call once at wiring time (cluster_init_phase2), after the
 * pool's liveness reservation is set. */
void commit_set_pool(commit_engine_t* ce, vtaskmgr_t* pool);

/* Register entry at append time. RETAIN returns 0 immediately.
 * WAIT blocks up to timeout_ms: 0 = committed; -ETIMEDOUT = not committed
 * (entry remains in log; outcome resolves per umbrella 2.5); -XCE_ERR_NOT_READY =
 * admission cap exhausted (only cause). An unmet own-term gate does NOT fail
 * fast: the gate is part of the completion predicate, so WAIT blocks within
 * timeout_ms until the gate opens, surfacing -ETIMEDOUT if it never does. */
int commit_submit(commit_engine_t* ce, uint64_t lsn, commit_mode_t mode,
                  uint32_t timeout_ms);

/* raft-apply-decoupling task 1.1: register a continuation for lsn instead of
 * parking the calling thread -- the event-driven equivalent of
 * commit_submit(..., COMMIT_MODE_WAIT, ...). Returns 0 on successful
 * registration; -EINVAL on bad args (ce/p NULL, lsn == 0, fn == NULL);
 * -XCE_ERR_NOT_READY when the admission cap is exhausted (the same shared
 * discipline commit_submit's WAIT mode uses -- see wait_admit_locked).
 * Never blocks: the calling thread returns immediately, before lsn's
 * outcome is known.
 *
 * raft-apply-decoupling task 1.3: registration also arms an implicit
 * DEFAULT_SYNC_TIMEOUT_MS deadline (identical constant and clock the
 * blocking WAIT path uses) -- if lsn has not resolved by then, the commit
 * engine's own timeout-sweep thread resolves it FAILED/-ETIMEDOUT with no
 * caller action required. A genuine majority ack that lands at or after the
 * deadline still wins: the sweep thread always checks the majority
 * predicate before the deadline in the same lock hold, mirroring
 * commit_submit's own last-look-before-timeout tie-break. There is no
 * per-call timeout parameter (unlike commit_submit) -- see design.md
 * §Commit Engine Resolve for the fixed signature. */
int commit_submit_async(commit_engine_t* ce, uint64_t lsn,
                        commit_continuation_fn fn, void* ctx);

/* raft-apply-decoupling task 1.2: resolve lsn's registered continuation (if
 * any) with the given outcome/rc, exactly once. Called internally on the
 * majority-commit path (commit_advance_locked and its callers, via the
 * wait_done_locked predicate -- mirrors COMMIT_MODE_WAIT's own completion
 * check) and is also the PUBLIC failure-path hook: a future caller detecting
 * timeout, demotion, or log-tail truncation for a pending lsn calls this
 * directly with COMMIT_OUTCOME_FAILED and the appropriate rc (the existing
 * error-code vocabulary -- e.g. -ETIMEDOUT, -XCE_ERR_STANDBY). Idempotent: a
 * second resolve() for an lsn with no registered (or already-resolved)
 * continuation is a silent no-op -- lsn == 0 is the free-slot sentinel, so a
 * cleared slot never matches again. Never invokes the continuation inline on
 * the calling thread: dispatch happens off-thread via vtask_enq_classified
 * (VTASK_LIVE_BLOCK -- never admission-gated, so delivery cannot fast-reject
 * the way VTASK_REQ_BLOCK could at the pool's blocking cap), falling back to
 * an inline call ONLY if dispatch is unavailable (OOM, or no vtaskmgr pool
 * wired -- commit-only unit fixtures), because a continuation must never be
 * silently left unresolved (Req 9.4; "controllers never self-terminate" —
 * fail loud and complete the work rather than drop it). Safe to call from any
 * thread that does NOT already hold ce->lock. */
void commit_resolve(commit_engine_t* ce, uint64_t lsn, commit_outcome_t outcome,
                    int rc);

/* iomgr path: record standby ACK, evaluate majority, advance committed_lsn
 * contiguously, wake satisfied waiters. Never blocks. */
void commit_on_ack(commit_engine_t* ce, int standby_idx, uint64_t acked_lsn);

/* Promotion: set own-term gate to the no-op entry's LSN.
 * (API ships here; no caller in this spec — unset = no gate.) */
void commit_set_writable_lsn(commit_engine_t* ce, uint64_t noop_lsn);

/* ha-term-safety commit-ceiling-optA: record the raft library's commit_index
 * for OBSERVABILITY / TRACE / conformance-oracle ONLY (monotonic max() — never
 * regresses). This does NOT gate commit-advance: the durable-majority barrier +
 * own-term writable gate are authoritative. Still fed by on_commit_advanced so
 * the library half keeps a sink. Re-evaluates the (commit_index-independent)
 * durable majority and wakes waiters as a convenient ACK-path wake point. Never
 * blocks. */
void commit_set_raft_commit_index(commit_engine_t* ce, uint64_t commit_index);

/* Quorum-membership change (standby add/remove/catch-up transition):
 * re-evaluate pending entries; standbys in catch-up are excluded. */
void commit_on_membership_change(commit_engine_t* ce);