feat(raft-drive-async): add drive-async config constants + liveness-reservation helper (task 1.1)

drive_async_liveness_reserved = max(1, min(n_peers-1, n_vcpus-1)) — single
source of truth for VTASK_LIVENESS_RESERVED (deadlock-freedom floor applied
last). New DEFAULT_CONNECT_TIMEOUT_MS / DEFAULT_SLOWLANE_* constants. Resolved
reservation logged at init; under-provisioned (n_vcpus<n_peers) clamp WARNs
loudly, never self-terminates. 8 unit cases, mutation-verified.
This commit is contained in:
Rongsong Shen 2026-06-26 04:58:07 +00:00
parent 295769d8ae
commit 729055a37f
8 changed files with 329 additions and 0 deletions

View file

@ -1342,6 +1342,29 @@ int cluster_init_phase2(cluster_t* c)
c->ha.repl = &c->persist.repl;
INFO("commit engine wired (n_peers=%d, n_vcpus=%d)",
c->ha.n_peers, c->vtaskmgr.n_vcpus);
/* raft-drive-async task 1.1: resolve and log the peer-scaled liveness
* reservation at init — both inputs (n_peers, n_vcpus) are final here.
* Single source of truth: task 1.6 (LivenessAdmission) consumes this
* value, it does NOT recompute the clamp/WARN. */
int liveness_reserved =
drive_async_liveness_reserved(c->ha.n_peers, c->vtaskmgr.n_vcpus);
INFO("drive-async: VTASK_LIVENESS_RESERVED=%d (n_peers=%d, n_vcpus=%d; "
"commit/REQ_BLOCK cap=%d)",
liveness_reserved, c->ha.n_peers, c->vtaskmgr.n_vcpus,
c->vtaskmgr.n_vcpus - liveness_reserved);
if (c->vtaskmgr.n_vcpus < c->ha.n_peers) {
/* Under-provisioned: the n_vcpus ceiling clamped the reservation
* below the ideal n_peers-1. Still correct (deadlock-free via fast
* worker cycling), but operator-visible per CLAUDE.md fail-loud.
* A controller NEVER self-terminates on this condition. */
WARN("drive-async: under-provisioned controller — n_vcpus=%d < "
"n_peers=%d; liveness reservation clamped to %d (ideal %d). "
"Per-peer liveness latency may rise under load; consider "
"raising --vcpus. Cluster remains correct and stays up.",
c->vtaskmgr.n_vcpus, c->ha.n_peers, liveness_reserved,
c->ha.n_peers - 1);
}
}
/* Role set by ha_elect_evaluate_and_signal() inside cluster_run() */

View file

@ -580,6 +580,24 @@ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
}
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
*
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
* applied LAST so deadlock-freedom (>=1 guaranteed-free liveness worker) holds
* even when the n_vcpus ceiling would otherwise drive the value to 0 (e.g.
* n_vcpus == 1). See ha.h for the full rationale; the caller WARNs on the
* under-provisioned (n_vcpus < n_peers) clamp — this helper never logs. */
int drive_async_liveness_reserved(int n_peers, int n_vcpus)
{
int desired = n_peers - 1; /* worst-case concurrent liveness work */
int ceiling = n_vcpus - 1; /* keep >=1 commit (REQ_BLOCK) slot */
int reserved = desired < ceiling ? desired : ceiling; /* min(desired, ceiling) */
if (reserved < 1) /* floor applied LAST — always wins */
reserved = 1;
return reserved;
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
*
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot

View file

@ -618,6 +618,34 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
* producing divergent WALs and a permanent prevote deadlock. */
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip);
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
*
* drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED —
* the number of vtask-pool workers kept free for liveness-critical handling
* (heartbeat/vote/ack) so they are never starved by blocking offloaded tasks.
*
* reserved = clamp(n_peers - 1, 1, n_vcpus - 1)
* = max(1, min(n_peers - 1, n_vcpus - 1))
*
* - n_peers is the TOTAL cluster size including self (matches ha->n_peers).
* Per-endpoint serialization (endpoint_block_tasks) caps in-flight work at
* one per peer connection, so worst-case concurrent liveness work is the
* n_peers-1 OTHER peers (one ack/vote each) — reserving n_peers-1 means no
* peer's liveness message queues behind a busy reserved worker (a LATENCY
* optimization).
* - Floor 1 is the hard CORRECTNESS invariant (deadlock-freedom): one
* guaranteed-free worker + liveness-never-gated + sub-ms cycling is
* deadlock-free regardless of peer count. Applied LAST so it always wins.
* - Ceiling n_vcpus-1 keeps >=1 REQ_BLOCK (commit) slot, so the commit cap
* (n_vcpus - reserved) is never 0.
*
* On an under-provisioned box (n_vcpus < n_peers) the ceiling clamps the
* result below n_peers-1; the result is still correct via fast cycling.
* The CALLER (controller init) is responsible for emitting a loud WARN in
* that case so the under-provisioning is operator-visible — this helper is
* pure and never logs. Controllers NEVER self-terminate on this condition. */
int drive_async_liveness_reserved(int n_peers, int n_vcpus);
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ──────────
*
* ha_promotion_gate_decision: PURE pre-promotion eligibility decision (Req 2.2,

View file

@ -139,6 +139,7 @@ controller_unit_test_src = [
'testsuite/test_ha_heartbeat_committed.c',
'testsuite/test_ha_vote_floor_heartbeat.c',
'testsuite/test_term_safety_adapter.c',
'testsuite/test_drive_async_config.c',
'testsuite/test_waldump.c',
'testsuite/waldump.c',
'testsuite/test_walcompare.c',

View file

@ -66,6 +66,7 @@ extern void test_ha_vote_floor_heartbeat(void);
extern void test_term_safety_adapter(void);
extern void test_waldump(void);
extern void test_walcompare(void);
extern void test_drive_async_config(void);
void setUp(void) { return; }
@ -201,6 +202,7 @@ int main(int argc, char* argv[])
TEST(test_term_safety_adapter);
TEST(test_waldump);
TEST(test_walcompare);
TEST(test_drive_async_config);
return UNITY_END();
}

View file

@ -0,0 +1,97 @@
#define _GNU_SOURCE
#include <stdio.h>
#include <unity/unity.h>
#include "config.h"
#include "ha.h"
/* ====================================================================
* raft-drive-async task 1.1 — drive-async configuration constants and
* the peer-scaled liveness-reservation clamp helper.
*
* Helper under test:
* int drive_async_liveness_reserved(int n_peers, int n_vcpus);
*
* Formula (design "LivenessAdmission → Sizing (peer-scaled)"):
* VTASK_LIVENESS_RESERVED = clamp(n_peers - 1, 1, n_vcpus - 1)
* = max(1, min(n_peers - 1, n_vcpus - 1))
* floor 1 -> deadlock-freedom (always >=1 guaranteed-free worker)
* ceiling n_vcpus - 1 -> keep >=1 REQ_BLOCK (commit) slot, cap never 0
* The floor is applied LAST so it always wins (the hard correctness invariant).
* ==================================================================== */
/* ── Regime 1: normal n_peers-1 (the un-clamped middle) ───────────── */
static void test_liveness_reserved_normal_three_peers(void)
{
/* 3-node cluster, 8 vcpus: n_peers-1 = 2, well under ceiling 7. */
TEST_ASSERT_EQUAL_INT(2, drive_async_liveness_reserved(3, 8));
}
static void test_liveness_reserved_normal_five_peers(void)
{
/* 5-node cluster, 8 vcpus: n_peers-1 = 4, under ceiling 7. */
TEST_ASSERT_EQUAL_INT(4, drive_async_liveness_reserved(5, 8));
}
/* ── Regime 2: floor hit (n_peers-1 would be < 1) ─────────────────── */
static void test_liveness_reserved_floor_single_node(void)
{
/* n_peers=1 -> n_peers-1 = 0 -> floor 1 must win. */
TEST_ASSERT_EQUAL_INT(1, drive_async_liveness_reserved(1, 8));
}
static void test_liveness_reserved_floor_zero_peers(void)
{
/* Degenerate n_peers=0 -> n_peers-1 = -1 -> floor 1 must win. */
TEST_ASSERT_EQUAL_INT(1, drive_async_liveness_reserved(0, 8));
}
/* ── Regime 3: ceiling / clamp hit (under-provisioned: n_vcpus < n_peers) ── */
static void test_liveness_reserved_ceiling_clamps_below_npeers(void)
{
/* n_peers=5, n_vcpus=3: n_peers-1 = 4 clamped to ceiling n_vcpus-1 = 2. */
TEST_ASSERT_EQUAL_INT(2, drive_async_liveness_reserved(5, 3));
}
static void test_liveness_reserved_ceiling_floor_wins_one_vcpu(void)
{
/* n_vcpus=1 -> ceiling n_vcpus-1 = 0, but the floor of 1 must win:
* max(1, min(n_peers-1, 0)) = max(1, 0) = 1. Never returns 0. */
TEST_ASSERT_EQUAL_INT(1, drive_async_liveness_reserved(5, 1));
}
static void test_liveness_reserved_exactly_npeers_vcpus(void)
{
/* n_vcpus == n_peers: ceiling = n_vcpus-1 = n_peers-1 -> no clamp.
* 3 peers, 3 vcpus -> min(2, 2) = 2. */
TEST_ASSERT_EQUAL_INT(2, drive_async_liveness_reserved(3, 3));
}
static void test_liveness_reserved_never_exceeds_vcpus_minus_one(void)
{
/* Invariant: result <= n_vcpus-1 keeps >=1 commit slot (cap never 0). */
for (int p = 0; p <= 8; p++) {
for (int v = 1; v <= 8; v++) {
int r = drive_async_liveness_reserved(p, v);
TEST_ASSERT_TRUE_MESSAGE(r >= 1, "reservation floor 1 violated");
TEST_ASSERT_TRUE_MESSAGE(r <= (v - 1 > 0 ? v - 1 : 1),
"reservation must keep >=1 commit slot");
}
}
}
void test_drive_async_config(void)
{
RUN_TEST(test_liveness_reserved_normal_three_peers);
RUN_TEST(test_liveness_reserved_normal_five_peers);
RUN_TEST(test_liveness_reserved_floor_single_node);
RUN_TEST(test_liveness_reserved_floor_zero_peers);
RUN_TEST(test_liveness_reserved_ceiling_clamps_below_npeers);
RUN_TEST(test_liveness_reserved_ceiling_floor_wins_one_vcpu);
RUN_TEST(test_liveness_reserved_exactly_npeers_vcpus);
RUN_TEST(test_liveness_reserved_never_exceeds_vcpus_minus_one);
}