feat(raft-drive-async): add drive-async config constants + liveness-reservation helper (task 1.1)
drive_async_liveness_reserved = max(1, min(n_peers-1, n_vcpus-1)) — single source of truth for VTASK_LIVENESS_RESERVED (deadlock-freedom floor applied last). New DEFAULT_CONNECT_TIMEOUT_MS / DEFAULT_SLOWLANE_* constants. Resolved reservation logged at init; under-provisioned (n_vcpus<n_peers) clamp WARNs loudly, never self-terminates. 8 unit cases, mutation-verified.
This commit is contained in:
parent
295769d8ae
commit
729055a37f
8 changed files with 329 additions and 0 deletions
|
|
@ -1342,6 +1342,29 @@ int cluster_init_phase2(cluster_t* c)
|
|||
c->ha.repl = &c->persist.repl;
|
||||
INFO("commit engine wired (n_peers=%d, n_vcpus=%d)",
|
||||
c->ha.n_peers, c->vtaskmgr.n_vcpus);
|
||||
|
||||
/* raft-drive-async task 1.1: resolve and log the peer-scaled liveness
|
||||
* reservation at init — both inputs (n_peers, n_vcpus) are final here.
|
||||
* Single source of truth: task 1.6 (LivenessAdmission) consumes this
|
||||
* value, it does NOT recompute the clamp/WARN. */
|
||||
int liveness_reserved =
|
||||
drive_async_liveness_reserved(c->ha.n_peers, c->vtaskmgr.n_vcpus);
|
||||
INFO("drive-async: VTASK_LIVENESS_RESERVED=%d (n_peers=%d, n_vcpus=%d; "
|
||||
"commit/REQ_BLOCK cap=%d)",
|
||||
liveness_reserved, c->ha.n_peers, c->vtaskmgr.n_vcpus,
|
||||
c->vtaskmgr.n_vcpus - liveness_reserved);
|
||||
if (c->vtaskmgr.n_vcpus < c->ha.n_peers) {
|
||||
/* Under-provisioned: the n_vcpus ceiling clamped the reservation
|
||||
* below the ideal n_peers-1. Still correct (deadlock-free via fast
|
||||
* worker cycling), but operator-visible per CLAUDE.md fail-loud.
|
||||
* A controller NEVER self-terminates on this condition. */
|
||||
WARN("drive-async: under-provisioned controller — n_vcpus=%d < "
|
||||
"n_peers=%d; liveness reservation clamped to %d (ideal %d). "
|
||||
"Per-peer liveness latency may rise under load; consider "
|
||||
"raising --vcpus. Cluster remains correct and stays up.",
|
||||
c->vtaskmgr.n_vcpus, c->ha.n_peers, liveness_reserved,
|
||||
c->ha.n_peers - 1);
|
||||
}
|
||||
}
|
||||
|
||||
/* Role set by ha_elect_evaluate_and_signal() inside cluster_run() */
|
||||
|
|
|
|||
|
|
@ -580,6 +580,24 @@ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
|
|||
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
|
||||
}
|
||||
|
||||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||||
*
|
||||
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
|
||||
* applied LAST so deadlock-freedom (>=1 guaranteed-free liveness worker) holds
|
||||
* even when the n_vcpus ceiling would otherwise drive the value to 0 (e.g.
|
||||
* n_vcpus == 1). See ha.h for the full rationale; the caller WARNs on the
|
||||
* under-provisioned (n_vcpus < n_peers) clamp — this helper never logs. */
|
||||
int drive_async_liveness_reserved(int n_peers, int n_vcpus)
|
||||
{
|
||||
int desired = n_peers - 1; /* worst-case concurrent liveness work */
|
||||
int ceiling = n_vcpus - 1; /* keep >=1 commit (REQ_BLOCK) slot */
|
||||
|
||||
int reserved = desired < ceiling ? desired : ceiling; /* min(desired, ceiling) */
|
||||
if (reserved < 1) /* floor applied LAST — always wins */
|
||||
reserved = 1;
|
||||
return reserved;
|
||||
}
|
||||
|
||||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
|
||||
*
|
||||
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
|
||||
|
|
|
|||
|
|
@ -618,6 +618,34 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
|||
* producing divergent WALs and a permanent prevote deadlock. */
|
||||
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip);
|
||||
|
||||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||||
*
|
||||
* drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED —
|
||||
* the number of vtask-pool workers kept free for liveness-critical handling
|
||||
* (heartbeat/vote/ack) so they are never starved by blocking offloaded tasks.
|
||||
*
|
||||
* reserved = clamp(n_peers - 1, 1, n_vcpus - 1)
|
||||
* = max(1, min(n_peers - 1, n_vcpus - 1))
|
||||
*
|
||||
* - n_peers is the TOTAL cluster size including self (matches ha->n_peers).
|
||||
* Per-endpoint serialization (endpoint_block_tasks) caps in-flight work at
|
||||
* one per peer connection, so worst-case concurrent liveness work is the
|
||||
* n_peers-1 OTHER peers (one ack/vote each) — reserving n_peers-1 means no
|
||||
* peer's liveness message queues behind a busy reserved worker (a LATENCY
|
||||
* optimization).
|
||||
* - Floor 1 is the hard CORRECTNESS invariant (deadlock-freedom): one
|
||||
* guaranteed-free worker + liveness-never-gated + sub-ms cycling is
|
||||
* deadlock-free regardless of peer count. Applied LAST so it always wins.
|
||||
* - Ceiling n_vcpus-1 keeps >=1 REQ_BLOCK (commit) slot, so the commit cap
|
||||
* (n_vcpus - reserved) is never 0.
|
||||
*
|
||||
* On an under-provisioned box (n_vcpus < n_peers) the ceiling clamps the
|
||||
* result below n_peers-1; the result is still correct via fast cycling.
|
||||
* The CALLER (controller init) is responsible for emitting a loud WARN in
|
||||
* that case so the under-provisioning is operator-visible — this helper is
|
||||
* pure and never logs. Controllers NEVER self-terminate on this condition. */
|
||||
int drive_async_liveness_reserved(int n_peers, int n_vcpus);
|
||||
|
||||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ──────────
|
||||
*
|
||||
* ha_promotion_gate_decision: PURE pre-promotion eligibility decision (Req 2.2,
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ controller_unit_test_src = [
|
|||
'testsuite/test_ha_heartbeat_committed.c',
|
||||
'testsuite/test_ha_vote_floor_heartbeat.c',
|
||||
'testsuite/test_term_safety_adapter.c',
|
||||
'testsuite/test_drive_async_config.c',
|
||||
'testsuite/test_waldump.c',
|
||||
'testsuite/waldump.c',
|
||||
'testsuite/test_walcompare.c',
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ extern void test_ha_vote_floor_heartbeat(void);
|
|||
extern void test_term_safety_adapter(void);
|
||||
extern void test_waldump(void);
|
||||
extern void test_walcompare(void);
|
||||
extern void test_drive_async_config(void);
|
||||
|
||||
void setUp(void) { return; }
|
||||
|
||||
|
|
@ -201,6 +202,7 @@ int main(int argc, char* argv[])
|
|||
TEST(test_term_safety_adapter);
|
||||
TEST(test_waldump);
|
||||
TEST(test_walcompare);
|
||||
TEST(test_drive_async_config);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
|
|
|
|||
97
src/controller/testsuite/test_drive_async_config.c
Normal file
97
src/controller/testsuite/test_drive_async_config.c
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
#define _GNU_SOURCE
|
||||
#include <stdio.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "ha.h"
|
||||
|
||||
/* ====================================================================
|
||||
* raft-drive-async task 1.1 — drive-async configuration constants and
|
||||
* the peer-scaled liveness-reservation clamp helper.
|
||||
*
|
||||
* Helper under test:
|
||||
* int drive_async_liveness_reserved(int n_peers, int n_vcpus);
|
||||
*
|
||||
* Formula (design "LivenessAdmission → Sizing (peer-scaled)"):
|
||||
* VTASK_LIVENESS_RESERVED = clamp(n_peers - 1, 1, n_vcpus - 1)
|
||||
* = max(1, min(n_peers - 1, n_vcpus - 1))
|
||||
* floor 1 -> deadlock-freedom (always >=1 guaranteed-free worker)
|
||||
* ceiling n_vcpus - 1 -> keep >=1 REQ_BLOCK (commit) slot, cap never 0
|
||||
* The floor is applied LAST so it always wins (the hard correctness invariant).
|
||||
* ==================================================================== */
|
||||
|
||||
/* ── Regime 1: normal n_peers-1 (the un-clamped middle) ───────────── */
|
||||
|
||||
static void test_liveness_reserved_normal_three_peers(void)
|
||||
{
|
||||
/* 3-node cluster, 8 vcpus: n_peers-1 = 2, well under ceiling 7. */
|
||||
TEST_ASSERT_EQUAL_INT(2, drive_async_liveness_reserved(3, 8));
|
||||
}
|
||||
|
||||
static void test_liveness_reserved_normal_five_peers(void)
|
||||
{
|
||||
/* 5-node cluster, 8 vcpus: n_peers-1 = 4, under ceiling 7. */
|
||||
TEST_ASSERT_EQUAL_INT(4, drive_async_liveness_reserved(5, 8));
|
||||
}
|
||||
|
||||
/* ── Regime 2: floor hit (n_peers-1 would be < 1) ─────────────────── */
|
||||
|
||||
static void test_liveness_reserved_floor_single_node(void)
|
||||
{
|
||||
/* n_peers=1 -> n_peers-1 = 0 -> floor 1 must win. */
|
||||
TEST_ASSERT_EQUAL_INT(1, drive_async_liveness_reserved(1, 8));
|
||||
}
|
||||
|
||||
static void test_liveness_reserved_floor_zero_peers(void)
|
||||
{
|
||||
/* Degenerate n_peers=0 -> n_peers-1 = -1 -> floor 1 must win. */
|
||||
TEST_ASSERT_EQUAL_INT(1, drive_async_liveness_reserved(0, 8));
|
||||
}
|
||||
|
||||
/* ── Regime 3: ceiling / clamp hit (under-provisioned: n_vcpus < n_peers) ── */
|
||||
|
||||
static void test_liveness_reserved_ceiling_clamps_below_npeers(void)
|
||||
{
|
||||
/* n_peers=5, n_vcpus=3: n_peers-1 = 4 clamped to ceiling n_vcpus-1 = 2. */
|
||||
TEST_ASSERT_EQUAL_INT(2, drive_async_liveness_reserved(5, 3));
|
||||
}
|
||||
|
||||
static void test_liveness_reserved_ceiling_floor_wins_one_vcpu(void)
|
||||
{
|
||||
/* n_vcpus=1 -> ceiling n_vcpus-1 = 0, but the floor of 1 must win:
|
||||
* max(1, min(n_peers-1, 0)) = max(1, 0) = 1. Never returns 0. */
|
||||
TEST_ASSERT_EQUAL_INT(1, drive_async_liveness_reserved(5, 1));
|
||||
}
|
||||
|
||||
static void test_liveness_reserved_exactly_npeers_vcpus(void)
|
||||
{
|
||||
/* n_vcpus == n_peers: ceiling = n_vcpus-1 = n_peers-1 -> no clamp.
|
||||
* 3 peers, 3 vcpus -> min(2, 2) = 2. */
|
||||
TEST_ASSERT_EQUAL_INT(2, drive_async_liveness_reserved(3, 3));
|
||||
}
|
||||
|
||||
static void test_liveness_reserved_never_exceeds_vcpus_minus_one(void)
|
||||
{
|
||||
/* Invariant: result <= n_vcpus-1 keeps >=1 commit slot (cap never 0). */
|
||||
for (int p = 0; p <= 8; p++) {
|
||||
for (int v = 1; v <= 8; v++) {
|
||||
int r = drive_async_liveness_reserved(p, v);
|
||||
TEST_ASSERT_TRUE_MESSAGE(r >= 1, "reservation floor 1 violated");
|
||||
TEST_ASSERT_TRUE_MESSAGE(r <= (v - 1 > 0 ? v - 1 : 1),
|
||||
"reservation must keep >=1 commit slot");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void test_drive_async_config(void)
|
||||
{
|
||||
RUN_TEST(test_liveness_reserved_normal_three_peers);
|
||||
RUN_TEST(test_liveness_reserved_normal_five_peers);
|
||||
RUN_TEST(test_liveness_reserved_floor_single_node);
|
||||
RUN_TEST(test_liveness_reserved_floor_zero_peers);
|
||||
RUN_TEST(test_liveness_reserved_ceiling_clamps_below_npeers);
|
||||
RUN_TEST(test_liveness_reserved_ceiling_floor_wins_one_vcpu);
|
||||
RUN_TEST(test_liveness_reserved_exactly_npeers_vcpus);
|
||||
RUN_TEST(test_liveness_reserved_never_exceeds_vcpus_minus_one);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue