lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(raft-drive-async): LivenessAdmission pool reservation + commit fold (tasks 1.6/1.7)

Browse source 
vtaskmgr gains vtask_class_t (LIVENESS/LIVE_BLOCK/REQ_BLOCK) + a pool-level
blocking reservation: REQ_BLOCK fast-rejects -XCE_ERR_NOT_READY past
n_vcpus-R; LIVENESS never gated (deadlock-freedom). commit_submit folds its
inflight_waits onto the pool via commit_set_pool; production wired in
cluster.c phase2 (vtaskmgr_set_liveness_reserved(R) + commit_set_pool).
Lock order p->lock->ce->lock->blk_lock. Dormant until 3.1. Commit suites
38/30/22/38 green; lib 282/0, controller 196/0, ASAN clean. Phase 1 done.
This commit is contained in:
Rongsong Shen 2026-06-26 07:43:19 +00:00
parent 0781c6c027
commit a36234a103
12 changed files with 721 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/controller/cluster.c
View file

@ -1353,6 +1353,15 @@ int cluster_init_phase2(cluster_t* c)
"commit/REQ_BLOCK cap=%d)",
liveness_reserved, c->ha.n_peers, c->vtaskmgr.n_vcpus,
c->vtaskmgr.n_vcpus - liveness_reserved);
/* raft-drive-async task 1.6: install the pool-level LivenessAdmission
* reservation and fold commit-wait admission onto it. The pool now owns
* the single reservation; commit_submit(WAIT) acquires a pool blocking
* slot (cap = n_vcpus - liveness_reserved) instead of the local
* COMMIT_WAIT_RESERVED cap. Dormant beyond the commit cap-value change
* until peer_ctrl dispatches classified tasks (task 3.1). */
vtaskmgr_set_liveness_reserved(&c->vtaskmgr, liveness_reserved);
commit_set_pool(&c->commit, &c->vtaskmgr);
if (c->vtaskmgr.n_vcpus < c->ha.n_peers) {
/* Under-provisioned: the n_vcpus ceiling clamped the reservation
* below the ideal n_peers-1. Still correct (deadlock-free via fast

80
src/controller/commit.c
View file

@ -191,10 +191,11 @@ static void commit_advance_locked(commit_engine_t* ce)
pthread_cond_broadcast(&ce->cond);
}
/* Admission cap for concurrent blocking waits, expressed only relative to
* the n_vcpus init argument: keep COMMIT_WAIT_RESERVED vtasks free of
* blocking commit waits, floored at 1 so one wait is always admittable
* (Req 7.1, 7.2). */
/* Admission cap for concurrent blocking waits, LOCAL (no-pool) fallback:
* keep COMMIT_WAIT_RESERVED vtasks free of blocking commit waits, floored at 1
* so one wait is always admittable (Req 7.1, 7.2). Used only when no vtaskmgr
* pool is wired (commit_set_pool); when a pool is wired the pool owns the
* single reservation (cap = n_vcpus - VTASK_LIVENESS_RESERVED). */
static int wait_cap(const commit_engine_t* ce)
{
int cap = ce->n_vcpus - COMMIT_WAIT_RESERVED;
@ -202,6 +203,40 @@ static int wait_cap(const commit_engine_t* ce)
return (cap < 1) ? 1 : cap;
}
/* raft-drive-async task 1.6: acquire a blocking-commit-wait admission slot.
* When a pool is wired, delegate to the pool's single LivenessAdmission
* reservation (the pool owns the cap); otherwise apply the local cap. Returns
* 0 (admitted) or -XCE_ERR_NOT_READY (cap exhausted — fast reject, never
* blocks). Caller holds ce->lock. inflight_waits is maintained in BOTH modes
* as the observable in-flight commit-wait count. */
static int wait_admit_locked(commit_engine_t* ce)
{
if (ce->pool != NULL) {
int rc = vtaskmgr_blocking_acquire(ce->pool);
if (rc != 0) {
return rc; /* -E_XCE_NOT_READY == -XCE_ERR_NOT_READY */
}
ce->inflight_waits++;
return 0;
}
/* Local fallback: cap = max(1, n_vcpus - COMMIT_WAIT_RESERVED). */
if (ce->inflight_waits >= wait_cap(ce)) {
return -XCE_ERR_NOT_READY;
}
ce->inflight_waits++;
return 0;
}
/* Release a blocking-commit-wait admission slot. Caller holds ce->lock. */
static void wait_release_locked(commit_engine_t* ce)
{
ce->inflight_waits--;
if (ce->pool != NULL) {
vtaskmgr_blocking_release(ce->pool);
}
}
/* WAIT completion predicate (Req 1.1): own entry committed AND the
* own-term gate (when set) reached. Caller holds ce->lock. */
static int wait_done_locked(const commit_engine_t* ce, uint64_t lsn)
@ -270,6 +305,7 @@ int commit_engine_init(commit_engine_t* ce, persist_t* p, int n_peers, int n_vcp
ce->p = p;
ce->n_peers = n_peers;
ce->n_vcpus = n_vcpus;
ce->pool = NULL; /* local cap until commit_set_pool wires the pool */
ce->inflight_waits = 0;
ce->writable_lsn = 0; /* no own-term gate until set by promotion */
ce->raft_commit_index = 0; /* advisory record only; does not gate commit (optA) */
@ -283,6 +319,20 @@ int commit_engine_init(commit_engine_t* ce, persist_t* p, int n_peers, int n_vcp
return 0;
}
/* raft-drive-async task 1.6: fold commit-wait admission onto the vtaskmgr
* pool's single LivenessAdmission reservation. Set-once at wiring time; safe
* to set NULL to restore the local fallback. The pool must outlive the engine
* (controller lifecycle guarantees this — vtaskmgr_fini after commit_engine_fini). */
void commit_set_pool(commit_engine_t* ce, vtaskmgr_t* pool)
{
if (ce == NULL) {
return;
}
pthread_mutex_lock(&ce->lock);
ce->pool = pool;
pthread_mutex_unlock(&ce->lock);
}
void commit_engine_fini(commit_engine_t* ce)
{
if (ce == NULL) {
@ -311,7 +361,7 @@ int commit_submit(commit_engine_t* ce, uint64_t lsn, commit_mode_t mode,
struct timespec dl;
uint64_t cur, gate;
int rc = 0;
int cap, wrc;
int wrc, arc;
if (ce == NULL || ce->p == NULL || lsn == 0) {
return -EINVAL;
@ -333,19 +383,19 @@ int commit_submit(commit_engine_t* ce, uint64_t lsn, commit_mode_t mode,
/* Admission check: cap exhaustion is the ONLY -XCE_ERR_NOT_READY
* cause (Req 7.1, 7.2; CORRECTED contract). Retriable fast-reject —
* never blocks. */
cap = wait_cap(ce);
if (ce->inflight_waits >= cap) {
* never blocks. When a vtaskmgr pool is wired (raft-drive-async task 1.6)
* the cap is the pool's single LivenessAdmission reservation; otherwise
* the local COMMIT_WAIT_RESERVED cap. */
arc = wait_admit_locked(ce);
if (arc != 0) {
pthread_mutex_unlock(&ce->lock);
WARN("commit: WAIT rejected, admission cap exhausted "
"(cap=%d n_vcpus=%d lsn=%lu)",
cap, ce->n_vcpus, lsn);
return -XCE_ERR_NOT_READY;
"(pool=%s n_vcpus=%d lsn=%lu)",
ce->pool ? "pool" : "local", ce->n_vcpus, lsn);
return arc;
}
ce->inflight_waits++;
#ifdef XCE_TESTING
assert(ce->inflight_waits > 0 && ce->inflight_waits <= cap);
assert(ce->inflight_waits > 0);
#endif
/* Standalone / single-member: self constitutes the quorum (Req 6.3).
@ -367,7 +417,7 @@ int commit_submit(commit_engine_t* ce, uint64_t lsn, commit_mode_t mode,
}
}
ce->inflight_waits--;
wait_release_locked(ce);
#ifdef XCE_TESTING
assert(ce->inflight_waits >= 0);
#endif

23
src/controller/include/commit.h
View file

@ -12,8 +12,13 @@
#include <stdint.h>
#include "persist.h" /* persist_t (owns committed_lsn atomic + replicator_t registry) */
#include "vtaskmgr.h" /* vtaskmgr_t — LivenessAdmission blocking-admission seam */
/* Headroom kept free of blocking commit waits; cap = max(1, n_vcpus - reserved) */
/* Headroom kept free of blocking commit waits; cap = max(1, n_vcpus - reserved).
* Used ONLY by the local (no-pool) admission fallback. When a vtaskmgr pool is
* wired (commit_set_pool, raft-drive-async task 1.6), the pool owns the single
* reservation (cap = max(1, n_vcpus - VTASK_LIVENESS_RESERVED)) and this
* constant is not consulted. */
#define COMMIT_WAIT_RESERVED 2
typedef enum { COMMIT_MODE_WAIT, COMMIT_MODE_RETAIN } commit_mode_t;
@ -29,7 +34,13 @@ typedef struct commit_engine {
persist_t* p; /* back-reference: committed_lsn store + repl registry */
int n_peers;
int n_vcpus;
int inflight_waits; /* <= max(1, n_vcpus - COMMIT_WAIT_RESERVED) */
/* raft-drive-async task 1.6: when non-NULL, commit-wait admission is folded
* onto the vtaskmgr pool's single LivenessAdmission reservation (the pool
* owns the cap). When NULL (commit-only unit fixtures), the legacy local
* cap max(1, n_vcpus - COMMIT_WAIT_RESERVED) is used. inflight_waits is
* maintained in BOTH modes as the observable in-flight commit-wait count. */
vtaskmgr_t* pool;
int inflight_waits; /* in-flight blocking commit waits */
uint64_t writable_lsn; /* own-term gate; 0 = no gate */
/* ha-term-safety commit-ceiling-optA: the raft library's commit_index,
* recorded for OBSERVABILITY / TRACE / conformance-oracle ONLY. It does NOT
@ -48,6 +59,14 @@ typedef struct commit_engine {
int commit_engine_init(commit_engine_t* ce, persist_t* p, int n_peers, int n_vcpus);
void commit_engine_fini(commit_engine_t* ce);
/* raft-drive-async task 1.6: fold commit-wait admission onto the vtaskmgr
* pool's single LivenessAdmission reservation. After this, commit_submit(WAIT)
* acquires a pool blocking-admission slot (cap = n_vcpus - VTASK_LIVENESS_RESERVED)
* instead of using the local COMMIT_WAIT_RESERVED cap. pool == NULL restores
* the local fallback. Call once at wiring time (cluster_init_phase2), after the
* pool's liveness reservation is set. */
void commit_set_pool(commit_engine_t* ce, vtaskmgr_t* pool);
/* Register entry at append time. RETAIN returns 0 immediately.
* WAIT blocks up to timeout_ms: 0 = committed; -ETIMEDOUT = not committed
* (entry remains in log; outcome resolves per umbrella 2.5); -XCE_ERR_NOT_READY =

1
src/controller/meson.build
View file

@ -142,6 +142,7 @@ controller_unit_test_src = [
'testsuite/test_term_safety_adapter.c',
'testsuite/test_drive_async_config.c',
'testsuite/test_slowlane.c',
'testsuite/test_commit_admission_pool.c',
'testsuite/test_waldump.c',
'testsuite/waldump.c',
'testsuite/test_walcompare.c',

2
src/controller/testsuite/controller_test.c
View file

@ -68,6 +68,7 @@ extern void test_waldump(void);
extern void test_walcompare(void);
extern void test_drive_async_config(void);
extern void test_slowlane(void);
extern void test_commit_admission_pool(void);
void setUp(void) { return; }
@ -205,6 +206,7 @@ int main(int argc, char* argv[])
TEST(test_walcompare);
TEST(test_drive_async_config);
TEST(test_slowlane);
TEST(test_commit_admission_pool);
return UNITY_END();
}

217
src/controller/testsuite/test_commit_admission_pool.c Normal file
View file

@ -0,0 +1,217 @@
/*
* test_commit_admission_pool.c — raft-drive-async task 1.6/1.7.
*
* The commit-wait admission cap is FOLDED onto the vtaskmgr pool's single
* LivenessAdmission reservation. Asserts:
*
* (1) with a pool wired and R = VTASK_LIVENESS_RESERVED set, commit_submit
* (WAIT) fast-rejects with -XCE_ERR_NOT_READY at the POOL cap
* (n_vcpus - R), not the old local COMMIT_WAIT_RESERVED cap. This is the
* intended cap-value change: n_peers=5 → R=4 → cap=1 (TIGHTER than the
* old n_vcpus-2=6).
* (2) commit-wait still fast-rejects (never blocks) at the cap — backpressure
* preserved.
* (3) releasing a pool slot (a waiter completing) re-admits the next wait.
* (4) no-pool fallback still uses the local COMMIT_WAIT_RESERVED cap
* (regression guard for the existing commit fixtures).
*
* Deterministic: a background thread parks in commit_submit(WAIT) on an LSN
* that cannot commit (no acks, n_peers>1) with a long timeout, holding the
* admission slot; the test polls ce->inflight_waits to confirm saturation
* before probing the next admit. No correctness-by-sleep on assertions.
*
* Requirements: 3.1, 3.2, 3.3, 4.5.
*/
#include <pthread.h>
#include <stdatomic.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "xce/errnox.h"
#include "commit.h"
#include "repl.h"
#include "vtaskmgr.h"
static persist_t g_p;
static commit_engine_t g_ce;
static vtaskmgr_t g_mgr;
static int g_eps[MAX_STANDBYS]; /* opaque repl_ep_t tokens */
static void pool_setup(int n_peers, int n_standbys, int n_vcpus, int reserved)
{
char name[16];
memset(&g_p, 0, sizeof(g_p));
TEST_ASSERT_EQUAL_INT(0, repl_init(&g_p.repl));
for (int i = 0; i < n_standbys; i++) {
snprintf(name, sizeof(name), "sb%d", i);
TEST_ASSERT_EQUAL_INT(
i, repl_add_standby(&g_p.repl, &g_eps[i], name, 7000 + (uint32_t)i));
}
/* Entry 1 appended + durable, but no acks → cannot commit (n_peers>1). */
g_p.current_lsn = 2;
atomic_store(&g_p.committed_lsn, 0);
atomic_store(&g_p.durable_lsn, 1);
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&g_mgr, n_vcpus));
vtaskmgr_set_liveness_reserved(&g_mgr, reserved);
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&g_ce, &g_p, n_peers, n_vcpus));
if (reserved >= 0) {
commit_set_pool(&g_ce, &g_mgr);
}
}
static void pool_teardown(void)
{
commit_engine_fini(&g_ce);
vtaskmgr_fini(&g_mgr);
repl_fini(&g_p.repl);
}
/* Background waiter: parks in commit_submit(WAIT) holding a slot. */
typedef struct {
uint64_t lsn;
uint32_t timeout_ms;
int rc;
} waiter_t;
static void* waiter_fn(void* a)
{
waiter_t* w = (waiter_t*)a;
w->rc = commit_submit(&g_ce, w->lsn, COMMIT_MODE_WAIT, w->timeout_ms);
return NULL;
}
static uint64_t mono_ms(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000 + ts.tv_nsec / 1000000;
}
static int poll_inflight(int want, uint32_t deadline_ms)
{
uint64_t t0 = mono_ms();
int v;
for (;;) {
pthread_mutex_lock(&g_ce.lock);
v = g_ce.inflight_waits;
pthread_mutex_unlock(&g_ce.lock);
if (v >= want || mono_ms() - t0 >= deadline_ms) {
return v;
}
usleep(500);
}
}
/* (1)+(2): the fold uses the POOL cap (n_vcpus - R), which differs from the
* old local COMMIT_WAIT_RESERVED cap. n_vcpus=4, R=3 → POOL cap = 4-3 = 1
* (TIGHTER than the old local n_vcpus-2 = 2). One parked waiter saturates the
* pool slot; the next commit_submit(WAIT) fast-rejects without blocking.
* (MAX_STANDBYS = 3 bounds the cluster, so a small cap is exercised here; the
* cap-value formula is what matters, not the absolute count.) */
static void test_commit_fold_uses_pool_cap(void)
{
const int N_VCPUS = 4;
const int RESERVED = 3; /* pool cap = 4 - 3 = 1 */
const int POOL_CAP = N_VCPUS - RESERVED; /* 1 */
const int OLD_LOCAL_CAP = N_VCPUS - COMMIT_WAIT_RESERVED; /* 2 — what the
old local cap would have allowed */
pthread_t tids[1];
waiter_t wargs[1];
int i;
/* Sanity: the pool cap is genuinely TIGHTER than the old local cap — this
* is the intended behavior change the fold introduces. */
TEST_ASSERT_TRUE(POOL_CAP < OLD_LOCAL_CAP);
pool_setup(/*n_peers=*/3, /*n_standbys=*/2, N_VCPUS, RESERVED);
TEST_ASSERT_EQUAL_INT_MESSAGE(POOL_CAP, vtaskmgr_blocking_cap(&g_mgr),
"fold must use the pool cap n_vcpus - VTASK_LIVENESS_RESERVED");
/* Saturate the pool cap with POOL_CAP parked waiters. */
for (i = 0; i < POOL_CAP; i++) {
wargs[i].lsn = 1;
wargs[i].timeout_ms = 4000; /* long — held until we release via ack */
wargs[i].rc = -9999;
TEST_ASSERT_EQUAL_INT(0,
pthread_create(&tids[i], NULL, waiter_fn, &wargs[i]));
}
TEST_ASSERT_EQUAL_INT_MESSAGE(POOL_CAP, poll_inflight(POOL_CAP, 3000),
"all POOL_CAP waiters must occupy admission slots");
/* The over-cap wait fast-rejects WITHOUT blocking (backpressure). Under the
* OLD local cap (2) this second wait would have been ADMITTED — proving the
* fold now governs admission. */
uint64_t t0 = mono_ms();
int rc = commit_submit(&g_ce, /*lsn=*/1, COMMIT_MODE_WAIT, /*to=*/4000);
uint64_t dt = mono_ms() - t0;
TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc,
"over-pool-cap commit-wait must fast-reject with -XCE_ERR_NOT_READY");
TEST_ASSERT_TRUE_MESSAGE(dt < 500,
"admission rejection must not block the caller");
/* Release the parked waiter: ack entry 1 from a majority so it commits.
* n_peers=3 → self + 1 standby = 2 > 3/2. */
repl_on_ack(&g_p.repl, &g_eps[0], 1);
commit_on_ack(&g_ce, 0, 1);
for (i = 0; i < POOL_CAP; i++) {
pthread_join(tids[i], NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, wargs[i].rc,
"admitted waiter must complete with success after majority ack");
}
/* (3) Slot released → a fresh wait is admitted again (commits immediately
* since entry 1 is now committed). */
rc = commit_submit(&g_ce, /*lsn=*/1, COMMIT_MODE_WAIT, /*to=*/1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"after the slot is released a fresh wait must be admitted (and commit)");
pool_teardown();
printf("test_commit_fold_uses_pool_cap: PASS\n");
}
/* (4) No pool wired → local COMMIT_WAIT_RESERVED cap still governs. n_vcpus=2,
* COMMIT_WAIT_RESERVED=2 → local cap = max(1, 0) = 1. One parked waiter
* saturates; the second fast-rejects. Regression guard for existing fixtures. */
static void test_commit_local_fallback_cap_preserved(void)
{
pthread_t tid;
waiter_t warg;
/* reserved = -1 sentinel → pool_setup skips commit_set_pool (no pool). */
pool_setup(/*n_peers=*/3, /*n_standbys=*/2, /*n_vcpus=*/2, /*reserved=*/-1);
/* Confirm no pool is wired: the engine uses the local cap path. */
warg.lsn = 1;
warg.timeout_ms = 4000;
warg.rc = -9999;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&tid, NULL, waiter_fn, &warg));
TEST_ASSERT_EQUAL_INT_MESSAGE(1, poll_inflight(1, 3000),
"local-cap waiter must occupy the only slot (cap=1)");
int rc = commit_submit(&g_ce, /*lsn=*/1, COMMIT_MODE_WAIT, /*to=*/4000);
TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_NOT_READY, rc,
"over-local-cap commit-wait must fast-reject (cap = n_vcpus-RESERVED)");
/* Release: majority ack (n_peers=3 → self + 1 standby = 2 > 3/2). */
repl_on_ack(&g_p.repl, &g_eps[0], 1);
commit_on_ack(&g_ce, 0, 1);
pthread_join(tid, NULL);
TEST_ASSERT_EQUAL_INT(0, warg.rc);
pool_teardown();
printf("test_commit_local_fallback_cap_preserved: PASS\n");
}
void test_commit_admission_pool(void)
{
RUN_TEST(test_commit_fold_uses_pool_cap);
RUN_TEST(test_commit_local_fallback_cap_preserved);
}