lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(raft-drive-async): LivenessAdmission pool reservation + commit fold (tasks 1.6/1.7)

Browse source 
vtaskmgr gains vtask_class_t (LIVENESS/LIVE_BLOCK/REQ_BLOCK) + a pool-level
blocking reservation: REQ_BLOCK fast-rejects -XCE_ERR_NOT_READY past
n_vcpus-R; LIVENESS never gated (deadlock-freedom). commit_submit folds its
inflight_waits onto the pool via commit_set_pool; production wired in
cluster.c phase2 (vtaskmgr_set_liveness_reserved(R) + commit_set_pool).
Lock order p->lock->ce->lock->blk_lock. Dormant until 3.1. Commit suites
38/30/22/38 green; lib 282/0, controller 196/0, ASAN clean. Phase 1 done.
This commit is contained in:
Rongsong Shen 2026-06-26 07:43:19 +00:00
parent 0781c6c027
commit a36234a103
12 changed files with 721 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/lib/meson.build
View file

@ -180,6 +180,7 @@ libxce_test_src = ['testsuite/test.c',
'testsuite/test_raft.c',
'testsuite/test_wal_wire.c',
'testsuite/test_ctrlstatus.c',
'testsuite/test_vtask_admission.c',
'raft/raft_sim.c',
]

2
src/lib/testsuite/test.c
View file

@ -43,6 +43,7 @@ extern void test_request_send_propagation(void);
extern void test_raft(void);
extern void test_wal_wire(void);
extern void test_ctrlstatus(void);
extern void test_vtask_admission(void);
void setUp(void) { return; }
@ -155,6 +156,7 @@ int main(int argc, char* argv[])
TEST(test_raft);
TEST(test_wal_wire);
TEST(test_ctrlstatus);
TEST(test_vtask_admission);
return UNITY_END();
}

221
src/lib/testsuite/test_vtask_admission.c Normal file
View file

@ -0,0 +1,221 @@
/*
* test_vtask_admission.c — raft-drive-async task 1.6/1.7: LivenessAdmission.
*
* Pool-level task-class reservation in vtaskmgr. Asserts:
* (a) under *_BLOCK saturation a VTASK_LIVENESS task still runs;
* (b) a VTASK_REQ_BLOCK over the cap returns -E_XCE_NOT_READY without
* blocking the caller (fast-reject, never parks a worker);
* (c) blocking-admission acquire/release (the seam commit.c folds onto)
* caps concurrent blocking at max(1, n_vcpus - reserved);
* (d) removing the reservation (reserved=0) lets liveness starve behind
* blockers (mutation intuition — the guarantee depends on the gate).
*
* Deterministic harness: blocking tasks park on a semaphore-equivalent
* (cbuf) gate that the test controls; a liveness task posts an atomic
* counter the test polls. No correctness-by-sleep on the assertion path.
*
* Requirements: 3.1, 3.2, 3.3, 4.5.
*/
#include <pthread.h>
#include <stdatomic.h>
#include <stdint.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "mylock.h"
#include "vtaskmgr.h"
#include "xce/errnox.h"
/* ── Deterministic gating primitives ───────────────────────────────────── */
/* A blocking task parks until the test releases it. We use a cbuf as a
* counting gate: the task does a blocking cbuf_get (parks holding its vcpu);
* the test cbuf_puts one token per task it wants to release. */
typedef struct {
cbuf_t* gate; /* park here (blocking get) */
atomic_int* started; /* ++ when the task body begins (occupies vcpu) */
atomic_int* finished; /* ++ when the task body returns */
} block_arg_t;
static int block_task_entry(void* p)
{
vtask_t* t = (vtask_t*)p;
block_arg_t* arg = (block_arg_t*)t->data->u.generic;
int tok = 0;
atomic_fetch_add(arg->started, 1);
cbuf_get(arg->gate, &tok); /* park, holding this vcpu, until released */
atomic_fetch_add(arg->finished, 1);
return 0;
}
typedef struct {
atomic_int* ran;
} live_arg_t;
static int live_task_entry(void* p)
{
vtask_t* t = (vtask_t*)p;
live_arg_t* arg = (live_arg_t*)t->data->u.generic;
atomic_fetch_add(arg->ran, 1);
return 0;
}
static vtask_t* make_task(vtask_func_t entry, void* generic)
{
vtask_data_t* d = calloc(1, sizeof(*d));
vtask_t* t = vtask_alloc();
d->vtype = VTASK_GENERIC;
d->u.generic = generic;
vtask_init(t, entry, d);
return t;
}
/* Spin-wait (bounded) for an atomic to reach >= want; returns 1 on success. */
static int wait_atomic_ge(atomic_int* a, int want, int max_ms)
{
for (int i = 0; i < max_ms * 10; i++) {
if (atomic_load(a) >= want) {
return 1;
}
usleep(100); /* 0.1ms poll — liveness, not a correctness sleep */
}
return atomic_load(a) >= want;
}
/* ── Test (a)+(b): liveness runs under REQ_BLOCK saturation; over-cap
* REQ_BLOCK fast-rejects ───────────────────────────────────────────── */
static void test_liveness_runs_under_block_saturation(void)
{
const int NVCPUS = 4;
const int RESERVED = 2; /* cap = n_vcpus - reserved = 2 */
const int CAP = NVCPUS - RESERVED;
vtaskmgr_t mgr;
cbuf_t gate;
atomic_int started = 0;
atomic_int finished = 0;
atomic_int live_ran = 0;
cbuf_init(&gate, NVCPUS + 4, sizeof(int));
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, NVCPUS));
vtaskmgr_set_liveness_reserved(&mgr, RESERVED);
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&mgr));
/* Saturate the REQ_BLOCK cap: CAP blocking tasks, all parked on the gate. */
block_arg_t barg = { .gate = &gate, .started = &started, .finished = &finished };
for (int i = 0; i < CAP; i++) {
vtask_t* t = make_task(block_task_entry, &barg);
TEST_ASSERT_EQUAL_INT(
0, vtask_enq_classified(&mgr, t, VTASK_REQ_BLOCK));
}
/* All CAP blocking tasks must reach their body (occupy a vcpu) and park. */
TEST_ASSERT_TRUE_MESSAGE(wait_atomic_ge(&started, CAP, 2000),
"blocking tasks did not all start");
/* (b) One more REQ_BLOCK is over the cap → fast-reject, NEVER parks. */
vtask_t* over = make_task(block_task_entry, &barg);
int rc = vtask_enq_classified(&mgr, over, VTASK_REQ_BLOCK);
TEST_ASSERT_EQUAL_INT_MESSAGE(-E_XCE_NOT_READY, rc,
"REQ_BLOCK over cap must fast-reject with -E_XCE_NOT_READY");
/* Rejected task was NOT enqueued — caller owns it. Free it. */
vtask_fini(over);
free(over);
/* (a) A LIVENESS task dispatched while *_BLOCK holds all CAP slots MUST
* still run — reserved headroom guarantees a free vcpu. */
live_arg_t larg = { .ran = &live_ran };
vtask_t* lt = make_task(live_task_entry, &larg);
TEST_ASSERT_EQUAL_INT(0, vtask_enq_classified(&mgr, lt, VTASK_LIVENESS));
TEST_ASSERT_TRUE_MESSAGE(wait_atomic_ge(&live_ran, 1, 2000),
"LIVENESS task starved while *_BLOCK saturated the cap");
/* Release all parked blockers so shutdown is clean. */
for (int i = 0; i < CAP; i++) {
int tok = 1;
cbuf_put(&gate, &tok);
}
TEST_ASSERT_TRUE(wait_atomic_ge(&finished, CAP, 2000));
vtaskmgr_fini(&mgr);
cbuf_fini(&gate);
}
/* ── Test (c): blocking acquire/release seam caps concurrency at the cap and
* fast-rejects past it (this is the seam commit.c folds onto). ───────── */
static void test_blocking_acquire_release_cap(void)
{
const int NVCPUS = 5;
const int RESERVED = 4; /* cap = 1 (n_peers=5 → R=4) */
vtaskmgr_t mgr;
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, NVCPUS));
vtaskmgr_set_liveness_reserved(&mgr, RESERVED);
TEST_ASSERT_EQUAL_INT(1, vtaskmgr_blocking_cap(&mgr)); /* 5 - 4 = 1 */
/* First acquire admitted; second over the cap of 1 fast-rejects. */
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&mgr));
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, vtaskmgr_blocking_acquire(&mgr));
/* Release frees the slot; next acquire admitted again. */
vtaskmgr_blocking_release(&mgr);
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&mgr));
vtaskmgr_blocking_release(&mgr);
vtaskmgr_fini(&mgr);
}
/* Cap is floored at 1 even when reserved >= n_vcpus (degenerate small-core). */
static void test_blocking_cap_floored_at_one(void)
{
vtaskmgr_t mgr;
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, 2));
vtaskmgr_set_liveness_reserved(&mgr, 5); /* reserved > n_vcpus */
TEST_ASSERT_EQUAL_INT(1, vtaskmgr_blocking_cap(&mgr));
/* One acquire always admittable (commit can still run). */
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&mgr));
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, vtaskmgr_blocking_acquire(&mgr));
vtaskmgr_blocking_release(&mgr);
vtaskmgr_fini(&mgr);
}
/* ── Test (d): MUTATION intuition — with reserved=0 the gate is disabled,
* so blocking is uncapped and a liveness task can be starved behind a
* full house of parked blockers. We assert the gate is OFF (acquire never
* rejects) which is exactly the unsafe pre-reservation behavior. ─────── */
static void test_reservation_disabled_does_not_gate(void)
{
vtaskmgr_t mgr;
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, 4));
vtaskmgr_set_liveness_reserved(&mgr, 0); /* gate disabled */
/* With the gate disabled, acquire NEVER rejects — blocking is uncapped,
* so nothing reserves a worker for liveness. This is the failure mode the
* reservation closes (mutation oracle): flip reserved back to >0 and the
* cap re-engages (covered by the tests above). */
for (int i = 0; i < 100; i++) {
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&mgr));
}
for (int i = 0; i < 100; i++) {
vtaskmgr_blocking_release(&mgr);
}
vtaskmgr_fini(&mgr);
}
void test_vtask_admission(void)
{
RUN_TEST(test_liveness_runs_under_block_saturation);
RUN_TEST(test_blocking_acquire_release_cap);
RUN_TEST(test_blocking_cap_floored_at_one);
RUN_TEST(test_reservation_disabled_does_not_gate);
}

127
src/lib/vtaskmgr.c
View file

@ -3,6 +3,8 @@
#include <stdlib.h>
#include <unistd.h>
#include "xce/errnox.h"
#include "mylock.h"
#include "netio.h"
#include "utils.h"
@ -24,6 +26,7 @@ int vtask_init(vtask_t* task, vtask_func_t entry, void* data)
task->vcpu = -1;
task->data = data;
task->owner = NULL;
task->cls = VTASK_LIVENESS; /* never-gated by default; legacy enq paths */
return 0;
}
@ -68,6 +71,14 @@ void run_task(vtask_t* task)
}
vtaskmgr = (vtaskmgr_t*)task->owner;
/* LivenessAdmission (task 1.6): a VTASK_REQ_BLOCK task held a
* blocking-admission slot from enqueue (vtask_enq_classified) through its
* run; release it now so the next REQ_BLOCK can be admitted. */
if (vtaskmgr != NULL && task->cls == VTASK_REQ_BLOCK) {
vtaskmgr_blocking_release(vtaskmgr);
}
if (vtaskmgr != NULL) {
/* wake up dispatcher */
vtaskmgr_wakeup(vtaskmgr);
@ -168,10 +179,14 @@ int vtaskmgr_init(vtaskmgr_t* vtaskmgr, int nvcpus)
return -EINVAL;
}
vtaskmgr->dispatcher = 0;
vtaskmgr->n_vcpus = nvcpus;
vtaskmgr->vcpus = (vcpu_t*)calloc(nvcpus, sizeof(vcpu_t));
vtaskmgr->dispatcher = 0;
vtaskmgr->n_vcpus = nvcpus;
vtaskmgr->liveness_reserved = 0; /* gate disabled until wired (task 1.6) */
vtaskmgr->blocking_inflight = 0;
pthread_mutex_init(&vtaskmgr->blk_lock, NULL);
vtaskmgr->vcpus = (vcpu_t*)calloc(nvcpus, sizeof(vcpu_t));
if (vtaskmgr->vcpus == NULL) {
pthread_mutex_destroy(&vtaskmgr->blk_lock);
return -ENOMEM;
}
for (i = 0; i < nvcpus; i++) {
@ -335,10 +350,116 @@ void vtaskmgr_fini(vtaskmgr_t* vtaskmgr)
dataq_fini(&(vtaskmgr->taskq));
cbuf_fini(&(vtaskmgr->wtbuf));
cbuf_fini(&(vtaskmgr->wbuf));
pthread_mutex_destroy(&vtaskmgr->blk_lock);
return;
}
/* ============================================================
* LivenessAdmission (raft-drive-async task 1.6)
* ============================================================ */
void vtaskmgr_set_liveness_reserved(vtaskmgr_t* vtaskmgr, int reserved)
{
if (vtaskmgr == NULL) {
return;
}
if (reserved < 0) {
reserved = 0;
}
pthread_mutex_lock(&vtaskmgr->blk_lock);
vtaskmgr->liveness_reserved = reserved;
pthread_mutex_unlock(&vtaskmgr->blk_lock);
}
int vtaskmgr_blocking_cap(const vtaskmgr_t* vtaskmgr)
{
int cap;
if (vtaskmgr == NULL) {
return 0;
}
if (vtaskmgr->liveness_reserved <= 0) {
/* Gate disabled: blocking is uncapped (legacy behavior). */
return vtaskmgr->n_vcpus;
}
cap = vtaskmgr->n_vcpus - vtaskmgr->liveness_reserved;
/* Floor at 1 so a commit (REQ_BLOCK) can always run, even on a degenerate
* small-core box where the reservation would otherwise drive the cap to 0
* (design Sizing note). */
return (cap < 1) ? 1 : cap;
}
int vtaskmgr_blocking_acquire(vtaskmgr_t* vtaskmgr)
{
int rc = 0;
if (vtaskmgr == NULL) {
return -EINVAL;
}
pthread_mutex_lock(&vtaskmgr->blk_lock);
if (vtaskmgr->liveness_reserved <= 0) {
/* Gate disabled: admit uncounted (legacy: nothing reserved). */
pthread_mutex_unlock(&vtaskmgr->blk_lock);
return 0;
}
int cap = vtaskmgr->n_vcpus - vtaskmgr->liveness_reserved;
if (cap < 1) {
cap = 1;
}
if (vtaskmgr->blocking_inflight >= cap) {
/* Reservation would be breached — fast reject (never parks). */
rc = -E_XCE_NOT_READY;
} else {
vtaskmgr->blocking_inflight++;
}
pthread_mutex_unlock(&vtaskmgr->blk_lock);
return rc;
}
void vtaskmgr_blocking_release(vtaskmgr_t* vtaskmgr)
{
if (vtaskmgr == NULL) {
return;
}
pthread_mutex_lock(&vtaskmgr->blk_lock);
if (vtaskmgr->liveness_reserved > 0 && vtaskmgr->blocking_inflight > 0) {
vtaskmgr->blocking_inflight--;
}
pthread_mutex_unlock(&vtaskmgr->blk_lock);
}
int vtask_enq_classified(vtaskmgr_t* vtaskmgr, vtask_t* task, vtask_class_t cls)
{
int rc;
if (vtaskmgr == NULL || task == NULL) {
return -EINVAL;
}
task->cls = cls;
if (cls == VTASK_REQ_BLOCK) {
/* Admission-gated: reserve a blocking slot BEFORE enqueue. On reject
* the task is NOT enqueued — the caller still owns it. The slot is
* released in run_task when this task completes. */
rc = vtaskmgr_blocking_acquire(vtaskmgr);
if (rc != 0) {
return rc;
}
rc = vtask_enq(vtaskmgr, task);
if (rc < 0) {
/* Enqueue failed after acquiring — give the slot back. */
vtaskmgr_blocking_release(vtaskmgr);
}
return rc;
}
/* VTASK_LIVENESS / VTASK_LIVE_BLOCK are never admission-gated. */
return vtask_enq(vtaskmgr, task);
}
int vtask_enq(vtaskmgr_t* vtaskmgr, vtask_t* task)
{
int rc = 0;