fix(controller): admission-gate OPCode_schedAdmin's blocking wait (Req 5 AC 10)

The schedAdmin dispatch case blocks its vtask handler thread for up to 5s
waiting on a cross-process reply while holding c->sched.lock -- bounded
blocking on a vtask thread, which ha-async-io.md permits only under an
admission reservation. The real risk isn't just the wait itself: since
c->sched.lock serializes concurrent requests one at a time, a burst of N
concurrent schedAdmin calls could already occupy N vtask threads for up to
N*5s each before this fix.

Now acquires vtaskmgr_blocking_acquire(&c->vtaskmgr) before ever locking
c->sched.lock, releasing on every exit path -- the same shared admission
budget commit.c's commit-wait barrier already gates on (wired via
commit_set_pool). Admission-exhausted requests fast-reject with
{ok=false, error="server busy, try again"} rather than queuing. New test:
test_schedadmin_fast_rejects_when_admission_exhausted, using a poison
c->sched.ep sentinel to prove the fast-reject happens before any
lock/forward attempt.
This commit is contained in:
Rongsong Shen 2026-08-19 11:00:49 +00:00
parent a13068dca6
commit e52c68f471
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
4 changed files with 103 additions and 3 deletions

View file

@ -860,6 +860,38 @@ static int myreq_handler(vtask_t* task)
break;
}
/* ha-async-io.md Rule 3 (design review follow-up, 2026-08-19):
* bounded blocking on a vtask thread is permitted only under an
* admission reservation. The blocking region below isn't just the
* pthread_cond_timedwait a few lines down -- mylock_lock(&c->sched.
* lock) immediately after this comment is ALSO a bounded wait (up
* to this same ~5s deadline, times however many requests are
* already queued ahead of this one), since c->sched.lock is
* deliberately held for the whole forward+wait duration (see that
* design note below). A burst of concurrent xceadmin schedAdmin
* calls could otherwise occupy N vtask threads for up to N*5s --
* exactly the starvation pattern this reservation exists to
* prevent. Reuses the SAME shared budget commit.c's wait_admit_
* locked()/wait_release_locked() already gate on c->vtaskmgr (wired
* via commit_set_pool(&c->commit, &c->vtaskmgr) below) -- the
* reservation protects a fixed number of vcpus from ANY class of
* admitted bounded-blocking work, not a per-caller quota, so
* schedAdmin and commit-wait traffic intentionally share it. Fast-
* reject (never touch c->sched.lock) rather than queue behind an
* exhausted cap. */
if (vtaskmgr_blocking_acquire(&(c->vtaskmgr)) != 0) {
sched_admin_reply_t busy_reply;
memset(&busy_reply, 0, sizeof(busy_reply));
busy_reply.error = "server busy, try again";
reply.status = 0;
reply.data_which = RequestReply_data_schedadminreplymsg;
reply.data.schedadminreplymsg = &busy_reply;
xce_reply(ep, c->myhost, &reply);
rc = 0;
break;
}
/* c->sched.lock is held for the FULL forward+wait duration below --
* an accepted trade-off (task 10.1 design notes): xceadmin admin
* commands are low-frequency, human-paced traffic, not a hot path.
@ -874,6 +906,7 @@ static int myreq_handler(vtask_t* task)
nc_reply.error = "scheduler not connected";
mylock_unlock(&(c->sched.lock));
vtaskmgr_blocking_release(&(c->vtaskmgr));
reply.status = 0;
reply.data_which = RequestReply_data_schedadminreplymsg;
@ -898,6 +931,7 @@ static int myreq_handler(vtask_t* task)
if (rc < 0) {
schedadmin_unregister(&pending);
mylock_unlock(&(c->sched.lock));
vtaskmgr_blocking_release(&(c->vtaskmgr));
reply.status = rc;
reply.data_which = RequestReply_data_nodata;
@ -982,6 +1016,7 @@ static int myreq_handler(vtask_t* task)
}
mylock_unlock(&(c->sched.lock));
vtaskmgr_blocking_release(&(c->vtaskmgr));
rc = 0;
break;
}

View file

@ -268,6 +268,67 @@ static void test_schedadmin_rejects_when_not_connected(void)
printf("test_schedadmin_rejects_when_not_connected: PASS\n");
}
/* ── Observable 2b (design review follow-up, 2026-08-19): admission-
* exhausted requests fast-reject instead of blocking a vtask thread ────────
*
* ha-async-io.md Rule 3: bounded blocking on a vtask thread requires an
* admission reservation. c->vtaskmgr starts gate-DISABLED by a bare
* sa_make_cluster() (liveness_reserved defaults to 0 on a calloc'd
* vtaskmgr_t -- vtaskmgr_blocking_acquire's own "Gate disabled: admit
* uncounted (legacy: nothing reserved)" branch, vtaskmgr.c), matching
* production behavior before vtaskmgr_set_liveness_reserved is called
* (cluster.c's real init path) -- this is why every OTHER test in this
* file, which never calls that setter, is completely unaffected by this
* feature. This test explicitly enables the gate with a 1-slot cap
* (vtaskmgr_set_liveness_reserved(..., 1) floors cap at 1 regardless of
* n_vcpus, vtaskmgr_blocking_cap's own doc comment) and pre-saturates that
* one slot -- simulating a concurrent commit-wait or another in-flight
* schedAdmin request already holding it -- before dispatching. The poison
* c->sched.ep sentinel (same technique test_schedadmin_rejects_when_not_
* primary uses above) proves the fast-reject happens BEFORE
* mylock_lock(&c->sched.lock)/any forwarding attempt, not merely that the
* eventual reply looks right. */
static void test_schedadmin_fast_rejects_when_admission_exhausted(void)
{
cluster_t* c = sa_make_cluster();
atomic_store(&c->ha.role, CTRL_PRIMARY);
endpoint_t* poison = (endpoint_t*)(uintptr_t)0x1;
c->sched.ep = poison;
vtaskmgr_set_liveness_reserved(&c->vtaskmgr, 1);
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&c->vtaskmgr));
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
ep.owner = c;
io_packet_t* pkt = sa_build_request_pkt();
int rc = myreq_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_TRUE_MESSAGE(
c->sched.ep == poison,
"c->sched.ep must be untouched -- admission-exhausted requests must "
"fast-reject before ever locking c->sched.lock or forwarding");
xce_request_t decoded;
sa_decode_queued_reply(&ep, &decoded);
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
xce_reply_t* rr = decoded.body.replymsg;
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_INT(0, rr->status);
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
TEST_ASSERT_NOT_NULL(rr->data.schedadminreplymsg);
TEST_ASSERT_EQUAL_STRING("server busy, try again",
rr->data.schedadminreplymsg->error);
xce_free_request(&decoded);
vtaskmgr_blocking_release(&c->vtaskmgr); /* release our pre-saturating slot */
sa_teardown(c);
printf("test_schedadmin_fast_rejects_when_admission_exhausted: PASS\n");
}
/* ── Basic round trip: forward -> scheduler reply wakes the waiter ──────────
*
* c->sched.ep points at a real (if unconnected) endpoint_t so the dispatch
@ -612,6 +673,7 @@ void test_sched_admin_dispatch(void)
UnitySetTestFile(__FILE__);
RUN_TEST(test_schedadmin_rejects_when_not_primary);
RUN_TEST(test_schedadmin_rejects_when_not_connected);
RUN_TEST(test_schedadmin_fast_rejects_when_admission_exhausted);
RUN_TEST(test_schedadmin_round_trip_success);
RUN_TEST(test_schedadmin_forward_timeout);
RUN_TEST(test_schedadmin_round_trip_rpc_status_only);