fix(controller): admission-gate OPCode_schedAdmin's blocking wait (Req 5 AC 10)
The schedAdmin dispatch case blocks its vtask handler thread for up to 5s
waiting on a cross-process reply while holding c->sched.lock -- bounded
blocking on a vtask thread, which ha-async-io.md permits only under an
admission reservation. The real risk isn't just the wait itself: since
c->sched.lock serializes concurrent requests one at a time, a burst of N
concurrent schedAdmin calls could already occupy N vtask threads for up to
N*5s each before this fix.
Now acquires vtaskmgr_blocking_acquire(&c->vtaskmgr) before ever locking
c->sched.lock, releasing on every exit path -- the same shared admission
budget commit.c's commit-wait barrier already gates on (wired via
commit_set_pool). Admission-exhausted requests fast-reject with
{ok=false, error="server busy, try again"} rather than queuing. New test:
test_schedadmin_fast_rejects_when_admission_exhausted, using a poison
c->sched.ep sentinel to prove the fast-reject happens before any
lock/forward attempt.
This commit is contained in:
parent
a13068dca6
commit
e52c68f471
4 changed files with 103 additions and 3 deletions
|
|
@ -860,6 +860,38 @@ static int myreq_handler(vtask_t* task)
|
|||
break;
|
||||
}
|
||||
|
||||
/* ha-async-io.md Rule 3 (design review follow-up, 2026-08-19):
|
||||
* bounded blocking on a vtask thread is permitted only under an
|
||||
* admission reservation. The blocking region below isn't just the
|
||||
* pthread_cond_timedwait a few lines down -- mylock_lock(&c->sched.
|
||||
* lock) immediately after this comment is ALSO a bounded wait (up
|
||||
* to this same ~5s deadline, times however many requests are
|
||||
* already queued ahead of this one), since c->sched.lock is
|
||||
* deliberately held for the whole forward+wait duration (see that
|
||||
* design note below). A burst of concurrent xceadmin schedAdmin
|
||||
* calls could otherwise occupy N vtask threads for up to N*5s --
|
||||
* exactly the starvation pattern this reservation exists to
|
||||
* prevent. Reuses the SAME shared budget commit.c's wait_admit_
|
||||
* locked()/wait_release_locked() already gate on c->vtaskmgr (wired
|
||||
* via commit_set_pool(&c->commit, &c->vtaskmgr) below) -- the
|
||||
* reservation protects a fixed number of vcpus from ANY class of
|
||||
* admitted bounded-blocking work, not a per-caller quota, so
|
||||
* schedAdmin and commit-wait traffic intentionally share it. Fast-
|
||||
* reject (never touch c->sched.lock) rather than queue behind an
|
||||
* exhausted cap. */
|
||||
if (vtaskmgr_blocking_acquire(&(c->vtaskmgr)) != 0) {
|
||||
sched_admin_reply_t busy_reply;
|
||||
memset(&busy_reply, 0, sizeof(busy_reply));
|
||||
busy_reply.error = "server busy, try again";
|
||||
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_schedadminreplymsg;
|
||||
reply.data.schedadminreplymsg = &busy_reply;
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
rc = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
/* c->sched.lock is held for the FULL forward+wait duration below --
|
||||
* an accepted trade-off (task 10.1 design notes): xceadmin admin
|
||||
* commands are low-frequency, human-paced traffic, not a hot path.
|
||||
|
|
@ -874,6 +906,7 @@ static int myreq_handler(vtask_t* task)
|
|||
nc_reply.error = "scheduler not connected";
|
||||
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
vtaskmgr_blocking_release(&(c->vtaskmgr));
|
||||
|
||||
reply.status = 0;
|
||||
reply.data_which = RequestReply_data_schedadminreplymsg;
|
||||
|
|
@ -898,6 +931,7 @@ static int myreq_handler(vtask_t* task)
|
|||
if (rc < 0) {
|
||||
schedadmin_unregister(&pending);
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
vtaskmgr_blocking_release(&(c->vtaskmgr));
|
||||
|
||||
reply.status = rc;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
|
|
@ -982,6 +1016,7 @@ static int myreq_handler(vtask_t* task)
|
|||
}
|
||||
|
||||
mylock_unlock(&(c->sched.lock));
|
||||
vtaskmgr_blocking_release(&(c->vtaskmgr));
|
||||
rc = 0;
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -268,6 +268,67 @@ static void test_schedadmin_rejects_when_not_connected(void)
|
|||
printf("test_schedadmin_rejects_when_not_connected: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Observable 2b (design review follow-up, 2026-08-19): admission-
|
||||
* exhausted requests fast-reject instead of blocking a vtask thread ────────
|
||||
*
|
||||
* ha-async-io.md Rule 3: bounded blocking on a vtask thread requires an
|
||||
* admission reservation. c->vtaskmgr starts gate-DISABLED by a bare
|
||||
* sa_make_cluster() (liveness_reserved defaults to 0 on a calloc'd
|
||||
* vtaskmgr_t -- vtaskmgr_blocking_acquire's own "Gate disabled: admit
|
||||
* uncounted (legacy: nothing reserved)" branch, vtaskmgr.c), matching
|
||||
* production behavior before vtaskmgr_set_liveness_reserved is called
|
||||
* (cluster.c's real init path) -- this is why every OTHER test in this
|
||||
* file, which never calls that setter, is completely unaffected by this
|
||||
* feature. This test explicitly enables the gate with a 1-slot cap
|
||||
* (vtaskmgr_set_liveness_reserved(..., 1) floors cap at 1 regardless of
|
||||
* n_vcpus, vtaskmgr_blocking_cap's own doc comment) and pre-saturates that
|
||||
* one slot -- simulating a concurrent commit-wait or another in-flight
|
||||
* schedAdmin request already holding it -- before dispatching. The poison
|
||||
* c->sched.ep sentinel (same technique test_schedadmin_rejects_when_not_
|
||||
* primary uses above) proves the fast-reject happens BEFORE
|
||||
* mylock_lock(&c->sched.lock)/any forwarding attempt, not merely that the
|
||||
* eventual reply looks right. */
|
||||
static void test_schedadmin_fast_rejects_when_admission_exhausted(void)
|
||||
{
|
||||
cluster_t* c = sa_make_cluster();
|
||||
atomic_store(&c->ha.role, CTRL_PRIMARY);
|
||||
|
||||
endpoint_t* poison = (endpoint_t*)(uintptr_t)0x1;
|
||||
c->sched.ep = poison;
|
||||
|
||||
vtaskmgr_set_liveness_reserved(&c->vtaskmgr, 1);
|
||||
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_blocking_acquire(&c->vtaskmgr));
|
||||
|
||||
endpoint_t ep;
|
||||
memset(&ep, 0, sizeof(ep));
|
||||
ep.owner = c;
|
||||
|
||||
io_packet_t* pkt = sa_build_request_pkt();
|
||||
int rc = myreq_handler_for_test(&ep, pkt);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
c->sched.ep == poison,
|
||||
"c->sched.ep must be untouched -- admission-exhausted requests must "
|
||||
"fast-reject before ever locking c->sched.lock or forwarding");
|
||||
|
||||
xce_request_t decoded;
|
||||
sa_decode_queued_reply(&ep, &decoded);
|
||||
TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
|
||||
xce_reply_t* rr = decoded.body.replymsg;
|
||||
TEST_ASSERT_NOT_NULL(rr);
|
||||
TEST_ASSERT_EQUAL_INT(0, rr->status);
|
||||
TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
|
||||
TEST_ASSERT_NOT_NULL(rr->data.schedadminreplymsg);
|
||||
TEST_ASSERT_EQUAL_STRING("server busy, try again",
|
||||
rr->data.schedadminreplymsg->error);
|
||||
xce_free_request(&decoded);
|
||||
|
||||
vtaskmgr_blocking_release(&c->vtaskmgr); /* release our pre-saturating slot */
|
||||
sa_teardown(c);
|
||||
printf("test_schedadmin_fast_rejects_when_admission_exhausted: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Basic round trip: forward -> scheduler reply wakes the waiter ──────────
|
||||
*
|
||||
* c->sched.ep points at a real (if unconnected) endpoint_t so the dispatch
|
||||
|
|
@ -612,6 +673,7 @@ void test_sched_admin_dispatch(void)
|
|||
UnitySetTestFile(__FILE__);
|
||||
RUN_TEST(test_schedadmin_rejects_when_not_primary);
|
||||
RUN_TEST(test_schedadmin_rejects_when_not_connected);
|
||||
RUN_TEST(test_schedadmin_fast_rejects_when_admission_exhausted);
|
||||
RUN_TEST(test_schedadmin_round_trip_success);
|
||||
RUN_TEST(test_schedadmin_forward_timeout);
|
||||
RUN_TEST(test_schedadmin_round_trip_rpc_status_only);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue