lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(libxce-failover): classify and propagate netio I/O errors on failover

Browse source 
Add io_classify_io_error and propagate endpoint/epoll errors through
endpoint_update_epoll and cbuf_get instead of swallowing them, so client
failover surfaces the real error rather than hanging. Adds netio arm-failure,
close-propagation, and request send-propagation tests.

Pre-existing uncommitted work, committed to establish a clean baseline before
ha-commit-durability implementation.
This commit is contained in:
Rongsong Shen 2026-06-18 04:17:07 +00:00
parent 749e194207
commit 80bb1d766e
7 changed files with 347 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

111
src/lib/netio.c
View file

@ -69,6 +69,41 @@ int io_fsm_start(io_fsm_t* fsm, io_packet_t* pkt)
return 0;
}
/*
* io_classify_io_error: map a failed SSL_read/SSL_write (io_rc <= 0) to an FSM
* step result, given SSL_get_error()'s value and the errno captured right after
* the SSL call.
*
* returns 0 → retriable would-block (WANT_READ/WANT_WRITE, or an
* EAGAIN/EWOULDBLOCK syscall): the FSM stays put and retries on
* the next readiness event.
* returns <0 → the peer closed or reset the connection: the caller MUST move
* the FSM to FSM_STATE_ERROR and propagate this so a blocked
* reply-waiter is woken. A clean EOF reports errno==0, and the
* old code returned `-errno == 0` here — making the close look
* like success, so handle_input never reached handle_error and a
* blocked cbuf_get() hung forever (bug-648). We therefore return
* -ECONNRESET whenever errno is 0.
*
* Pure (no SSL object, no endpoint state) so it is unit-testable.
*/
int io_classify_io_error(int ssl_err, int ecode, int io_rc)
{
switch (ssl_err) {
case SSL_ERROR_WANT_READ:
case SSL_ERROR_WANT_WRITE:
return 0;
case SSL_ERROR_SYSCALL:
if ((io_rc < 0) && ((ecode == EAGAIN) || (ecode == EWOULDBLOCK)))
return 0;
return (ecode != 0) ? -ecode : -ECONNRESET;
default:
/* SSL_ERROR_ZERO_RETURN (clean TLS close_notify) and any other SSL
* error: the session is closed — never report success. */
return (ecode != 0) ? -ecode : -ECONNRESET;
}
}
static int io_fsm_do(io_fsm_t* fsm, char* data, size_t len)
{
int rc = 0;
@ -87,32 +122,15 @@ static int io_fsm_do(io_fsm_t* fsm, char* data, size_t len)
}
if (rc <= 0) {
int err = 0;
int ecode = 0;
int ecode = errno;
int err = SSL_get_error(ep->ssl, rc);
ecode = errno;
err = SSL_get_error(ep->ssl, rc);
switch (err) {
case SSL_ERROR_WANT_READ:
case SSL_ERROR_WANT_WRITE:
rc = 0;
break;
case SSL_ERROR_SYSCALL:
if ((rc < 0) && ((ecode == EAGAIN) || (ecode == EWOULDBLOCK))) {
rc = 0;
} else {
fsm->state = FSM_STATE_ERROR;
rc = -errno;
}
if (rc == 0) {
/* we meet EOF */
fsm->state = FSM_STATE_ERROR;
}
break;
default:
/* Classify the failure. A negative result means the connection is
* closed/errored — move to ERROR and propagate (never return 0 on a
* close, or a blocked reply-waiter is never woken: bug-648). */
rc = io_classify_io_error(err, ecode, rc);
if (rc < 0) {
fsm->state = FSM_STATE_ERROR;
rc = -errno;
break;
}
return rc;
@ -411,7 +429,13 @@ static int handle_output(endpoint_t* ep)
ep->fsm_out.pkt = NULL;
endpoint_update_epoll(ep);
/* Re-arm (drop EPOLLOUT now the queue is drained). A failure here means
* the fd is no longer registered — surface it so io_handler runs the
* error path rather than leaving a silently-dead endpoint (bug-646). */
rc = endpoint_update_epoll(ep);
if (rc < 0) {
return rc;
}
}
return 0;
@ -558,27 +582,41 @@ static int endpoint_calc_flags(endpoint_t* ep)
static int endpoint_update_epoll(endpoint_t* ep)
{
int flags;
int rc = 0;
mylock_lock(&(ep->lock));
flags = endpoint_calc_flags(ep);
if (flags != ep->flags) {
if (ep->iomgr != NULL) {
int rc1;
int saved_errno;
mylock_lock(&(ep->iomgr->lock));
rc1 = iomgr_update_poll(ep->iomgr, ep, flags);
rc1 = iomgr_update_poll(ep->iomgr, ep, flags);
saved_errno = errno;
mylock_unlock(&(ep->iomgr->lock));
if (rc1 >= 0) {
ep->flags = flags;
} else {
WARN("rc1=%d < 0", rc1);
/* Do NOT swallow this: an unarmed poll interest silently
* strands all I/O on the endpoint — outbound bytes never flush
* and inbound replies never wake a blocked cbuf_get(), so a
* synchronous caller hangs forever while io_worker sleeps in
* epoll_wait(). EPOLL_CTL_MOD fails with ENOENT when the fd is
* no longer registered (e.g. after a failed reconnect).
* Surface the failure so the caller tears the endpoint down and
* the failover loop rotates instead of hanging (bug-646). */
rc = (saved_errno != 0) ? -saved_errno : -EIO;
WARN("endpoint_update_epoll: iomgr_update_poll failed "
"(rc1=%d errno=%d) — endpoint marked dead",
rc1, saved_errno);
}
}
}
mylock_unlock(&(ep->lock));
return 0;
return rc;
}
static void outq_item_killer(data_item_t* it)
@ -807,7 +845,22 @@ int endpoint_enq(endpoint_t* ep, data_item_t* it)
rc = dataq_enq(&(ep->outq), it);
mylock_unlock(&(ep->lock));
endpoint_update_epoll(ep);
/* Arm the output interest. If the poll registration cannot be updated
* (e.g. EPOLL_CTL_MOD on an fd no longer registered after a failed
* reconnect), the enqueued bytes will never flush and the endpoint is dead.
* Surface a RETRIABLE transport error so the caller propagates it (the
* request.c senders return it; the client failover loop classifies
* -ECONNRESET as retriable and rotates — bug-650).
*
* Do NOT invoke handle_error()/the err_handler here. endpoint_enq() runs on
* the *caller's* thread, which may already hold a lock the err_handler needs
* — e.g. the controller's primary_heartbeat_fn holds ha->lock and
* vote_ep_err_handler re-acquires it, self-deadlocking the controller
* (bug-652). The real teardown stays on the io_worker thread, which drives
* handle_error() off the next epoll error event holding no caller locks. */
if (endpoint_update_epoll(ep) < 0) {
return -ECONNRESET;
}
return rc;
}