fix(libxce-failover): classify and propagate netio I/O errors on failover

Add io_classify_io_error and propagate endpoint/epoll errors through
endpoint_update_epoll and cbuf_get instead of swallowing them, so client
failover surfaces the real error rather than hanging. Adds netio arm-failure,
close-propagation, and request send-propagation tests.

Pre-existing uncommitted work, committed to establish a clean baseline before
ha-commit-durability implementation.

src/lib/testsuite/test.c

@@ -36,6 +36,9 @@ extern void test_cluster(void);
 extern void test_cluster_parse(void);
 extern void test_libxce_retry(void);
 extern void test_netio_on_connected(void);
+extern void test_netio_arm_failure(void);
+extern void test_netio_close_propagation(void);
+extern void test_request_send_propagation(void);
 extern void test_raft(void);
 extern void test_wal_wire(void);
 
@@ -143,6 +146,9 @@ int main(int argc, char* argv[])
     TEST(test_cluster_parse);
     TEST(test_libxce_retry);
     TEST(test_netio_on_connected);
+    TEST(test_netio_arm_failure);
+    TEST(test_netio_close_propagation);
+    TEST(test_request_send_propagation);
     TEST(test_raft);
     TEST(test_wal_wire);
 

src/lib/testsuite/test_netio_arm_failure.c

@@ -0,0 +1,95 @@
+/*
+ * test_netio_arm_failure.c — libxce-failover: a poll-arm failure must fail LOUD.
+ *
+ * Root cause (bug-646): endpoint_update_epoll() swallowed iomgr_update_poll()
+ * (EPOLL_CTL_MOD) failures — it logged "rc1=-1 < 0" but returned 0 and left the
+ * poll interest unarmed.  On the synchronous client send path (endpoint_enq),
+ * a failed arm meant the request bytes were never flushed, no reply ever
+ * arrived, and the caller blocked forever in cbuf_get() while a perfectly
+ * healthy io_worker slept in epoll_wait().  (EPOLL_CTL_MOD fails with ENOENT
+ * when the fd is no longer registered — e.g. after a failed reconnect.)
+ *
+ * Contract under test: when the poll interest cannot be armed, endpoint_enq()
+ * must (1) return a negative, RETRIABLE error so the caller propagates it
+ * (request.c senders return it; the client failover loop rotates — bug-650),
+ * and (2) must NOT synchronously invoke the endpoint's err_handler.  endpoint_enq
+ * runs on the caller's thread, which may already hold a lock the err_handler
+ * needs (the controller's primary_heartbeat_fn holds ha->lock and
+ * vote_ep_err_handler re-acquires it → self-deadlock, bug-652).  The real
+ * teardown stays on the io_worker thread, off the next epoll error event.
+ */
+
+#define _GNU_SOURCE
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+#include "netio.h"
+
+/* ── err_handler invocation tracker ──────────────────────────────── */
+
+static int g_err_count = 0;
+
+static int err_handler_stub(void *ep)
+{
+    (void)ep;
+    g_err_count++;
+    return 0;
+}
+
+/*
+ * Arming EPOLL_CTL_MOD on an fd that was never EPOLL_CTL_ADD'd fails with
+ * ENOENT — the exact post-failed-reconnect condition.  endpoint_enq() must
+ * surface it (negative rc) WITHOUT calling the err_handler (no re-entrancy).
+ */
+static void test_enq_arm_failure_is_loud_no_errhandler(void)
+{
+    iomgr_t iomgr;
+    memset(&iomgr, 0, sizeof(iomgr));
+    TEST_ASSERT_EQUAL_INT(0, iomgr_init(&iomgr));
+
+    endpoint_t *ep = endpoint_get();
+    TEST_ASSERT_NOT_NULL(ep);
+
+    /* A real socket fd that is NOT registered in iomgr's epoll: EPOLL_CTL_MOD
+     * on it fails with ENOENT, mirroring a send after a failed reconnect. */
+    ep->fd = socket(AF_INET, SOCK_STREAM, 0);
+    TEST_ASSERT_TRUE(ep->fd >= 0);
+    ep->iomgr              = &iomgr;
+    ep->ssl_handshake_done = 1; /* past handshake: calc_flags arms EPOLLIN     */
+    ep->flags              = 0; /* force flags != calc_flags → iomgr_update_poll */
+    endpoint_set_err_handler(ep, (err_handler_t)err_handler_stub);
+
+    g_err_count = 0;
+
+    /* One packet on the outq so there is real outbound work to strand. The
+     * endpoint owns it after enq; endpoint_fini drains it via outq_item_killer. */
+    io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
+    TEST_ASSERT_NOT_NULL(pkt);
+
+    int rc = endpoint_enq(ep, &pkt->link);
+
+    /* Loud failure (negative, retriable) so the caller propagates and rotates —
+     * but the err_handler must NOT be invoked from this synchronous send path
+     * (re-entrancy would self-deadlock the controller, bug-652). */
+    TEST_ASSERT_TRUE(rc < 0);
+    TEST_ASSERT_EQUAL_INT(0, g_err_count);
+
+    endpoint_fini(ep);
+    free(ep);
+    close(iomgr.epoll_fd);
+    printf("test_enq_arm_failure_is_loud_no_errhandler: PASS\n");
+}
+
+/* ── suite entry point ───────────────────────────────────────────── */
+
+void test_netio_arm_failure(void)
+{
+    printf("\n>> netio poll-arm failure tests\n\n");
+
+    RUN_TEST(test_enq_arm_failure_is_loud_no_errhandler);
+}

src/lib/testsuite/test_netio_close_propagation.c

@@ -0,0 +1,77 @@
+/*
+ * test_netio_close_propagation.c — bug-648: a peer close/EOF must propagate as a
+ * negative error, never as rc==0.
+ *
+ * io_fsm_do() moved the read FSM to FSM_STATE_ERROR on a peer close but returned
+ * rc = -errno, which on a clean EOF (errno==0) is 0. handle_input() then read
+ * rc>=0, never called handle_error, and a blocked cbuf_get() reply-waiter was
+ * never woken. io_classify_io_error() is the extracted, pure decision; it must
+ * return <0 for every close/error case and 0 only for retriable would-block.
+ */
+
+#define _GNU_SOURCE
+#include <errno.h>
+#include <stdio.h>
+
+#include <openssl/ssl.h>
+#include <unity/unity.h>
+
+/* Pure classifier defined (non-static) in netio.c, linked into this binary. */
+extern int io_classify_io_error(int ssl_err, int ecode, int io_rc);
+
+/* WANT_READ / WANT_WRITE are retriable would-block → rc==0, FSM stays put. */
+static void test_wantread_writeready_is_retriable_zero(void)
+{
+    TEST_ASSERT_EQUAL_INT(0, io_classify_io_error(SSL_ERROR_WANT_READ, 0, -1));
+    TEST_ASSERT_EQUAL_INT(0, io_classify_io_error(SSL_ERROR_WANT_WRITE, 0, -1));
+    printf("test_wantread_writeready_is_retriable_zero: PASS\n");
+}
+
+/* Non-blocking EAGAIN/EWOULDBLOCK syscall (io_rc<0) is retriable → rc==0. */
+static void test_syscall_eagain_is_retriable_zero(void)
+{
+    TEST_ASSERT_EQUAL_INT(0, io_classify_io_error(SSL_ERROR_SYSCALL, EAGAIN, -1));
+    TEST_ASSERT_EQUAL_INT(0,
+        io_classify_io_error(SSL_ERROR_SYSCALL, EWOULDBLOCK, -1));
+    printf("test_syscall_eagain_is_retriable_zero: PASS\n");
+}
+
+/*
+ * THE BUG: a clean EOF — SSL_read returns 0 with errno==0 — must yield a
+ * NEGATIVE error (not 0) so the close propagates and the waiter is woken.
+ */
+static void test_clean_eof_is_negative_not_zero(void)
+{
+    int rc = io_classify_io_error(SSL_ERROR_SYSCALL, 0 /*errno*/, 0 /*io_rc EOF*/);
+    TEST_ASSERT_TRUE(rc < 0);
+    TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc);
+    printf("test_clean_eof_is_negative_not_zero: PASS\n");
+}
+
+/* Clean TLS close_notify (ZERO_RETURN) with errno==0 must also be negative. */
+static void test_zero_return_is_negative_not_zero(void)
+{
+    int rc = io_classify_io_error(SSL_ERROR_ZERO_RETURN, 0, 0);
+    TEST_ASSERT_TRUE(rc < 0);
+    TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc);
+    printf("test_zero_return_is_negative_not_zero: PASS\n");
+}
+
+/* A hard syscall error preserves its real errno (still negative). */
+static void test_hard_syscall_error_preserves_errno(void)
+{
+    TEST_ASSERT_EQUAL_INT(-ECONNRESET,
+        io_classify_io_error(SSL_ERROR_SYSCALL, ECONNRESET, -1));
+    printf("test_hard_syscall_error_preserves_errno: PASS\n");
+}
+
+void test_netio_close_propagation(void)
+{
+    printf("\n>> netio peer-close propagation tests\n\n");
+
+    RUN_TEST(test_wantread_writeready_is_retriable_zero);
+    RUN_TEST(test_syscall_eagain_is_retriable_zero);
+    RUN_TEST(test_clean_eof_is_negative_not_zero);
+    RUN_TEST(test_zero_return_is_negative_not_zero);
+    RUN_TEST(test_hard_syscall_error_preserves_errno);
+}

src/lib/testsuite/test_request_send_propagation.c

@@ -0,0 +1,72 @@
+/*
+ * test_request_send_propagation.c — bug-650: xce_request() must PROPAGATE an
+ * endpoint_enq() send failure, not swallow it.
+ *
+ * xce_request() (and the other request.c senders) ignored endpoint_enq()'s
+ * return and always returned 0.  So when the transport was dead — endpoint_enq()
+ * returns <0 and marks the endpoint dead (bug-646) — xce_request() still
+ * reported success.  xce_jobsubmit() then fell through to a BLOCKING cbuf_get()
+ * and hung forever waiting for a reply to a request that was never transmitted
+ * (server idle, client fd unconnected).  xce_request() must return the negative
+ * enq error so the failover loop rotates instead of blocking.
+ */
+
+#define _GNU_SOURCE
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+#include "netio.h"
+#include "request.h"
+
+/*
+ * A request that ENCODES successfully but whose send fails because the endpoint
+ * is dead (EPOLL_CTL_MOD on an fd that is not registered → ENOENT) must make
+ * xce_request() return a negative error, not 0.
+ */
+static void test_xce_request_propagates_send_failure(void)
+{
+    iomgr_t iomgr;
+    memset(&iomgr, 0, sizeof(iomgr));
+    TEST_ASSERT_EQUAL_INT(0, iomgr_init(&iomgr));
+
+    endpoint_t *ep = endpoint_get();
+    TEST_ASSERT_NOT_NULL(ep);
+
+    ep->fd = socket(AF_INET, SOCK_STREAM, 0); /* valid fd, NOT in the epoll */
+    TEST_ASSERT_TRUE(ep->fd >= 0);
+    ep->iomgr              = &iomgr;
+    ep->ssl_handshake_done = 1;
+    ep->flags              = 0; /* force iomgr_update_poll MOD on the unregistered fd */
+
+    char           nm[] = "tester";
+    char           ky[] = "key";
+    xce_peerinfo_t peer = { .name = nm, .key = ky };
+    ep->peer            = &peer;
+
+    xce_request_t req;
+    memset(&req, 0, sizeof(req));
+    req.opcode = OPCode_nop;
+    req.seq    = 42;
+
+    int rc = xce_request(ep, &req);
+
+    /* Old code: always 0 (swallowed). Fixed: the negative enq error propagates. */
+    TEST_ASSERT_TRUE(rc < 0);
+
+    endpoint_fini(ep);
+    free(ep);
+    close(iomgr.epoll_fd);
+    printf("test_xce_request_propagates_send_failure: PASS\n");
+}
+
+void test_request_send_propagation(void)
+{
+    printf("\n>> request send-failure propagation tests\n\n");
+
+    RUN_TEST(test_xce_request_propagates_send_failure);
+}