myxce/src/lib/testsuite/test_libxce_retry.c

/*
 * test_libxce_retry.c — RED-phase tests for xce_request_with_retry() (tasks 10.1/10.2).
 *
 * Uses xce_retry_send_hook to inject mock round-trip outcomes without real
 * network connections. Each mock call returns a configured status/redirect.
 *
 * Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6
 */
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include <unity/unity.h>
#include "request.h"
#include "xce/errnox.h"

/* ── Forward declarations for functions-under-test ───────────────── */
/* xce_request_with_retry is not yet implemented — linker will fail (RED). */
extern int xce_request_with_retry(endpoint_t *ep,
                                   xce_request_t *req,
                                   const xce_retry_config_t *cfg);

/*
 * xce_retry_send_hook: test seam for synchronous request-reply.
 * When set, xce_request_with_retry calls this instead of the real network path.
 * The hook fills in *reply_out and returns 0 on success, -errno on failure.
 */
extern int (*xce_retry_send_hook)(endpoint_t *ep, xce_request_t *req,
                                   xce_reply_t *reply_out);

/* ── Default retry config for tests (fast backoff) ──────────────── */
static const xce_retry_config_t g_fast_cfg = {
    .max_retries        = 5,
    .initial_retry_ms   = 1,
    .max_retry_ms       = 10,
    .backoff_multiplier = 2.0,
};

/* ── Mock state ─────────────────────────────────────────────────── */

#define MAX_MOCK_REPLIES 32

struct mock_reply {
    int32_t  status;         /* xce_reply_t.status to return */
    int      send_rc;        /* return code from the hook (0 or -errno) */
    char    *redirect_host;  /* non-NULL if status == -E_XCE_STANDBY */
    uint16_t redirect_port;
};

static struct mock_reply g_replies[MAX_MOCK_REPLIES];
static int               g_reply_count  = 0;
static int               g_reply_idx    = 0;
static int               g_call_count   = 0;

static void reset_mock(void)
{
    for (int i = 0; i < g_reply_count; i++) {
        free(g_replies[i].redirect_host);
        g_replies[i].redirect_host = NULL;
    }
    g_reply_count = 0;
    g_reply_idx   = 0;
    g_call_count  = 0;
    memset(g_replies, 0, sizeof(g_replies));
}

static void add_reply(int32_t status, int send_rc,
                      const char *rhost, uint16_t rport)
{
    if (g_reply_count >= MAX_MOCK_REPLIES) return;
    int i = g_reply_count++;
    g_replies[i].status        = status;
    g_replies[i].send_rc       = send_rc;
    g_replies[i].redirect_host = rhost ? strdup(rhost) : NULL;
    g_replies[i].redirect_port = rport;
}

static int mock_send(endpoint_t *ep, xce_request_t *req, xce_reply_t *reply_out)
{
    (void)ep; (void)req;
    g_call_count++;

    if (g_reply_idx >= g_reply_count) {
        /* Default: return success after mocks exhausted */
        memset(reply_out, 0, sizeof(*reply_out));
        reply_out->status = 0;
        return 0;
    }

    struct mock_reply *m = &g_replies[g_reply_idx++];
    if (m->send_rc != 0)
        return m->send_rc;

    memset(reply_out, 0, sizeof(*reply_out));
    reply_out->status        = m->status;
    reply_out->redirect_host = m->redirect_host;
    reply_out->redirect_port = m->redirect_port;
    return 0;
}

static void install_mock(void)
{
    xce_retry_send_hook = mock_send;
}

/* ── Tests ──────────────────────────────────────────────────────── */

/* Req 7.1: E_XCE_NOT_READY → retry with backoff, eventually succeed */
static void test_not_ready_retries_then_succeeds(void)
{
    install_mock();
    reset_mock();

    /* 3 NOT_READY responses, then success */
    add_reply(-E_XCE_NOT_READY, 0, NULL, 0);
    add_reply(-E_XCE_NOT_READY, 0, NULL, 0);
    add_reply(-E_XCE_NOT_READY, 0, NULL, 0);
    add_reply(0, 0, NULL, 0);  /* success */

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    TEST_ASSERT_EQUAL_INT(0, rc);
    TEST_ASSERT_EQUAL_INT(4, g_call_count);

    printf("test_not_ready_retries_then_succeeds: PASS\n");
}

/* Req 7.2: E_XCE_STANDBY with redirect → reconnect to redirect, retry */
static void test_standby_with_redirect_reconnects_and_retries(void)
{
    install_mock();
    reset_mock();

    /* Standby response with redirect */
    add_reply(-E_XCE_STANDBY, 0, "primary.host", 9990);
    add_reply(0, 0, NULL, 0);  /* success on redirect target */

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    TEST_ASSERT_EQUAL_INT(0, rc);
    TEST_ASSERT_EQUAL_INT(2, g_call_count);

    printf("test_standby_with_redirect_reconnects_and_retries: PASS\n");
}

/* Req 7.2 fallback: E_XCE_STANDBY without redirect → backoff, retry same */
static void test_standby_without_redirect_falls_back_to_backoff(void)
{
    install_mock();
    reset_mock();

    /* STANDBY without redirect, then success */
    add_reply(-E_XCE_STANDBY, 0, NULL, 0);
    add_reply(0, 0, NULL, 0);

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    TEST_ASSERT_EQUAL_INT(0, rc);
    TEST_ASSERT_EQUAL_INT(2, g_call_count);

    printf("test_standby_without_redirect_falls_back_to_backoff: PASS\n");
}

/* Req 7.4: non-retriable error (EPERM) → surface immediately, no retry */
static void test_non_retriable_error_surfaces_immediately(void)
{
    install_mock();
    reset_mock();

    /* Auth/perm failure — must NOT retry */
    add_reply(-EPERM, 0, NULL, 0);
    /* Add a success after — it should never be reached */
    add_reply(0, 0, NULL, 0);

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    TEST_ASSERT_EQUAL_INT(-EPERM, rc);
    TEST_ASSERT_EQUAL_INT(1, g_call_count);  /* only 1 attempt */

    printf("test_non_retriable_error_surfaces_immediately: PASS\n");
}

/* Req 7.5: max_retries exhausted → surface last error */
static void test_max_retries_exhausted_returns_last_error(void)
{
    install_mock();
    reset_mock();

    /* Fill with NOT_READY beyond max_retries */
    for (int i = 0; i <= g_fast_cfg.max_retries + 1; i++)
        add_reply(-E_XCE_NOT_READY, 0, NULL, 0);

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    /* Must return the last error, not 0 */
    TEST_ASSERT_NOT_EQUAL(0, rc);
    /* Must have tried max_retries+1 times total */
    TEST_ASSERT_EQUAL_INT(g_fast_cfg.max_retries + 1, g_call_count);

    printf("test_max_retries_exhausted_returns_last_error: PASS\n");
}

/* Req 7.3: ECONNREFUSED → reconnect, retry */
static void test_conn_refused_retries(void)
{
    install_mock();
    reset_mock();

    add_reply(0, -ECONNREFUSED, NULL, 0);
    add_reply(0, 0, NULL, 0);

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    TEST_ASSERT_EQUAL_INT(0, rc);
    TEST_ASSERT_EQUAL_INT(2, g_call_count);

    printf("test_conn_refused_retries: PASS\n");
}

/* NULL cfg → uses defaults (max_retries=20, etc.) */
static void test_null_cfg_uses_defaults(void)
{
    install_mock();
    reset_mock();

    add_reply(0, 0, NULL, 0);  /* immediate success */

    xce_request_t req;
    memset(&req, 0, sizeof(req));

    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, NULL);
    TEST_ASSERT_EQUAL_INT(0, rc);

    printf("test_null_cfg_uses_defaults: PASS\n");
}

/* Req 7.6: 50 concurrent simulated clients all succeed (backoff prevents starvation) */

struct concurrent_client_args {
    int          id;
    int          rc;
    atomic_int  *success_count;
};

static void *concurrent_client_fn(void *arg)
{
    struct concurrent_client_args *a = arg;
    xce_request_t req;
    memset(&req, 0, sizeof(req));

    /* Each client sees NOT_READY twice, then success */
    int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
    a->rc = rc;
    if (rc == 0)
        atomic_fetch_add(a->success_count, 1);
    return NULL;
}

static int concurrent_mock_send(endpoint_t *ep, xce_request_t *req,
                                 xce_reply_t *reply_out)
{
    (void)ep; (void)req;
    static atomic_int s_call_count = 0;
    int n = atomic_fetch_add(&s_call_count, 1);

    memset(reply_out, 0, sizeof(*reply_out));

    /* First 3 calls per "client" return NOT_READY, then succeed.
     * Since we can't track per-client state here, use a simple global
     * threshold: first 100 calls NOT_READY, rest succeed. */
    if (n < 100)
        reply_out->status = -E_XCE_NOT_READY;
    else
        reply_out->status = 0;
    return 0;
}

static void test_50_concurrent_clients_all_succeed(void)
{
    xce_retry_send_hook = concurrent_mock_send;

    const int N = 50;
    pthread_t threads[50];
    struct concurrent_client_args args[50];
    atomic_int success_count;
    atomic_init(&success_count, 0);

    for (int i = 0; i < N; i++) {
        args[i].id            = i;
        args[i].rc            = -1;
        args[i].success_count = &success_count;
        pthread_create(&threads[i], NULL, concurrent_client_fn, &args[i]);
    }

    /* Wait for all clients (with 30s overall deadline from Unity timeout) */
    for (int i = 0; i < N; i++)
        pthread_join(threads[i], NULL);

    TEST_ASSERT_EQUAL_INT(N, atomic_load(&success_count));

    printf("test_50_concurrent_clients_all_succeed: PASS (%d/%d)\n",
           atomic_load(&success_count), N);
}

/* ── Suite entry point ──────────────────────────────────────────── */

void test_libxce_retry(void)
{
    UNITY_BEGIN();
    RUN_TEST(test_not_ready_retries_then_succeeds);
    RUN_TEST(test_standby_with_redirect_reconnects_and_retries);
    RUN_TEST(test_standby_without_redirect_falls_back_to_backoff);
    RUN_TEST(test_non_retriable_error_surfaces_immediately);
    RUN_TEST(test_max_retries_exhausted_returns_last_error);
    RUN_TEST(test_conn_refused_retries);
    RUN_TEST(test_null_cfg_uses_defaults);
    RUN_TEST(test_50_concurrent_clients_all_succeed);
    UNITY_END();
}