test(ha-config-bootstrap): add RED-phase tests for xce_request_with_retry (task 10.1)
This commit is contained in:
parent
97a76617e6
commit
bb8e531a12
4 changed files with 338 additions and 1 deletions
|
|
@ -236,7 +236,7 @@ Dependency order: startup validation (D) unblocks vtaskmgr before election (A);
|
|||
- _Depends: 9.2_
|
||||
|
||||
- [ ] 10. Change G: libxce client-side transparent retry (P)
|
||||
- [ ] 10.1 Write failing tests for xce_request_with_retry()
|
||||
- [x] 10.1 Write failing tests for xce_request_with_retry()
|
||||
- In `src/lib/testsuite/test_libxce_retry.c`: test `XCE_ERR_NOT_READY` → backoff + retry same host; test `XCE_ERR_STANDBY` with `redirect_host/port` → reconnect to redirect + retry; test `XCE_ERR_STANDBY` without redirect → fallback to backoff; test `XCE_ERR_AUTH` → surface immediately (no retry); test `max_retries` exhausted → surface last error; test 50 concurrent simulated clients → all succeed within 30s (backoff prevents starvation)
|
||||
- `make test` fails
|
||||
- _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6_
|
||||
|
|
|
|||
|
|
@ -169,6 +169,7 @@ libxce_test_src = ['testsuite/test.c',
|
|||
'testsuite/test_hooks.c',
|
||||
'testsuite/test_cluster.c',
|
||||
'testsuite/test_cluster_parse.c',
|
||||
'testsuite/test_libxce_retry.c',
|
||||
]
|
||||
|
||||
libxce_test_deps = [libxce_deps, librtti_dep, unity_dep, libxce_dep]
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ extern void test_kvmgr_capture(void);
|
|||
extern void test_hooks(void);
|
||||
extern void test_cluster(void);
|
||||
extern void test_cluster_parse(void);
|
||||
extern void test_libxce_retry(void);
|
||||
|
||||
void setUp(void) { return; }
|
||||
|
||||
|
|
@ -137,6 +138,7 @@ int main(int argc, char* argv[])
|
|||
TEST(test_hooks);
|
||||
TEST(test_cluster);
|
||||
TEST(test_cluster_parse);
|
||||
TEST(test_libxce_retry);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
|
|
|
|||
334
src/lib/testsuite/test_libxce_retry.c
Normal file
334
src/lib/testsuite/test_libxce_retry.c
Normal file
|
|
@ -0,0 +1,334 @@
|
|||
/*
|
||||
* test_libxce_retry.c — RED-phase tests for xce_request_with_retry() (tasks 10.1/10.2).
|
||||
*
|
||||
* Uses xce_retry_send_hook to inject mock round-trip outcomes without real
|
||||
* network connections. Each mock call returns a configured status/redirect.
|
||||
*
|
||||
* Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
#include "request.h"
|
||||
#include "xce/errnox.h"
|
||||
|
||||
/* ── Forward declarations for functions-under-test ───────────────── */
|
||||
/* xce_request_with_retry is not yet implemented — linker will fail (RED). */
|
||||
extern int xce_request_with_retry(endpoint_t *ep,
|
||||
xce_request_t *req,
|
||||
const xce_retry_config_t *cfg);
|
||||
|
||||
/*
|
||||
* xce_retry_send_hook: test seam for synchronous request-reply.
|
||||
* When set, xce_request_with_retry calls this instead of the real network path.
|
||||
* The hook fills in *reply_out and returns 0 on success, -errno on failure.
|
||||
*/
|
||||
extern int (*xce_retry_send_hook)(endpoint_t *ep, xce_request_t *req,
|
||||
xce_reply_t *reply_out);
|
||||
|
||||
/* ── Default retry config for tests (fast backoff) ──────────────── */
|
||||
static const xce_retry_config_t g_fast_cfg = {
|
||||
.max_retries = 5,
|
||||
.initial_retry_ms = 1,
|
||||
.max_retry_ms = 10,
|
||||
.backoff_multiplier = 2.0,
|
||||
};
|
||||
|
||||
/* ── Mock state ─────────────────────────────────────────────────── */
|
||||
|
||||
#define MAX_MOCK_REPLIES 32
|
||||
|
||||
struct mock_reply {
|
||||
int32_t status; /* xce_reply_t.status to return */
|
||||
int send_rc; /* return code from the hook (0 or -errno) */
|
||||
char *redirect_host; /* non-NULL if status == -XCE_ERR_STANDBY */
|
||||
uint16_t redirect_port;
|
||||
};
|
||||
|
||||
static struct mock_reply g_replies[MAX_MOCK_REPLIES];
|
||||
static int g_reply_count = 0;
|
||||
static int g_reply_idx = 0;
|
||||
static int g_call_count = 0;
|
||||
|
||||
static void reset_mock(void)
|
||||
{
|
||||
g_reply_count = 0;
|
||||
g_reply_idx = 0;
|
||||
g_call_count = 0;
|
||||
memset(g_replies, 0, sizeof(g_replies));
|
||||
}
|
||||
|
||||
static void add_reply(int32_t status, int send_rc,
|
||||
const char *rhost, uint16_t rport)
|
||||
{
|
||||
if (g_reply_count >= MAX_MOCK_REPLIES) return;
|
||||
int i = g_reply_count++;
|
||||
g_replies[i].status = status;
|
||||
g_replies[i].send_rc = send_rc;
|
||||
g_replies[i].redirect_host = rhost ? strdup(rhost) : NULL;
|
||||
g_replies[i].redirect_port = rport;
|
||||
}
|
||||
|
||||
static int mock_send(endpoint_t *ep, xce_request_t *req, xce_reply_t *reply_out)
|
||||
{
|
||||
(void)ep; (void)req;
|
||||
g_call_count++;
|
||||
|
||||
if (g_reply_idx >= g_reply_count) {
|
||||
/* Default: return success after mocks exhausted */
|
||||
memset(reply_out, 0, sizeof(*reply_out));
|
||||
reply_out->status = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct mock_reply *m = &g_replies[g_reply_idx++];
|
||||
if (m->send_rc != 0)
|
||||
return m->send_rc;
|
||||
|
||||
memset(reply_out, 0, sizeof(*reply_out));
|
||||
reply_out->status = m->status;
|
||||
reply_out->redirect_host = m->redirect_host;
|
||||
reply_out->redirect_port = m->redirect_port;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void install_mock(void)
|
||||
{
|
||||
xce_retry_send_hook = mock_send;
|
||||
}
|
||||
|
||||
/* ── Tests ──────────────────────────────────────────────────────── */
|
||||
|
||||
/* Req 7.1: XCE_ERR_NOT_READY → retry with backoff, eventually succeed */
|
||||
static void test_not_ready_retries_then_succeeds(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
/* 3 NOT_READY responses, then success */
|
||||
add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0);
|
||||
add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0);
|
||||
add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0);
|
||||
add_reply(0, 0, NULL, 0); /* success */
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT(4, g_call_count);
|
||||
|
||||
printf("test_not_ready_retries_then_succeeds: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 7.2: XCE_ERR_STANDBY with redirect → reconnect to redirect, retry */
|
||||
static void test_standby_with_redirect_reconnects_and_retries(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
/* Standby response with redirect */
|
||||
add_reply(-XCE_ERR_STANDBY, 0, "primary.host", 9990);
|
||||
add_reply(0, 0, NULL, 0); /* success on redirect target */
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT(2, g_call_count);
|
||||
|
||||
printf("test_standby_with_redirect_reconnects_and_retries: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 7.2 fallback: XCE_ERR_STANDBY without redirect → backoff, retry same */
|
||||
static void test_standby_without_redirect_falls_back_to_backoff(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
/* STANDBY without redirect, then success */
|
||||
add_reply(-XCE_ERR_STANDBY, 0, NULL, 0);
|
||||
add_reply(0, 0, NULL, 0);
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT(2, g_call_count);
|
||||
|
||||
printf("test_standby_without_redirect_falls_back_to_backoff: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 7.4: non-retriable error (EPERM) → surface immediately, no retry */
|
||||
static void test_non_retriable_error_surfaces_immediately(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
/* Auth/perm failure — must NOT retry */
|
||||
add_reply(-EPERM, 0, NULL, 0);
|
||||
/* Add a success after — it should never be reached */
|
||||
add_reply(0, 0, NULL, 0);
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
TEST_ASSERT_EQUAL_INT(-EPERM, rc);
|
||||
TEST_ASSERT_EQUAL_INT(1, g_call_count); /* only 1 attempt */
|
||||
|
||||
printf("test_non_retriable_error_surfaces_immediately: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 7.5: max_retries exhausted → surface last error */
|
||||
static void test_max_retries_exhausted_returns_last_error(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
/* Fill with NOT_READY beyond max_retries */
|
||||
for (int i = 0; i <= g_fast_cfg.max_retries + 1; i++)
|
||||
add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0);
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
/* Must return the last error, not 0 */
|
||||
TEST_ASSERT_NOT_EQUAL(0, rc);
|
||||
/* Must have tried max_retries+1 times total */
|
||||
TEST_ASSERT_EQUAL_INT(g_fast_cfg.max_retries + 1, g_call_count);
|
||||
|
||||
printf("test_max_retries_exhausted_returns_last_error: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 7.3: ECONNREFUSED → reconnect, retry */
|
||||
static void test_conn_refused_retries(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
add_reply(0, -ECONNREFUSED, NULL, 0);
|
||||
add_reply(0, 0, NULL, 0);
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_EQUAL_INT(2, g_call_count);
|
||||
|
||||
printf("test_conn_refused_retries: PASS\n");
|
||||
}
|
||||
|
||||
/* NULL cfg → uses defaults (max_retries=20, etc.) */
|
||||
static void test_null_cfg_uses_defaults(void)
|
||||
{
|
||||
install_mock();
|
||||
reset_mock();
|
||||
|
||||
add_reply(0, 0, NULL, 0); /* immediate success */
|
||||
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, NULL);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
printf("test_null_cfg_uses_defaults: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 7.6: 50 concurrent simulated clients all succeed (backoff prevents starvation) */
|
||||
|
||||
struct concurrent_client_args {
|
||||
int id;
|
||||
int rc;
|
||||
atomic_int *success_count;
|
||||
};
|
||||
|
||||
static void *concurrent_client_fn(void *arg)
|
||||
{
|
||||
struct concurrent_client_args *a = arg;
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
|
||||
/* Each client sees NOT_READY twice, then success */
|
||||
int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg);
|
||||
a->rc = rc;
|
||||
if (rc == 0)
|
||||
atomic_fetch_add(a->success_count, 1);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int concurrent_mock_send(endpoint_t *ep, xce_request_t *req,
|
||||
xce_reply_t *reply_out)
|
||||
{
|
||||
(void)ep; (void)req;
|
||||
static atomic_int s_call_count = 0;
|
||||
int n = atomic_fetch_add(&s_call_count, 1);
|
||||
|
||||
memset(reply_out, 0, sizeof(*reply_out));
|
||||
|
||||
/* First 3 calls per "client" return NOT_READY, then succeed.
|
||||
* Since we can't track per-client state here, use a simple global
|
||||
* threshold: first 100 calls NOT_READY, rest succeed. */
|
||||
if (n < 100)
|
||||
reply_out->status = -XCE_ERR_NOT_READY;
|
||||
else
|
||||
reply_out->status = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void test_50_concurrent_clients_all_succeed(void)
|
||||
{
|
||||
xce_retry_send_hook = concurrent_mock_send;
|
||||
|
||||
const int N = 50;
|
||||
pthread_t threads[50];
|
||||
struct concurrent_client_args args[50];
|
||||
atomic_int success_count;
|
||||
atomic_init(&success_count, 0);
|
||||
|
||||
for (int i = 0; i < N; i++) {
|
||||
args[i].id = i;
|
||||
args[i].rc = -1;
|
||||
args[i].success_count = &success_count;
|
||||
pthread_create(&threads[i], NULL, concurrent_client_fn, &args[i]);
|
||||
}
|
||||
|
||||
/* Wait for all clients (with 30s overall deadline from Unity timeout) */
|
||||
for (int i = 0; i < N; i++)
|
||||
pthread_join(threads[i], NULL);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(N, atomic_load(&success_count));
|
||||
|
||||
printf("test_50_concurrent_clients_all_succeed: PASS (%d/%d)\n",
|
||||
atomic_load(&success_count), N);
|
||||
}
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────── */
|
||||
|
||||
void test_libxce_retry(void)
|
||||
{
|
||||
UNITY_BEGIN();
|
||||
RUN_TEST(test_not_ready_retries_then_succeeds);
|
||||
RUN_TEST(test_standby_with_redirect_reconnects_and_retries);
|
||||
RUN_TEST(test_standby_without_redirect_falls_back_to_backoff);
|
||||
RUN_TEST(test_non_retriable_error_surfaces_immediately);
|
||||
RUN_TEST(test_max_retries_exhausted_returns_last_error);
|
||||
RUN_TEST(test_conn_refused_retries);
|
||||
RUN_TEST(test_null_cfg_uses_defaults);
|
||||
RUN_TEST(test_50_concurrent_clients_all_succeed);
|
||||
UNITY_END();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue