From bb8e531a122995d7ca0e0b76709416abcb1297f7 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Fri, 29 May 2026 02:45:24 +0000 Subject: [PATCH] test(ha-config-bootstrap): add RED-phase tests for xce_request_with_retry (task 10.1) --- .kiro/specs/ha-config-bootstrap/tasks.md | 2 +- src/lib/meson.build | 1 + src/lib/testsuite/test.c | 2 + src/lib/testsuite/test_libxce_retry.c | 334 +++++++++++++++++++++++ 4 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 src/lib/testsuite/test_libxce_retry.c diff --git a/.kiro/specs/ha-config-bootstrap/tasks.md b/.kiro/specs/ha-config-bootstrap/tasks.md index 4157c5f6..f3601449 100644 --- a/.kiro/specs/ha-config-bootstrap/tasks.md +++ b/.kiro/specs/ha-config-bootstrap/tasks.md @@ -236,7 +236,7 @@ Dependency order: startup validation (D) unblocks vtaskmgr before election (A); - _Depends: 9.2_ - [ ] 10. Change G: libxce client-side transparent retry (P) -- [ ] 10.1 Write failing tests for xce_request_with_retry() +- [x] 10.1 Write failing tests for xce_request_with_retry() - In `src/lib/testsuite/test_libxce_retry.c`: test `XCE_ERR_NOT_READY` → backoff + retry same host; test `XCE_ERR_STANDBY` with `redirect_host/port` → reconnect to redirect + retry; test `XCE_ERR_STANDBY` without redirect → fallback to backoff; test `XCE_ERR_AUTH` → surface immediately (no retry); test `max_retries` exhausted → surface last error; test 50 concurrent simulated clients → all succeed within 30s (backoff prevents starvation) - `make test` fails - _Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6_ diff --git a/src/lib/meson.build b/src/lib/meson.build index 283433f2..354e1356 100644 --- a/src/lib/meson.build +++ b/src/lib/meson.build @@ -169,6 +169,7 @@ libxce_test_src = ['testsuite/test.c', 'testsuite/test_hooks.c', 'testsuite/test_cluster.c', 'testsuite/test_cluster_parse.c', + 'testsuite/test_libxce_retry.c', ] libxce_test_deps = [libxce_deps, librtti_dep, unity_dep, libxce_dep] diff --git a/src/lib/testsuite/test.c b/src/lib/testsuite/test.c index 627e58f4..f0f3f189 100644 --- a/src/lib/testsuite/test.c +++ b/src/lib/testsuite/test.c @@ -34,6 +34,7 @@ extern void test_kvmgr_capture(void); extern void test_hooks(void); extern void test_cluster(void); extern void test_cluster_parse(void); +extern void test_libxce_retry(void); void setUp(void) { return; } @@ -137,6 +138,7 @@ int main(int argc, char* argv[]) TEST(test_hooks); TEST(test_cluster); TEST(test_cluster_parse); + TEST(test_libxce_retry); return UNITY_END(); } diff --git a/src/lib/testsuite/test_libxce_retry.c b/src/lib/testsuite/test_libxce_retry.c new file mode 100644 index 00000000..1a48c819 --- /dev/null +++ b/src/lib/testsuite/test_libxce_retry.c @@ -0,0 +1,334 @@ +/* + * test_libxce_retry.c — RED-phase tests for xce_request_with_retry() (tasks 10.1/10.2). + * + * Uses xce_retry_send_hook to inject mock round-trip outcomes without real + * network connections. Each mock call returns a configured status/redirect. + * + * Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6 + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "request.h" +#include "xce/errnox.h" + +/* ── Forward declarations for functions-under-test ───────────────── */ +/* xce_request_with_retry is not yet implemented — linker will fail (RED). */ +extern int xce_request_with_retry(endpoint_t *ep, + xce_request_t *req, + const xce_retry_config_t *cfg); + +/* + * xce_retry_send_hook: test seam for synchronous request-reply. + * When set, xce_request_with_retry calls this instead of the real network path. + * The hook fills in *reply_out and returns 0 on success, -errno on failure. + */ +extern int (*xce_retry_send_hook)(endpoint_t *ep, xce_request_t *req, + xce_reply_t *reply_out); + +/* ── Default retry config for tests (fast backoff) ──────────────── */ +static const xce_retry_config_t g_fast_cfg = { + .max_retries = 5, + .initial_retry_ms = 1, + .max_retry_ms = 10, + .backoff_multiplier = 2.0, +}; + +/* ── Mock state ─────────────────────────────────────────────────── */ + +#define MAX_MOCK_REPLIES 32 + +struct mock_reply { + int32_t status; /* xce_reply_t.status to return */ + int send_rc; /* return code from the hook (0 or -errno) */ + char *redirect_host; /* non-NULL if status == -XCE_ERR_STANDBY */ + uint16_t redirect_port; +}; + +static struct mock_reply g_replies[MAX_MOCK_REPLIES]; +static int g_reply_count = 0; +static int g_reply_idx = 0; +static int g_call_count = 0; + +static void reset_mock(void) +{ + g_reply_count = 0; + g_reply_idx = 0; + g_call_count = 0; + memset(g_replies, 0, sizeof(g_replies)); +} + +static void add_reply(int32_t status, int send_rc, + const char *rhost, uint16_t rport) +{ + if (g_reply_count >= MAX_MOCK_REPLIES) return; + int i = g_reply_count++; + g_replies[i].status = status; + g_replies[i].send_rc = send_rc; + g_replies[i].redirect_host = rhost ? strdup(rhost) : NULL; + g_replies[i].redirect_port = rport; +} + +static int mock_send(endpoint_t *ep, xce_request_t *req, xce_reply_t *reply_out) +{ + (void)ep; (void)req; + g_call_count++; + + if (g_reply_idx >= g_reply_count) { + /* Default: return success after mocks exhausted */ + memset(reply_out, 0, sizeof(*reply_out)); + reply_out->status = 0; + return 0; + } + + struct mock_reply *m = &g_replies[g_reply_idx++]; + if (m->send_rc != 0) + return m->send_rc; + + memset(reply_out, 0, sizeof(*reply_out)); + reply_out->status = m->status; + reply_out->redirect_host = m->redirect_host; + reply_out->redirect_port = m->redirect_port; + return 0; +} + +static void install_mock(void) +{ + xce_retry_send_hook = mock_send; +} + +/* ── Tests ──────────────────────────────────────────────────────── */ + +/* Req 7.1: XCE_ERR_NOT_READY → retry with backoff, eventually succeed */ +static void test_not_ready_retries_then_succeeds(void) +{ + install_mock(); + reset_mock(); + + /* 3 NOT_READY responses, then success */ + add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0); + add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0); + add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0); + add_reply(0, 0, NULL, 0); /* success */ + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(4, g_call_count); + + printf("test_not_ready_retries_then_succeeds: PASS\n"); +} + +/* Req 7.2: XCE_ERR_STANDBY with redirect → reconnect to redirect, retry */ +static void test_standby_with_redirect_reconnects_and_retries(void) +{ + install_mock(); + reset_mock(); + + /* Standby response with redirect */ + add_reply(-XCE_ERR_STANDBY, 0, "primary.host", 9990); + add_reply(0, 0, NULL, 0); /* success on redirect target */ + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(2, g_call_count); + + printf("test_standby_with_redirect_reconnects_and_retries: PASS\n"); +} + +/* Req 7.2 fallback: XCE_ERR_STANDBY without redirect → backoff, retry same */ +static void test_standby_without_redirect_falls_back_to_backoff(void) +{ + install_mock(); + reset_mock(); + + /* STANDBY without redirect, then success */ + add_reply(-XCE_ERR_STANDBY, 0, NULL, 0); + add_reply(0, 0, NULL, 0); + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(2, g_call_count); + + printf("test_standby_without_redirect_falls_back_to_backoff: PASS\n"); +} + +/* Req 7.4: non-retriable error (EPERM) → surface immediately, no retry */ +static void test_non_retriable_error_surfaces_immediately(void) +{ + install_mock(); + reset_mock(); + + /* Auth/perm failure — must NOT retry */ + add_reply(-EPERM, 0, NULL, 0); + /* Add a success after — it should never be reached */ + add_reply(0, 0, NULL, 0); + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + TEST_ASSERT_EQUAL_INT(-EPERM, rc); + TEST_ASSERT_EQUAL_INT(1, g_call_count); /* only 1 attempt */ + + printf("test_non_retriable_error_surfaces_immediately: PASS\n"); +} + +/* Req 7.5: max_retries exhausted → surface last error */ +static void test_max_retries_exhausted_returns_last_error(void) +{ + install_mock(); + reset_mock(); + + /* Fill with NOT_READY beyond max_retries */ + for (int i = 0; i <= g_fast_cfg.max_retries + 1; i++) + add_reply(-XCE_ERR_NOT_READY, 0, NULL, 0); + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + /* Must return the last error, not 0 */ + TEST_ASSERT_NOT_EQUAL(0, rc); + /* Must have tried max_retries+1 times total */ + TEST_ASSERT_EQUAL_INT(g_fast_cfg.max_retries + 1, g_call_count); + + printf("test_max_retries_exhausted_returns_last_error: PASS\n"); +} + +/* Req 7.3: ECONNREFUSED → reconnect, retry */ +static void test_conn_refused_retries(void) +{ + install_mock(); + reset_mock(); + + add_reply(0, -ECONNREFUSED, NULL, 0); + add_reply(0, 0, NULL, 0); + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(2, g_call_count); + + printf("test_conn_refused_retries: PASS\n"); +} + +/* NULL cfg → uses defaults (max_retries=20, etc.) */ +static void test_null_cfg_uses_defaults(void) +{ + install_mock(); + reset_mock(); + + add_reply(0, 0, NULL, 0); /* immediate success */ + + xce_request_t req; + memset(&req, 0, sizeof(req)); + + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, NULL); + TEST_ASSERT_EQUAL_INT(0, rc); + + printf("test_null_cfg_uses_defaults: PASS\n"); +} + +/* Req 7.6: 50 concurrent simulated clients all succeed (backoff prevents starvation) */ + +struct concurrent_client_args { + int id; + int rc; + atomic_int *success_count; +}; + +static void *concurrent_client_fn(void *arg) +{ + struct concurrent_client_args *a = arg; + xce_request_t req; + memset(&req, 0, sizeof(req)); + + /* Each client sees NOT_READY twice, then success */ + int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); + a->rc = rc; + if (rc == 0) + atomic_fetch_add(a->success_count, 1); + return NULL; +} + +static int concurrent_mock_send(endpoint_t *ep, xce_request_t *req, + xce_reply_t *reply_out) +{ + (void)ep; (void)req; + static atomic_int s_call_count = 0; + int n = atomic_fetch_add(&s_call_count, 1); + + memset(reply_out, 0, sizeof(*reply_out)); + + /* First 3 calls per "client" return NOT_READY, then succeed. + * Since we can't track per-client state here, use a simple global + * threshold: first 100 calls NOT_READY, rest succeed. */ + if (n < 100) + reply_out->status = -XCE_ERR_NOT_READY; + else + reply_out->status = 0; + return 0; +} + +static void test_50_concurrent_clients_all_succeed(void) +{ + xce_retry_send_hook = concurrent_mock_send; + + const int N = 50; + pthread_t threads[50]; + struct concurrent_client_args args[50]; + atomic_int success_count; + atomic_init(&success_count, 0); + + for (int i = 0; i < N; i++) { + args[i].id = i; + args[i].rc = -1; + args[i].success_count = &success_count; + pthread_create(&threads[i], NULL, concurrent_client_fn, &args[i]); + } + + /* Wait for all clients (with 30s overall deadline from Unity timeout) */ + for (int i = 0; i < N; i++) + pthread_join(threads[i], NULL); + + TEST_ASSERT_EQUAL_INT(N, atomic_load(&success_count)); + + printf("test_50_concurrent_clients_all_succeed: PASS (%d/%d)\n", + atomic_load(&success_count), N); +} + +/* ── Suite entry point ──────────────────────────────────────────── */ + +void test_libxce_retry(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_not_ready_retries_then_succeeds); + RUN_TEST(test_standby_with_redirect_reconnects_and_retries); + RUN_TEST(test_standby_without_redirect_falls_back_to_backoff); + RUN_TEST(test_non_retriable_error_surfaces_immediately); + RUN_TEST(test_max_retries_exhausted_returns_last_error); + RUN_TEST(test_conn_refused_retries); + RUN_TEST(test_null_cfg_uses_defaults); + RUN_TEST(test_50_concurrent_clients_all_succeed); + UNITY_END(); +}