1169 lines
47 KiB
C
1169 lines
47 KiB
C
/*
|
|
* test_libxce_retry.c — reusable libxce-failover test harness + harness
|
|
* self-tests (spec libxce-failover, task 1.2 ADAPTED).
|
|
*
|
|
* This file builds the GREEN, reusable harness that the later component tasks
|
|
* consume to write their own RED→GREEN tests:
|
|
* - task 2.1 (StatusClassifier) → drives FO_STATUS_TABLE
|
|
* - task 2.2 (EndpointRotator) → drives the fake-master cursor
|
|
* - task 2.3 (reply-capture) → drives the redirect-hint rows
|
|
* - task 3.1/3.2 (decision helper) → drives the table + cursor
|
|
* - task 5.1/6.1 (concurrency) → drive fo_run_threads()
|
|
*
|
|
* It deliberately references NO not-yet-existing failover production symbols
|
|
* (xce_failover_classify / xcecli_retry_next / xce_failover_rotate); those are
|
|
* added by the component tasks. The lib build+test MUST stay GREEN after this
|
|
* task, so every fixture here is exercised by at least one real assertion.
|
|
*
|
|
* Requirements covered by the harness: 2.5, 4.1, 4.2, 4.3, 4.4, 4.5, 5.2, 8.3,
|
|
* 8.4 (status taxonomy); 2.1, 2.4 (master cursor); 6.3 (concurrency reuse).
|
|
* The retained retry-defaults test still covers 6.1, 6.2.
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <errno.h>
|
|
#include <pthread.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <stdint.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "client_internal.h" /* api_ctx_t, xce_failover_state_t */
|
|
#include "failover_harness.h"
|
|
#include "request.h"
|
|
#include "xce/client_failover.h"
|
|
#include "xce/errnox.h"
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Harness implementation
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* ── 1. Fake multi-master config + cursor ─────────────────────────────── */
|
|
|
|
int fo_make_fake_masters(xce_cluster_config_t *cfg)
|
|
{
|
|
static const struct {
|
|
const char *host;
|
|
int port;
|
|
} masters[FO_FAKE_MASTERS] = {
|
|
{ "10.20.0.1", 7001 },
|
|
{ "10.20.0.2", 7002 },
|
|
{ "10.20.0.3", 7003 },
|
|
};
|
|
|
|
memset(cfg, 0, sizeof(*cfg));
|
|
for (int i = 0; i < FO_FAKE_MASTERS && i < MAX_HA_CANDIDATES; i++) {
|
|
strncpy(cfg->candidates[i].hostname, masters[i].host,
|
|
XCE_MAXHOSTNAME_LEN - 1);
|
|
cfg->candidates[i].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
|
|
cfg->candidates[i].port = masters[i].port;
|
|
}
|
|
cfg->candidates_count =
|
|
(FO_FAKE_MASTERS < MAX_HA_CANDIDATES) ? FO_FAKE_MASTERS
|
|
: MAX_HA_CANDIDATES;
|
|
return cfg->candidates_count;
|
|
}
|
|
|
|
void fo_cursor_init(fo_master_cursor_t *cur, const xce_cluster_config_t *cfg)
|
|
{
|
|
cur->cfg = cfg;
|
|
cur->cur = 0;
|
|
}
|
|
|
|
int fo_cursor_advance(fo_master_cursor_t *cur)
|
|
{
|
|
if (!cur->cfg || cur->cfg->candidates_count <= 0)
|
|
return 0;
|
|
cur->cur = (cur->cur + 1) % cur->cfg->candidates_count;
|
|
return cur->cur;
|
|
}
|
|
|
|
const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur)
|
|
{
|
|
if (!cur->cfg || cur->cfg->candidates_count <= 0)
|
|
return NULL;
|
|
return &cur->cfg->candidates[cur->cur];
|
|
}
|
|
|
|
/* ── 2. Controller-status taxonomy table ──────────────────────────────── */
|
|
|
|
const fo_status_row_t FO_STATUS_TABLE[] = {
|
|
/* success */
|
|
{ "OK", 0, false, false, FO_ACT_SUCCESS,
|
|
"request served by primary" },
|
|
|
|
/* redirect (a primary exists elsewhere) — rotate now, hint first if any */
|
|
{ "E_XCE_REDIRECT+hint", E_XCE_REDIRECT, true, false, FO_ACT_RETRY_NOW,
|
|
"standby redirects with primary address hint (8.3)" },
|
|
{ "E_XCE_REDIRECT", E_XCE_REDIRECT, false, false, FO_ACT_RETRY_NOW,
|
|
"standby redirects without address — rotate master list (8.4)" },
|
|
|
|
/* standby (op is primary-only) — treat as redirect, rotate now (4.3) */
|
|
{ "E_XCE_STANDBY", E_XCE_STANDBY, false, false, FO_ACT_RETRY_NOW,
|
|
"controller in standby; op is primary-only (4.3)" },
|
|
|
|
/* no-primary / not-ready — rotate after backoff (4.1) */
|
|
{ "E_XCE_NOMASTER", E_XCE_NOMASTER, false, false, FO_ACT_RETRY_BACKOFF,
|
|
"standby knows no primary yet — backoff + rotate (4.1)" },
|
|
{ "E_XCE_NOT_READY", E_XCE_NOT_READY, false, false, FO_ACT_RETRY_BACKOFF,
|
|
"controller initializing/electing — backoff + rotate (4.1)" },
|
|
|
|
/* idempotent-replay duplicate (write replay) — DUP/EEXIST (5.2) */
|
|
{ "E_XCE_DUP_JOB", E_XCE_DUP_JOB, false, false, FO_ACT_TERMINAL,
|
|
"duplicate job; SUCCESS only on a re-sent write (5.2, attempt-aware)" },
|
|
{ "-EEXIST", -EEXIST, false, true, FO_ACT_TERMINAL,
|
|
"duplicate via -EEXIST; SUCCESS only on a re-sent write (5.2)" },
|
|
|
|
/* transport errors — reconnect/rotate now (4.4) */
|
|
{ "-ECONNREFUSED", -ECONNREFUSED, false, true, FO_ACT_RETRY_NOW,
|
|
"connection refused — reconnect/rotate (4.4)" },
|
|
{ "-ETIMEDOUT", -ETIMEDOUT, false, true, FO_ACT_RETRY_NOW,
|
|
"connection timeout — reconnect/rotate (4.4)" },
|
|
{ "-ECONNRESET", -ECONNRESET, false, true, FO_ACT_RETRY_NOW,
|
|
"peer reset — reconnect/rotate (4.4)" },
|
|
|
|
/* terminal / non-retriable — surface immediately (4.5) */
|
|
{ "-EINVAL", -EINVAL, false, true, FO_ACT_TERMINAL,
|
|
"invalid request — surface, no retry (4.5)" },
|
|
};
|
|
|
|
const size_t FO_STATUS_TABLE_LEN =
|
|
sizeof(FO_STATUS_TABLE) / sizeof(FO_STATUS_TABLE[0]);
|
|
|
|
const fo_status_row_t *fo_status_lookup(int status)
|
|
{
|
|
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
|
|
if (FO_STATUS_TABLE[i].status == status)
|
|
return &FO_STATUS_TABLE[i];
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
/* ── 3. Generic N-thread concurrency harness ──────────────────────────── */
|
|
|
|
typedef struct {
|
|
fo_thread_fn fn;
|
|
void *ctx;
|
|
int idx;
|
|
} fo_thread_arg_t;
|
|
|
|
static void *fo_thread_trampoline(void *p)
|
|
{
|
|
fo_thread_arg_t *a = (fo_thread_arg_t *)p;
|
|
a->fn(a->ctx, a->idx);
|
|
return NULL;
|
|
}
|
|
|
|
int fo_run_threads(int n, fo_thread_fn fn, void *ctx)
|
|
{
|
|
if (n <= 0 || !fn)
|
|
return -EINVAL;
|
|
|
|
pthread_t *tids = calloc((size_t)n, sizeof(*tids));
|
|
fo_thread_arg_t *args = calloc((size_t)n, sizeof(*args));
|
|
if (!tids || !args) {
|
|
free(tids);
|
|
free(args);
|
|
return -ENOMEM;
|
|
}
|
|
|
|
int spawned = 0;
|
|
int rc = 0;
|
|
for (int i = 0; i < n; i++) {
|
|
args[i].fn = fn;
|
|
args[i].ctx = ctx;
|
|
args[i].idx = i;
|
|
int e = pthread_create(&tids[i], NULL, fo_thread_trampoline, &args[i]);
|
|
if (e != 0) {
|
|
rc = -e;
|
|
break;
|
|
}
|
|
spawned++;
|
|
}
|
|
|
|
for (int i = 0; i < spawned; i++)
|
|
pthread_join(tids[i], NULL);
|
|
|
|
free(tids);
|
|
free(args);
|
|
return rc;
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Harness self-tests (GREEN) — each fixture proven by real assertions
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* Req 6.1, 6.2: the retry-tunables struct + safe production defaults are
|
|
* retained and reachable for the failover loop. */
|
|
static void test_retry_defaults_are_reachable(void)
|
|
{
|
|
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
|
|
|
|
TEST_ASSERT_EQUAL_INT(20, cfg.max_retries);
|
|
TEST_ASSERT_EQUAL_INT(50, cfg.initial_retry_ms);
|
|
TEST_ASSERT_EQUAL_INT(5000, cfg.max_retry_ms);
|
|
TEST_ASSERT_TRUE(cfg.backoff_multiplier > 1.0);
|
|
|
|
printf("test_retry_defaults_are_reachable: PASS\n");
|
|
}
|
|
|
|
/* Fixture 1: fake multi-master config holds 3 distinct candidates. */
|
|
static void test_fake_masters_hold_three_candidates(void)
|
|
{
|
|
xce_cluster_config_t cfg;
|
|
int n = fo_make_fake_masters(&cfg);
|
|
|
|
TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, n);
|
|
TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, cfg.candidates_count);
|
|
TEST_ASSERT_EQUAL_STRING("10.20.0.1", cfg.candidates[0].hostname);
|
|
TEST_ASSERT_EQUAL_INT(7001, cfg.candidates[0].port);
|
|
TEST_ASSERT_EQUAL_STRING("10.20.0.3", cfg.candidates[2].hostname);
|
|
TEST_ASSERT_EQUAL_INT(7003, cfg.candidates[2].port);
|
|
|
|
printf("test_fake_masters_hold_three_candidates: PASS\n");
|
|
}
|
|
|
|
/* Fixture 1: cursor advances through masters and wraps (2.1, 2.4, 8.4). */
|
|
static void test_master_cursor_advances_and_wraps(void)
|
|
{
|
|
xce_cluster_config_t cfg;
|
|
fo_make_fake_masters(&cfg);
|
|
|
|
fo_master_cursor_t cur;
|
|
fo_cursor_init(&cur, &cfg);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, cur.cur);
|
|
TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname);
|
|
|
|
TEST_ASSERT_EQUAL_INT(1, fo_cursor_advance(&cur));
|
|
TEST_ASSERT_EQUAL_STRING("10.20.0.2", fo_cursor_current(&cur)->hostname);
|
|
|
|
TEST_ASSERT_EQUAL_INT(2, fo_cursor_advance(&cur));
|
|
TEST_ASSERT_EQUAL_STRING("10.20.0.3", fo_cursor_current(&cur)->hostname);
|
|
|
|
/* wrap back to master 0 */
|
|
TEST_ASSERT_EQUAL_INT(0, fo_cursor_advance(&cur));
|
|
TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname);
|
|
|
|
printf("test_master_cursor_advances_and_wraps: PASS\n");
|
|
}
|
|
|
|
/* Fixture 2: taxonomy table is complete — every status the classifier must
|
|
* handle is present, lookups resolve, and the status codes equal the real
|
|
* errnox.h values (2.5, 4.1-4.5, 5.2, 8.3, 8.4). */
|
|
static void test_status_table_is_complete_and_resolves(void)
|
|
{
|
|
/* every mandated controller status + transport error is represented */
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(0));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_REDIRECT));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_STANDBY));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOMASTER));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOT_READY));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_DUP_JOB));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(-EEXIST));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNREFUSED));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ETIMEDOUT));
|
|
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNRESET));
|
|
|
|
/* status codes resolve to the real errnox.h values (ordered enum) */
|
|
TEST_ASSERT_EQUAL_INT(XCE_ERRNO_BASE, E_XCE_BAD_JOB);
|
|
TEST_ASSERT_TRUE(E_XCE_REDIRECT == E_XCE_NOMASTER + 1);
|
|
TEST_ASSERT_TRUE(E_XCE_NOT_READY == E_XCE_REDIRECT + 1);
|
|
TEST_ASSERT_TRUE(E_XCE_STANDBY == E_XCE_NOT_READY + 1);
|
|
|
|
/* expected actions line up with the design's StatusClassifier (first send) */
|
|
TEST_ASSERT_EQUAL_INT(FO_ACT_SUCCESS, fo_status_lookup(0)->expect);
|
|
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
|
|
fo_status_lookup(E_XCE_REDIRECT)->expect);
|
|
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
|
|
fo_status_lookup(E_XCE_STANDBY)->expect);
|
|
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
|
|
fo_status_lookup(E_XCE_NOMASTER)->expect);
|
|
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
|
|
fo_status_lookup(E_XCE_NOT_READY)->expect);
|
|
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
|
|
fo_status_lookup(-ECONNRESET)->expect);
|
|
|
|
printf("test_status_table_is_complete_and_resolves: PASS\n");
|
|
}
|
|
|
|
/* Fixture 2: the redirect-hint flag distinguishes hint vs. no-hint rows (8.3). */
|
|
static void test_status_table_redirect_hint_flag(void)
|
|
{
|
|
bool saw_hint = false;
|
|
bool saw_no_hint = false;
|
|
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
|
|
if (FO_STATUS_TABLE[i].status == E_XCE_REDIRECT) {
|
|
if (FO_STATUS_TABLE[i].has_hint)
|
|
saw_hint = true;
|
|
else
|
|
saw_no_hint = true;
|
|
}
|
|
}
|
|
TEST_ASSERT_TRUE(saw_hint); /* REDIRECT+hint row present (8.3) */
|
|
TEST_ASSERT_TRUE(saw_no_hint); /* REDIRECT no-hint row present (8.4) */
|
|
|
|
printf("test_status_table_redirect_hint_flag: PASS\n");
|
|
}
|
|
|
|
/* Fixture 3: concurrency harness runs N threads to completion (6.3 reuse). */
|
|
#define FO_TEST_NTHREADS 16
|
|
|
|
typedef struct {
|
|
pthread_mutex_t lock;
|
|
int counter;
|
|
} fo_selftest_ctx_t;
|
|
|
|
static void fo_selftest_worker(void *ctx, int idx)
|
|
{
|
|
(void)idx;
|
|
fo_selftest_ctx_t *c = (fo_selftest_ctx_t *)ctx;
|
|
pthread_mutex_lock(&c->lock);
|
|
c->counter++;
|
|
pthread_mutex_unlock(&c->lock);
|
|
}
|
|
|
|
static void test_thread_harness_runs_all_threads(void)
|
|
{
|
|
fo_selftest_ctx_t ctx;
|
|
pthread_mutex_init(&ctx.lock, NULL);
|
|
ctx.counter = 0;
|
|
|
|
int rc = fo_run_threads(FO_TEST_NTHREADS, fo_selftest_worker, &ctx);
|
|
pthread_mutex_destroy(&ctx.lock);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(FO_TEST_NTHREADS, ctx.counter);
|
|
|
|
printf("test_thread_harness_runs_all_threads: PASS\n");
|
|
}
|
|
|
|
/* Fixture 3: harness rejects bad arguments. */
|
|
static void test_thread_harness_rejects_bad_args(void)
|
|
{
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(0, fo_selftest_worker, NULL));
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(4, NULL, NULL));
|
|
|
|
printf("test_thread_harness_rejects_bad_args: PASS\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Task 2.1 — StatusClassifier (xce_failover_classify) RED→GREEN tests
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* Map the harness' expected-action category to the real classifier enum. The
|
|
* two enums share order by design, but we map explicitly so a future reorder of
|
|
* either enum is caught here rather than silently passing. */
|
|
static xce_fo_action_t fo_expected_to_action(fo_expected_action_t e)
|
|
{
|
|
switch (e) {
|
|
case FO_ACT_SUCCESS: return XCE_FO_SUCCESS;
|
|
case FO_ACT_RETRY_NOW: return XCE_FO_RETRY_NOW;
|
|
case FO_ACT_RETRY_BACKOFF: return XCE_FO_RETRY_BACKOFF;
|
|
case FO_ACT_TERMINAL: return XCE_FO_TERMINAL;
|
|
}
|
|
return XCE_FO_TERMINAL;
|
|
}
|
|
|
|
/* Req 2.5, 4.1, 4.2, 4.3, 4.5: every taxonomy row's first-send classification
|
|
* (is_write=false, attempt=0) matches the design's StatusClassifier mapping. */
|
|
static void test_classify_matches_taxonomy_table(void)
|
|
{
|
|
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
|
|
const fo_status_row_t *row = &FO_STATUS_TABLE[i];
|
|
xce_fo_action_t want = fo_expected_to_action(row->expect);
|
|
xce_fo_action_t got =
|
|
xce_failover_classify(row->status, /*is_write=*/false,
|
|
/*attempt=*/0);
|
|
char msg[128];
|
|
snprintf(msg, sizeof(msg), "row '%s' (status=%d): want %d got %d",
|
|
row->name, row->status, (int)want, (int)got);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(want, got, msg);
|
|
}
|
|
|
|
printf("test_classify_matches_taxonomy_table: PASS\n");
|
|
}
|
|
|
|
/* Req 2.5: REDIRECT/STANDBY/NOMASTER/NOT_READY are never reported as success,
|
|
* regardless of is_write/attempt. */
|
|
static void test_classify_control_codes_never_success(void)
|
|
{
|
|
const int codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, E_XCE_NOMASTER,
|
|
E_XCE_NOT_READY };
|
|
for (size_t i = 0; i < sizeof(codes) / sizeof(codes[0]); i++) {
|
|
TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS,
|
|
xce_failover_classify(codes[i], false, 0));
|
|
TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS,
|
|
xce_failover_classify(codes[i], true, 1));
|
|
/* accepted in either sign */
|
|
TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS,
|
|
xce_failover_classify(-codes[i], true, 1));
|
|
}
|
|
|
|
printf("test_classify_control_codes_never_success: PASS\n");
|
|
}
|
|
|
|
/* Req 5.2: idempotent-replay rule — DUP_JOB/EEXIST on a RE-SENT WRITE is a
|
|
* success; the first send (or a read) is a genuine duplicate → terminal. */
|
|
static void test_classify_dup_replay_rule(void)
|
|
{
|
|
/* re-sent write (is_write && attempt>0) → SUCCESS */
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
|
|
xce_failover_classify(-E_XCE_DUP_JOB, true, 1));
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
|
|
xce_failover_classify(E_XCE_DUP_JOB, true, 1));
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
|
|
xce_failover_classify(-EEXIST, true, 1));
|
|
|
|
/* first send → genuine duplicate → TERMINAL */
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
|
|
xce_failover_classify(-E_XCE_DUP_JOB, true, 0));
|
|
/* non-write replay → TERMINAL */
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
|
|
xce_failover_classify(-E_XCE_DUP_JOB, false, 1));
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
|
|
xce_failover_classify(-EEXIST, false, 1));
|
|
|
|
printf("test_classify_dup_replay_rule: PASS\n");
|
|
}
|
|
|
|
/* Req 3.2 / general: status 0 is always success. */
|
|
static void test_classify_zero_is_success(void)
|
|
{
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
|
|
xce_failover_classify(0, false, 0));
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
|
|
xce_failover_classify(0, true, 3));
|
|
|
|
printf("test_classify_zero_is_success: PASS\n");
|
|
}
|
|
|
|
/* Req 4.5: unknown / unrecognized statuses are conservatively terminal. */
|
|
static void test_classify_unknown_is_terminal(void)
|
|
{
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
|
|
xce_failover_classify(-EACCES, false, 0));
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
|
|
xce_failover_classify(-ENOENT, true, 1));
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
|
|
xce_failover_classify(-12345, false, 0));
|
|
|
|
printf("test_classify_unknown_is_terminal: PASS\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Task 2.2 — EndpointRotator target selection (xce_failover_next_target)
|
|
* RED→GREEN tests. Pure, socket-free target-selection logic; the
|
|
* live connect path is covered by E2E task 6.2.
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* Build a 3-master client config (distinct host:port) the rotator selects over.
|
|
* Strings are static-duration so the api_ctx may borrow them by pointer. */
|
|
static void fo_make_client_masters(xce_client_config_t *conf)
|
|
{
|
|
static char *hosts[N_MAX_MASTERS] = { "10.30.0.1", "10.30.0.2",
|
|
"10.30.0.3" };
|
|
static int ports[N_MAX_MASTERS] = { 8001, 8002, 8003 };
|
|
|
|
memset(conf, 0, sizeof(*conf));
|
|
conf->nmasters = N_MAX_MASTERS;
|
|
for (int i = 0; i < N_MAX_MASTERS; i++) {
|
|
conf->masters[i] = hosts[i];
|
|
conf->master_ports[i] = ports[i];
|
|
}
|
|
}
|
|
|
|
/* Minimal api_ctx for target-selection tests: only conf, the failover cursor,
|
|
* and the ctx redirect-hint fields are exercised. No endpoint/socket touched. */
|
|
static void fo_make_api_ctx(api_ctx_t *ac, xce_client_config_t *conf)
|
|
{
|
|
memset(ac, 0, sizeof(*ac));
|
|
ac->conf = conf;
|
|
ac->failover.cur_master = 0;
|
|
}
|
|
|
|
static void fo_set_hint(api_ctx_t *ac, const char *host, uint16_t port)
|
|
{
|
|
strncpy(ac->ctx.redirect_hint_host, host, XCE_MAXHOSTNAME_LEN - 1);
|
|
ac->ctx.redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1] = '\0';
|
|
ac->ctx.redirect_hint_port = port;
|
|
}
|
|
|
|
/* Req 2.2, 8.4: with a redirect hint set, next_target returns the hint
|
|
* host/port and CONSUMES (clears) it; the following call ignores the stale
|
|
* hint and advances the master cursor instead. */
|
|
static void test_next_target_hint_first_then_consumed(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
fo_set_hint(&ac, "10.99.0.7", 9100);
|
|
|
|
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
|
|
uint16_t port = 0;
|
|
|
|
int rc = xce_failover_next_target(&ac, host, sizeof(host), &port);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_STRING("10.99.0.7", host);
|
|
TEST_ASSERT_EQUAL_INT(9100, port);
|
|
|
|
/* hint consumed: cleared on the ctx, cursor untouched */
|
|
TEST_ASSERT_EQUAL_STRING("", ac.ctx.redirect_hint_host);
|
|
TEST_ASSERT_EQUAL_INT(0, ac.failover.cur_master);
|
|
|
|
/* next call has no hint → advances cursor to master 1 */
|
|
rc = xce_failover_next_target(&ac, host, sizeof(host), &port);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_STRING("10.30.0.2", host);
|
|
TEST_ASSERT_EQUAL_INT(8002, port);
|
|
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
|
|
|
|
printf("test_next_target_hint_first_then_consumed: PASS\n");
|
|
}
|
|
|
|
/* Req 2.1, 8.4: with no hint, successive next_target calls advance through the
|
|
* configured masters and WRAP around; the cursor lives on failover.cur_master. */
|
|
static void test_next_target_rotates_and_wraps(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
|
|
uint16_t port = 0;
|
|
|
|
/* from cur_master=0, first rotation advances to master 1 */
|
|
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_STRING("10.30.0.2", host);
|
|
TEST_ASSERT_EQUAL_INT(8002, port);
|
|
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_STRING("10.30.0.3", host);
|
|
TEST_ASSERT_EQUAL_INT(8003, port);
|
|
TEST_ASSERT_EQUAL_INT(2, ac.failover.cur_master);
|
|
|
|
/* wrap back to master 0 */
|
|
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_STRING("10.30.0.1", host);
|
|
TEST_ASSERT_EQUAL_INT(8001, port);
|
|
TEST_ASSERT_EQUAL_INT(0, ac.failover.cur_master);
|
|
|
|
printf("test_next_target_rotates_and_wraps: PASS\n");
|
|
}
|
|
|
|
/* Req 4.4, 8.4: after a hint is consumed, the very next call advances to the
|
|
* next master — correctness is independent of the (one-shot) hint. */
|
|
static void test_next_target_hint_then_fallback_rotation(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
/* start the cursor at master 1 to prove fallback advances from current */
|
|
ac.failover.cur_master = 1;
|
|
fo_set_hint(&ac, "10.99.0.9", 9200);
|
|
|
|
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
|
|
uint16_t port = 0;
|
|
|
|
/* hint first */
|
|
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_STRING("10.99.0.9", host);
|
|
TEST_ASSERT_EQUAL_INT(9200, port);
|
|
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); /* cursor untouched */
|
|
|
|
/* fallback: advances from master 1 → master 2 */
|
|
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_STRING("10.30.0.3", host);
|
|
TEST_ASSERT_EQUAL_INT(8003, port);
|
|
TEST_ASSERT_EQUAL_INT(2, ac.failover.cur_master);
|
|
|
|
printf("test_next_target_hint_then_fallback_rotation: PASS\n");
|
|
}
|
|
|
|
/* Guard: NULL/invalid arguments are rejected without touching state. */
|
|
static void test_next_target_rejects_bad_args(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
|
|
uint16_t port = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
|
xce_failover_next_target(NULL, host, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
|
xce_failover_next_target(&ac, NULL, sizeof(host),
|
|
&port));
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
|
xce_failover_next_target(&ac, host, 0, &port));
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL,
|
|
xce_failover_next_target(&ac, host, sizeof(host),
|
|
NULL));
|
|
|
|
printf("test_next_target_rejects_bad_args: PASS\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Task 2.3 — RedirectCapture (xce_ctx_capture_redirect_hint) RED→GREEN tests.
|
|
*
|
|
* handle_reply() copies the redirect address carried on a redirect reply into
|
|
* the ctx's redirect_hint_host/port BEFORE the capnp-owned reply is freed. The
|
|
* copy logic is factored into a non-static seam, xce_ctx_capture_redirect_hint,
|
|
* so it can be unit-tested without a live server.
|
|
*
|
|
* Requirements: 2.2 (capture redirect addr as a best-effort hint), 8.3 (try the
|
|
* hinted address first — capture is the prerequisite).
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* Build a bare xce_ctx_t with an empty redirect hint, without touching any
|
|
* iomgr/ssl/endpoint state (capture is pure ctx-field manipulation). */
|
|
static void fo_ctx_clear_hint(xce_ctx_t *ctx)
|
|
{
|
|
memset(ctx, 0, sizeof(*ctx));
|
|
ctx->redirect_hint_host[0] = '\0';
|
|
ctx->redirect_hint_port = 0;
|
|
}
|
|
|
|
/* Req 2.2, 8.3: a present host+port is copied into the ctx hint fields. */
|
|
static void test_capture_redirect_hint_present(void)
|
|
{
|
|
xce_ctx_t ctx;
|
|
fo_ctx_clear_hint(&ctx);
|
|
|
|
xce_ctx_capture_redirect_hint(&ctx, "10.40.0.5", 9300);
|
|
|
|
TEST_ASSERT_EQUAL_STRING("10.40.0.5", ctx.redirect_hint_host);
|
|
TEST_ASSERT_EQUAL_INT(9300, ctx.redirect_hint_port);
|
|
|
|
printf("test_capture_redirect_hint_present: PASS\n");
|
|
}
|
|
|
|
/* Req 2.2: when no address is present (host NULL/empty or port 0) the hint is
|
|
* left empty — capture must not invent an address. */
|
|
static void test_capture_redirect_hint_absent_stays_empty(void)
|
|
{
|
|
xce_ctx_t ctx;
|
|
|
|
/* NULL host → hint untouched */
|
|
fo_ctx_clear_hint(&ctx);
|
|
xce_ctx_capture_redirect_hint(&ctx, NULL, 9300);
|
|
TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host);
|
|
TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port);
|
|
|
|
/* empty host → hint untouched */
|
|
fo_ctx_clear_hint(&ctx);
|
|
xce_ctx_capture_redirect_hint(&ctx, "", 9300);
|
|
TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host);
|
|
TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port);
|
|
|
|
/* port 0 → hint untouched (no usable target) */
|
|
fo_ctx_clear_hint(&ctx);
|
|
xce_ctx_capture_redirect_hint(&ctx, "10.40.0.5", 0);
|
|
TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host);
|
|
TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port);
|
|
|
|
printf("test_capture_redirect_hint_absent_stays_empty: PASS\n");
|
|
}
|
|
|
|
/* Req 2.2: an over-long host is bounded-copied (NUL-terminated, no overflow);
|
|
* the capnp-owned string the controller may send is not length-checked here. */
|
|
static void test_capture_redirect_hint_truncates_long_host(void)
|
|
{
|
|
xce_ctx_t ctx;
|
|
fo_ctx_clear_hint(&ctx);
|
|
|
|
char longhost[XCE_MAXHOSTNAME_LEN * 2];
|
|
memset(longhost, 'h', sizeof(longhost) - 1);
|
|
longhost[sizeof(longhost) - 1] = '\0';
|
|
|
|
xce_ctx_capture_redirect_hint(&ctx, longhost, 9400);
|
|
|
|
/* copied, NUL-terminated, and never longer than the field bound */
|
|
TEST_ASSERT_EQUAL_INT(XCE_MAXHOSTNAME_LEN - 1,
|
|
(int)strlen(ctx.redirect_hint_host));
|
|
TEST_ASSERT_EQUAL_INT('\0',
|
|
ctx.redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1]);
|
|
TEST_ASSERT_EQUAL_INT(9400, ctx.redirect_hint_port);
|
|
|
|
printf("test_capture_redirect_hint_truncates_long_host: PASS\n");
|
|
}
|
|
|
|
/* Guard: a NULL ctx is rejected without crashing. */
|
|
static void test_capture_redirect_hint_null_ctx_safe(void)
|
|
{
|
|
/* must not crash */
|
|
xce_ctx_capture_redirect_hint(NULL, "10.40.0.5", 9300);
|
|
printf("test_capture_redirect_hint_null_ctx_safe: PASS\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Task 2.3 — ReplyWaitWake RED→GREEN test.
|
|
*
|
|
* On an endpoint error/peer reset, the existing disconnect path injects a
|
|
* transport error into ctx->wbuf so a blocked cbuf_get returns instead of
|
|
* hanging. This test proves the wake mechanism end-to-end on a real wbuf: a
|
|
* thread blocks in cbuf_get, the "disconnect" injects -ECONNRESET, and the
|
|
* blocked waiter wakes with exactly that transport error (Req 4.4).
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
typedef struct {
|
|
cbuf_t wbuf;
|
|
int woke_status; /* what the blocked cbuf_get observed */
|
|
int got; /* cbuf_get return code */
|
|
} fo_wake_ctx_t;
|
|
|
|
static void fo_wake_waiter(void *p, int idx)
|
|
{
|
|
fo_wake_ctx_t *w = (fo_wake_ctx_t *)p;
|
|
if (idx == 0) {
|
|
/* waiter: block until the disconnect injects an error */
|
|
w->got = cbuf_get(&w->wbuf, &w->woke_status);
|
|
} else {
|
|
/* injector: mirror handle_disconnect's wbuf error injection. A short
|
|
* spin lets the waiter reach cbuf_get first so we exercise the wake,
|
|
* not a pre-filled buffer. */
|
|
for (volatile int i = 0; i < 2000000; i++) {
|
|
}
|
|
int err = -ECONNRESET;
|
|
cbuf_tryput(&w->wbuf, &err);
|
|
}
|
|
}
|
|
|
|
/* Req 4.4: a pending cbuf_get is released with the injected transport error so
|
|
* an in-flight xce_* call returns instead of hanging across an endpoint reset. */
|
|
static void test_reply_wait_wake_releases_blocked_get(void)
|
|
{
|
|
fo_wake_ctx_t w;
|
|
memset(&w, 0, sizeof(w));
|
|
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&w.wbuf, 1, sizeof(int)));
|
|
|
|
int rc = fo_run_threads(2, fo_wake_waiter, &w);
|
|
cbuf_fini(&w.wbuf);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_TRUE(w.got >= 0); /* cbuf_get returned, did not hang */
|
|
TEST_ASSERT_EQUAL_INT(-ECONNRESET, w.woke_status);
|
|
|
|
printf("test_reply_wait_wake_releases_blocked_get: PASS\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Task 3.1 — decision loop (xcecli_retry_decide) RED→GREEN tests.
|
|
*
|
|
* xcecli_retry_decide is the PURE core of the failover decision loop: classify
|
|
* → apply SUCCESS/DUP/TERMINAL to *rc → advance attempt → compute backoff →
|
|
* handle budget exhaustion → report the chosen action. It does NOT sleep, open
|
|
* sockets, rotate, or log. The cursor-advancing side (xce_failover_next_target)
|
|
* is driven separately to prove the become-primary path advances the cursor.
|
|
*
|
|
* Requirements: 1.2, 3.1, 3.2 (budget), 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2.
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* A small, deterministic retry config for the decide tests: a low max_retries
|
|
* so budget exhaustion is reached quickly, a tiny initial delay, a modest cap. */
|
|
static xce_retry_config_t fo_test_retry_cfg(void)
|
|
{
|
|
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
|
|
cfg.max_retries = 3;
|
|
cfg.initial_retry_ms = 10;
|
|
cfg.max_retry_ms = 100;
|
|
cfg.backoff_multiplier = 2.0;
|
|
return cfg;
|
|
}
|
|
|
|
/* Req 3.3, 3.4: a controller that keeps reporting no-primary (NOMASTER /
|
|
* NOT_READY) is retried with backoff until the budget is exhausted; decide then
|
|
* returns false with *rc == -E_XCE_NOT_READY — the distinct cluster-not-ready
|
|
* code, never an opaque -EINVAL. */
|
|
static void test_decide_no_primary_exhausts_to_not_ready(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
xce_retry_config_t cfg = fo_test_retry_cfg();
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
|
|
int rc;
|
|
xce_fo_action_t action;
|
|
bool again;
|
|
int retries = 0;
|
|
|
|
do {
|
|
rc = E_XCE_NOMASTER; /* every attempt: no primary known */
|
|
again = xcecli_retry_decide(&ac, &rc, /*is_write=*/false, &cfg, &st,
|
|
&action);
|
|
if (again) {
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
|
|
retries++;
|
|
}
|
|
} while (again && retries < 100); /* hard cap so a bug can't spin forever */
|
|
|
|
TEST_ASSERT_FALSE(again);
|
|
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
|
|
/* exactly max_retries retries happened before exhaustion */
|
|
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
|
|
|
|
printf("test_decide_no_primary_exhausts_to_not_ready: PASS\n");
|
|
}
|
|
|
|
/* Req 3.2 / become-primary: no-primary then a served reply (status 0) → the
|
|
* loop stops with success and the master cursor advanced across the retries
|
|
* (the rotation side runs in the wrapper; here we advance it explicitly to
|
|
* mirror that and assert the sticky cursor moved). */
|
|
static void test_decide_no_primary_then_becomes_primary(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
xce_retry_config_t cfg = fo_test_retry_cfg();
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
|
|
int rc;
|
|
xce_fo_action_t action;
|
|
|
|
/* attempt 0: no primary → retry (decide says backoff). */
|
|
rc = E_XCE_NOMASTER;
|
|
bool a0 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
|
TEST_ASSERT_TRUE(a0);
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
|
|
/* mirror the wrapper: rotate advances the cursor (no hint set) */
|
|
char host[XCE_MAXHOSTNAME_LEN];
|
|
uint16_t port;
|
|
xce_failover_next_target(&ac, host, sizeof(host), &port);
|
|
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
|
|
|
|
/* attempt 1: served by the new primary → success, loop stops. */
|
|
rc = 0;
|
|
bool a1 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
|
TEST_ASSERT_FALSE(a1);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
/* cursor stayed advanced (sticky primary = where success landed) */
|
|
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
|
|
|
|
printf("test_decide_no_primary_then_becomes_primary: PASS\n");
|
|
}
|
|
|
|
/* Req 4.2, 4.4: redirect / standby / transport causes are RETRY_NOW with NO
|
|
* backoff delay applied (st->delay_ms == 0), so the wrapper rotates promptly. */
|
|
static void test_decide_retry_now_has_no_backoff_delay(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
const int now_codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, -ECONNRESET,
|
|
-ECONNREFUSED, -ETIMEDOUT };
|
|
for (size_t i = 0; i < sizeof(now_codes) / sizeof(now_codes[0]); i++) {
|
|
xce_retry_config_t cfg = fo_test_retry_cfg();
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
int rc = now_codes[i];
|
|
xce_fo_action_t action;
|
|
|
|
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
|
TEST_ASSERT_TRUE(again);
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
|
|
TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* no backoff for RETRY_NOW */
|
|
}
|
|
|
|
printf("test_decide_retry_now_has_no_backoff_delay: PASS\n");
|
|
}
|
|
|
|
/* Req 4.1, 6.3: no-primary causes get a jittered exponential backoff — the
|
|
* delay grows across retries and never exceeds max_retry_ms. */
|
|
static void test_decide_backoff_grows_and_is_bounded(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
|
|
cfg.max_retries = 12;
|
|
cfg.initial_retry_ms = 10;
|
|
cfg.max_retry_ms = 200;
|
|
cfg.backoff_multiplier = 2.0;
|
|
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
int rc;
|
|
xce_fo_action_t action;
|
|
|
|
int prev_base = 0;
|
|
for (int i = 0; i < 8; i++) {
|
|
rc = E_XCE_NOT_READY;
|
|
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
|
TEST_ASSERT_TRUE(again);
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
|
|
|
|
/* delay is positive and never exceeds the cap */
|
|
TEST_ASSERT_TRUE(st.delay_ms > 0);
|
|
TEST_ASSERT_TRUE(st.delay_ms <= cfg.max_retry_ms);
|
|
|
|
/* the un-jittered base (initial * mult^attempt, capped) is monotone
|
|
* non-decreasing; once we hit the cap it stays there. The delay floor
|
|
* grows: by attempt 5+ it must exceed the very first delay. */
|
|
if (i >= 5)
|
|
TEST_ASSERT_TRUE(st.delay_ms >= prev_base);
|
|
if (i == 0)
|
|
prev_base = st.delay_ms;
|
|
}
|
|
|
|
printf("test_decide_backoff_grows_and_is_bounded: PASS\n");
|
|
}
|
|
|
|
/* Req 4.5: a terminal (non-retriable) status stops the loop immediately with
|
|
* *rc unchanged — no retry, no -E_XCE_NOT_READY rewrite. */
|
|
static void test_decide_terminal_surfaces_unchanged(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
const int term_codes[] = { -EACCES, -EPERM, -EINVAL, -ENOENT };
|
|
for (size_t i = 0; i < sizeof(term_codes) / sizeof(term_codes[0]); i++) {
|
|
xce_retry_config_t cfg = fo_test_retry_cfg();
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
int rc = term_codes[i];
|
|
xce_fo_action_t action;
|
|
|
|
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
|
TEST_ASSERT_FALSE(again);
|
|
TEST_ASSERT_EQUAL_INT(term_codes[i], rc); /* surfaced unchanged */
|
|
}
|
|
|
|
printf("test_decide_terminal_surfaces_unchanged: PASS\n");
|
|
}
|
|
|
|
/* Req 5.2: DUP on a re-sent write (is_write && attempt>0) → success (*rc = 0),
|
|
* loop stops. The controller's client-jobid replay guard means the write was
|
|
* already applied, so the retry is a success not a duplicate failure. */
|
|
static void test_decide_dup_on_resent_write_is_success(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
xce_retry_config_t cfg = fo_test_retry_cfg();
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
st.attempt = 1; /* a re-sent write */
|
|
|
|
int rc = -E_XCE_DUP_JOB;
|
|
xce_fo_action_t action;
|
|
bool again = xcecli_retry_decide(&ac, &rc, /*is_write=*/true, &cfg, &st,
|
|
&action);
|
|
TEST_ASSERT_FALSE(again);
|
|
TEST_ASSERT_EQUAL_INT(0, rc); /* replay → success */
|
|
|
|
/* contrast: DUP on the FIRST send (attempt 0) is a genuine duplicate →
|
|
* terminal, *rc surfaced unchanged. */
|
|
xcecli_retry_t st0 = XCECLI_RETRY_INIT(&ac);
|
|
int rc0 = -E_XCE_DUP_JOB;
|
|
bool a0 =
|
|
xcecli_retry_decide(&ac, &rc0, true, &cfg, &st0, &action);
|
|
TEST_ASSERT_FALSE(a0);
|
|
TEST_ASSERT_EQUAL_INT(-E_XCE_DUP_JOB, rc0);
|
|
|
|
printf("test_decide_dup_on_resent_write_is_success: PASS\n");
|
|
}
|
|
|
|
/* Req 3.4 (D6): a transport-only cause that outlasts the budget keeps the LAST
|
|
* transport rc (not -E_XCE_NOT_READY — that mapping is reserved for no-primary
|
|
* exhaustion). Proves the two exhaustion causes are distinguished. */
|
|
static void test_decide_transport_exhaustion_keeps_last_rc(void)
|
|
{
|
|
xce_client_config_t conf;
|
|
api_ctx_t ac;
|
|
fo_make_client_masters(&conf);
|
|
fo_make_api_ctx(&ac, &conf);
|
|
|
|
xce_retry_config_t cfg = fo_test_retry_cfg();
|
|
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
|
|
|
int rc;
|
|
xce_fo_action_t action;
|
|
bool again;
|
|
int retries = 0;
|
|
|
|
do {
|
|
rc = -ECONNREFUSED; /* every attempt: transport failure */
|
|
again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
|
if (again) {
|
|
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
|
|
retries++;
|
|
}
|
|
} while (again && retries < 100);
|
|
|
|
TEST_ASSERT_FALSE(again);
|
|
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
|
|
/* transport cause → last transport rc preserved, NOT rewritten */
|
|
TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc);
|
|
|
|
printf("test_decide_transport_exhaustion_keeps_last_rc: PASS\n");
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Task 3.2 — Reconnect recovery (xcecli_ctx_recover) RED→GREEN tests.
|
|
*
|
|
* After a transport failure the disconnect path leaves ctx->state == XCE_CTX_ERR
|
|
* (xcectx.c:174) and may have injected a stale transport status into the single-
|
|
* slot ctx->wbuf (xcectx.c:180). xcecli_ctx_recover repairs the ctx after a
|
|
* successful reconnect, with NO socket I/O so it is unit-testable:
|
|
* - CI2 state reset: ERR → RUN (only ERR; RUN/DOWN/INIT left as-is) so the
|
|
* next xce_* op passes the xceapi XCE_CTX_RUN guard instead of -EINVAL.
|
|
* - CI1 stale-reply drain: any entry already sitting in wbuf is drained with a
|
|
* non-blocking get, so the next attempt's cbuf_get cannot consume a prior-
|
|
* attempt reply.
|
|
*
|
|
* Requirements: 1.2 (complete next request against new primary, no caller
|
|
* change), 4.4 (transport → reconnect/rotate and resume).
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* CI2: an ERR-state ctx is restored to RUN (the reported -EINVAL bug fix). */
|
|
static void test_recover_resets_err_state_to_run(void)
|
|
{
|
|
api_ctx_t ac;
|
|
memset(&ac, 0, sizeof(ac));
|
|
ac.ctx.state = XCE_CTX_ERR;
|
|
/* wbuf left un-init: recover must tolerate a recover that has nothing to
|
|
* drain (init it so the drain path is also exercised safely). */
|
|
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
|
|
|
|
int rc = xcecli_ctx_recover(&ac);
|
|
cbuf_fini(&ac.ctx.wbuf);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state);
|
|
|
|
printf("test_recover_resets_err_state_to_run: PASS\n");
|
|
}
|
|
|
|
/* CI2: a ctx already in RUN is left in RUN; only ERR→RUN is performed. */
|
|
static void test_recover_leaves_run_state_untouched(void)
|
|
{
|
|
api_ctx_t ac;
|
|
memset(&ac, 0, sizeof(ac));
|
|
ac.ctx.state = XCE_CTX_RUN;
|
|
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
|
|
|
|
int rc = xcecli_ctx_recover(&ac);
|
|
cbuf_fini(&ac.ctx.wbuf);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state);
|
|
|
|
printf("test_recover_leaves_run_state_untouched: PASS\n");
|
|
}
|
|
|
|
/* CI2: a DOWN ctx is NOT flipped to RUN — only an ERR transport state is
|
|
* repaired; a deliberately-stopped ctx stays down. */
|
|
static void test_recover_leaves_down_state_untouched(void)
|
|
{
|
|
api_ctx_t ac;
|
|
memset(&ac, 0, sizeof(ac));
|
|
ac.ctx.state = XCE_CTX_DOWN;
|
|
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
|
|
|
|
int rc = xcecli_ctx_recover(&ac);
|
|
cbuf_fini(&ac.ctx.wbuf);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(XCE_CTX_DOWN, ac.ctx.state);
|
|
|
|
printf("test_recover_leaves_down_state_untouched: PASS\n");
|
|
}
|
|
|
|
/* CI1: a stale status already sitting in wbuf is drained, so a subsequent
|
|
* non-blocking get finds the channel EMPTY (the prior-attempt reply was
|
|
* discarded, not handed to the next attempt's cbuf_get). */
|
|
static void test_recover_drains_stale_wbuf_entry(void)
|
|
{
|
|
api_ctx_t ac;
|
|
memset(&ac, 0, sizeof(ac));
|
|
ac.ctx.state = XCE_CTX_ERR;
|
|
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
|
|
|
|
/* mirror handle_disconnect: a stale transport status sits in the channel */
|
|
int stale = -ECONNRESET;
|
|
TEST_ASSERT_EQUAL_INT(0, cbuf_tryput(&ac.ctx.wbuf, &stale));
|
|
|
|
int rc = xcecli_ctx_recover(&ac);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state);
|
|
|
|
/* the stale entry was drained: a non-blocking get now finds nothing */
|
|
int drained = 0;
|
|
int g = cbuf_tryget(&ac.ctx.wbuf, &drained);
|
|
cbuf_fini(&ac.ctx.wbuf);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EAGAIN, g); /* channel empty after recover */
|
|
|
|
printf("test_recover_drains_stale_wbuf_entry: PASS\n");
|
|
}
|
|
|
|
/* Guard: a NULL api_ctx is rejected without crashing. */
|
|
static void test_recover_null_ctx_safe(void)
|
|
{
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_ctx_recover(NULL));
|
|
printf("test_recover_null_ctx_safe: PASS\n");
|
|
}
|
|
|
|
/* ── Suite entry point ──────────────────────────────────────────────── */
|
|
|
|
void test_libxce_retry(void)
|
|
{
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_retry_defaults_are_reachable);
|
|
RUN_TEST(test_fake_masters_hold_three_candidates);
|
|
RUN_TEST(test_master_cursor_advances_and_wraps);
|
|
RUN_TEST(test_status_table_is_complete_and_resolves);
|
|
RUN_TEST(test_status_table_redirect_hint_flag);
|
|
RUN_TEST(test_thread_harness_runs_all_threads);
|
|
RUN_TEST(test_thread_harness_rejects_bad_args);
|
|
RUN_TEST(test_classify_matches_taxonomy_table);
|
|
RUN_TEST(test_classify_control_codes_never_success);
|
|
RUN_TEST(test_classify_dup_replay_rule);
|
|
RUN_TEST(test_classify_zero_is_success);
|
|
RUN_TEST(test_classify_unknown_is_terminal);
|
|
RUN_TEST(test_next_target_hint_first_then_consumed);
|
|
RUN_TEST(test_next_target_rotates_and_wraps);
|
|
RUN_TEST(test_next_target_hint_then_fallback_rotation);
|
|
RUN_TEST(test_next_target_rejects_bad_args);
|
|
RUN_TEST(test_capture_redirect_hint_present);
|
|
RUN_TEST(test_capture_redirect_hint_absent_stays_empty);
|
|
RUN_TEST(test_capture_redirect_hint_truncates_long_host);
|
|
RUN_TEST(test_capture_redirect_hint_null_ctx_safe);
|
|
RUN_TEST(test_reply_wait_wake_releases_blocked_get);
|
|
RUN_TEST(test_decide_no_primary_exhausts_to_not_ready);
|
|
RUN_TEST(test_decide_no_primary_then_becomes_primary);
|
|
RUN_TEST(test_decide_retry_now_has_no_backoff_delay);
|
|
RUN_TEST(test_decide_backoff_grows_and_is_bounded);
|
|
RUN_TEST(test_decide_terminal_surfaces_unchanged);
|
|
RUN_TEST(test_decide_dup_on_resent_write_is_success);
|
|
RUN_TEST(test_decide_transport_exhaustion_keeps_last_rc);
|
|
RUN_TEST(test_recover_resets_err_state_to_run);
|
|
RUN_TEST(test_recover_leaves_run_state_untouched);
|
|
RUN_TEST(test_recover_leaves_down_state_untouched);
|
|
RUN_TEST(test_recover_drains_stale_wbuf_entry);
|
|
RUN_TEST(test_recover_null_ctx_safe);
|
|
UNITY_END();
|
|
}
|