lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/lib/testsuite/test_libxce_retry.c
Rongsong Shen fa65e794ec fix(libxce-failover): known-dead ctx (XCE_CTX_ERR) triggers reconnect instead of surfacing -EINVAL 
Live 3-node E2E revealed the reported 'Invalid argument' bug was still
reachable: after the primary dies, handle_disconnect leaves ctx.state=
XCE_CTX_ERR, so a FRESH xce_* call returns -EINVAL from the xceapi guard
before reaching a primary. The classifier mapped -EINVAL to TERMINAL, so the
loop bailed without rotating/recovering. decide() now detects the known-dead
ctx and treats it as a prompt reconnect cause (RETRY_NOW), surfacing the
distinct cluster-not-ready code on budget exhaustion, never -EINVAL.
Regression test: test_decide_known_dead_ctx_retries_not_terminal.
2026-06-17 04:46:01 +00:00

1720 lines
70 KiB
C
Raw Blame History

/*
* test_libxce_retry.c — reusable libxce-failover test harness + harness
* self-tests (spec libxce-failover, task 1.2 ADAPTED).
*
* This file builds the GREEN, reusable harness that the later component tasks
* consume to write their own RED→GREEN tests:
* - task 2.1 (StatusClassifier) → drives FO_STATUS_TABLE
* - task 2.2 (EndpointRotator) → drives the fake-master cursor
* - task 2.3 (reply-capture) → drives the redirect-hint rows
* - task 3.1/3.2 (decision helper) → drives the table + cursor
* - task 5.1/6.1 (concurrency) → drive fo_run_threads()
*
* It deliberately references NO not-yet-existing failover production symbols
* (xce_failover_classify / xcecli_retry_next / xce_failover_rotate); those are
* added by the component tasks. The lib build+test MUST stay GREEN after this
* task, so every fixture here is exercised by at least one real assertion.
*
* Requirements covered by the harness: 2.5, 4.1, 4.2, 4.3, 4.4, 4.5, 5.2, 8.3,
* 8.4 (status taxonomy); 2.1, 2.4 (master cursor); 6.3 (concurrency reuse).
* The retained retry-defaults test still covers 6.1, 6.2.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unity/unity.h>
#include "client_internal.h" /* api_ctx_t, xce_failover_state_t */
#include "failover_harness.h"
#include "mylock.h"
#include "request.h"
#include "xce/client_failover.h"
#include "xce/errnox.h"
/* ════════════════════════════════════════════════════════════════════════
* Harness implementation
* ════════════════════════════════════════════════════════════════════════ */
/* ── 1. Fake multi-master config + cursor ─────────────────────────────── */
int fo_make_fake_masters(xce_cluster_config_t *cfg)
{
static const struct {
const char *host;
int port;
} masters[FO_FAKE_MASTERS] = {
{ "10.20.0.1", 7001 },
{ "10.20.0.2", 7002 },
{ "10.20.0.3", 7003 },
};
memset(cfg, 0, sizeof(*cfg));
for (int i = 0; i < FO_FAKE_MASTERS && i < MAX_HA_CANDIDATES; i++) {
strncpy(cfg->candidates[i].hostname, masters[i].host,
XCE_MAXHOSTNAME_LEN - 1);
cfg->candidates[i].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
cfg->candidates[i].port = masters[i].port;
}
cfg->candidates_count =
(FO_FAKE_MASTERS < MAX_HA_CANDIDATES) ? FO_FAKE_MASTERS
: MAX_HA_CANDIDATES;
return cfg->candidates_count;
}
void fo_cursor_init(fo_master_cursor_t *cur, const xce_cluster_config_t *cfg)
{
cur->cfg = cfg;
cur->cur = 0;
}
int fo_cursor_advance(fo_master_cursor_t *cur)
{
if (!cur->cfg || cur->cfg->candidates_count <= 0)
return 0;
cur->cur = (cur->cur + 1) % cur->cfg->candidates_count;
return cur->cur;
}
const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur)
{
if (!cur->cfg || cur->cfg->candidates_count <= 0)
return NULL;
return &cur->cfg->candidates[cur->cur];
}
/* ── 2. Controller-status taxonomy table ──────────────────────────────── */
const fo_status_row_t FO_STATUS_TABLE[] = {
/* success */
{ "OK", 0, false, false, FO_ACT_SUCCESS,
"request served by primary" },
/* redirect (a primary exists elsewhere) — rotate now, hint first if any */
{ "E_XCE_REDIRECT+hint", E_XCE_REDIRECT, true, false, FO_ACT_RETRY_NOW,
"standby redirects with primary address hint (8.3)" },
{ "E_XCE_REDIRECT", E_XCE_REDIRECT, false, false, FO_ACT_RETRY_NOW,
"standby redirects without address — rotate master list (8.4)" },
/* standby (op is primary-only) — treat as redirect, rotate now (4.3) */
{ "E_XCE_STANDBY", E_XCE_STANDBY, false, false, FO_ACT_RETRY_NOW,
"controller in standby; op is primary-only (4.3)" },
/* no-primary / not-ready — rotate after backoff (4.1) */
{ "E_XCE_NOMASTER", E_XCE_NOMASTER, false, false, FO_ACT_RETRY_BACKOFF,
"standby knows no primary yet — backoff + rotate (4.1)" },
{ "E_XCE_NOT_READY", E_XCE_NOT_READY, false, false, FO_ACT_RETRY_BACKOFF,
"controller initializing/electing — backoff + rotate (4.1)" },
/* idempotent-replay duplicate (write replay) — DUP/EEXIST (5.2) */
{ "E_XCE_DUP_JOB", E_XCE_DUP_JOB, false, false, FO_ACT_TERMINAL,
"duplicate job; SUCCESS only on a re-sent write (5.2, attempt-aware)" },
{ "-EEXIST", -EEXIST, false, true, FO_ACT_TERMINAL,
"duplicate via -EEXIST; SUCCESS only on a re-sent write (5.2)" },
/* transport errors — reconnect/rotate now (4.4) */
{ "-ECONNREFUSED", -ECONNREFUSED, false, true, FO_ACT_RETRY_NOW,
"connection refused — reconnect/rotate (4.4)" },
{ "-ETIMEDOUT", -ETIMEDOUT, false, true, FO_ACT_RETRY_NOW,
"connection timeout — reconnect/rotate (4.4)" },
{ "-ECONNRESET", -ECONNRESET, false, true, FO_ACT_RETRY_NOW,
"peer reset — reconnect/rotate (4.4)" },
/* terminal / non-retriable — surface immediately (4.5) */
{ "-EINVAL", -EINVAL, false, true, FO_ACT_TERMINAL,
"invalid request — surface, no retry (4.5)" },
};
const size_t FO_STATUS_TABLE_LEN =
sizeof(FO_STATUS_TABLE) / sizeof(FO_STATUS_TABLE[0]);
const fo_status_row_t *fo_status_lookup(int status)
{
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
if (FO_STATUS_TABLE[i].status == status)
return &FO_STATUS_TABLE[i];
}
return NULL;
}
/* ── 3. Generic N-thread concurrency harness ──────────────────────────── */
typedef struct {
fo_thread_fn fn;
void *ctx;
int idx;
} fo_thread_arg_t;
static void *fo_thread_trampoline(void *p)
{
fo_thread_arg_t *a = (fo_thread_arg_t *)p;
a->fn(a->ctx, a->idx);
return NULL;
}
int fo_run_threads(int n, fo_thread_fn fn, void *ctx)
{
if (n <= 0 || !fn)
return -EINVAL;
pthread_t *tids = calloc((size_t)n, sizeof(*tids));
fo_thread_arg_t *args = calloc((size_t)n, sizeof(*args));
if (!tids || !args) {
free(tids);
free(args);
return -ENOMEM;
}
int spawned = 0;
int rc = 0;
for (int i = 0; i < n; i++) {
args[i].fn = fn;
args[i].ctx = ctx;
args[i].idx = i;
int e = pthread_create(&tids[i], NULL, fo_thread_trampoline, &args[i]);
if (e != 0) {
rc = -e;
break;
}
spawned++;
}
for (int i = 0; i < spawned; i++)
pthread_join(tids[i], NULL);
free(tids);
free(args);
return rc;
}
/* ════════════════════════════════════════════════════════════════════════
* Harness self-tests (GREEN) — each fixture proven by real assertions
* ════════════════════════════════════════════════════════════════════════ */
/* Req 6.1, 6.2: the retry-tunables struct + safe production defaults are
* retained and reachable for the failover loop. */
static void test_retry_defaults_are_reachable(void)
{
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
TEST_ASSERT_EQUAL_INT(20, cfg.max_retries);
TEST_ASSERT_EQUAL_INT(50, cfg.initial_retry_ms);
TEST_ASSERT_EQUAL_INT(5000, cfg.max_retry_ms);
TEST_ASSERT_TRUE(cfg.backoff_multiplier > 1.0);
printf("test_retry_defaults_are_reachable: PASS\n");
}
/* Fixture 1: fake multi-master config holds 3 distinct candidates. */
static void test_fake_masters_hold_three_candidates(void)
{
xce_cluster_config_t cfg;
int n = fo_make_fake_masters(&cfg);
TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, n);
TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("10.20.0.1", cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(7001, cfg.candidates[0].port);
TEST_ASSERT_EQUAL_STRING("10.20.0.3", cfg.candidates[2].hostname);
TEST_ASSERT_EQUAL_INT(7003, cfg.candidates[2].port);
printf("test_fake_masters_hold_three_candidates: PASS\n");
}
/* Fixture 1: cursor advances through masters and wraps (2.1, 2.4, 8.4). */
static void test_master_cursor_advances_and_wraps(void)
{
xce_cluster_config_t cfg;
fo_make_fake_masters(&cfg);
fo_master_cursor_t cur;
fo_cursor_init(&cur, &cfg);
TEST_ASSERT_EQUAL_INT(0, cur.cur);
TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname);
TEST_ASSERT_EQUAL_INT(1, fo_cursor_advance(&cur));
TEST_ASSERT_EQUAL_STRING("10.20.0.2", fo_cursor_current(&cur)->hostname);
TEST_ASSERT_EQUAL_INT(2, fo_cursor_advance(&cur));
TEST_ASSERT_EQUAL_STRING("10.20.0.3", fo_cursor_current(&cur)->hostname);
/* wrap back to master 0 */
TEST_ASSERT_EQUAL_INT(0, fo_cursor_advance(&cur));
TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname);
printf("test_master_cursor_advances_and_wraps: PASS\n");
}
/* Fixture 2: taxonomy table is complete — every status the classifier must
* handle is present, lookups resolve, and the status codes equal the real
* errnox.h values (2.5, 4.1-4.5, 5.2, 8.3, 8.4). */
static void test_status_table_is_complete_and_resolves(void)
{
/* every mandated controller status + transport error is represented */
TEST_ASSERT_NOT_NULL(fo_status_lookup(0));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_REDIRECT));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_STANDBY));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOMASTER));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOT_READY));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_DUP_JOB));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-EEXIST));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNREFUSED));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ETIMEDOUT));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNRESET));
/* status codes resolve to the real errnox.h values (ordered enum) */
TEST_ASSERT_EQUAL_INT(XCE_ERRNO_BASE, E_XCE_BAD_JOB);
TEST_ASSERT_TRUE(E_XCE_REDIRECT == E_XCE_NOMASTER + 1);
TEST_ASSERT_TRUE(E_XCE_NOT_READY == E_XCE_REDIRECT + 1);
TEST_ASSERT_TRUE(E_XCE_STANDBY == E_XCE_NOT_READY + 1);
/* expected actions line up with the design's StatusClassifier (first send) */
TEST_ASSERT_EQUAL_INT(FO_ACT_SUCCESS, fo_status_lookup(0)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(E_XCE_REDIRECT)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(E_XCE_STANDBY)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
fo_status_lookup(E_XCE_NOMASTER)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
fo_status_lookup(E_XCE_NOT_READY)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(-ECONNRESET)->expect);
printf("test_status_table_is_complete_and_resolves: PASS\n");
}
/* Fixture 2: the redirect-hint flag distinguishes hint vs. no-hint rows (8.3). */
static void test_status_table_redirect_hint_flag(void)
{
bool saw_hint = false;
bool saw_no_hint = false;
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
if (FO_STATUS_TABLE[i].status == E_XCE_REDIRECT) {
if (FO_STATUS_TABLE[i].has_hint)
saw_hint = true;
else
saw_no_hint = true;
}
}
TEST_ASSERT_TRUE(saw_hint); /* REDIRECT+hint row present (8.3) */
TEST_ASSERT_TRUE(saw_no_hint); /* REDIRECT no-hint row present (8.4) */
printf("test_status_table_redirect_hint_flag: PASS\n");
}
/* Fixture 3: concurrency harness runs N threads to completion (6.3 reuse). */
#define FO_TEST_NTHREADS 16
typedef struct {
pthread_mutex_t lock;
int counter;
} fo_selftest_ctx_t;
static void fo_selftest_worker(void *ctx, int idx)
{
(void)idx;
fo_selftest_ctx_t *c = (fo_selftest_ctx_t *)ctx;
pthread_mutex_lock(&c->lock);
c->counter++;
pthread_mutex_unlock(&c->lock);
}
static void test_thread_harness_runs_all_threads(void)
{
fo_selftest_ctx_t ctx;
pthread_mutex_init(&ctx.lock, NULL);
ctx.counter = 0;
int rc = fo_run_threads(FO_TEST_NTHREADS, fo_selftest_worker, &ctx);
pthread_mutex_destroy(&ctx.lock);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(FO_TEST_NTHREADS, ctx.counter);
printf("test_thread_harness_runs_all_threads: PASS\n");
}
/* Fixture 3: harness rejects bad arguments. */
static void test_thread_harness_rejects_bad_args(void)
{
TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(0, fo_selftest_worker, NULL));
TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(4, NULL, NULL));
printf("test_thread_harness_rejects_bad_args: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.1 — StatusClassifier (xce_failover_classify) RED→GREEN tests
* ════════════════════════════════════════════════════════════════════════ */
/* Map the harness' expected-action category to the real classifier enum. The
* two enums share order by design, but we map explicitly so a future reorder of
* either enum is caught here rather than silently passing. */
static xce_fo_action_t fo_expected_to_action(fo_expected_action_t e)
{
switch (e) {
case FO_ACT_SUCCESS: return XCE_FO_SUCCESS;
case FO_ACT_RETRY_NOW: return XCE_FO_RETRY_NOW;
case FO_ACT_RETRY_BACKOFF: return XCE_FO_RETRY_BACKOFF;
case FO_ACT_TERMINAL: return XCE_FO_TERMINAL;
}
return XCE_FO_TERMINAL;
}
/* Req 2.5, 4.1, 4.2, 4.3, 4.5: every taxonomy row's first-send classification
* (is_write=false, attempt=0) matches the design's StatusClassifier mapping. */
static void test_classify_matches_taxonomy_table(void)
{
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
const fo_status_row_t *row = &FO_STATUS_TABLE[i];
xce_fo_action_t want = fo_expected_to_action(row->expect);
xce_fo_action_t got =
xce_failover_classify(row->status, /*is_write=*/false,
/*attempt=*/0);
char msg[128];
snprintf(msg, sizeof(msg), "row '%s' (status=%d): want %d got %d",
row->name, row->status, (int)want, (int)got);
TEST_ASSERT_EQUAL_INT_MESSAGE(want, got, msg);
}
printf("test_classify_matches_taxonomy_table: PASS\n");
}
/* Req 2.5: REDIRECT/STANDBY/NOMASTER/NOT_READY are never reported as success,
* regardless of is_write/attempt. */
static void test_classify_control_codes_never_success(void)
{
const int codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, E_XCE_NOMASTER,
E_XCE_NOT_READY };
for (size_t i = 0; i < sizeof(codes) / sizeof(codes[0]); i++) {
TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS,
xce_failover_classify(codes[i], false, 0));
TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS,
xce_failover_classify(codes[i], true, 1));
/* accepted in either sign */
TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS,
xce_failover_classify(-codes[i], true, 1));
}
printf("test_classify_control_codes_never_success: PASS\n");
}
/* Req 5.2: idempotent-replay rule — DUP_JOB/EEXIST on a RE-SENT WRITE is a
* success; the first send (or a read) is a genuine duplicate → terminal. */
static void test_classify_dup_replay_rule(void)
{
/* re-sent write (is_write && attempt>0) → SUCCESS */
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
xce_failover_classify(-E_XCE_DUP_JOB, true, 1));
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
xce_failover_classify(E_XCE_DUP_JOB, true, 1));
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
xce_failover_classify(-EEXIST, true, 1));
/* first send → genuine duplicate → TERMINAL */
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
xce_failover_classify(-E_XCE_DUP_JOB, true, 0));
/* non-write replay → TERMINAL */
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
xce_failover_classify(-E_XCE_DUP_JOB, false, 1));
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
xce_failover_classify(-EEXIST, false, 1));
printf("test_classify_dup_replay_rule: PASS\n");
}
/* Req 3.2 / general: status 0 is always success. */
static void test_classify_zero_is_success(void)
{
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
xce_failover_classify(0, false, 0));
TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS,
xce_failover_classify(0, true, 3));
printf("test_classify_zero_is_success: PASS\n");
}
/* Req 4.5: unknown / unrecognized statuses are conservatively terminal. */
static void test_classify_unknown_is_terminal(void)
{
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
xce_failover_classify(-EACCES, false, 0));
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
xce_failover_classify(-ENOENT, true, 1));
TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL,
xce_failover_classify(-12345, false, 0));
printf("test_classify_unknown_is_terminal: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.2 — EndpointRotator target selection (xce_failover_next_target)
* RED→GREEN tests. Pure, socket-free target-selection logic; the
* live connect path is covered by E2E task 6.2.
* ════════════════════════════════════════════════════════════════════════ */
/* Build a 3-master client config (distinct host:port) the rotator selects over.
* Strings are static-duration so the api_ctx may borrow them by pointer. */
static void fo_make_client_masters(xce_client_config_t *conf)
{
static char *hosts[N_MAX_MASTERS] = { "10.30.0.1", "10.30.0.2",
"10.30.0.3" };
static int ports[N_MAX_MASTERS] = { 8001, 8002, 8003 };
memset(conf, 0, sizeof(*conf));
conf->nmasters = N_MAX_MASTERS;
for (int i = 0; i < N_MAX_MASTERS; i++) {
conf->masters[i] = hosts[i];
conf->master_ports[i] = ports[i];
}
}
/* Minimal api_ctx for target-selection tests: only conf, the failover cursor,
* and the ctx redirect-hint fields are exercised. No endpoint/socket touched. */
static void fo_make_api_ctx(api_ctx_t *ac, xce_client_config_t *conf)
{
memset(ac, 0, sizeof(*ac));
ac->conf = conf;
ac->failover.cur_master = 0;
}
static void fo_set_hint(api_ctx_t *ac, const char *host, uint16_t port)
{
strncpy(ac->ctx.redirect_hint_host, host, XCE_MAXHOSTNAME_LEN - 1);
ac->ctx.redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1] = '\0';
ac->ctx.redirect_hint_port = port;
}
/* Req 2.2, 8.4: with a redirect hint set, next_target returns the hint
* host/port and CONSUMES (clears) it; the following call ignores the stale
* hint and advances the master cursor instead. */
static void test_next_target_hint_first_then_consumed(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
fo_set_hint(&ac, "10.99.0.7", 9100);
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
uint16_t port = 0;
int rc = xce_failover_next_target(&ac, host, sizeof(host), &port);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_STRING("10.99.0.7", host);
TEST_ASSERT_EQUAL_INT(9100, port);
/* hint consumed: cleared on the ctx, cursor untouched */
TEST_ASSERT_EQUAL_STRING("", ac.ctx.redirect_hint_host);
TEST_ASSERT_EQUAL_INT(0, ac.failover.cur_master);
/* next call has no hint → advances cursor to master 1 */
rc = xce_failover_next_target(&ac, host, sizeof(host), &port);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_STRING("10.30.0.2", host);
TEST_ASSERT_EQUAL_INT(8002, port);
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
printf("test_next_target_hint_first_then_consumed: PASS\n");
}
/* Req 2.1, 8.4: with no hint, successive next_target calls advance through the
* configured masters and WRAP around; the cursor lives on failover.cur_master. */
static void test_next_target_rotates_and_wraps(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
uint16_t port = 0;
/* from cur_master=0, first rotation advances to master 1 */
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
&port));
TEST_ASSERT_EQUAL_STRING("10.30.0.2", host);
TEST_ASSERT_EQUAL_INT(8002, port);
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
&port));
TEST_ASSERT_EQUAL_STRING("10.30.0.3", host);
TEST_ASSERT_EQUAL_INT(8003, port);
TEST_ASSERT_EQUAL_INT(2, ac.failover.cur_master);
/* wrap back to master 0 */
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
&port));
TEST_ASSERT_EQUAL_STRING("10.30.0.1", host);
TEST_ASSERT_EQUAL_INT(8001, port);
TEST_ASSERT_EQUAL_INT(0, ac.failover.cur_master);
printf("test_next_target_rotates_and_wraps: PASS\n");
}
/* Req 4.4, 8.4: after a hint is consumed, the very next call advances to the
* next master — correctness is independent of the (one-shot) hint. */
static void test_next_target_hint_then_fallback_rotation(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
/* start the cursor at master 1 to prove fallback advances from current */
ac.failover.cur_master = 1;
fo_set_hint(&ac, "10.99.0.9", 9200);
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
uint16_t port = 0;
/* hint first */
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
&port));
TEST_ASSERT_EQUAL_STRING("10.99.0.9", host);
TEST_ASSERT_EQUAL_INT(9200, port);
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); /* cursor untouched */
/* fallback: advances from master 1 → master 2 */
TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host),
&port));
TEST_ASSERT_EQUAL_STRING("10.30.0.3", host);
TEST_ASSERT_EQUAL_INT(8003, port);
TEST_ASSERT_EQUAL_INT(2, ac.failover.cur_master);
printf("test_next_target_hint_then_fallback_rotation: PASS\n");
}
/* Guard: NULL/invalid arguments are rejected without touching state. */
static void test_next_target_rejects_bad_args(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
uint16_t port = 0;
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_failover_next_target(NULL, host, sizeof(host),
&port));
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_failover_next_target(&ac, NULL, sizeof(host),
&port));
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_failover_next_target(&ac, host, 0, &port));
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_failover_next_target(&ac, host, sizeof(host),
NULL));
printf("test_next_target_rejects_bad_args: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — RedirectCapture (xce_ctx_capture_redirect_hint) RED→GREEN tests.
*
* handle_reply() copies the redirect address carried on a redirect reply into
* the ctx's redirect_hint_host/port BEFORE the capnp-owned reply is freed. The
* copy logic is factored into a non-static seam, xce_ctx_capture_redirect_hint,
* so it can be unit-tested without a live server.
*
* Requirements: 2.2 (capture redirect addr as a best-effort hint), 8.3 (try the
* hinted address first — capture is the prerequisite).
* ════════════════════════════════════════════════════════════════════════ */
/* Build a bare xce_ctx_t with an empty redirect hint, without touching any
* iomgr/ssl/endpoint state (capture is pure ctx-field manipulation). */
static void fo_ctx_clear_hint(xce_ctx_t *ctx)
{
memset(ctx, 0, sizeof(*ctx));
ctx->redirect_hint_host[0] = '\0';
ctx->redirect_hint_port = 0;
}
/* Req 2.2, 8.3: a present host+port is copied into the ctx hint fields. */
static void test_capture_redirect_hint_present(void)
{
xce_ctx_t ctx;
fo_ctx_clear_hint(&ctx);
xce_ctx_capture_redirect_hint(&ctx, "10.40.0.5", 9300);
TEST_ASSERT_EQUAL_STRING("10.40.0.5", ctx.redirect_hint_host);
TEST_ASSERT_EQUAL_INT(9300, ctx.redirect_hint_port);
printf("test_capture_redirect_hint_present: PASS\n");
}
/* Req 2.2: when no address is present (host NULL/empty or port 0) the hint is
* left empty — capture must not invent an address. */
static void test_capture_redirect_hint_absent_stays_empty(void)
{
xce_ctx_t ctx;
/* NULL host → hint untouched */
fo_ctx_clear_hint(&ctx);
xce_ctx_capture_redirect_hint(&ctx, NULL, 9300);
TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host);
TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port);
/* empty host → hint untouched */
fo_ctx_clear_hint(&ctx);
xce_ctx_capture_redirect_hint(&ctx, "", 9300);
TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host);
TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port);
/* port 0 → hint untouched (no usable target) */
fo_ctx_clear_hint(&ctx);
xce_ctx_capture_redirect_hint(&ctx, "10.40.0.5", 0);
TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host);
TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port);
printf("test_capture_redirect_hint_absent_stays_empty: PASS\n");
}
/* Req 2.2: an over-long host is bounded-copied (NUL-terminated, no overflow);
* the capnp-owned string the controller may send is not length-checked here. */
static void test_capture_redirect_hint_truncates_long_host(void)
{
xce_ctx_t ctx;
fo_ctx_clear_hint(&ctx);
char longhost[XCE_MAXHOSTNAME_LEN * 2];
memset(longhost, 'h', sizeof(longhost) - 1);
longhost[sizeof(longhost) - 1] = '\0';
xce_ctx_capture_redirect_hint(&ctx, longhost, 9400);
/* copied, NUL-terminated, and never longer than the field bound */
TEST_ASSERT_EQUAL_INT(XCE_MAXHOSTNAME_LEN - 1,
(int)strlen(ctx.redirect_hint_host));
TEST_ASSERT_EQUAL_INT('\0',
ctx.redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1]);
TEST_ASSERT_EQUAL_INT(9400, ctx.redirect_hint_port);
printf("test_capture_redirect_hint_truncates_long_host: PASS\n");
}
/* Guard: a NULL ctx is rejected without crashing. */
static void test_capture_redirect_hint_null_ctx_safe(void)
{
/* must not crash */
xce_ctx_capture_redirect_hint(NULL, "10.40.0.5", 9300);
printf("test_capture_redirect_hint_null_ctx_safe: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 2.3 — ReplyWaitWake RED→GREEN test.
*
* On an endpoint error/peer reset, the existing disconnect path injects a
* transport error into ctx->wbuf so a blocked cbuf_get returns instead of
* hanging. This test proves the wake mechanism end-to-end on a real wbuf: a
* thread blocks in cbuf_get, the "disconnect" injects -ECONNRESET, and the
* blocked waiter wakes with exactly that transport error (Req 4.4).
* ════════════════════════════════════════════════════════════════════════ */
typedef struct {
cbuf_t wbuf;
int woke_status; /* what the blocked cbuf_get observed */
int got; /* cbuf_get return code */
} fo_wake_ctx_t;
static void fo_wake_waiter(void *p, int idx)
{
fo_wake_ctx_t *w = (fo_wake_ctx_t *)p;
if (idx == 0) {
/* waiter: block until the disconnect injects an error */
w->got = cbuf_get(&w->wbuf, &w->woke_status);
} else {
/* injector: mirror handle_disconnect's wbuf error injection. A short
* spin lets the waiter reach cbuf_get first so we exercise the wake,
* not a pre-filled buffer. */
for (volatile int i = 0; i < 2000000; i++) {
}
int err = -ECONNRESET;
cbuf_tryput(&w->wbuf, &err);
}
}
/* Req 4.4: a pending cbuf_get is released with the injected transport error so
* an in-flight xce_* call returns instead of hanging across an endpoint reset. */
static void test_reply_wait_wake_releases_blocked_get(void)
{
fo_wake_ctx_t w;
memset(&w, 0, sizeof(w));
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&w.wbuf, 1, sizeof(int)));
int rc = fo_run_threads(2, fo_wake_waiter, &w);
cbuf_fini(&w.wbuf);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_TRUE(w.got >= 0); /* cbuf_get returned, did not hang */
TEST_ASSERT_EQUAL_INT(-ECONNRESET, w.woke_status);
printf("test_reply_wait_wake_releases_blocked_get: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 3.1 — decision loop (xcecli_retry_decide) RED→GREEN tests.
*
* xcecli_retry_decide is the PURE core of the failover decision loop: classify
* → apply SUCCESS/DUP/TERMINAL to *rc → advance attempt → compute backoff →
* handle budget exhaustion → report the chosen action. It does NOT sleep, open
* sockets, rotate, or log. The cursor-advancing side (xce_failover_next_target)
* is driven separately to prove the become-primary path advances the cursor.
*
* Requirements: 1.2, 3.1, 3.2 (budget), 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2.
* ════════════════════════════════════════════════════════════════════════ */
/* A small, deterministic retry config for the decide tests: a low max_retries
* so budget exhaustion is reached quickly, a tiny initial delay, a modest cap. */
static xce_retry_config_t fo_test_retry_cfg(void)
{
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 3;
cfg.initial_retry_ms = 10;
cfg.max_retry_ms = 100;
cfg.backoff_multiplier = 2.0;
return cfg;
}
/* Req 3.3, 3.4: a controller that keeps reporting no-primary (NOMASTER /
* NOT_READY) is retried with backoff until the budget is exhausted; decide then
* returns false with *rc == -E_XCE_NOT_READY — the distinct cluster-not-ready
* code, never an opaque -EINVAL. */
static void test_decide_no_primary_exhausts_to_not_ready(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
bool again;
int retries = 0;
do {
rc = E_XCE_NOMASTER; /* every attempt: no primary known */
again = xcecli_retry_decide(&ac, &rc, /*is_write=*/false, &cfg, &st,
&action);
if (again) {
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
retries++;
}
} while (again && retries < 100); /* hard cap so a bug can't spin forever */
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
/* exactly max_retries retries happened before exhaustion */
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
printf("test_decide_no_primary_exhausts_to_not_ready: PASS\n");
}
/* Req 3.2 / become-primary: no-primary then a served reply (status 0) → the
* loop stops with success and the master cursor advanced across the retries
* (the rotation side runs in the wrapper; here we advance it explicitly to
* mirror that and assert the sticky cursor moved). */
static void test_decide_no_primary_then_becomes_primary(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
/* attempt 0: no primary → retry (decide says backoff). */
rc = E_XCE_NOMASTER;
bool a0 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(a0);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
/* mirror the wrapper: rotate advances the cursor (no hint set) */
char host[XCE_MAXHOSTNAME_LEN];
uint16_t port;
xce_failover_next_target(&ac, host, sizeof(host), &port);
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
/* attempt 1: served by the new primary → success, loop stops. */
rc = 0;
bool a1 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_FALSE(a1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* cursor stayed advanced (sticky primary = where success landed) */
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
printf("test_decide_no_primary_then_becomes_primary: PASS\n");
}
/* Req 4.2, 4.4: redirect / standby / transport causes are RETRY_NOW with NO
* backoff delay applied (st->delay_ms == 0), so the wrapper rotates promptly. */
static void test_decide_retry_now_has_no_backoff_delay(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
const int now_codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, -ECONNRESET,
-ECONNREFUSED, -ETIMEDOUT };
for (size_t i = 0; i < sizeof(now_codes) / sizeof(now_codes[0]); i++) {
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = now_codes[i];
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* no backoff for RETRY_NOW */
}
printf("test_decide_retry_now_has_no_backoff_delay: PASS\n");
}
/* Req 4.1, 6.3: no-primary causes get a jittered exponential backoff — the
* delay grows across retries and never exceeds max_retry_ms. */
static void test_decide_backoff_grows_and_is_bounded(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 12;
cfg.initial_retry_ms = 10;
cfg.max_retry_ms = 200;
cfg.backoff_multiplier = 2.0;
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
int prev_base = 0;
for (int i = 0; i < 8; i++) {
rc = E_XCE_NOT_READY;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
/* delay is positive and never exceeds the cap */
TEST_ASSERT_TRUE(st.delay_ms > 0);
TEST_ASSERT_TRUE(st.delay_ms <= cfg.max_retry_ms);
/* the un-jittered base (initial * mult^attempt, capped) is monotone
* non-decreasing; once we hit the cap it stays there. The delay floor
* grows: by attempt 5+ it must exceed the very first delay. */
if (i >= 5)
TEST_ASSERT_TRUE(st.delay_ms >= prev_base);
if (i == 0)
prev_base = st.delay_ms;
}
printf("test_decide_backoff_grows_and_is_bounded: PASS\n");
}
/* Req 4.5: a terminal (non-retriable) status stops the loop immediately with
* *rc unchanged — no retry, no -E_XCE_NOT_READY rewrite. */
static void test_decide_terminal_surfaces_unchanged(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
const int term_codes[] = { -EACCES, -EPERM, -EINVAL, -ENOENT };
for (size_t i = 0; i < sizeof(term_codes) / sizeof(term_codes[0]); i++) {
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = term_codes[i];
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(term_codes[i], rc); /* surfaced unchanged */
}
printf("test_decide_terminal_surfaces_unchanged: PASS\n");
}
/* Req 5.2: DUP on a re-sent write (is_write && attempt>0) → success (*rc = 0),
* loop stops. The controller's client-jobid replay guard means the write was
* already applied, so the retry is a success not a duplicate failure. */
static void test_decide_dup_on_resent_write_is_success(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
st.attempt = 1; /* a re-sent write */
int rc = -E_XCE_DUP_JOB;
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, /*is_write=*/true, &cfg, &st,
&action);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(0, rc); /* replay → success */
/* contrast: DUP on the FIRST send (attempt 0) is a genuine duplicate →
* terminal, *rc surfaced unchanged. */
xcecli_retry_t st0 = XCECLI_RETRY_INIT(&ac);
int rc0 = -E_XCE_DUP_JOB;
bool a0 =
xcecli_retry_decide(&ac, &rc0, true, &cfg, &st0, &action);
TEST_ASSERT_FALSE(a0);
TEST_ASSERT_EQUAL_INT(-E_XCE_DUP_JOB, rc0);
printf("test_decide_dup_on_resent_write_is_success: PASS\n");
}
/* Req 3.4 (D6): a transport-only cause that outlasts the budget keeps the LAST
* transport rc (not -E_XCE_NOT_READY — that mapping is reserved for no-primary
* exhaustion). Proves the two exhaustion causes are distinguished. */
static void test_decide_transport_exhaustion_keeps_last_rc(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
bool again;
int retries = 0;
do {
rc = -ECONNREFUSED; /* every attempt: transport failure */
again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
if (again) {
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
retries++;
}
} while (again && retries < 100);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
/* transport cause → last transport rc preserved, NOT rewritten */
TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc);
printf("test_decide_transport_exhaustion_keeps_last_rc: PASS\n");
}
/*
* Regression for the live-E2E bug: after the connected primary dies,
* handle_disconnect leaves ctx.state == XCE_CTX_ERR, so a FRESH xce_* call
* returns -EINVAL from the per-op xceapi guard BEFORE reaching a primary.
* decide() must treat the known-dead ctx as a prompt reconnect cause
* (RETRY_NOW, no backoff) rather than classifying -EINVAL as TERMINAL, and on
* budget exhaustion surface the distinct cluster-not-ready code, never -EINVAL.
*/
static void test_decide_known_dead_ctx_retries_not_terminal(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
xce_fo_action_t action;
/* Known-dead ctx + the guard's -EINVAL must RETRY (reconnect), not surface. */
ac.ctx.state = XCE_CTX_ERR;
int rc = -EINVAL;
bool a = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(a);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action); /* prompt rotate, no backoff */
TEST_ASSERT_EQUAL_INT(0, st.delay_ms);
/* ctx stays ERR (no real reconnect in this unit) → keep retrying until the
* budget is exhausted, then surface -E_XCE_NOT_READY (never -EINVAL). */
int guard = 0;
do {
rc = -EINVAL;
a = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
} while (a && ++guard < 100);
TEST_ASSERT_FALSE(a);
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
printf("test_decide_known_dead_ctx_retries_not_terminal: PASS\n");
}
/* Req 6.3: when MANY clients fail over simultaneously they must NOT retry in
* lockstep — randomized (jittered) backoff staggers their delays. Each client
* carries its OWN retry carrier (seeded from its own api_ctx pointer, exactly as
* XCECLI_RETRY_INIT does in production), so N independent carriers driven through
* the SAME no-primary backoff attempt must produce delays that VARY across
* clients (not a single shared value) while every delay stays within the
* [base/2, max_retry_ms] envelope. This drives the pure xcecli_retry_decide path
* — socket-free — for many concurrent carriers and proves the anti-thundering-
* herd property the single-carrier growth/bounds test does not. */
#define FO_JITTER_NCLIENTS 64
typedef struct {
api_ctx_t *ac; /* shared cfg source; seeds are per-carrier */
const xce_retry_config_t *cfg;
int delays[FO_JITTER_NCLIENTS]; /* one client's computed delay */
pthread_mutex_t lock;
} fo_jitter_ctx_t;
/* Each "client" runs one no-primary attempt through a FRESH retry carrier seeded
* from a distinct value (its thread index), mirroring how distinct api_ctx
* pointers seed distinct carriers. We record the jittered RETRY_BACKOFF delay it
* computed for the first no-primary retry. */
static void fo_jitter_worker(void *ctx, int idx)
{
fo_jitter_ctx_t *c = (fo_jitter_ctx_t *)ctx;
/* a distinct carrier per client: same init shape as XCECLI_RETRY_INIT but a
* distinct seed per client (production seeds from the distinct ctx pointer;
* here each thread stands in for a distinct client). */
xcecli_retry_t st = XCECLI_RETRY_INIT(c->ac);
st.rand_seed = (unsigned int)(idx + 1) * 2654435761u; /* distinct/client */
int rc = E_XCE_NOMASTER; /* no primary → RETRY_BACKOFF */
xce_fo_action_t action;
bool again = xcecli_retry_decide(c->ac, &rc, /*is_write=*/false, c->cfg, &st,
&action);
/* every client agrees on the action and stays within budget on attempt 1 */
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
TEST_ASSERT_TRUE(st.delay_ms > 0);
TEST_ASSERT_TRUE(st.delay_ms <= c->cfg->max_retry_ms);
pthread_mutex_lock(&c->lock);
c->delays[idx] = st.delay_ms;
pthread_mutex_unlock(&c->lock);
}
static void test_decide_jitter_staggers_many_clients(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
/* a wide backoff window so jitter has room to spread: base after the first
* retry is initial_retry_ms, jitter ∈ [base/2, base] = [100, 200]. */
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 20;
cfg.initial_retry_ms = 200;
cfg.max_retry_ms = 5000;
cfg.backoff_multiplier = 2.0;
fo_jitter_ctx_t jc;
memset(&jc, 0, sizeof(jc));
jc.ac = &ac;
jc.cfg = &cfg;
TEST_ASSERT_EQUAL_INT(0, pthread_mutex_init(&jc.lock, NULL));
int rc = fo_run_threads(FO_JITTER_NCLIENTS, fo_jitter_worker, &jc);
pthread_mutex_destroy(&jc.lock);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Count distinct delay values: lock-step retry would yield a SINGLE value
* across all clients. Jitter must spread them across many distinct delays. */
int distinct = 0;
int min_d = jc.delays[0];
int max_d = jc.delays[0];
for (int i = 0; i < FO_JITTER_NCLIENTS; i++) {
bool seen = false;
for (int j = 0; j < i; j++) {
if (jc.delays[j] == jc.delays[i]) {
seen = true;
break;
}
}
if (!seen)
distinct++;
if (jc.delays[i] < min_d)
min_d = jc.delays[i];
if (jc.delays[i] > max_d)
max_d = jc.delays[i];
/* every client's delay stays within the jittered envelope [base/2, base]
* for the first retry: base == initial_retry_ms == 200 here. */
TEST_ASSERT_TRUE(jc.delays[i] >= cfg.initial_retry_ms / 2);
TEST_ASSERT_TRUE(jc.delays[i] <= cfg.initial_retry_ms);
}
/* the decisive anti-thundering-herd assertion: the clients did NOT retry in
* lockstep — their delays span many distinct values, not one shared delay. */
TEST_ASSERT_TRUE(distinct > 1);
/* with 64 clients over a ~100ms jitter span the spread must be substantial */
TEST_ASSERT_TRUE(distinct >= FO_JITTER_NCLIENTS / 4);
TEST_ASSERT_TRUE(max_d > min_d); /* a real spread, not a constant */
printf("test_decide_jitter_staggers_many_clients: PASS "
"(%d distinct delays across %d clients, span [%d,%d]ms)\n",
distinct, FO_JITTER_NCLIENTS, min_d, max_d);
}
/* ════════════════════════════════════════════════════════════════════════
* Task 3.2 — Reconnect recovery (xcecli_ctx_recover) RED→GREEN tests.
*
* After a transport failure the disconnect path leaves ctx->state == XCE_CTX_ERR
* (xcectx.c:174) and may have injected a stale transport status into the single-
* slot ctx->wbuf (xcectx.c:180). xcecli_ctx_recover repairs the ctx after a
* successful reconnect, with NO socket I/O so it is unit-testable:
* - CI2 state reset: ERR → RUN (only ERR; RUN/DOWN/INIT left as-is) so the
* next xce_* op passes the xceapi XCE_CTX_RUN guard instead of -EINVAL.
* - CI1 stale-reply drain: any entry already sitting in wbuf is drained with a
* non-blocking get, so the next attempt's cbuf_get cannot consume a prior-
* attempt reply.
*
* Requirements: 1.2 (complete next request against new primary, no caller
* change), 4.4 (transport → reconnect/rotate and resume).
* ════════════════════════════════════════════════════════════════════════ */
/* CI2: an ERR-state ctx is restored to RUN (the reported -EINVAL bug fix). */
static void test_recover_resets_err_state_to_run(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_ERR;
/* wbuf left un-init: recover must tolerate a recover that has nothing to
* drain (init it so the drain path is also exercised safely). */
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
int rc = xcecli_ctx_recover(&ac);
cbuf_fini(&ac.ctx.wbuf);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state);
printf("test_recover_resets_err_state_to_run: PASS\n");
}
/* CI2: a ctx already in RUN is left in RUN; only ERR→RUN is performed. */
static void test_recover_leaves_run_state_untouched(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_RUN;
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
int rc = xcecli_ctx_recover(&ac);
cbuf_fini(&ac.ctx.wbuf);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state);
printf("test_recover_leaves_run_state_untouched: PASS\n");
}
/* CI2: a DOWN ctx is NOT flipped to RUN — only an ERR transport state is
* repaired; a deliberately-stopped ctx stays down. */
static void test_recover_leaves_down_state_untouched(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_DOWN;
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
int rc = xcecli_ctx_recover(&ac);
cbuf_fini(&ac.ctx.wbuf);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(XCE_CTX_DOWN, ac.ctx.state);
printf("test_recover_leaves_down_state_untouched: PASS\n");
}
/* CI1: a stale status already sitting in wbuf is drained, so a subsequent
* non-blocking get finds the channel EMPTY (the prior-attempt reply was
* discarded, not handed to the next attempt's cbuf_get). */
static void test_recover_drains_stale_wbuf_entry(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_ERR;
TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int)));
/* mirror handle_disconnect: a stale transport status sits in the channel */
int stale = -ECONNRESET;
TEST_ASSERT_EQUAL_INT(0, cbuf_tryput(&ac.ctx.wbuf, &stale));
int rc = xcecli_ctx_recover(&ac);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state);
/* the stale entry was drained: a non-blocking get now finds nothing */
int drained = 0;
int g = cbuf_tryget(&ac.ctx.wbuf, &drained);
cbuf_fini(&ac.ctx.wbuf);
TEST_ASSERT_EQUAL_INT(-EAGAIN, g); /* channel empty after recover */
printf("test_recover_drains_stale_wbuf_entry: PASS\n");
}
/* Guard: a NULL api_ctx is rejected without crashing. */
static void test_recover_null_ctx_safe(void)
{
TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_ctx_recover(NULL));
printf("test_recover_null_ctx_safe: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 4.1 — ReconnectRejoinHook (xcecli_set_reconnect_handler +
* xcecli_invoke_rejoin) RED→GREEN tests.
*
* libxce owns the hook MECHANISM + ORDERING (reconnect → hook → resume); the
* callback CONTENT is the consumer's. The failover machinery invokes the
* registered callback once per successful reconnect, AFTER reconnect/recovery
* and BEFORE the interrupted op resumes. A hook returning <0 means the new
* primary rejected the daemon's join → treat it as a FAILED attempt (do not
* resume; the loop rotates/backs off again). Pure clients register NO hook
* (NULL) → invoke is a no-op returning 0 and the loop resumes directly.
*
* xcecli_invoke_rejoin is socket-free (it only calls the stored cb), so it is
* unit-testable directly here without a live server.
*
* Requirements: 1.2 (re-establish session against new primary before resume).
* ════════════════════════════════════════════════════════════════════════ */
/* A test rejoin callback: records that it ran (and the ctx/arg it saw) and
* returns a caller-controlled rc so we can drive the success and failed-join
* paths. */
typedef struct {
int called; /* number of times the hook fired */
xcecli_ctx_t seen_ctx; /* the ctx the hook observed */
void *seen_arg; /* the opaque arg the hook observed */
int ret; /* rc the hook returns (0 ok, <0 failed rejoin) */
} fo_rejoin_probe_t;
static int fo_rejoin_cb(xcecli_ctx_t ctx, void *arg)
{
fo_rejoin_probe_t *p = (fo_rejoin_probe_t *)arg;
p->called++;
p->seen_ctx = ctx;
p->seen_arg = arg;
return p->ret;
}
/* Req 1.2: the setter stores cb+arg into the failover state; invoke then calls
* the registered cb exactly once, passing the opaque ctx and arg, and returns
* the cb's rc (0 on a successful rejoin). */
static void test_invoke_rejoin_calls_registered_hook(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
fo_rejoin_probe_t probe = { 0 };
probe.ret = 0;
/* setter stores cb+arg on the failover state */
int src = xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, fo_rejoin_cb,
&probe);
TEST_ASSERT_EQUAL_INT(0, src);
TEST_ASSERT_TRUE(ac.failover.rejoin_cb == fo_rejoin_cb);
TEST_ASSERT_TRUE(ac.failover.rejoin_arg == &probe);
/* invoke fires the hook once, returns its rc (0), passes ctx + arg */
int rc = xcecli_invoke_rejoin(&ac);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, probe.called);
TEST_ASSERT_TRUE(probe.seen_ctx == (xcecli_ctx_t)&ac);
TEST_ASSERT_TRUE(probe.seen_arg == &probe);
printf("test_invoke_rejoin_calls_registered_hook: PASS\n");
}
/* Req 1.2: a hook that returns <0 (the new primary rejected the join) makes
* invoke return that <0 — the wrapper treats it as a failed attempt and does
* NOT resume the interrupted op. */
static void test_invoke_rejoin_propagates_failure(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
fo_rejoin_probe_t probe = { 0 };
probe.ret = -ECONNREFUSED; /* join rejected by new primary */
TEST_ASSERT_EQUAL_INT(
0, xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, fo_rejoin_cb,
&probe));
int rc = xcecli_invoke_rejoin(&ac);
TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc); /* failure surfaced to the loop */
TEST_ASSERT_EQUAL_INT(1, probe.called);
printf("test_invoke_rejoin_propagates_failure: PASS\n");
}
/* Req 1.2: a pure client registers NO hook (NULL) → invoke is a no-op that
* returns 0 and calls nothing, so the loop resumes directly. */
static void test_invoke_rejoin_no_hook_is_noop(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf); /* leaves rejoin_cb == NULL */
TEST_ASSERT_NULL(ac.failover.rejoin_cb);
int rc = xcecli_invoke_rejoin(&ac);
TEST_ASSERT_EQUAL_INT(0, rc); /* no hook → no-op success */
printf("test_invoke_rejoin_no_hook_is_noop: PASS\n");
}
/* Req 1.2: the setter is NULL-safe on a NULL ctx, and can clear a previously
* registered hook by passing NULL (back to the pure-client no-op path). */
static void test_set_reconnect_handler_null_safe_and_clears(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
/* NULL ctx rejected without crashing */
TEST_ASSERT_EQUAL_INT(-EINVAL,
xcecli_set_reconnect_handler(NULL, fo_rejoin_cb,
NULL));
fo_rejoin_probe_t probe = { 0 };
TEST_ASSERT_EQUAL_INT(
0, xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, fo_rejoin_cb,
&probe));
TEST_ASSERT_TRUE(ac.failover.rejoin_cb == fo_rejoin_cb);
/* clearing with a NULL cb restores the no-hook no-op path */
TEST_ASSERT_EQUAL_INT(
0, xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, NULL, NULL));
TEST_ASSERT_NULL(ac.failover.rejoin_cb);
TEST_ASSERT_EQUAL_INT(0, xcecli_invoke_rejoin(&ac));
TEST_ASSERT_EQUAL_INT(0, probe.called); /* cleared hook never fired */
printf("test_set_reconnect_handler_null_safe_and_clears: PASS\n");
}
/* Guard: invoke on a NULL api_ctx is rejected without crashing. */
static void test_invoke_rejoin_null_ctx_safe(void)
{
TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_invoke_rejoin(NULL));
printf("test_invoke_rejoin_null_ctx_safe: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 5.1 — XcecliRetryLoop wrapping + per-ctx lock (CI2 concurrency).
*
* Each reply-bearing xcecli_* op wraps its one-shot xce_* call in the failover
* do/while loop AND holds ac->failover.lock for the loop's full duration so
* concurrent ops on ONE shared ctx are serialised (the single-slot wbuf already
* implies one in-flight request per ctx; the lock makes that explicit and covers
* the reconnect/rotate window). Full failover behaviour against a live primary
* is validated by E2E task 6.2; here we prove the serialisation contract.
*
* The XCECLI_FAILOVER_CALL macro (client_internal.h) is the shared wrapper that
* every op body uses: lock → init retry carrier → do/while(xcecli_retry_next) →
* unlock → return rc. We drive that exact macro here with a trivial EXPR that
* mutates shared state under the held lock, so N threads contending on one ctx's
* failover.lock must not lose updates (mutual exclusion), and a non-failing EXPR
* runs exactly once (the success path returns immediately).
*
* Requirements: 1.1, 1.3, 2.4, 5.1, 5.3, and design "FailoverState → Concurrency
* (CI2)".
* ════════════════════════════════════════════════════════════════════════ */
#define FO_LOCK_NTHREADS 32
#define FO_LOCK_ITERS 2000
typedef struct {
api_ctx_t *ac; /* the ONE shared ctx all threads contend on */
long counter; /* mutated under failover.lock via the wrapper macro */
int max_inflight;/* peak concurrent in-section (must stay 1) */
int inflight; /* current in-section count */
} fo_lock_ctx_t;
/* A trivial reply-bearing EXPR that always "succeeds" (rc==0) but, as a side
* effect, performs a non-atomic read-modify-write on the shared counter and
* tracks the peak in-section concurrency. If the wrapper macro did NOT hold
* ac->failover.lock for the section, concurrent threads would interleave the
* RMW (lost updates) and/or observe inflight > 1. */
static int fo_lock_body(fo_lock_ctx_t *c)
{
c->inflight++;
if (c->inflight > c->max_inflight)
c->max_inflight = c->inflight;
/* non-atomic RMW: a data race here would lose increments */
long v = c->counter;
for (volatile int s = 0; s < 50; s++) {
} /* widen the race window */
c->counter = v + 1;
c->inflight--;
return 0; /* success → wrapper returns immediately, no retry */
}
/* The per-thread worker drives FO_LOCK_ITERS wrapped "ops" on the SHARED ctx.
* Each op uses the real XCECLI_FAILOVER_CALL wrapper, so it acquires/releases
* ac->failover.lock exactly as a production xcecli_* op does. */
static void fo_lock_worker(void *ctx, int idx)
{
(void)idx;
fo_lock_ctx_t *c = (fo_lock_ctx_t *)ctx;
api_ctx_t *ac = c->ac;
for (int i = 0; i < FO_LOCK_ITERS; i++) {
int rc = XCECLI_FAILOVER_CALL(ac, /*is_write=*/false, fo_lock_body(c));
TEST_ASSERT_EQUAL_INT(0, rc);
}
}
/* Req 5.1, 2.4 (CI2): N threads sharing ONE ctx are serialised by the per-ctx
* failover lock the wrapper holds — no lost updates and peak in-section
* concurrency stays 1. The total counter equals N * ITERS exactly. */
static void test_failover_call_serialises_shared_ctx(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
/* the wrapper takes ac->failover.lock; init it (xcecli_start would in prod) */
TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock));
ac.failover.retry = XCE_RETRY_DEFAULTS;
fo_lock_ctx_t lc;
memset(&lc, 0, sizeof(lc));
lc.ac = &ac;
int rc = fo_run_threads(FO_LOCK_NTHREADS, fo_lock_worker, &lc);
mylock_fini(&ac.failover.lock);
TEST_ASSERT_EQUAL_INT(0, rc);
/* mutual exclusion held: every increment landed, none lost */
TEST_ASSERT_EQUAL_INT((long)FO_LOCK_NTHREADS * FO_LOCK_ITERS, lc.counter);
/* the critical section was never entered by two threads at once */
TEST_ASSERT_EQUAL_INT(1, lc.max_inflight);
printf("test_failover_call_serialises_shared_ctx: PASS\n");
}
/* Req 1.1, 5.3: the wrapper returns the op's success rc on the first try (no
* spurious retry) for a status-0 EXPR, and the per-op retry carrier starts at
* attempt 0 — i.e. a success path does not consume the retry budget. */
static void test_failover_call_success_runs_once(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock));
ac.failover.retry = XCE_RETRY_DEFAULTS;
fo_lock_ctx_t lc;
memset(&lc, 0, sizeof(lc));
lc.ac = &ac;
int rc = XCECLI_FAILOVER_CALL(&ac, /*is_write=*/false, fo_lock_body(&lc));
mylock_fini(&ac.failover.lock);
TEST_ASSERT_EQUAL_INT(0, rc); /* success surfaced verbatim */
TEST_ASSERT_EQUAL_INT(1, lc.counter); /* body ran exactly once, no replay */
printf("test_failover_call_success_runs_once: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 5.2 — Best-effort reconnect for fire-and-forget ops.
*
* Fire-and-forget ops (notify_*, heartbeat, nop) send a *_req only — there is
* NO reply to classify, so they do NOT use the full failover loop (no status
* classification, no no-primary backoff, no retry budget). Putting a heartbeat
* through the retry loop would let it block on the no-primary backoff budget,
* which is explicitly forbidden. Instead the FF path makes AT MOST ONE
* best-effort transport reconnect when the connection is KNOWN-DEAD, then does
* a single-shot send (which may be lost — acceptable, redelivery is deferred to
* worker-rejoin-resync).
*
* The "known-dead" signal is ctx->state == XCE_CTX_ERR (set by the disconnect
* path after a detected transport failure). xcecli_ff_needs_reconnect is the
* pure predicate factored out so this decision is unit-testable socket-free:
* ERR → true (reconnect once, best-effort)
* RUN → false (connection live, send directly)
* DOWN → false (deliberately stopped — do not resurrect)
* INIT → false (not yet started)
*
* Requirements: 1.1 (all ops go through failover machinery — FF via the
* best-effort transport-reconnect variant), 5.3 (no silent drop on a path that
* CAN reconnect; a known-dead connection is repaired before the single shot).
* Full live reconnect-then-send is E2E task 6.2.
* ════════════════════════════════════════════════════════════════════════ */
/* Req 1.1/5.3: a known-dead (ERR) connection needs a best-effort reconnect. */
static void test_ff_needs_reconnect_on_err(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_ERR;
TEST_ASSERT_TRUE(xcecli_ff_needs_reconnect(&ac));
printf("test_ff_needs_reconnect_on_err: PASS\n");
}
/* Req 1.1: a live (RUN) connection needs NO reconnect — send directly. */
static void test_ff_no_reconnect_on_run(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_RUN;
TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(&ac));
printf("test_ff_no_reconnect_on_run: PASS\n");
}
/* Req 1.1: a DOWN (deliberately stopped) or INIT (not yet started) ctx is NOT
* resurrected by a fire-and-forget op — only a transport-ERR connection is. */
static void test_ff_no_reconnect_on_down_or_init(void)
{
api_ctx_t ac;
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_DOWN;
TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(&ac));
memset(&ac, 0, sizeof(ac));
ac.ctx.state = XCE_CTX_INIT;
TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(&ac));
printf("test_ff_no_reconnect_on_down_or_init: PASS\n");
}
/* Guard: a NULL api_ctx is rejected as "no reconnect" without crashing. */
static void test_ff_needs_reconnect_null_safe(void)
{
TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(NULL));
printf("test_ff_needs_reconnect_null_safe: PASS\n");
}
/* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT
* MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget /
* sleeps. We prove this structurally: the FF macro drives the body exactly once
* regardless of the rc it returns (a "lost" send rc is surfaced verbatim, NOT
* retried), and on a known-dead ctx the single reconnect attempt fires exactly
* once before that single body invocation. */
typedef struct {
api_ctx_t *ac;
int body_calls; /* number of times the FF body ran */
int body_rc; /* rc the body returns each call */
} fo_ff_probe_t;
static int fo_ff_body(fo_ff_probe_t *p)
{
p->body_calls++;
return p->body_rc;
}
static void test_ff_call_is_single_shot_no_budget(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock));
ac.failover.retry = XCE_RETRY_DEFAULTS;
/* a live RUN ctx: no reconnect, just the single-shot send */
ac.ctx.state = XCE_CTX_RUN;
fo_ff_probe_t p;
memset(&p, 0, sizeof(p));
p.ac = &ac;
/* a "lost" fire-and-forget send (e.g. -ECONNRESET) is surfaced verbatim and
* is NOT retried — the body must run exactly once. */
p.body_rc = -ECONNRESET;
int rc = XCECLI_FF_CALL(&ac, fo_ff_body(&p));
mylock_fini(&ac.failover.lock);
TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc); /* best-effort: surfaced as-is */
TEST_ASSERT_EQUAL_INT(1, p.body_calls); /* single shot, no retry loop */
printf("test_ff_call_is_single_shot_no_budget: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────── */
void test_libxce_retry(void)
{
UNITY_BEGIN();
RUN_TEST(test_retry_defaults_are_reachable);
RUN_TEST(test_fake_masters_hold_three_candidates);
RUN_TEST(test_master_cursor_advances_and_wraps);
RUN_TEST(test_status_table_is_complete_and_resolves);
RUN_TEST(test_status_table_redirect_hint_flag);
RUN_TEST(test_thread_harness_runs_all_threads);
RUN_TEST(test_thread_harness_rejects_bad_args);
RUN_TEST(test_classify_matches_taxonomy_table);
RUN_TEST(test_classify_control_codes_never_success);
RUN_TEST(test_classify_dup_replay_rule);
RUN_TEST(test_classify_zero_is_success);
RUN_TEST(test_classify_unknown_is_terminal);
RUN_TEST(test_next_target_hint_first_then_consumed);
RUN_TEST(test_next_target_rotates_and_wraps);
RUN_TEST(test_next_target_hint_then_fallback_rotation);
RUN_TEST(test_next_target_rejects_bad_args);
RUN_TEST(test_capture_redirect_hint_present);
RUN_TEST(test_capture_redirect_hint_absent_stays_empty);
RUN_TEST(test_capture_redirect_hint_truncates_long_host);
RUN_TEST(test_capture_redirect_hint_null_ctx_safe);
RUN_TEST(test_reply_wait_wake_releases_blocked_get);
RUN_TEST(test_decide_no_primary_exhausts_to_not_ready);
RUN_TEST(test_decide_no_primary_then_becomes_primary);
RUN_TEST(test_decide_retry_now_has_no_backoff_delay);
RUN_TEST(test_decide_backoff_grows_and_is_bounded);
RUN_TEST(test_decide_terminal_surfaces_unchanged);
RUN_TEST(test_decide_dup_on_resent_write_is_success);
RUN_TEST(test_decide_transport_exhaustion_keeps_last_rc);
RUN_TEST(test_decide_known_dead_ctx_retries_not_terminal);
RUN_TEST(test_decide_jitter_staggers_many_clients);
RUN_TEST(test_recover_resets_err_state_to_run);
RUN_TEST(test_recover_leaves_run_state_untouched);
RUN_TEST(test_recover_leaves_down_state_untouched);
RUN_TEST(test_recover_drains_stale_wbuf_entry);
RUN_TEST(test_recover_null_ctx_safe);
RUN_TEST(test_invoke_rejoin_calls_registered_hook);
RUN_TEST(test_invoke_rejoin_propagates_failure);
RUN_TEST(test_invoke_rejoin_no_hook_is_noop);
RUN_TEST(test_set_reconnect_handler_null_safe_and_clears);
RUN_TEST(test_invoke_rejoin_null_ctx_safe);
RUN_TEST(test_failover_call_serialises_shared_ctx);
RUN_TEST(test_failover_call_success_runs_once);
RUN_TEST(test_ff_needs_reconnect_on_err);
RUN_TEST(test_ff_no_reconnect_on_run);
RUN_TEST(test_ff_no_reconnect_on_down_or_init);
RUN_TEST(test_ff_needs_reconnect_null_safe);
RUN_TEST(test_ff_call_is_single_shot_no_budget);
UNITY_END();
}
Reference in a new issue View git blame Copy permalink