lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(libxce-failover): reusable failover test harness — fake masters, status taxonomy, concurrency runner (task 1.2)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-17 02:32:19 +00:00
parent 387ccfa8dc
commit d11c6e1d89
5 changed files with 760 additions and 228 deletions
Download patch file Download diff file Expand all files Collapse all files

107
src/lib/testsuite/failover_harness.h Normal file
View file

@ -0,0 +1,107 @@
/*
* failover_harness.h — reusable, test-only fixtures for the libxce-failover
* spec (test_libxce_retry.c and the later component tasks 2.1/2.2/2.3/3.1/3.2,
* plus the concurrency tasks 5.1/6.1).
*
* This header is HARNESS ONLY. It references NO not-yet-existing failover
* production symbols (e.g. xce_failover_classify, xcecli_retry_next,
* xce_failover_rotate) — those arrive in later tasks. Everything here compiles
* and links against the current libxce surface so the lib test suite stays
* GREEN between this task (1.2) and the component tasks.
*
* What it provides:
* 1. A fake multi-master client config (3 masters) + a master-list cursor
* that the rotation logic will later consume (advance/wrap).
* 2. A controller-status taxonomy table mapping every status the failover
* classifier must handle (REDIRECT with/without hint, NOMASTER, NOT_READY,
* STANDBY, DUP_JOB/EEXIST) and the transport errors (ECONNREFUSED,
* ETIMEDOUT, ECONNRESET) to a stable description + expected failover
* action category — so later tasks can table-drive the classifier RED/GREEN.
* 3. A generic N-pthread concurrency harness so 5.1/6.1 can spawn N threads
* against a shared ctx without re-rolling pthread plumbing.
*
* Requirements laid down for: 2.5, 4.1-4.5, 5.2, 8.3, 8.4 (taxonomy table);
* 2.1, 2.4, 8.4 (master cursor); 6.3 / concurrency reuse (thread harness).
*/
#ifndef XCE_FAILOVER_HARNESS_H
#define XCE_FAILOVER_HARNESS_H
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdbool.h>
#include <stddef.h>
#include "xce/cluster.h" /* xce_cluster_config_t, xce_candidate_t, MAX_HA_CANDIDATES */
#include "xce/errnox.h" /* E_XCE_REDIRECT / NOMASTER / NOT_READY / STANDBY / DUP_JOB */
/* ── 1. Fake multi-master config + cursor ─────────────────────────────── */
/* Number of fake masters the rotation fixtures stand up. Kept <= MAX_HA_CANDIDATES. */
#define FO_FAKE_MASTERS 3
/* A master-list cursor: mirrors the "advance to next master / wrap" behaviour
* the real EndpointRotator (task 2.2) will drive over api_ctx->conf. Pure value
* type — no I/O, no production dependency. */
typedef struct {
const xce_cluster_config_t *cfg;
int cur; /* index into cfg->candidates[] */
} fo_master_cursor_t;
/* Populate `cfg` with FO_FAKE_MASTERS distinct host:port candidates. Returns the
* candidate count written. */
int fo_make_fake_masters(xce_cluster_config_t *cfg);
/* Initialise a cursor at master 0. */
void fo_cursor_init(fo_master_cursor_t *cur, const xce_cluster_config_t *cfg);
/* Advance the cursor to the next master, wrapping at the end. Returns the new
* index. */
int fo_cursor_advance(fo_master_cursor_t *cur);
/* The candidate the cursor currently points at (NULL if cfg empty). */
const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur);
/* ── 2. Controller-status taxonomy table ──────────────────────────────── */
/* Expected failover action for each status — mirrors design.md StatusClassifier
* (XCE_FO_SUCCESS / RETRY_NOW / RETRY_BACKOFF / TERMINAL). This is the HARNESS'
* expectation table; the real xce_fo_action_t enum is introduced by task 2.1,
* and its tests will assert the real classifier matches these expectations. */
typedef enum {
FO_ACT_SUCCESS = 0, /* status 0, or idempotent DUP replay on a re-sent write */
FO_ACT_RETRY_NOW, /* REDIRECT / STANDBY / transport error — rotate, no backoff */
FO_ACT_RETRY_BACKOFF,/* NOMASTER / NOT_READY — rotate after jittered backoff */
FO_ACT_TERMINAL /* non-retriable — surface to caller */
} fo_expected_action_t;
typedef struct {
const char *name; /* human label, e.g. "E_XCE_REDIRECT+hint" */
int status; /* status/rc as the classifier sees it */
bool has_hint; /* redirect carried an address hint? */
bool is_transport;/* negative -errno transport error? */
fo_expected_action_t expect; /* expected first-send classification */
const char *desc; /* one-line description */
} fo_status_row_t;
/* The complete taxonomy table and its length. Defined in test_libxce_retry.c.
* Covers: REDIRECT (with + without hint), NOMASTER, NOT_READY, STANDBY,
* DUP_JOB, -EEXIST, and the three transport errors. */
extern const fo_status_row_t FO_STATUS_TABLE[];
extern const size_t FO_STATUS_TABLE_LEN;
/* Look up a row by its `status` field (first match), or NULL. */
const fo_status_row_t *fo_status_lookup(int status);
/* ── 3. Generic N-thread concurrency harness ──────────────────────────── */
/* A worker fn run on each spawned thread. `ctx` is a shared context the caller
* owns; `idx` is the thread's 0-based index. */
typedef void (*fo_thread_fn)(void *ctx, int idx);
/* Spawn `n` pthreads, each invoking `fn(ctx, idx)`, and join them all. Returns 0
* if every thread was created and joined, -errno on the first failure. Generic
* so 5.1 (shared-ctx safety) and 6.1 (50-client jitter) reuse it unchanged. */
int fo_run_threads(int n, fo_thread_fn fn, void *ctx);
#endif /* XCE_FAILOVER_HARNESS_H */

338
src/lib/testsuite/test_libxce_retry.c
View file

@ -1,23 +1,195 @@
/*
* test_libxce_retry.c — placeholder for the libxce failover retry tests.
* test_libxce_retry.c — reusable libxce-failover test harness + harness
* self-tests (spec libxce-failover, task 1.2 ADAPTED).
*
* The orphaned xce_request_with_retry() primitive and its xce_retry_send_hook
* test seam were removed (superseded by the xcecli_* failover loop). Task 1.2
* rewrites this suite against the new failover model (StatusClassifier, master
* rotation, redirect-hint, budget → cluster-not-ready).
* This file builds the GREEN, reusable harness that the later component tasks
* consume to write their own RED→GREEN tests:
* - task 2.1 (StatusClassifier) → drives FO_STATUS_TABLE
* - task 2.2 (EndpointRotator) → drives the fake-master cursor
* - task 2.3 (reply-capture) → drives the redirect-hint rows
* - task 3.1/3.2 (decision helper) → drives the table + cursor
* - task 5.1/6.1 (concurrency) → drive fo_run_threads()
*
* For now this is a minimal compiling placeholder that asserts the retained
* retry-tunables scaffolding (xce_retry_config_t + XCE_RETRY_DEFAULTS) is
* reachable, keeping the lib test suite GREEN between tasks 1.1 and 1.2.
* It deliberately references NO not-yet-existing failover production symbols
* (xce_failover_classify / xcecli_retry_next / xce_failover_rotate); those are
* added by the component tasks. The lib build+test MUST stay GREEN after this
* task, so every fixture here is exercised by at least one real assertion.
*
* Requirements: 6.1, 6.2
* Requirements covered by the harness: 2.5, 4.1, 4.2, 4.3, 4.4, 4.5, 5.2, 8.3,
* 8.4 (status taxonomy); 2.1, 2.4 (master cursor); 6.3 (concurrency reuse).
* The retained retry-defaults test still covers 6.1, 6.2.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "failover_harness.h"
#include "request.h"
/* ════════════════════════════════════════════════════════════════════════
* Harness implementation
* ════════════════════════════════════════════════════════════════════════ */
/* ── 1. Fake multi-master config + cursor ─────────────────────────────── */
int fo_make_fake_masters(xce_cluster_config_t *cfg)
{
static const struct {
const char *host;
int port;
} masters[FO_FAKE_MASTERS] = {
{ "10.20.0.1", 7001 },
{ "10.20.0.2", 7002 },
{ "10.20.0.3", 7003 },
};
memset(cfg, 0, sizeof(*cfg));
for (int i = 0; i < FO_FAKE_MASTERS && i < MAX_HA_CANDIDATES; i++) {
strncpy(cfg->candidates[i].hostname, masters[i].host,
XCE_MAXHOSTNAME_LEN - 1);
cfg->candidates[i].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
cfg->candidates[i].port = masters[i].port;
}
cfg->candidates_count =
(FO_FAKE_MASTERS < MAX_HA_CANDIDATES) ? FO_FAKE_MASTERS
: MAX_HA_CANDIDATES;
return cfg->candidates_count;
}
void fo_cursor_init(fo_master_cursor_t *cur, const xce_cluster_config_t *cfg)
{
cur->cfg = cfg;
cur->cur = 0;
}
int fo_cursor_advance(fo_master_cursor_t *cur)
{
if (!cur->cfg || cur->cfg->candidates_count <= 0)
return 0;
cur->cur = (cur->cur + 1) % cur->cfg->candidates_count;
return cur->cur;
}
const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur)
{
if (!cur->cfg || cur->cfg->candidates_count <= 0)
return NULL;
return &cur->cfg->candidates[cur->cur];
}
/* ── 2. Controller-status taxonomy table ──────────────────────────────── */
const fo_status_row_t FO_STATUS_TABLE[] = {
/* success */
{ "OK", 0, false, false, FO_ACT_SUCCESS,
"request served by primary" },
/* redirect (a primary exists elsewhere) — rotate now, hint first if any */
{ "E_XCE_REDIRECT+hint", E_XCE_REDIRECT, true, false, FO_ACT_RETRY_NOW,
"standby redirects with primary address hint (8.3)" },
{ "E_XCE_REDIRECT", E_XCE_REDIRECT, false, false, FO_ACT_RETRY_NOW,
"standby redirects without address — rotate master list (8.4)" },
/* standby (op is primary-only) — treat as redirect, rotate now (4.3) */
{ "E_XCE_STANDBY", E_XCE_STANDBY, false, false, FO_ACT_RETRY_NOW,
"controller in standby; op is primary-only (4.3)" },
/* no-primary / not-ready — rotate after backoff (4.1) */
{ "E_XCE_NOMASTER", E_XCE_NOMASTER, false, false, FO_ACT_RETRY_BACKOFF,
"standby knows no primary yet — backoff + rotate (4.1)" },
{ "E_XCE_NOT_READY", E_XCE_NOT_READY, false, false, FO_ACT_RETRY_BACKOFF,
"controller initializing/electing — backoff + rotate (4.1)" },
/* idempotent-replay duplicate (write replay) — DUP/EEXIST (5.2) */
{ "E_XCE_DUP_JOB", E_XCE_DUP_JOB, false, false, FO_ACT_TERMINAL,
"duplicate job; SUCCESS only on a re-sent write (5.2, attempt-aware)" },
{ "-EEXIST", -EEXIST, false, true, FO_ACT_TERMINAL,
"duplicate via -EEXIST; SUCCESS only on a re-sent write (5.2)" },
/* transport errors — reconnect/rotate now (4.4) */
{ "-ECONNREFUSED", -ECONNREFUSED, false, true, FO_ACT_RETRY_NOW,
"connection refused — reconnect/rotate (4.4)" },
{ "-ETIMEDOUT", -ETIMEDOUT, false, true, FO_ACT_RETRY_NOW,
"connection timeout — reconnect/rotate (4.4)" },
{ "-ECONNRESET", -ECONNRESET, false, true, FO_ACT_RETRY_NOW,
"peer reset — reconnect/rotate (4.4)" },
/* terminal / non-retriable — surface immediately (4.5) */
{ "-EINVAL", -EINVAL, false, true, FO_ACT_TERMINAL,
"invalid request — surface, no retry (4.5)" },
};
const size_t FO_STATUS_TABLE_LEN =
sizeof(FO_STATUS_TABLE) / sizeof(FO_STATUS_TABLE[0]);
const fo_status_row_t *fo_status_lookup(int status)
{
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
if (FO_STATUS_TABLE[i].status == status)
return &FO_STATUS_TABLE[i];
}
return NULL;
}
/* ── 3. Generic N-thread concurrency harness ──────────────────────────── */
typedef struct {
fo_thread_fn fn;
void *ctx;
int idx;
} fo_thread_arg_t;
static void *fo_thread_trampoline(void *p)
{
fo_thread_arg_t *a = (fo_thread_arg_t *)p;
a->fn(a->ctx, a->idx);
return NULL;
}
int fo_run_threads(int n, fo_thread_fn fn, void *ctx)
{
if (n <= 0 || !fn)
return -EINVAL;
pthread_t *tids = calloc((size_t)n, sizeof(*tids));
fo_thread_arg_t *args = calloc((size_t)n, sizeof(*args));
if (!tids || !args) {
free(tids);
free(args);
return -ENOMEM;
}
int spawned = 0;
int rc = 0;
for (int i = 0; i < n; i++) {
args[i].fn = fn;
args[i].ctx = ctx;
args[i].idx = i;
int e = pthread_create(&tids[i], NULL, fo_thread_trampoline, &args[i]);
if (e != 0) {
rc = -e;
break;
}
spawned++;
}
for (int i = 0; i < spawned; i++)
pthread_join(tids[i], NULL);
free(tids);
free(args);
return rc;
}
/* ════════════════════════════════════════════════════════════════════════
* Harness self-tests (GREEN) — each fixture proven by real assertions
* ════════════════════════════════════════════════════════════════════════ */
/* Req 6.1, 6.2: the retry-tunables struct + safe production defaults are
* retained and reachable for the failover loop. */
static void test_retry_defaults_are_reachable(void)
@ -32,11 +204,157 @@ static void test_retry_defaults_are_reachable(void)
printf("test_retry_defaults_are_reachable: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────── */
/* Fixture 1: fake multi-master config holds 3 distinct candidates. */
static void test_fake_masters_hold_three_candidates(void)
{
xce_cluster_config_t cfg;
int n = fo_make_fake_masters(&cfg);
TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, n);
TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("10.20.0.1", cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(7001, cfg.candidates[0].port);
TEST_ASSERT_EQUAL_STRING("10.20.0.3", cfg.candidates[2].hostname);
TEST_ASSERT_EQUAL_INT(7003, cfg.candidates[2].port);
printf("test_fake_masters_hold_three_candidates: PASS\n");
}
/* Fixture 1: cursor advances through masters and wraps (2.1, 2.4, 8.4). */
static void test_master_cursor_advances_and_wraps(void)
{
xce_cluster_config_t cfg;
fo_make_fake_masters(&cfg);
fo_master_cursor_t cur;
fo_cursor_init(&cur, &cfg);
TEST_ASSERT_EQUAL_INT(0, cur.cur);
TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname);
TEST_ASSERT_EQUAL_INT(1, fo_cursor_advance(&cur));
TEST_ASSERT_EQUAL_STRING("10.20.0.2", fo_cursor_current(&cur)->hostname);
TEST_ASSERT_EQUAL_INT(2, fo_cursor_advance(&cur));
TEST_ASSERT_EQUAL_STRING("10.20.0.3", fo_cursor_current(&cur)->hostname);
/* wrap back to master 0 */
TEST_ASSERT_EQUAL_INT(0, fo_cursor_advance(&cur));
TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname);
printf("test_master_cursor_advances_and_wraps: PASS\n");
}
/* Fixture 2: taxonomy table is complete — every status the classifier must
* handle is present, lookups resolve, and the status codes equal the real
* errnox.h values (2.5, 4.1-4.5, 5.2, 8.3, 8.4). */
static void test_status_table_is_complete_and_resolves(void)
{
/* every mandated controller status + transport error is represented */
TEST_ASSERT_NOT_NULL(fo_status_lookup(0));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_REDIRECT));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_STANDBY));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOMASTER));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOT_READY));
TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_DUP_JOB));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-EEXIST));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNREFUSED));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ETIMEDOUT));
TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNRESET));
/* status codes resolve to the real errnox.h values (ordered enum) */
TEST_ASSERT_EQUAL_INT(XCE_ERRNO_BASE, E_XCE_BAD_JOB);
TEST_ASSERT_TRUE(E_XCE_REDIRECT == E_XCE_NOMASTER + 1);
TEST_ASSERT_TRUE(E_XCE_NOT_READY == E_XCE_REDIRECT + 1);
TEST_ASSERT_TRUE(E_XCE_STANDBY == E_XCE_NOT_READY + 1);
/* expected actions line up with the design's StatusClassifier (first send) */
TEST_ASSERT_EQUAL_INT(FO_ACT_SUCCESS, fo_status_lookup(0)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(E_XCE_REDIRECT)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(E_XCE_STANDBY)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
fo_status_lookup(E_XCE_NOMASTER)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
fo_status_lookup(E_XCE_NOT_READY)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(-ECONNRESET)->expect);
printf("test_status_table_is_complete_and_resolves: PASS\n");
}
/* Fixture 2: the redirect-hint flag distinguishes hint vs. no-hint rows (8.3). */
static void test_status_table_redirect_hint_flag(void)
{
bool saw_hint = false;
bool saw_no_hint = false;
for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) {
if (FO_STATUS_TABLE[i].status == E_XCE_REDIRECT) {
if (FO_STATUS_TABLE[i].has_hint)
saw_hint = true;
else
saw_no_hint = true;
}
}
TEST_ASSERT_TRUE(saw_hint); /* REDIRECT+hint row present (8.3) */
TEST_ASSERT_TRUE(saw_no_hint); /* REDIRECT no-hint row present (8.4) */
printf("test_status_table_redirect_hint_flag: PASS\n");
}
/* Fixture 3: concurrency harness runs N threads to completion (6.3 reuse). */
#define FO_TEST_NTHREADS 16
typedef struct {
pthread_mutex_t lock;
int counter;
} fo_selftest_ctx_t;
static void fo_selftest_worker(void *ctx, int idx)
{
(void)idx;
fo_selftest_ctx_t *c = (fo_selftest_ctx_t *)ctx;
pthread_mutex_lock(&c->lock);
c->counter++;
pthread_mutex_unlock(&c->lock);
}
static void test_thread_harness_runs_all_threads(void)
{
fo_selftest_ctx_t ctx;
pthread_mutex_init(&ctx.lock, NULL);
ctx.counter = 0;
int rc = fo_run_threads(FO_TEST_NTHREADS, fo_selftest_worker, &ctx);
pthread_mutex_destroy(&ctx.lock);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(FO_TEST_NTHREADS, ctx.counter);
printf("test_thread_harness_runs_all_threads: PASS\n");
}
/* Fixture 3: harness rejects bad arguments. */
static void test_thread_harness_rejects_bad_args(void)
{
TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(0, fo_selftest_worker, NULL));
TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(4, NULL, NULL));
printf("test_thread_harness_rejects_bad_args: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────── */
void test_libxce_retry(void)
{
UNITY_BEGIN();
RUN_TEST(test_retry_defaults_are_reachable);
RUN_TEST(test_fake_masters_hold_three_candidates);
RUN_TEST(test_master_cursor_advances_and_wraps);
RUN_TEST(test_status_table_is_complete_and_resolves);
RUN_TEST(test_status_table_redirect_hint_flag);
RUN_TEST(test_thread_harness_runs_all_threads);
RUN_TEST(test_thread_harness_rejects_bad_args);
UNITY_END();
}