feat(libxce-failover): retry decision loop — pure decide (budget/backoff/jitter, D6 exhaustion) + thin next wrapper (task 3.1)

This commit is contained in:
Rongsong Shen 2026-06-17 03:27:38 +00:00
parent cec2e647dc
commit 84c5e32c2c
4 changed files with 481 additions and 2 deletions

View file

@ -15,12 +15,15 @@
#include "xce/client_failover.h"
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <time.h>
#include "client_internal.h" /* api_ctx_t + EndpointRotator declarations */
#include "client_internal.h" /* api_ctx_t + EndpointRotator + retry decls */
#include "config.h" /* XCE_MAXHOSTNAME_LEN */
#include "netio.h" /* endpoint_init / endpoint_connect / endpoint_fini */
#include "utils.h" /* INFO / WARN logging macros */
#include "xce/errnox.h"
/*
@ -155,3 +158,152 @@ int xce_failover_rotate(api_ctx_t *ac)
rc = endpoint_connect(&ac->ep, host, (int)port);
return rc < 0 ? rc : 0;
}
/* ════════════════════════════════════════════════════════════════════════
* XcecliRetryLoop — decision core (pure) + side-effecting wrapper
* ════════════════════════════════════════════════════════════════════════ */
/*
* Compute the next no-primary backoff delay (ms) for retry number `attempt`
* (1-based: the delay applied BEFORE the attempt'th re-send). Base is a capped
* exponential (initial_retry_ms * multiplier^(attempt-1), clamped to
* max_retry_ms); jitter spreads the actual delay within [base/2, base] using
* rand_r on the per-op seed so concurrent clients do not retry in lockstep
* (6.3). The base is monotone non-decreasing and never exceeds max_retry_ms.
*/
static int fo_backoff_delay_ms(const xce_retry_config_t* cfg, int attempt,
unsigned int* seed)
{
double base = (double)cfg->initial_retry_ms;
for (int i = 1; i < attempt; i++) {
base *= cfg->backoff_multiplier;
if (base >= (double)cfg->max_retry_ms) {
base = (double)cfg->max_retry_ms;
break;
}
}
if (base > (double)cfg->max_retry_ms)
base = (double)cfg->max_retry_ms;
if (base < 1.0)
base = 1.0;
int ibase = (int)base;
/* jitter within the lower half: delay ∈ [ibase/2, ibase]. */
int half = ibase / 2;
int jitter = (half > 0) ? (int)(rand_r(seed) % (unsigned int)(half + 1)) : 0;
int delay = ibase - jitter;
if (delay < 1)
delay = 1;
if (delay > cfg->max_retry_ms)
delay = cfg->max_retry_ms;
return delay;
}
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
const xce_retry_config_t* cfg, xcecli_retry_t* st,
xce_fo_action_t* action_out)
{
if (ac == NULL || rc == NULL || cfg == NULL || st == NULL)
return false;
xce_fo_action_t action =
xce_failover_classify(*rc, is_write, st->attempt);
if (action_out != NULL)
*action_out = action;
switch (action) {
case XCE_FO_SUCCESS:
/* DUP-replay success surfaces as a clean 0; a real status-0 is already
* 0. Either way the op succeeded — normalise and stop. */
*rc = 0;
return false;
case XCE_FO_TERMINAL:
/* Non-retriable — surface *rc unchanged. */
return false;
case XCE_FO_RETRY_NOW:
case XCE_FO_RETRY_BACKOFF:
break;
}
/* A retriable outcome consumes one retry from the budget. */
st->attempt++;
if (st->attempt > cfg->max_retries) {
/* Budget exhausted. Distinguish the cause (design D6): a no-primary
* window maps to the distinct cluster-not-ready code; a transport-only
* outage keeps the last transport rc so the caller still sees it. */
if (action == XCE_FO_RETRY_BACKOFF)
*rc = -E_XCE_NOT_READY;
return false;
}
/* Compute the pre-attempt delay: none for prompt rotation (RETRY_NOW),
* a jittered exponential backoff for the no-primary case (RETRY_BACKOFF). */
st->delay_ms = (action == XCE_FO_RETRY_BACKOFF)
? fo_backoff_delay_ms(cfg, st->attempt, &st->rand_seed)
: 0;
return true;
}
/* Sleep for `ms` milliseconds, tolerating EINTR. Isolated here so the pure
* decision core stays sleep-free and unit-testable. */
static void fo_sleep_ms(int ms)
{
if (ms <= 0)
return;
struct timespec ts = { .tv_sec = ms / 1000,
.tv_nsec = (long)(ms % 1000) * 1000000L };
while (nanosleep(&ts, &ts) != 0 && errno == EINTR)
; /* finish the remaining interval */
}
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
xcecli_retry_t* st)
{
if (ac == NULL || rc == NULL || st == NULL)
return false;
xce_fo_action_t action;
bool retry = xcecli_retry_decide(ac, rc, is_write, &ac->failover.retry, st,
&action);
if (!retry) {
/* Stopped. If the budget was exhausted on a no-primary window, decide
* has already set *rc to -E_XCE_NOT_READY (7.2 observable). */
if (*rc == -E_XCE_NOT_READY)
WARN("xcecli failover: cluster not ready after %d attempts "
"(last status=%d)",
st->attempt, *rc);
return false;
}
/* RETRY_BACKOFF: honour the jittered pre-attempt delay (4.1, 6.3). */
if (action == XCE_FO_RETRY_BACKOFF)
fo_sleep_ms(st->delay_ms);
/* Re-point the endpoint to the redirect hint or the next master and
* reconnect (2.x, 4.2, 4.4). Log the rotation target (7.1). A rotate
* failure is not fatal here: the next xce_* attempt observes the dead
* connection and the loop reclassifies it as a transport error. */
int rrc = xce_failover_rotate(ac);
if (rrc < 0) {
WARN("xcecli failover: rotate failed (rc=%d); retrying anyway", rrc);
} else {
/* Log the rotation target (7.1). xce_failover_rotate consumes the
* redirect hint when present (so the post-rotate target is not always
* the sticky master); report the sticky master cursor as a best-effort
* endpoint label, which is exact for the configured-master rotation. */
int cur = ac->failover.cur_master;
const char* host =
(ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
? ac->conf->masters[cur]
: "(unknown)";
int port = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
? ac->conf->master_ports[cur]
: 0;
INFO("xcecli failover: attempt %d → %s:%d", st->attempt,
host ? host : "(null)", port);
}
return true;
}

View file

@ -2,6 +2,7 @@
#define _CLIENT_INTERNAL_H_ 1
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
@ -9,6 +10,7 @@
#include "mylock.h"
#include "netio.h"
#include "request.h"
#include "xce/client_failover.h" /* xce_fo_action_t */
#include "xcectx.h"
/*
@ -80,4 +82,61 @@ int xce_failover_next_target(api_ctx_t* ac, char* host_out, size_t host_sz,
uint16_t* port_out);
int xce_failover_rotate(api_ctx_t* ac);
/*
* XcecliRetryLoop / decision helper (task 3.1) — see design.md
* "Components and Interfaces → xcecli_retry_next".
*
* xcecli_retry_t: per-op retry carrier. One instance lives on the caller's
* stack for the duration of a single xcecli_* op's do/while loop.
* attempt — number of re-sends so far (0 on the first send).
* delay_ms — backoff to apply BEFORE the next attempt (0 for RETRY_NOW).
* rand_seed — per-op rand_r() state for jittered backoff (6.3).
*/
typedef struct {
int attempt;
int delay_ms;
unsigned int rand_seed;
} xcecli_retry_t;
/*
* XCECLI_RETRY_INIT(ac): initialise a retry carrier from an api_ctx_t. The seed
* mixes the ctx pointer so concurrent ops on distinct ctxs jitter independently.
*/
#define XCECLI_RETRY_INIT(ac) \
((xcecli_retry_t){ .attempt = 0, \
.delay_ms = 0, \
.rand_seed = (unsigned int)(uintptr_t)(ac) })
/*
* xcecli_retry_decide(): the PURE decision core (no I/O, no sleep, no socket,
* no logging) so it is unit-testable. Given the rc returned by the last xce_*
* attempt, it:
* - classifies *rc via xce_failover_classify(*rc, is_write, st->attempt);
* - on SUCCESS applies the DUP-replay rule (sets *rc = 0) and returns false;
* - on TERMINAL returns false leaving *rc as-is;
* - on a retriable action advances st->attempt, computes st->delay_ms
* (0 for RETRY_NOW; a jittered exponential backoff bounded by
* cfg->max_retry_ms for RETRY_BACKOFF), reports the action in *action_out,
* and returns true — UNLESS the retry budget (cfg->max_retries) is now
* exhausted, in which case it returns false and, for a no-primary cause,
* sets *rc = -E_XCE_NOT_READY (a transport-only cause keeps the last rc).
*
* It does NOT reset ctx state, drain wbuf, sleep, rotate, or invoke the rejoin
* hook — those are the side-effecting wrapper's job (xcecli_retry_next) and
* later tasks (3.2 state-reset/drain, 4.1 rejoin hook).
*/
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
const xce_retry_config_t* cfg, xcecli_retry_t* st,
xce_fo_action_t* action_out);
/*
* xcecli_retry_next(): the side-effecting wrapper the xcecli_* do/while loops
* call. = xcecli_retry_decide() + (RETRY_BACKOFF: a real st->delay_ms sleep) +
* (any retry: xce_failover_rotate(ac) + INFO log of the rotation target, 7.1) +
* (budget exhaustion: WARN log of the attempt count, 7.2). Returns true → the
* caller re-invokes its xce_* op; false → stop, *rc is final.
*/
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
xcecli_retry_t* st);
#endif

View file

@ -761,6 +761,266 @@ static void test_reply_wait_wake_releases_blocked_get(void)
printf("test_reply_wait_wake_releases_blocked_get: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 3.1 — decision loop (xcecli_retry_decide) RED→GREEN tests.
*
* xcecli_retry_decide is the PURE core of the failover decision loop: classify
* → apply SUCCESS/DUP/TERMINAL to *rc → advance attempt → compute backoff →
* handle budget exhaustion → report the chosen action. It does NOT sleep, open
* sockets, rotate, or log. The cursor-advancing side (xce_failover_next_target)
* is driven separately to prove the become-primary path advances the cursor.
*
* Requirements: 1.2, 3.1, 3.2 (budget), 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2.
* ════════════════════════════════════════════════════════════════════════ */
/* A small, deterministic retry config for the decide tests: a low max_retries
* so budget exhaustion is reached quickly, a tiny initial delay, a modest cap. */
static xce_retry_config_t fo_test_retry_cfg(void)
{
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 3;
cfg.initial_retry_ms = 10;
cfg.max_retry_ms = 100;
cfg.backoff_multiplier = 2.0;
return cfg;
}
/* Req 3.3, 3.4: a controller that keeps reporting no-primary (NOMASTER /
* NOT_READY) is retried with backoff until the budget is exhausted; decide then
* returns false with *rc == -E_XCE_NOT_READY — the distinct cluster-not-ready
* code, never an opaque -EINVAL. */
static void test_decide_no_primary_exhausts_to_not_ready(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
bool again;
int retries = 0;
do {
rc = E_XCE_NOMASTER; /* every attempt: no primary known */
again = xcecli_retry_decide(&ac, &rc, /*is_write=*/false, &cfg, &st,
&action);
if (again) {
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
retries++;
}
} while (again && retries < 100); /* hard cap so a bug can't spin forever */
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
/* exactly max_retries retries happened before exhaustion */
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
printf("test_decide_no_primary_exhausts_to_not_ready: PASS\n");
}
/* Req 3.2 / become-primary: no-primary then a served reply (status 0) → the
* loop stops with success and the master cursor advanced across the retries
* (the rotation side runs in the wrapper; here we advance it explicitly to
* mirror that and assert the sticky cursor moved). */
static void test_decide_no_primary_then_becomes_primary(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
/* attempt 0: no primary → retry (decide says backoff). */
rc = E_XCE_NOMASTER;
bool a0 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(a0);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
/* mirror the wrapper: rotate advances the cursor (no hint set) */
char host[XCE_MAXHOSTNAME_LEN];
uint16_t port;
xce_failover_next_target(&ac, host, sizeof(host), &port);
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
/* attempt 1: served by the new primary → success, loop stops. */
rc = 0;
bool a1 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_FALSE(a1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* cursor stayed advanced (sticky primary = where success landed) */
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
printf("test_decide_no_primary_then_becomes_primary: PASS\n");
}
/* Req 4.2, 4.4: redirect / standby / transport causes are RETRY_NOW with NO
* backoff delay applied (st->delay_ms == 0), so the wrapper rotates promptly. */
static void test_decide_retry_now_has_no_backoff_delay(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
const int now_codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, -ECONNRESET,
-ECONNREFUSED, -ETIMEDOUT };
for (size_t i = 0; i < sizeof(now_codes) / sizeof(now_codes[0]); i++) {
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = now_codes[i];
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* no backoff for RETRY_NOW */
}
printf("test_decide_retry_now_has_no_backoff_delay: PASS\n");
}
/* Req 4.1, 6.3: no-primary causes get a jittered exponential backoff — the
* delay grows across retries and never exceeds max_retry_ms. */
static void test_decide_backoff_grows_and_is_bounded(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 12;
cfg.initial_retry_ms = 10;
cfg.max_retry_ms = 200;
cfg.backoff_multiplier = 2.0;
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
int prev_base = 0;
for (int i = 0; i < 8; i++) {
rc = E_XCE_NOT_READY;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
/* delay is positive and never exceeds the cap */
TEST_ASSERT_TRUE(st.delay_ms > 0);
TEST_ASSERT_TRUE(st.delay_ms <= cfg.max_retry_ms);
/* the un-jittered base (initial * mult^attempt, capped) is monotone
* non-decreasing; once we hit the cap it stays there. The delay floor
* grows: by attempt 5+ it must exceed the very first delay. */
if (i >= 5)
TEST_ASSERT_TRUE(st.delay_ms >= prev_base);
if (i == 0)
prev_base = st.delay_ms;
}
printf("test_decide_backoff_grows_and_is_bounded: PASS\n");
}
/* Req 4.5: a terminal (non-retriable) status stops the loop immediately with
* *rc unchanged — no retry, no -E_XCE_NOT_READY rewrite. */
static void test_decide_terminal_surfaces_unchanged(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
const int term_codes[] = { -EACCES, -EPERM, -EINVAL, -ENOENT };
for (size_t i = 0; i < sizeof(term_codes) / sizeof(term_codes[0]); i++) {
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = term_codes[i];
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(term_codes[i], rc); /* surfaced unchanged */
}
printf("test_decide_terminal_surfaces_unchanged: PASS\n");
}
/* Req 5.2: DUP on a re-sent write (is_write && attempt>0) → success (*rc = 0),
* loop stops. The controller's client-jobid replay guard means the write was
* already applied, so the retry is a success not a duplicate failure. */
static void test_decide_dup_on_resent_write_is_success(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
st.attempt = 1; /* a re-sent write */
int rc = -E_XCE_DUP_JOB;
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, /*is_write=*/true, &cfg, &st,
&action);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(0, rc); /* replay → success */
/* contrast: DUP on the FIRST send (attempt 0) is a genuine duplicate →
* terminal, *rc surfaced unchanged. */
xcecli_retry_t st0 = XCECLI_RETRY_INIT(&ac);
int rc0 = -E_XCE_DUP_JOB;
bool a0 =
xcecli_retry_decide(&ac, &rc0, true, &cfg, &st0, &action);
TEST_ASSERT_FALSE(a0);
TEST_ASSERT_EQUAL_INT(-E_XCE_DUP_JOB, rc0);
printf("test_decide_dup_on_resent_write_is_success: PASS\n");
}
/* Req 3.4 (D6): a transport-only cause that outlasts the budget keeps the LAST
* transport rc (not -E_XCE_NOT_READY — that mapping is reserved for no-primary
* exhaustion). Proves the two exhaustion causes are distinguished. */
static void test_decide_transport_exhaustion_keeps_last_rc(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
bool again;
int retries = 0;
do {
rc = -ECONNREFUSED; /* every attempt: transport failure */
again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
if (again) {
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
retries++;
}
} while (again && retries < 100);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
/* transport cause → last transport rc preserved, NOT rewritten */
TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc);
printf("test_decide_transport_exhaustion_keeps_last_rc: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────── */
void test_libxce_retry(void)
@ -787,5 +1047,12 @@ void test_libxce_retry(void)
RUN_TEST(test_capture_redirect_hint_truncates_long_host);
RUN_TEST(test_capture_redirect_hint_null_ctx_safe);
RUN_TEST(test_reply_wait_wake_releases_blocked_get);
RUN_TEST(test_decide_no_primary_exhausts_to_not_ready);
RUN_TEST(test_decide_no_primary_then_becomes_primary);
RUN_TEST(test_decide_retry_now_has_no_backoff_delay);
RUN_TEST(test_decide_backoff_grows_and_is_bounded);
RUN_TEST(test_decide_terminal_surfaces_unchanged);
RUN_TEST(test_decide_dup_on_resent_write_is_success);
RUN_TEST(test_decide_transport_exhaustion_keeps_last_rc);
UNITY_END();
}