forked from lei.chen/myxce
feat(libxce-failover): retry decision loop — pure decide (budget/backoff/jitter, D6 exhaustion) + thin next wrapper (task 3.1)
This commit is contained in:
parent
cec2e647dc
commit
84c5e32c2c
4 changed files with 481 additions and 2 deletions
|
|
@ -39,7 +39,7 @@
|
|||
- _Depends: 1.1_
|
||||
|
||||
- [ ] 3. Core: the retry/failover decision helper
|
||||
- [ ] 3.1 Implement the decision loop (classify, rotate, backoff, budget)
|
||||
- [x] 3.1 Implement the decision loop (classify, rotate, backoff, budget)
|
||||
- Combine classification, rotation, and the captured redirect into one decision: success and terminal stop the loop; redirect, standby, and transport results rotate without backoff; no-primary and not-ready apply jittered backoff
|
||||
- Track the retry budget; on exhaustion with no primary, return the distinct cluster-not-ready code
|
||||
- Update the sticky current-master on a successful served-as-primary reply; log the target on each rotation and the attempt count on budget exhaustion
|
||||
|
|
@ -104,3 +104,4 @@
|
|||
- 1.2 (ADAPTED): RED-against-undefined-symbols would break linking of the shared lib test binary across tasks, so 1.2 builds a GREEN reusable harness (`src/lib/testsuite/failover_harness.h`): fake multi-master config+cursor, an 11-row controller-status taxonomy table (resolves to real errnox.h codes), and an N-pthread runner. Component RED→GREEN tests live in 2.1/2.2/2.3/3.1/3.2. The harness defines `fo_expected_action_t` (FO_ACT_*) mirroring the design `xce_fo_action_t` — 2.1 asserts the real classifier against these rows.
|
||||
- 2.2: EndpointRotator `xce_failover_next_target` (pure, socket-free: hint-first/consume, else cursor advance+wrap over conf->masters[]) + thin `xce_failover_rotate` (close-old/reconnect same endpoint). Live connect path is E2E (6.2). LAYERING: functions taking the private `api_ctx_t` are declared in `src/lib/client_internal.h`, NOT the public `xce/client_failover.h` (which keeps only the pure int-typed classifier) — per src/lib/CLAUDE.md (public headers in src/include/xce/ must not pull private state).
|
||||
- 2.3: RedirectCapture via non-static seam `xce_ctx_capture_redirect_hint()` called in handle_reply BEFORE the capnp free AND before the `status<0` early return (so negative E_XCE_REDIRECT still captures). Bounded strncpy into redirect_hint_host. ReplyWaitWake needed NO production change — existing `handle_disconnect` (err_handler, xcectx.c) already injects -ECONNRESET into wbuf + sets XCE_CTX_ERR; full err_handler path is covered by E2E (6.2).
|
||||
- 3.1: pure `xcecli_retry_decide` (no I/O — classify+budget+jittered-backoff calc, D6 exhaustion: no-primary→-E_XCE_NOT_READY, transport→keep last rc) + thin `xcecli_retry_next` (decide + RETRY_BACKOFF nanosleep + xce_failover_rotate + tlog_info/tlog_warn). Logger = utils.h tlog_* macros. State-reset/drain deferred to 3.2; rejoin hook to 4.1.
|
||||
|
|
|
|||
|
|
@ -15,12 +15,15 @@
|
|||
#include "xce/client_failover.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
#include <time.h>
|
||||
|
||||
#include "client_internal.h" /* api_ctx_t + EndpointRotator declarations */
|
||||
#include "client_internal.h" /* api_ctx_t + EndpointRotator + retry decls */
|
||||
#include "config.h" /* XCE_MAXHOSTNAME_LEN */
|
||||
#include "netio.h" /* endpoint_init / endpoint_connect / endpoint_fini */
|
||||
#include "utils.h" /* INFO / WARN logging macros */
|
||||
#include "xce/errnox.h"
|
||||
|
||||
/*
|
||||
|
|
@ -155,3 +158,152 @@ int xce_failover_rotate(api_ctx_t *ac)
|
|||
rc = endpoint_connect(&ac->ep, host, (int)port);
|
||||
return rc < 0 ? rc : 0;
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* XcecliRetryLoop — decision core (pure) + side-effecting wrapper
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/*
|
||||
* Compute the next no-primary backoff delay (ms) for retry number `attempt`
|
||||
* (1-based: the delay applied BEFORE the attempt'th re-send). Base is a capped
|
||||
* exponential (initial_retry_ms * multiplier^(attempt-1), clamped to
|
||||
* max_retry_ms); jitter spreads the actual delay within [base/2, base] using
|
||||
* rand_r on the per-op seed so concurrent clients do not retry in lockstep
|
||||
* (6.3). The base is monotone non-decreasing and never exceeds max_retry_ms.
|
||||
*/
|
||||
static int fo_backoff_delay_ms(const xce_retry_config_t* cfg, int attempt,
|
||||
unsigned int* seed)
|
||||
{
|
||||
double base = (double)cfg->initial_retry_ms;
|
||||
for (int i = 1; i < attempt; i++) {
|
||||
base *= cfg->backoff_multiplier;
|
||||
if (base >= (double)cfg->max_retry_ms) {
|
||||
base = (double)cfg->max_retry_ms;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (base > (double)cfg->max_retry_ms)
|
||||
base = (double)cfg->max_retry_ms;
|
||||
if (base < 1.0)
|
||||
base = 1.0;
|
||||
|
||||
int ibase = (int)base;
|
||||
/* jitter within the lower half: delay ∈ [ibase/2, ibase]. */
|
||||
int half = ibase / 2;
|
||||
int jitter = (half > 0) ? (int)(rand_r(seed) % (unsigned int)(half + 1)) : 0;
|
||||
int delay = ibase - jitter;
|
||||
if (delay < 1)
|
||||
delay = 1;
|
||||
if (delay > cfg->max_retry_ms)
|
||||
delay = cfg->max_retry_ms;
|
||||
return delay;
|
||||
}
|
||||
|
||||
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
|
||||
const xce_retry_config_t* cfg, xcecli_retry_t* st,
|
||||
xce_fo_action_t* action_out)
|
||||
{
|
||||
if (ac == NULL || rc == NULL || cfg == NULL || st == NULL)
|
||||
return false;
|
||||
|
||||
xce_fo_action_t action =
|
||||
xce_failover_classify(*rc, is_write, st->attempt);
|
||||
if (action_out != NULL)
|
||||
*action_out = action;
|
||||
|
||||
switch (action) {
|
||||
case XCE_FO_SUCCESS:
|
||||
/* DUP-replay success surfaces as a clean 0; a real status-0 is already
|
||||
* 0. Either way the op succeeded — normalise and stop. */
|
||||
*rc = 0;
|
||||
return false;
|
||||
|
||||
case XCE_FO_TERMINAL:
|
||||
/* Non-retriable — surface *rc unchanged. */
|
||||
return false;
|
||||
|
||||
case XCE_FO_RETRY_NOW:
|
||||
case XCE_FO_RETRY_BACKOFF:
|
||||
break;
|
||||
}
|
||||
|
||||
/* A retriable outcome consumes one retry from the budget. */
|
||||
st->attempt++;
|
||||
if (st->attempt > cfg->max_retries) {
|
||||
/* Budget exhausted. Distinguish the cause (design D6): a no-primary
|
||||
* window maps to the distinct cluster-not-ready code; a transport-only
|
||||
* outage keeps the last transport rc so the caller still sees it. */
|
||||
if (action == XCE_FO_RETRY_BACKOFF)
|
||||
*rc = -E_XCE_NOT_READY;
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Compute the pre-attempt delay: none for prompt rotation (RETRY_NOW),
|
||||
* a jittered exponential backoff for the no-primary case (RETRY_BACKOFF). */
|
||||
st->delay_ms = (action == XCE_FO_RETRY_BACKOFF)
|
||||
? fo_backoff_delay_ms(cfg, st->attempt, &st->rand_seed)
|
||||
: 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Sleep for `ms` milliseconds, tolerating EINTR. Isolated here so the pure
|
||||
* decision core stays sleep-free and unit-testable. */
|
||||
static void fo_sleep_ms(int ms)
|
||||
{
|
||||
if (ms <= 0)
|
||||
return;
|
||||
struct timespec ts = { .tv_sec = ms / 1000,
|
||||
.tv_nsec = (long)(ms % 1000) * 1000000L };
|
||||
while (nanosleep(&ts, &ts) != 0 && errno == EINTR)
|
||||
; /* finish the remaining interval */
|
||||
}
|
||||
|
||||
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
|
||||
xcecli_retry_t* st)
|
||||
{
|
||||
if (ac == NULL || rc == NULL || st == NULL)
|
||||
return false;
|
||||
|
||||
xce_fo_action_t action;
|
||||
bool retry = xcecli_retry_decide(ac, rc, is_write, &ac->failover.retry, st,
|
||||
&action);
|
||||
if (!retry) {
|
||||
/* Stopped. If the budget was exhausted on a no-primary window, decide
|
||||
* has already set *rc to -E_XCE_NOT_READY (7.2 observable). */
|
||||
if (*rc == -E_XCE_NOT_READY)
|
||||
WARN("xcecli failover: cluster not ready after %d attempts "
|
||||
"(last status=%d)",
|
||||
st->attempt, *rc);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* RETRY_BACKOFF: honour the jittered pre-attempt delay (4.1, 6.3). */
|
||||
if (action == XCE_FO_RETRY_BACKOFF)
|
||||
fo_sleep_ms(st->delay_ms);
|
||||
|
||||
/* Re-point the endpoint to the redirect hint or the next master and
|
||||
* reconnect (2.x, 4.2, 4.4). Log the rotation target (7.1). A rotate
|
||||
* failure is not fatal here: the next xce_* attempt observes the dead
|
||||
* connection and the loop reclassifies it as a transport error. */
|
||||
int rrc = xce_failover_rotate(ac);
|
||||
if (rrc < 0) {
|
||||
WARN("xcecli failover: rotate failed (rc=%d); retrying anyway", rrc);
|
||||
} else {
|
||||
/* Log the rotation target (7.1). xce_failover_rotate consumes the
|
||||
* redirect hint when present (so the post-rotate target is not always
|
||||
* the sticky master); report the sticky master cursor as a best-effort
|
||||
* endpoint label, which is exact for the configured-master rotation. */
|
||||
int cur = ac->failover.cur_master;
|
||||
const char* host =
|
||||
(ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
|
||||
? ac->conf->masters[cur]
|
||||
: "(unknown)";
|
||||
int port = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
|
||||
? ac->conf->master_ports[cur]
|
||||
: 0;
|
||||
INFO("xcecli failover: attempt %d → %s:%d", st->attempt,
|
||||
host ? host : "(null)", port);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#define _CLIENT_INTERNAL_H_ 1
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
|
|
@ -9,6 +10,7 @@
|
|||
#include "mylock.h"
|
||||
#include "netio.h"
|
||||
#include "request.h"
|
||||
#include "xce/client_failover.h" /* xce_fo_action_t */
|
||||
#include "xcectx.h"
|
||||
|
||||
/*
|
||||
|
|
@ -80,4 +82,61 @@ int xce_failover_next_target(api_ctx_t* ac, char* host_out, size_t host_sz,
|
|||
uint16_t* port_out);
|
||||
int xce_failover_rotate(api_ctx_t* ac);
|
||||
|
||||
/*
|
||||
* XcecliRetryLoop / decision helper (task 3.1) — see design.md
|
||||
* "Components and Interfaces → xcecli_retry_next".
|
||||
*
|
||||
* xcecli_retry_t: per-op retry carrier. One instance lives on the caller's
|
||||
* stack for the duration of a single xcecli_* op's do/while loop.
|
||||
* attempt — number of re-sends so far (0 on the first send).
|
||||
* delay_ms — backoff to apply BEFORE the next attempt (0 for RETRY_NOW).
|
||||
* rand_seed — per-op rand_r() state for jittered backoff (6.3).
|
||||
*/
|
||||
typedef struct {
|
||||
int attempt;
|
||||
int delay_ms;
|
||||
unsigned int rand_seed;
|
||||
} xcecli_retry_t;
|
||||
|
||||
/*
|
||||
* XCECLI_RETRY_INIT(ac): initialise a retry carrier from an api_ctx_t. The seed
|
||||
* mixes the ctx pointer so concurrent ops on distinct ctxs jitter independently.
|
||||
*/
|
||||
#define XCECLI_RETRY_INIT(ac) \
|
||||
((xcecli_retry_t){ .attempt = 0, \
|
||||
.delay_ms = 0, \
|
||||
.rand_seed = (unsigned int)(uintptr_t)(ac) })
|
||||
|
||||
/*
|
||||
* xcecli_retry_decide(): the PURE decision core (no I/O, no sleep, no socket,
|
||||
* no logging) so it is unit-testable. Given the rc returned by the last xce_*
|
||||
* attempt, it:
|
||||
* - classifies *rc via xce_failover_classify(*rc, is_write, st->attempt);
|
||||
* - on SUCCESS applies the DUP-replay rule (sets *rc = 0) and returns false;
|
||||
* - on TERMINAL returns false leaving *rc as-is;
|
||||
* - on a retriable action advances st->attempt, computes st->delay_ms
|
||||
* (0 for RETRY_NOW; a jittered exponential backoff bounded by
|
||||
* cfg->max_retry_ms for RETRY_BACKOFF), reports the action in *action_out,
|
||||
* and returns true — UNLESS the retry budget (cfg->max_retries) is now
|
||||
* exhausted, in which case it returns false and, for a no-primary cause,
|
||||
* sets *rc = -E_XCE_NOT_READY (a transport-only cause keeps the last rc).
|
||||
*
|
||||
* It does NOT reset ctx state, drain wbuf, sleep, rotate, or invoke the rejoin
|
||||
* hook — those are the side-effecting wrapper's job (xcecli_retry_next) and
|
||||
* later tasks (3.2 state-reset/drain, 4.1 rejoin hook).
|
||||
*/
|
||||
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
|
||||
const xce_retry_config_t* cfg, xcecli_retry_t* st,
|
||||
xce_fo_action_t* action_out);
|
||||
|
||||
/*
|
||||
* xcecli_retry_next(): the side-effecting wrapper the xcecli_* do/while loops
|
||||
* call. = xcecli_retry_decide() + (RETRY_BACKOFF: a real st->delay_ms sleep) +
|
||||
* (any retry: xce_failover_rotate(ac) + INFO log of the rotation target, 7.1) +
|
||||
* (budget exhaustion: WARN log of the attempt count, 7.2). Returns true → the
|
||||
* caller re-invokes its xce_* op; false → stop, *rc is final.
|
||||
*/
|
||||
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
|
||||
xcecli_retry_t* st);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -761,6 +761,266 @@ static void test_reply_wait_wake_releases_blocked_get(void)
|
|||
printf("test_reply_wait_wake_releases_blocked_get: PASS\n");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Task 3.1 — decision loop (xcecli_retry_decide) RED→GREEN tests.
|
||||
*
|
||||
* xcecli_retry_decide is the PURE core of the failover decision loop: classify
|
||||
* → apply SUCCESS/DUP/TERMINAL to *rc → advance attempt → compute backoff →
|
||||
* handle budget exhaustion → report the chosen action. It does NOT sleep, open
|
||||
* sockets, rotate, or log. The cursor-advancing side (xce_failover_next_target)
|
||||
* is driven separately to prove the become-primary path advances the cursor.
|
||||
*
|
||||
* Requirements: 1.2, 3.1, 3.2 (budget), 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* A small, deterministic retry config for the decide tests: a low max_retries
|
||||
* so budget exhaustion is reached quickly, a tiny initial delay, a modest cap. */
|
||||
static xce_retry_config_t fo_test_retry_cfg(void)
|
||||
{
|
||||
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
|
||||
cfg.max_retries = 3;
|
||||
cfg.initial_retry_ms = 10;
|
||||
cfg.max_retry_ms = 100;
|
||||
cfg.backoff_multiplier = 2.0;
|
||||
return cfg;
|
||||
}
|
||||
|
||||
/* Req 3.3, 3.4: a controller that keeps reporting no-primary (NOMASTER /
|
||||
* NOT_READY) is retried with backoff until the budget is exhausted; decide then
|
||||
* returns false with *rc == -E_XCE_NOT_READY — the distinct cluster-not-ready
|
||||
* code, never an opaque -EINVAL. */
|
||||
static void test_decide_no_primary_exhausts_to_not_ready(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
xce_retry_config_t cfg = fo_test_retry_cfg();
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
|
||||
int rc;
|
||||
xce_fo_action_t action;
|
||||
bool again;
|
||||
int retries = 0;
|
||||
|
||||
do {
|
||||
rc = E_XCE_NOMASTER; /* every attempt: no primary known */
|
||||
again = xcecli_retry_decide(&ac, &rc, /*is_write=*/false, &cfg, &st,
|
||||
&action);
|
||||
if (again) {
|
||||
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
|
||||
retries++;
|
||||
}
|
||||
} while (again && retries < 100); /* hard cap so a bug can't spin forever */
|
||||
|
||||
TEST_ASSERT_FALSE(again);
|
||||
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
|
||||
/* exactly max_retries retries happened before exhaustion */
|
||||
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
|
||||
|
||||
printf("test_decide_no_primary_exhausts_to_not_ready: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 3.2 / become-primary: no-primary then a served reply (status 0) → the
|
||||
* loop stops with success and the master cursor advanced across the retries
|
||||
* (the rotation side runs in the wrapper; here we advance it explicitly to
|
||||
* mirror that and assert the sticky cursor moved). */
|
||||
static void test_decide_no_primary_then_becomes_primary(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
xce_retry_config_t cfg = fo_test_retry_cfg();
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
|
||||
int rc;
|
||||
xce_fo_action_t action;
|
||||
|
||||
/* attempt 0: no primary → retry (decide says backoff). */
|
||||
rc = E_XCE_NOMASTER;
|
||||
bool a0 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
||||
TEST_ASSERT_TRUE(a0);
|
||||
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
|
||||
/* mirror the wrapper: rotate advances the cursor (no hint set) */
|
||||
char host[XCE_MAXHOSTNAME_LEN];
|
||||
uint16_t port;
|
||||
xce_failover_next_target(&ac, host, sizeof(host), &port);
|
||||
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
|
||||
|
||||
/* attempt 1: served by the new primary → success, loop stops. */
|
||||
rc = 0;
|
||||
bool a1 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
||||
TEST_ASSERT_FALSE(a1);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
/* cursor stayed advanced (sticky primary = where success landed) */
|
||||
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
|
||||
|
||||
printf("test_decide_no_primary_then_becomes_primary: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 4.2, 4.4: redirect / standby / transport causes are RETRY_NOW with NO
|
||||
* backoff delay applied (st->delay_ms == 0), so the wrapper rotates promptly. */
|
||||
static void test_decide_retry_now_has_no_backoff_delay(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
const int now_codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, -ECONNRESET,
|
||||
-ECONNREFUSED, -ETIMEDOUT };
|
||||
for (size_t i = 0; i < sizeof(now_codes) / sizeof(now_codes[0]); i++) {
|
||||
xce_retry_config_t cfg = fo_test_retry_cfg();
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
int rc = now_codes[i];
|
||||
xce_fo_action_t action;
|
||||
|
||||
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
||||
TEST_ASSERT_TRUE(again);
|
||||
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
|
||||
TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* no backoff for RETRY_NOW */
|
||||
}
|
||||
|
||||
printf("test_decide_retry_now_has_no_backoff_delay: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 4.1, 6.3: no-primary causes get a jittered exponential backoff — the
|
||||
* delay grows across retries and never exceeds max_retry_ms. */
|
||||
static void test_decide_backoff_grows_and_is_bounded(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
|
||||
cfg.max_retries = 12;
|
||||
cfg.initial_retry_ms = 10;
|
||||
cfg.max_retry_ms = 200;
|
||||
cfg.backoff_multiplier = 2.0;
|
||||
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
int rc;
|
||||
xce_fo_action_t action;
|
||||
|
||||
int prev_base = 0;
|
||||
for (int i = 0; i < 8; i++) {
|
||||
rc = E_XCE_NOT_READY;
|
||||
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
||||
TEST_ASSERT_TRUE(again);
|
||||
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
|
||||
|
||||
/* delay is positive and never exceeds the cap */
|
||||
TEST_ASSERT_TRUE(st.delay_ms > 0);
|
||||
TEST_ASSERT_TRUE(st.delay_ms <= cfg.max_retry_ms);
|
||||
|
||||
/* the un-jittered base (initial * mult^attempt, capped) is monotone
|
||||
* non-decreasing; once we hit the cap it stays there. The delay floor
|
||||
* grows: by attempt 5+ it must exceed the very first delay. */
|
||||
if (i >= 5)
|
||||
TEST_ASSERT_TRUE(st.delay_ms >= prev_base);
|
||||
if (i == 0)
|
||||
prev_base = st.delay_ms;
|
||||
}
|
||||
|
||||
printf("test_decide_backoff_grows_and_is_bounded: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 4.5: a terminal (non-retriable) status stops the loop immediately with
|
||||
* *rc unchanged — no retry, no -E_XCE_NOT_READY rewrite. */
|
||||
static void test_decide_terminal_surfaces_unchanged(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
const int term_codes[] = { -EACCES, -EPERM, -EINVAL, -ENOENT };
|
||||
for (size_t i = 0; i < sizeof(term_codes) / sizeof(term_codes[0]); i++) {
|
||||
xce_retry_config_t cfg = fo_test_retry_cfg();
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
int rc = term_codes[i];
|
||||
xce_fo_action_t action;
|
||||
|
||||
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
||||
TEST_ASSERT_FALSE(again);
|
||||
TEST_ASSERT_EQUAL_INT(term_codes[i], rc); /* surfaced unchanged */
|
||||
}
|
||||
|
||||
printf("test_decide_terminal_surfaces_unchanged: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 5.2: DUP on a re-sent write (is_write && attempt>0) → success (*rc = 0),
|
||||
* loop stops. The controller's client-jobid replay guard means the write was
|
||||
* already applied, so the retry is a success not a duplicate failure. */
|
||||
static void test_decide_dup_on_resent_write_is_success(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
xce_retry_config_t cfg = fo_test_retry_cfg();
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
st.attempt = 1; /* a re-sent write */
|
||||
|
||||
int rc = -E_XCE_DUP_JOB;
|
||||
xce_fo_action_t action;
|
||||
bool again = xcecli_retry_decide(&ac, &rc, /*is_write=*/true, &cfg, &st,
|
||||
&action);
|
||||
TEST_ASSERT_FALSE(again);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc); /* replay → success */
|
||||
|
||||
/* contrast: DUP on the FIRST send (attempt 0) is a genuine duplicate →
|
||||
* terminal, *rc surfaced unchanged. */
|
||||
xcecli_retry_t st0 = XCECLI_RETRY_INIT(&ac);
|
||||
int rc0 = -E_XCE_DUP_JOB;
|
||||
bool a0 =
|
||||
xcecli_retry_decide(&ac, &rc0, true, &cfg, &st0, &action);
|
||||
TEST_ASSERT_FALSE(a0);
|
||||
TEST_ASSERT_EQUAL_INT(-E_XCE_DUP_JOB, rc0);
|
||||
|
||||
printf("test_decide_dup_on_resent_write_is_success: PASS\n");
|
||||
}
|
||||
|
||||
/* Req 3.4 (D6): a transport-only cause that outlasts the budget keeps the LAST
|
||||
* transport rc (not -E_XCE_NOT_READY — that mapping is reserved for no-primary
|
||||
* exhaustion). Proves the two exhaustion causes are distinguished. */
|
||||
static void test_decide_transport_exhaustion_keeps_last_rc(void)
|
||||
{
|
||||
xce_client_config_t conf;
|
||||
api_ctx_t ac;
|
||||
fo_make_client_masters(&conf);
|
||||
fo_make_api_ctx(&ac, &conf);
|
||||
|
||||
xce_retry_config_t cfg = fo_test_retry_cfg();
|
||||
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
|
||||
|
||||
int rc;
|
||||
xce_fo_action_t action;
|
||||
bool again;
|
||||
int retries = 0;
|
||||
|
||||
do {
|
||||
rc = -ECONNREFUSED; /* every attempt: transport failure */
|
||||
again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
|
||||
if (again) {
|
||||
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
|
||||
retries++;
|
||||
}
|
||||
} while (again && retries < 100);
|
||||
|
||||
TEST_ASSERT_FALSE(again);
|
||||
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
|
||||
/* transport cause → last transport rc preserved, NOT rewritten */
|
||||
TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc);
|
||||
|
||||
printf("test_decide_transport_exhaustion_keeps_last_rc: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────── */
|
||||
|
||||
void test_libxce_retry(void)
|
||||
|
|
@ -787,5 +1047,12 @@ void test_libxce_retry(void)
|
|||
RUN_TEST(test_capture_redirect_hint_truncates_long_host);
|
||||
RUN_TEST(test_capture_redirect_hint_null_ctx_safe);
|
||||
RUN_TEST(test_reply_wait_wake_releases_blocked_get);
|
||||
RUN_TEST(test_decide_no_primary_exhausts_to_not_ready);
|
||||
RUN_TEST(test_decide_no_primary_then_becomes_primary);
|
||||
RUN_TEST(test_decide_retry_now_has_no_backoff_delay);
|
||||
RUN_TEST(test_decide_backoff_grows_and_is_bounded);
|
||||
RUN_TEST(test_decide_terminal_surfaces_unchanged);
|
||||
RUN_TEST(test_decide_dup_on_resent_write_is_success);
|
||||
RUN_TEST(test_decide_transport_exhaustion_keeps_last_rc);
|
||||
UNITY_END();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue