lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(libxce-failover): retry decision loop — pure decide (budget/backoff/jitter, D6 exhaustion) + thin next wrapper (task 3.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-17 03:27:38 +00:00
parent cec2e647dc
commit 84c5e32c2c
4 changed files with 481 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

267
src/lib/testsuite/test_libxce_retry.c
View file

@ -761,6 +761,266 @@ static void test_reply_wait_wake_releases_blocked_get(void)
printf("test_reply_wait_wake_releases_blocked_get: PASS\n");
}
/* ════════════════════════════════════════════════════════════════════════
* Task 3.1 — decision loop (xcecli_retry_decide) RED→GREEN tests.
*
* xcecli_retry_decide is the PURE core of the failover decision loop: classify
* → apply SUCCESS/DUP/TERMINAL to *rc → advance attempt → compute backoff →
* handle budget exhaustion → report the chosen action. It does NOT sleep, open
* sockets, rotate, or log. The cursor-advancing side (xce_failover_next_target)
* is driven separately to prove the become-primary path advances the cursor.
*
* Requirements: 1.2, 3.1, 3.2 (budget), 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2.
* ════════════════════════════════════════════════════════════════════════ */
/* A small, deterministic retry config for the decide tests: a low max_retries
* so budget exhaustion is reached quickly, a tiny initial delay, a modest cap. */
static xce_retry_config_t fo_test_retry_cfg(void)
{
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 3;
cfg.initial_retry_ms = 10;
cfg.max_retry_ms = 100;
cfg.backoff_multiplier = 2.0;
return cfg;
}
/* Req 3.3, 3.4: a controller that keeps reporting no-primary (NOMASTER /
* NOT_READY) is retried with backoff until the budget is exhausted; decide then
* returns false with *rc == -E_XCE_NOT_READY — the distinct cluster-not-ready
* code, never an opaque -EINVAL. */
static void test_decide_no_primary_exhausts_to_not_ready(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
bool again;
int retries = 0;
do {
rc = E_XCE_NOMASTER; /* every attempt: no primary known */
again = xcecli_retry_decide(&ac, &rc, /*is_write=*/false, &cfg, &st,
&action);
if (again) {
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
retries++;
}
} while (again && retries < 100); /* hard cap so a bug can't spin forever */
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
/* exactly max_retries retries happened before exhaustion */
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
printf("test_decide_no_primary_exhausts_to_not_ready: PASS\n");
}
/* Req 3.2 / become-primary: no-primary then a served reply (status 0) → the
* loop stops with success and the master cursor advanced across the retries
* (the rotation side runs in the wrapper; here we advance it explicitly to
* mirror that and assert the sticky cursor moved). */
static void test_decide_no_primary_then_becomes_primary(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
/* attempt 0: no primary → retry (decide says backoff). */
rc = E_XCE_NOMASTER;
bool a0 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(a0);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
/* mirror the wrapper: rotate advances the cursor (no hint set) */
char host[XCE_MAXHOSTNAME_LEN];
uint16_t port;
xce_failover_next_target(&ac, host, sizeof(host), &port);
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
/* attempt 1: served by the new primary → success, loop stops. */
rc = 0;
bool a1 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_FALSE(a1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* cursor stayed advanced (sticky primary = where success landed) */
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
printf("test_decide_no_primary_then_becomes_primary: PASS\n");
}
/* Req 4.2, 4.4: redirect / standby / transport causes are RETRY_NOW with NO
* backoff delay applied (st->delay_ms == 0), so the wrapper rotates promptly. */
static void test_decide_retry_now_has_no_backoff_delay(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
const int now_codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, -ECONNRESET,
-ECONNREFUSED, -ETIMEDOUT };
for (size_t i = 0; i < sizeof(now_codes) / sizeof(now_codes[0]); i++) {
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = now_codes[i];
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* no backoff for RETRY_NOW */
}
printf("test_decide_retry_now_has_no_backoff_delay: PASS\n");
}
/* Req 4.1, 6.3: no-primary causes get a jittered exponential backoff — the
* delay grows across retries and never exceeds max_retry_ms. */
static void test_decide_backoff_grows_and_is_bounded(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = XCE_RETRY_DEFAULTS;
cfg.max_retries = 12;
cfg.initial_retry_ms = 10;
cfg.max_retry_ms = 200;
cfg.backoff_multiplier = 2.0;
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
int prev_base = 0;
for (int i = 0; i < 8; i++) {
rc = E_XCE_NOT_READY;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
/* delay is positive and never exceeds the cap */
TEST_ASSERT_TRUE(st.delay_ms > 0);
TEST_ASSERT_TRUE(st.delay_ms <= cfg.max_retry_ms);
/* the un-jittered base (initial * mult^attempt, capped) is monotone
* non-decreasing; once we hit the cap it stays there. The delay floor
* grows: by attempt 5+ it must exceed the very first delay. */
if (i >= 5)
TEST_ASSERT_TRUE(st.delay_ms >= prev_base);
if (i == 0)
prev_base = st.delay_ms;
}
printf("test_decide_backoff_grows_and_is_bounded: PASS\n");
}
/* Req 4.5: a terminal (non-retriable) status stops the loop immediately with
* *rc unchanged — no retry, no -E_XCE_NOT_READY rewrite. */
static void test_decide_terminal_surfaces_unchanged(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
const int term_codes[] = { -EACCES, -EPERM, -EINVAL, -ENOENT };
for (size_t i = 0; i < sizeof(term_codes) / sizeof(term_codes[0]); i++) {
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = term_codes[i];
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(term_codes[i], rc); /* surfaced unchanged */
}
printf("test_decide_terminal_surfaces_unchanged: PASS\n");
}
/* Req 5.2: DUP on a re-sent write (is_write && attempt>0) → success (*rc = 0),
* loop stops. The controller's client-jobid replay guard means the write was
* already applied, so the retry is a success not a duplicate failure. */
static void test_decide_dup_on_resent_write_is_success(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
st.attempt = 1; /* a re-sent write */
int rc = -E_XCE_DUP_JOB;
xce_fo_action_t action;
bool again = xcecli_retry_decide(&ac, &rc, /*is_write=*/true, &cfg, &st,
&action);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(0, rc); /* replay → success */
/* contrast: DUP on the FIRST send (attempt 0) is a genuine duplicate →
* terminal, *rc surfaced unchanged. */
xcecli_retry_t st0 = XCECLI_RETRY_INIT(&ac);
int rc0 = -E_XCE_DUP_JOB;
bool a0 =
xcecli_retry_decide(&ac, &rc0, true, &cfg, &st0, &action);
TEST_ASSERT_FALSE(a0);
TEST_ASSERT_EQUAL_INT(-E_XCE_DUP_JOB, rc0);
printf("test_decide_dup_on_resent_write_is_success: PASS\n");
}
/* Req 3.4 (D6): a transport-only cause that outlasts the budget keeps the LAST
* transport rc (not -E_XCE_NOT_READY — that mapping is reserved for no-primary
* exhaustion). Proves the two exhaustion causes are distinguished. */
static void test_decide_transport_exhaustion_keeps_last_rc(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc;
xce_fo_action_t action;
bool again;
int retries = 0;
do {
rc = -ECONNREFUSED; /* every attempt: transport failure */
again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
if (again) {
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action);
retries++;
}
} while (again && retries < 100);
TEST_ASSERT_FALSE(again);
TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries);
/* transport cause → last transport rc preserved, NOT rewritten */
TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc);
printf("test_decide_transport_exhaustion_keeps_last_rc: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────── */
void test_libxce_retry(void)
@ -787,5 +1047,12 @@ void test_libxce_retry(void)
RUN_TEST(test_capture_redirect_hint_truncates_long_host);
RUN_TEST(test_capture_redirect_hint_null_ctx_safe);
RUN_TEST(test_reply_wait_wake_releases_blocked_get);
RUN_TEST(test_decide_no_primary_exhausts_to_not_ready);
RUN_TEST(test_decide_no_primary_then_becomes_primary);
RUN_TEST(test_decide_retry_now_has_no_backoff_delay);
RUN_TEST(test_decide_backoff_grows_and_is_bounded);
RUN_TEST(test_decide_terminal_surfaces_unchanged);
RUN_TEST(test_decide_dup_on_resent_write_is_success);
RUN_TEST(test_decide_transport_exhaustion_keeps_last_rc);
UNITY_END();
}