lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(lib): retry a NOT_READY controller in place instead of failing over (bug-1850)

Browse source 
xce_failover_classify() mapped E_XCE_NOT_READY with NOMASTER to
RETRY_BACKOFF, so xcecli_retry_next() rotated to the next master,
reconnected and fired the rejoin hook. But a NOT_READY reply comes from a
controller that answered on a working connection -- here a healthy primary
that had briefly lost its write quorum (both standbys NACKing, bug-1864).
The scheduler abandoned it for the standbys, rejoined, and got a full
re-push built from lagging committed state (21:39:24-27 in data/logs).

New action XCE_FO_RETRY_SAME (appended; existing values unchanged):
NOT_READY backs off (same jittered schedule, same -E_XCE_NOT_READY on
budget exhaustion) and re-sends on the current connection -- no rotate,
reconnect or rejoin. NOMASTER keeps rotating after backoff. If the
controller is really a standby it answers REDIRECT/STANDBY/NOMASTER next
time, and if it dies the transport error rotates, as before.

Tests (test_libxce_retry.c): taxonomy row NOT_READY -> RETRY_SAME;
decide backs off and exhausts to -E_XCE_NOT_READY; the real
xcecli_retry_next keeps cur_master on NOT_READY (rotated before) while
STANDBY still rotates; backoff-growth test moved to NOMASTER. Failover
tests 56/56; lib, controller 5/5, worker 10/10, scheduler 87/87,
harness 5/5.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-25 15:04:47 +00:00
parent 910b43cd37
commit 6b1862f8d3
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
4 changed files with 100 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/lib/testsuite/failover_harness.h
View file

@ -71,8 +71,9 @@ const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur);
typedef enum {
FO_ACT_SUCCESS = 0, /* status 0, or idempotent DUP replay on a re-sent write */
FO_ACT_RETRY_NOW, /* REDIRECT / STANDBY / transport error — rotate, no backoff */
FO_ACT_RETRY_BACKOFF,/* NOMASTER / NOT_READY — rotate after jittered backoff */
FO_ACT_TERMINAL /* non-retriable — surface to caller */
FO_ACT_RETRY_BACKOFF,/* NOMASTER — rotate after jittered backoff */
FO_ACT_TERMINAL, /* non-retriable — surface to caller */
FO_ACT_RETRY_SAME /* NOT_READY — backoff, retry the SAME controller */
} fo_expected_action_t;
typedef struct {

70
src/lib/testsuite/test_libxce_retry.c
View file

@ -116,8 +116,12 @@ const fo_status_row_t FO_STATUS_TABLE[] = {
/* no-primary / not-ready — rotate after backoff (4.1) */
{ "E_XCE_NOMASTER", E_XCE_NOMASTER, false, false, FO_ACT_RETRY_BACKOFF,
"standby knows no primary yet — backoff + rotate (4.1)" },
{ "E_XCE_NOT_READY", E_XCE_NOT_READY, false, false, FO_ACT_RETRY_BACKOFF,
"controller initializing/electing — backoff + rotate (4.1)" },
/* bug-1850: the controller answered, so the connection works -- it is
* briefly not ready (electing, or a primary that lost its write quorum).
* Back off and retry the same controller; rotating away from a healthy
* primary forced a scheduler rejoin + full re-push. */
{ "E_XCE_NOT_READY", E_XCE_NOT_READY, false, false, FO_ACT_RETRY_SAME,
"controller not ready — backoff, retry the same controller" },
/* idempotent-replay duplicate (write replay) — DUP/EEXIST (5.2) */
{ "E_XCE_DUP_JOB", E_XCE_DUP_JOB, false, false, FO_ACT_TERMINAL,
@ -290,7 +294,7 @@ static void test_status_table_is_complete_and_resolves(void)
fo_status_lookup(E_XCE_STANDBY)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
fo_status_lookup(E_XCE_NOMASTER)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF,
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_SAME,
fo_status_lookup(E_XCE_NOT_READY)->expect);
TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW,
fo_status_lookup(-ECONNRESET)->expect);
@ -372,6 +376,7 @@ static xce_fo_action_t fo_expected_to_action(fo_expected_action_t e)
case FO_ACT_RETRY_NOW: return XCE_FO_RETRY_NOW;
case FO_ACT_RETRY_BACKOFF: return XCE_FO_RETRY_BACKOFF;
case FO_ACT_TERMINAL: return XCE_FO_TERMINAL;
case FO_ACT_RETRY_SAME: return XCE_FO_RETRY_SAME;
}
return XCE_FO_TERMINAL;
}
@ -867,6 +872,61 @@ static void test_decide_no_primary_exhausts_to_not_ready(void)
* loop stops with success and the master cursor advanced across the retries
* (the rotation side runs in the wrapper; here we advance it explicitly to
* mirror that and assert the sticky cursor moved). */
/* bug-1850: NOT_READY still backs off (and exhausts to -E_XCE_NOT_READY),
* but as RETRY_SAME -- the caller must not rotate. */
static void test_decide_not_ready_backs_off_on_same_controller(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
xce_retry_config_t cfg = fo_test_retry_cfg();
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = E_XCE_NOT_READY;
xce_fo_action_t action;
TEST_ASSERT_TRUE(xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action));
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_SAME, action);
TEST_ASSERT_TRUE(st.delay_ms > 0);
for (int i = 0; i < cfg.max_retries + 1; i++) {
rc = E_XCE_NOT_READY;
if (!xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action))
break;
}
TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc);
printf("test_decide_not_ready_backs_off_on_same_controller: PASS\n");
}
/* bug-1850: the live retry loop must not rotate (reconnect to the next
* master, fire the rejoin hook) on NOT_READY from a controller that
* answered -- a healthy primary briefly without write quorum was abandoned
* for the standbys, forcing a scheduler rejoin and a full re-push. */
static void test_retry_next_not_ready_does_not_rotate(void)
{
xce_client_config_t conf;
api_ctx_t ac;
fo_make_client_masters(&conf);
fo_make_api_ctx(&ac, &conf);
ac.failover.retry = fo_test_retry_cfg();
ac.failover.retry.initial_retry_ms = 1;
ac.failover.retry.max_retry_ms = 2;
xcecli_retry_t st = XCECLI_RETRY_INIT(&ac);
int rc = E_XCE_NOT_READY;
TEST_ASSERT_TRUE(xcecli_retry_next(&ac, &rc, false, &st));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ac.failover.cur_master,
"NOT_READY must be retried on the same controller, not rotated");
/* a genuine redirect still rotates */
rc = E_XCE_STANDBY;
TEST_ASSERT_TRUE(xcecli_retry_next(&ac, &rc, false, &st));
TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master);
printf("test_retry_next_not_ready_does_not_rotate: PASS\n");
}
static void test_decide_no_primary_then_becomes_primary(void)
{
xce_client_config_t conf;
@ -949,7 +1009,7 @@ static void test_decide_backoff_grows_and_is_bounded(void)
int prev_base = 0;
for (int i = 0; i < 8; i++) {
rc = E_XCE_NOT_READY;
rc = E_XCE_NOMASTER;
bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action);
TEST_ASSERT_TRUE(again);
TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action);
@ -1844,5 +1904,7 @@ void test_libxce_retry(void)
RUN_TEST(test_is_connection_broken_false_on_run);
RUN_TEST(test_is_connection_broken_null_safe);
RUN_TEST(test_ff_call_is_single_shot_no_budget);
RUN_TEST(test_decide_not_ready_backs_off_on_same_controller);
RUN_TEST(test_retry_next_not_ready_does_not_rotate);
UNITY_END();
}