feat(component-failover-adoption): fix worker startup ordering, add rejoin retry-on-failure
Thread cluster_cfg by value through xcecd_config_t -> xcecd_context_t -> cc_ctx so it's populated before controller_connector_start() fires the first join, instead of after xcecd_start() returns. checking_connection() now retries a failed post-reconnect xcecli_join() on the next liveness probe instead of leaving needs_rejoin cleared on mere attempt.
This commit is contained in:
parent
ce9f734a38
commit
331fa02423
SSH key fingerprint:
SHA256:8Vf2pZ3h6rJa7EhGK+WaAQ9Mta/apZvteW6g9U/WKKk
8 changed files with 663 additions and 12 deletions
|
|
@ -105,6 +105,9 @@ void setUp(void)
|
|||
cc_ctx = NULL;
|
||||
xcecli_ctx = NULL;
|
||||
needs_rejoin = false;
|
||||
/* Restore the real production call by default (task 2.2) -- individual
|
||||
* rejoin-retry tests below override this to a controllable double. */
|
||||
rejoin_call = perform_pending_rejoin;
|
||||
}
|
||||
|
||||
void tearDown(void) { return; }
|
||||
|
|
@ -307,6 +310,171 @@ static void test_checking_connection_never_permanently_disables_worker(void)
|
|||
elapsed_ms);
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* Rejoin retry-on-failure (task 2.2, Req 2.5): after a transparent
|
||||
* reconnect sets needs_rejoin, checking_connection() must issue the actual
|
||||
* rejoin as a separate call and clear the flag ONLY on that call's success
|
||||
* -- on failure the flag must stay set so the NEXT checking_connection()
|
||||
* tick retries it, rather than leaving the worker transport-connected but
|
||||
* never actually re-registered.
|
||||
*
|
||||
* These tests override the rejoin_call function-pointer seam (controller_
|
||||
* connector.c) with a controllable double instead of exercising the real
|
||||
* xcecli_join(): a real failure goes through XCECLI_FAILOVER_CALL's full
|
||||
* retry/backoff budget (up to ~100s) before returning, and a real success
|
||||
* requires a live capnp-speaking controller -- neither is practical from a
|
||||
* fast unit test. Production always calls through the same seam via its
|
||||
* default (perform_pending_rejoin(), verified wired correctly by
|
||||
* test_rejoin_callback_registered_via_public_api's sibling tests above and
|
||||
* by direct reading of checking_connection()'s body); only these tests
|
||||
* substitute the pointer.
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static int g_rejoin_call_count = 0;
|
||||
static int g_rejoin_call_results[4];
|
||||
static bool g_rejoin_use_scripted_results = false;
|
||||
|
||||
static int fake_rejoin_always_fails(void)
|
||||
{
|
||||
g_rejoin_call_count++;
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int fake_rejoin_always_succeeds(void)
|
||||
{
|
||||
g_rejoin_call_count++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Returns g_rejoin_call_results[call index] on each successive call --
|
||||
* lets a test script "fails once, then succeeds" or similar sequences. */
|
||||
static int fake_rejoin_scripted(void)
|
||||
{
|
||||
int idx = g_rejoin_call_count;
|
||||
g_rejoin_call_count++;
|
||||
if (!g_rejoin_use_scripted_results
|
||||
|| idx >= (int)(sizeof(g_rejoin_call_results)
|
||||
/ sizeof(g_rejoin_call_results[0]))) {
|
||||
return -1;
|
||||
}
|
||||
return g_rejoin_call_results[idx];
|
||||
}
|
||||
|
||||
static void test_checking_connection_skips_rejoin_when_not_pending(void)
|
||||
{
|
||||
controller_connector_ctx_t fake_cc = { 0 };
|
||||
fake_cc.running = true;
|
||||
cc_ctx = &fake_cc;
|
||||
xcecli_ctx = NULL; /* xcecli_hello fails fast */
|
||||
|
||||
needs_rejoin = false;
|
||||
g_rejoin_call_count = 0;
|
||||
rejoin_call = fake_rejoin_always_succeeds;
|
||||
|
||||
checking_connection();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_rejoin_call_count,
|
||||
"checking_connection() must not attempt a rejoin when none is "
|
||||
"pending");
|
||||
TEST_ASSERT_FALSE(needs_rejoin);
|
||||
|
||||
printf("test_checking_connection_skips_rejoin_when_not_pending: PASS\n");
|
||||
}
|
||||
|
||||
static void test_checking_connection_rejoin_failure_keeps_flag_set(void)
|
||||
{
|
||||
controller_connector_ctx_t fake_cc = { 0 };
|
||||
fake_cc.running = true;
|
||||
cc_ctx = &fake_cc;
|
||||
xcecli_ctx = NULL;
|
||||
|
||||
needs_rejoin = true;
|
||||
g_rejoin_call_count = 0;
|
||||
rejoin_call = fake_rejoin_always_fails;
|
||||
|
||||
checking_connection();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rejoin_call_count,
|
||||
"rejoin must have been attempted");
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
needs_rejoin,
|
||||
"a failed rejoin call must leave needs_rejoin set so the next "
|
||||
"liveness probe retries it (Req 2.5) -- it must NOT be cleared "
|
||||
"merely because the call was attempted");
|
||||
|
||||
printf("test_checking_connection_rejoin_failure_keeps_flag_set: PASS\n");
|
||||
}
|
||||
|
||||
static void test_checking_connection_rejoin_success_clears_flag(void)
|
||||
{
|
||||
controller_connector_ctx_t fake_cc = { 0 };
|
||||
fake_cc.running = true;
|
||||
cc_ctx = &fake_cc;
|
||||
xcecli_ctx = NULL;
|
||||
|
||||
needs_rejoin = true;
|
||||
g_rejoin_call_count = 0;
|
||||
rejoin_call = fake_rejoin_always_succeeds;
|
||||
|
||||
checking_connection();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(1, g_rejoin_call_count);
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
needs_rejoin,
|
||||
"a successful rejoin call must clear needs_rejoin (Req 2.5)");
|
||||
|
||||
printf("test_checking_connection_rejoin_success_clears_flag: PASS\n");
|
||||
}
|
||||
|
||||
/* The core Req 2.5 / task 2.2 observable-completion scenario: "a rejoin
|
||||
* call that fails once before succeeding is retried on the next
|
||||
* liveness-probe cycle rather than abandoned" -- exercised as TWO separate
|
||||
* checking_connection() calls (two liveness-probe ticks), not two inner
|
||||
* attempts within one call, matching design.md's "checking_connection()'s
|
||||
* existing periodic cadence is what makes this retry naturally". */
|
||||
static void
|
||||
test_checking_connection_rejoin_retried_on_next_probe_after_failure(void)
|
||||
{
|
||||
controller_connector_ctx_t fake_cc = { 0 };
|
||||
fake_cc.running = true;
|
||||
cc_ctx = &fake_cc;
|
||||
xcecli_ctx = NULL;
|
||||
|
||||
needs_rejoin = true;
|
||||
g_rejoin_call_count = 0;
|
||||
g_rejoin_use_scripted_results = true;
|
||||
g_rejoin_call_results[0] = -E_XCE_NOT_READY; /* 1st tick: fails */
|
||||
g_rejoin_call_results[1] = 0; /* 2nd tick: succeeds */
|
||||
rejoin_call = fake_rejoin_scripted;
|
||||
|
||||
/* Liveness-probe tick 1: rejoin attempted, fails, flag must stay set. */
|
||||
checking_connection();
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rejoin_call_count,
|
||||
"tick 1 must attempt exactly one rejoin");
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
needs_rejoin,
|
||||
"flag must remain set after tick 1's failed rejoin -- not "
|
||||
"abandoned");
|
||||
|
||||
/* Liveness-probe tick 2 (checking_connection()'s own periodic cadence,
|
||||
* no new timer): rejoin retried automatically because the flag is
|
||||
* still set, this time succeeds, flag clears. */
|
||||
checking_connection();
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
2, g_rejoin_call_count,
|
||||
"tick 2 must retry the rejoin since it is still pending");
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
needs_rejoin,
|
||||
"flag must clear once the retried rejoin succeeds on tick 2");
|
||||
|
||||
g_rejoin_use_scripted_results = false;
|
||||
|
||||
printf(
|
||||
"test_checking_connection_rejoin_retried_on_next_probe_after_"
|
||||
"failure: PASS\n");
|
||||
}
|
||||
|
||||
/* ════════════════════════════════════════════════════════════════════════
|
||||
* main
|
||||
* ════════════════════════════════════════════════════════════════════════ */
|
||||
|
|
@ -324,6 +492,11 @@ int main(void)
|
|||
RUN_TEST(test_rejoin_callback_sets_flag_and_returns_immediately);
|
||||
RUN_TEST(test_rejoin_callback_registered_via_public_api);
|
||||
RUN_TEST(test_checking_connection_never_permanently_disables_worker);
|
||||
RUN_TEST(test_checking_connection_skips_rejoin_when_not_pending);
|
||||
RUN_TEST(test_checking_connection_rejoin_failure_keeps_flag_set);
|
||||
RUN_TEST(test_checking_connection_rejoin_success_clears_flag);
|
||||
RUN_TEST(
|
||||
test_checking_connection_rejoin_retried_on_next_probe_after_failure);
|
||||
|
||||
return UNITY_END();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue