/* * test_libxce_retry.c — reusable libxce-failover test harness + harness * self-tests (spec libxce-failover, task 1.2 ADAPTED). * * This file builds the GREEN, reusable harness that the later component tasks * consume to write their own RED→GREEN tests: * - task 2.1 (StatusClassifier) → drives FO_STATUS_TABLE * - task 2.2 (EndpointRotator) → drives the fake-master cursor * - task 2.3 (reply-capture) → drives the redirect-hint rows * - task 3.1/3.2 (decision helper) → drives the table + cursor * - task 5.1/6.1 (concurrency) → drive fo_run_threads() * * It deliberately references NO not-yet-existing failover production symbols * (xce_failover_classify / xcecli_retry_next / xce_failover_rotate); those are * added by the component tasks. The lib build+test MUST stay GREEN after this * task, so every fixture here is exercised by at least one real assertion. * * Requirements covered by the harness: 2.5, 4.1, 4.2, 4.3, 4.4, 4.5, 5.2, 8.3, * 8.4 (status taxonomy); 2.1, 2.4 (master cursor); 6.3 (concurrency reuse). * The retained retry-defaults test still covers 6.1, 6.2. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include "client_internal.h" /* api_ctx_t, xce_failover_state_t */ #include "failover_harness.h" #include "mylock.h" #include "request.h" #include "xce/client_failover.h" #include "xce/errnox.h" /* ════════════════════════════════════════════════════════════════════════ * Harness implementation * ════════════════════════════════════════════════════════════════════════ */ /* ── 1. Fake multi-master config + cursor ─────────────────────────────── */ int fo_make_fake_masters(xce_cluster_config_t *cfg) { static const struct { const char *host; int port; } masters[FO_FAKE_MASTERS] = { { "10.20.0.1", 7001 }, { "10.20.0.2", 7002 }, { "10.20.0.3", 7003 }, }; memset(cfg, 0, sizeof(*cfg)); for (int i = 0; i < FO_FAKE_MASTERS && i < MAX_HA_CANDIDATES; i++) { strncpy(cfg->candidates[i].hostname, masters[i].host, XCE_MAXHOSTNAME_LEN - 1); cfg->candidates[i].hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0'; cfg->candidates[i].port = masters[i].port; } cfg->candidates_count = (FO_FAKE_MASTERS < MAX_HA_CANDIDATES) ? FO_FAKE_MASTERS : MAX_HA_CANDIDATES; return cfg->candidates_count; } void fo_cursor_init(fo_master_cursor_t *cur, const xce_cluster_config_t *cfg) { cur->cfg = cfg; cur->cur = 0; } int fo_cursor_advance(fo_master_cursor_t *cur) { if (!cur->cfg || cur->cfg->candidates_count <= 0) return 0; cur->cur = (cur->cur + 1) % cur->cfg->candidates_count; return cur->cur; } const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur) { if (!cur->cfg || cur->cfg->candidates_count <= 0) return NULL; return &cur->cfg->candidates[cur->cur]; } /* ── 2. Controller-status taxonomy table ──────────────────────────────── */ const fo_status_row_t FO_STATUS_TABLE[] = { /* success */ { "OK", 0, false, false, FO_ACT_SUCCESS, "request served by primary" }, /* redirect (a primary exists elsewhere) — rotate now, hint first if any */ { "E_XCE_REDIRECT+hint", E_XCE_REDIRECT, true, false, FO_ACT_RETRY_NOW, "standby redirects with primary address hint (8.3)" }, { "E_XCE_REDIRECT", E_XCE_REDIRECT, false, false, FO_ACT_RETRY_NOW, "standby redirects without address — rotate master list (8.4)" }, /* standby (op is primary-only) — treat as redirect, rotate now (4.3) */ { "E_XCE_STANDBY", E_XCE_STANDBY, false, false, FO_ACT_RETRY_NOW, "controller in standby; op is primary-only (4.3)" }, /* no-primary / not-ready — rotate after backoff (4.1) */ { "E_XCE_NOMASTER", E_XCE_NOMASTER, false, false, FO_ACT_RETRY_BACKOFF, "standby knows no primary yet — backoff + rotate (4.1)" }, { "E_XCE_NOT_READY", E_XCE_NOT_READY, false, false, FO_ACT_RETRY_BACKOFF, "controller initializing/electing — backoff + rotate (4.1)" }, /* idempotent-replay duplicate (write replay) — DUP/EEXIST (5.2) */ { "E_XCE_DUP_JOB", E_XCE_DUP_JOB, false, false, FO_ACT_TERMINAL, "duplicate job; SUCCESS only on a re-sent write (5.2, attempt-aware)" }, { "-EEXIST", -EEXIST, false, true, FO_ACT_TERMINAL, "duplicate via -EEXIST; SUCCESS only on a re-sent write (5.2)" }, /* transport errors — reconnect/rotate now (4.4) */ { "-ECONNREFUSED", -ECONNREFUSED, false, true, FO_ACT_RETRY_NOW, "connection refused — reconnect/rotate (4.4)" }, { "-ETIMEDOUT", -ETIMEDOUT, false, true, FO_ACT_RETRY_NOW, "connection timeout — reconnect/rotate (4.4)" }, { "-ECONNRESET", -ECONNRESET, false, true, FO_ACT_RETRY_NOW, "peer reset — reconnect/rotate (4.4)" }, /* terminal / non-retriable — surface immediately (4.5) */ { "-EINVAL", -EINVAL, false, true, FO_ACT_TERMINAL, "invalid request — surface, no retry (4.5)" }, }; const size_t FO_STATUS_TABLE_LEN = sizeof(FO_STATUS_TABLE) / sizeof(FO_STATUS_TABLE[0]); const fo_status_row_t *fo_status_lookup(int status) { for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) { if (FO_STATUS_TABLE[i].status == status) return &FO_STATUS_TABLE[i]; } return NULL; } /* ── 3. Generic N-thread concurrency harness ──────────────────────────── */ typedef struct { fo_thread_fn fn; void *ctx; int idx; } fo_thread_arg_t; static void *fo_thread_trampoline(void *p) { fo_thread_arg_t *a = (fo_thread_arg_t *)p; a->fn(a->ctx, a->idx); return NULL; } int fo_run_threads(int n, fo_thread_fn fn, void *ctx) { if (n <= 0 || !fn) return -EINVAL; pthread_t *tids = calloc((size_t)n, sizeof(*tids)); fo_thread_arg_t *args = calloc((size_t)n, sizeof(*args)); if (!tids || !args) { free(tids); free(args); return -ENOMEM; } int spawned = 0; int rc = 0; for (int i = 0; i < n; i++) { args[i].fn = fn; args[i].ctx = ctx; args[i].idx = i; int e = pthread_create(&tids[i], NULL, fo_thread_trampoline, &args[i]); if (e != 0) { rc = -e; break; } spawned++; } for (int i = 0; i < spawned; i++) pthread_join(tids[i], NULL); free(tids); free(args); return rc; } /* ════════════════════════════════════════════════════════════════════════ * Harness self-tests (GREEN) — each fixture proven by real assertions * ════════════════════════════════════════════════════════════════════════ */ /* Req 6.1, 6.2: the retry-tunables struct + safe production defaults are * retained and reachable for the failover loop. */ static void test_retry_defaults_are_reachable(void) { xce_retry_config_t cfg = XCE_RETRY_DEFAULTS; TEST_ASSERT_EQUAL_INT(20, cfg.max_retries); TEST_ASSERT_EQUAL_INT(50, cfg.initial_retry_ms); TEST_ASSERT_EQUAL_INT(5000, cfg.max_retry_ms); TEST_ASSERT_TRUE(cfg.backoff_multiplier > 1.0); printf("test_retry_defaults_are_reachable: PASS\n"); } /* Fixture 1: fake multi-master config holds 3 distinct candidates. */ static void test_fake_masters_hold_three_candidates(void) { xce_cluster_config_t cfg; int n = fo_make_fake_masters(&cfg); TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, n); TEST_ASSERT_EQUAL_INT(FO_FAKE_MASTERS, cfg.candidates_count); TEST_ASSERT_EQUAL_STRING("10.20.0.1", cfg.candidates[0].hostname); TEST_ASSERT_EQUAL_INT(7001, cfg.candidates[0].port); TEST_ASSERT_EQUAL_STRING("10.20.0.3", cfg.candidates[2].hostname); TEST_ASSERT_EQUAL_INT(7003, cfg.candidates[2].port); printf("test_fake_masters_hold_three_candidates: PASS\n"); } /* Fixture 1: cursor advances through masters and wraps (2.1, 2.4, 8.4). */ static void test_master_cursor_advances_and_wraps(void) { xce_cluster_config_t cfg; fo_make_fake_masters(&cfg); fo_master_cursor_t cur; fo_cursor_init(&cur, &cfg); TEST_ASSERT_EQUAL_INT(0, cur.cur); TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname); TEST_ASSERT_EQUAL_INT(1, fo_cursor_advance(&cur)); TEST_ASSERT_EQUAL_STRING("10.20.0.2", fo_cursor_current(&cur)->hostname); TEST_ASSERT_EQUAL_INT(2, fo_cursor_advance(&cur)); TEST_ASSERT_EQUAL_STRING("10.20.0.3", fo_cursor_current(&cur)->hostname); /* wrap back to master 0 */ TEST_ASSERT_EQUAL_INT(0, fo_cursor_advance(&cur)); TEST_ASSERT_EQUAL_STRING("10.20.0.1", fo_cursor_current(&cur)->hostname); printf("test_master_cursor_advances_and_wraps: PASS\n"); } /* Fixture 2: taxonomy table is complete — every status the classifier must * handle is present, lookups resolve, and the status codes equal the real * errnox.h values (2.5, 4.1-4.5, 5.2, 8.3, 8.4). */ static void test_status_table_is_complete_and_resolves(void) { /* every mandated controller status + transport error is represented */ TEST_ASSERT_NOT_NULL(fo_status_lookup(0)); TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_REDIRECT)); TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_STANDBY)); TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOMASTER)); TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_NOT_READY)); TEST_ASSERT_NOT_NULL(fo_status_lookup(E_XCE_DUP_JOB)); TEST_ASSERT_NOT_NULL(fo_status_lookup(-EEXIST)); TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNREFUSED)); TEST_ASSERT_NOT_NULL(fo_status_lookup(-ETIMEDOUT)); TEST_ASSERT_NOT_NULL(fo_status_lookup(-ECONNRESET)); /* status codes resolve to the real errnox.h values (ordered enum) */ TEST_ASSERT_EQUAL_INT(XCE_ERRNO_BASE, E_XCE_BAD_JOB); TEST_ASSERT_TRUE(E_XCE_REDIRECT == E_XCE_NOMASTER + 1); TEST_ASSERT_TRUE(E_XCE_NOT_READY == E_XCE_REDIRECT + 1); TEST_ASSERT_TRUE(E_XCE_STANDBY == E_XCE_NOT_READY + 1); /* expected actions line up with the design's StatusClassifier (first send) */ TEST_ASSERT_EQUAL_INT(FO_ACT_SUCCESS, fo_status_lookup(0)->expect); TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW, fo_status_lookup(E_XCE_REDIRECT)->expect); TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW, fo_status_lookup(E_XCE_STANDBY)->expect); TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF, fo_status_lookup(E_XCE_NOMASTER)->expect); TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_BACKOFF, fo_status_lookup(E_XCE_NOT_READY)->expect); TEST_ASSERT_EQUAL_INT(FO_ACT_RETRY_NOW, fo_status_lookup(-ECONNRESET)->expect); printf("test_status_table_is_complete_and_resolves: PASS\n"); } /* Fixture 2: the redirect-hint flag distinguishes hint vs. no-hint rows (8.3). */ static void test_status_table_redirect_hint_flag(void) { bool saw_hint = false; bool saw_no_hint = false; for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) { if (FO_STATUS_TABLE[i].status == E_XCE_REDIRECT) { if (FO_STATUS_TABLE[i].has_hint) saw_hint = true; else saw_no_hint = true; } } TEST_ASSERT_TRUE(saw_hint); /* REDIRECT+hint row present (8.3) */ TEST_ASSERT_TRUE(saw_no_hint); /* REDIRECT no-hint row present (8.4) */ printf("test_status_table_redirect_hint_flag: PASS\n"); } /* Fixture 3: concurrency harness runs N threads to completion (6.3 reuse). */ #define FO_TEST_NTHREADS 16 typedef struct { pthread_mutex_t lock; int counter; } fo_selftest_ctx_t; static void fo_selftest_worker(void *ctx, int idx) { (void)idx; fo_selftest_ctx_t *c = (fo_selftest_ctx_t *)ctx; pthread_mutex_lock(&c->lock); c->counter++; pthread_mutex_unlock(&c->lock); } static void test_thread_harness_runs_all_threads(void) { fo_selftest_ctx_t ctx; pthread_mutex_init(&ctx.lock, NULL); ctx.counter = 0; int rc = fo_run_threads(FO_TEST_NTHREADS, fo_selftest_worker, &ctx); pthread_mutex_destroy(&ctx.lock); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(FO_TEST_NTHREADS, ctx.counter); printf("test_thread_harness_runs_all_threads: PASS\n"); } /* Fixture 3: harness rejects bad arguments. */ static void test_thread_harness_rejects_bad_args(void) { TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(0, fo_selftest_worker, NULL)); TEST_ASSERT_EQUAL_INT(-EINVAL, fo_run_threads(4, NULL, NULL)); printf("test_thread_harness_rejects_bad_args: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.1 — StatusClassifier (xce_failover_classify) RED→GREEN tests * ════════════════════════════════════════════════════════════════════════ */ /* Map the harness' expected-action category to the real classifier enum. The * two enums share order by design, but we map explicitly so a future reorder of * either enum is caught here rather than silently passing. */ static xce_fo_action_t fo_expected_to_action(fo_expected_action_t e) { switch (e) { case FO_ACT_SUCCESS: return XCE_FO_SUCCESS; case FO_ACT_RETRY_NOW: return XCE_FO_RETRY_NOW; case FO_ACT_RETRY_BACKOFF: return XCE_FO_RETRY_BACKOFF; case FO_ACT_TERMINAL: return XCE_FO_TERMINAL; } return XCE_FO_TERMINAL; } /* Req 2.5, 4.1, 4.2, 4.3, 4.5: every taxonomy row's first-send classification * (is_write=false, attempt=0) matches the design's StatusClassifier mapping. */ static void test_classify_matches_taxonomy_table(void) { for (size_t i = 0; i < FO_STATUS_TABLE_LEN; i++) { const fo_status_row_t *row = &FO_STATUS_TABLE[i]; xce_fo_action_t want = fo_expected_to_action(row->expect); xce_fo_action_t got = xce_failover_classify(row->status, /*is_write=*/false, /*attempt=*/0); char msg[128]; snprintf(msg, sizeof(msg), "row '%s' (status=%d): want %d got %d", row->name, row->status, (int)want, (int)got); TEST_ASSERT_EQUAL_INT_MESSAGE(want, got, msg); } printf("test_classify_matches_taxonomy_table: PASS\n"); } /* Req 2.5: REDIRECT/STANDBY/NOMASTER/NOT_READY are never reported as success, * regardless of is_write/attempt. */ static void test_classify_control_codes_never_success(void) { const int codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, E_XCE_NOMASTER, E_XCE_NOT_READY }; for (size_t i = 0; i < sizeof(codes) / sizeof(codes[0]); i++) { TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS, xce_failover_classify(codes[i], false, 0)); TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS, xce_failover_classify(codes[i], true, 1)); /* accepted in either sign */ TEST_ASSERT_NOT_EQUAL(XCE_FO_SUCCESS, xce_failover_classify(-codes[i], true, 1)); } printf("test_classify_control_codes_never_success: PASS\n"); } /* Req 5.2: idempotent-replay rule — DUP_JOB/EEXIST on a RE-SENT WRITE is a * success; the first send (or a read) is a genuine duplicate → terminal. */ static void test_classify_dup_replay_rule(void) { /* re-sent write (is_write && attempt>0) → SUCCESS */ TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS, xce_failover_classify(-E_XCE_DUP_JOB, true, 1)); TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS, xce_failover_classify(E_XCE_DUP_JOB, true, 1)); TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS, xce_failover_classify(-EEXIST, true, 1)); /* first send → genuine duplicate → TERMINAL */ TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL, xce_failover_classify(-E_XCE_DUP_JOB, true, 0)); /* non-write replay → TERMINAL */ TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL, xce_failover_classify(-E_XCE_DUP_JOB, false, 1)); TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL, xce_failover_classify(-EEXIST, false, 1)); printf("test_classify_dup_replay_rule: PASS\n"); } /* Req 3.2 / general: status 0 is always success. */ static void test_classify_zero_is_success(void) { TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS, xce_failover_classify(0, false, 0)); TEST_ASSERT_EQUAL_INT(XCE_FO_SUCCESS, xce_failover_classify(0, true, 3)); printf("test_classify_zero_is_success: PASS\n"); } /* Req 4.5: unknown / unrecognized statuses are conservatively terminal. */ static void test_classify_unknown_is_terminal(void) { TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL, xce_failover_classify(-EACCES, false, 0)); TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL, xce_failover_classify(-ENOENT, true, 1)); TEST_ASSERT_EQUAL_INT(XCE_FO_TERMINAL, xce_failover_classify(-12345, false, 0)); printf("test_classify_unknown_is_terminal: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.2 — EndpointRotator target selection (xce_failover_next_target) * RED→GREEN tests. Pure, socket-free target-selection logic; the * live connect path is covered by E2E task 6.2. * ════════════════════════════════════════════════════════════════════════ */ /* Build a 3-master client config (distinct host:port) the rotator selects over. * Strings are static-duration so the api_ctx may borrow them by pointer. */ static void fo_make_client_masters(xce_client_config_t *conf) { static char *hosts[N_MAX_MASTERS] = { "10.30.0.1", "10.30.0.2", "10.30.0.3" }; static int ports[N_MAX_MASTERS] = { 8001, 8002, 8003 }; memset(conf, 0, sizeof(*conf)); conf->nmasters = N_MAX_MASTERS; for (int i = 0; i < N_MAX_MASTERS; i++) { conf->masters[i] = hosts[i]; conf->master_ports[i] = ports[i]; } } /* Minimal api_ctx for target-selection tests: only conf, the failover cursor, * and the ctx redirect-hint fields are exercised. No endpoint/socket touched. */ static void fo_make_api_ctx(api_ctx_t *ac, xce_client_config_t *conf) { memset(ac, 0, sizeof(*ac)); ac->conf = conf; ac->failover.cur_master = 0; } static void fo_set_hint(api_ctx_t *ac, const char *host, uint16_t port) { strncpy(ac->ctx.redirect_hint_host, host, XCE_MAXHOSTNAME_LEN - 1); ac->ctx.redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1] = '\0'; ac->ctx.redirect_hint_port = port; } /* Req 2.2, 8.4: with a redirect hint set, next_target returns the hint * host/port and CONSUMES (clears) it; the following call ignores the stale * hint and advances the master cursor instead. */ static void test_next_target_hint_first_then_consumed(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); fo_set_hint(&ac, "10.99.0.7", 9100); char host[XCE_MAXHOSTNAME_LEN] = { 0 }; uint16_t port = 0; int rc = xce_failover_next_target(&ac, host, sizeof(host), &port); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_STRING("10.99.0.7", host); TEST_ASSERT_EQUAL_INT(9100, port); /* hint consumed: cleared on the ctx, cursor untouched */ TEST_ASSERT_EQUAL_STRING("", ac.ctx.redirect_hint_host); TEST_ASSERT_EQUAL_INT(0, ac.failover.cur_master); /* next call has no hint → advances cursor to master 1 */ rc = xce_failover_next_target(&ac, host, sizeof(host), &port); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_STRING("10.30.0.2", host); TEST_ASSERT_EQUAL_INT(8002, port); TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); printf("test_next_target_hint_first_then_consumed: PASS\n"); } /* Req 2.1, 8.4: with no hint, successive next_target calls advance through the * configured masters and WRAP around; the cursor lives on failover.cur_master. */ static void test_next_target_rotates_and_wraps(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); char host[XCE_MAXHOSTNAME_LEN] = { 0 }; uint16_t port = 0; /* from cur_master=0, first rotation advances to master 1 */ TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host), &port)); TEST_ASSERT_EQUAL_STRING("10.30.0.2", host); TEST_ASSERT_EQUAL_INT(8002, port); TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host), &port)); TEST_ASSERT_EQUAL_STRING("10.30.0.3", host); TEST_ASSERT_EQUAL_INT(8003, port); TEST_ASSERT_EQUAL_INT(2, ac.failover.cur_master); /* wrap back to master 0 */ TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host), &port)); TEST_ASSERT_EQUAL_STRING("10.30.0.1", host); TEST_ASSERT_EQUAL_INT(8001, port); TEST_ASSERT_EQUAL_INT(0, ac.failover.cur_master); printf("test_next_target_rotates_and_wraps: PASS\n"); } /* Req 4.4, 8.4: after a hint is consumed, the very next call advances to the * next master — correctness is independent of the (one-shot) hint. */ static void test_next_target_hint_then_fallback_rotation(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); /* start the cursor at master 1 to prove fallback advances from current */ ac.failover.cur_master = 1; fo_set_hint(&ac, "10.99.0.9", 9200); char host[XCE_MAXHOSTNAME_LEN] = { 0 }; uint16_t port = 0; /* hint first */ TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host), &port)); TEST_ASSERT_EQUAL_STRING("10.99.0.9", host); TEST_ASSERT_EQUAL_INT(9200, port); TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); /* cursor untouched */ /* fallback: advances from master 1 → master 2 */ TEST_ASSERT_EQUAL_INT(0, xce_failover_next_target(&ac, host, sizeof(host), &port)); TEST_ASSERT_EQUAL_STRING("10.30.0.3", host); TEST_ASSERT_EQUAL_INT(8003, port); TEST_ASSERT_EQUAL_INT(2, ac.failover.cur_master); printf("test_next_target_hint_then_fallback_rotation: PASS\n"); } /* Guard: NULL/invalid arguments are rejected without touching state. */ static void test_next_target_rejects_bad_args(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); char host[XCE_MAXHOSTNAME_LEN] = { 0 }; uint16_t port = 0; TEST_ASSERT_EQUAL_INT(-EINVAL, xce_failover_next_target(NULL, host, sizeof(host), &port)); TEST_ASSERT_EQUAL_INT(-EINVAL, xce_failover_next_target(&ac, NULL, sizeof(host), &port)); TEST_ASSERT_EQUAL_INT(-EINVAL, xce_failover_next_target(&ac, host, 0, &port)); TEST_ASSERT_EQUAL_INT(-EINVAL, xce_failover_next_target(&ac, host, sizeof(host), NULL)); printf("test_next_target_rejects_bad_args: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.3 — RedirectCapture (xce_ctx_capture_redirect_hint) RED→GREEN tests. * * handle_reply() copies the redirect address carried on a redirect reply into * the ctx's redirect_hint_host/port BEFORE the capnp-owned reply is freed. The * copy logic is factored into a non-static seam, xce_ctx_capture_redirect_hint, * so it can be unit-tested without a live server. * * Requirements: 2.2 (capture redirect addr as a best-effort hint), 8.3 (try the * hinted address first — capture is the prerequisite). * ════════════════════════════════════════════════════════════════════════ */ /* Build a bare xce_ctx_t with an empty redirect hint, without touching any * iomgr/ssl/endpoint state (capture is pure ctx-field manipulation). */ static void fo_ctx_clear_hint(xce_ctx_t *ctx) { memset(ctx, 0, sizeof(*ctx)); ctx->redirect_hint_host[0] = '\0'; ctx->redirect_hint_port = 0; } /* Req 2.2, 8.3: a present host+port is copied into the ctx hint fields. */ static void test_capture_redirect_hint_present(void) { xce_ctx_t ctx; fo_ctx_clear_hint(&ctx); xce_ctx_capture_redirect_hint(&ctx, "10.40.0.5", 9300); TEST_ASSERT_EQUAL_STRING("10.40.0.5", ctx.redirect_hint_host); TEST_ASSERT_EQUAL_INT(9300, ctx.redirect_hint_port); printf("test_capture_redirect_hint_present: PASS\n"); } /* Req 2.2: when no address is present (host NULL/empty or port 0) the hint is * left empty — capture must not invent an address. */ static void test_capture_redirect_hint_absent_stays_empty(void) { xce_ctx_t ctx; /* NULL host → hint untouched */ fo_ctx_clear_hint(&ctx); xce_ctx_capture_redirect_hint(&ctx, NULL, 9300); TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host); TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port); /* empty host → hint untouched */ fo_ctx_clear_hint(&ctx); xce_ctx_capture_redirect_hint(&ctx, "", 9300); TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host); TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port); /* port 0 → hint untouched (no usable target) */ fo_ctx_clear_hint(&ctx); xce_ctx_capture_redirect_hint(&ctx, "10.40.0.5", 0); TEST_ASSERT_EQUAL_STRING("", ctx.redirect_hint_host); TEST_ASSERT_EQUAL_INT(0, ctx.redirect_hint_port); printf("test_capture_redirect_hint_absent_stays_empty: PASS\n"); } /* Req 2.2: an over-long host is bounded-copied (NUL-terminated, no overflow); * the capnp-owned string the controller may send is not length-checked here. */ static void test_capture_redirect_hint_truncates_long_host(void) { xce_ctx_t ctx; fo_ctx_clear_hint(&ctx); char longhost[XCE_MAXHOSTNAME_LEN * 2]; memset(longhost, 'h', sizeof(longhost) - 1); longhost[sizeof(longhost) - 1] = '\0'; xce_ctx_capture_redirect_hint(&ctx, longhost, 9400); /* copied, NUL-terminated, and never longer than the field bound */ TEST_ASSERT_EQUAL_INT(XCE_MAXHOSTNAME_LEN - 1, (int)strlen(ctx.redirect_hint_host)); TEST_ASSERT_EQUAL_INT('\0', ctx.redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1]); TEST_ASSERT_EQUAL_INT(9400, ctx.redirect_hint_port); printf("test_capture_redirect_hint_truncates_long_host: PASS\n"); } /* Guard: a NULL ctx is rejected without crashing. */ static void test_capture_redirect_hint_null_ctx_safe(void) { /* must not crash */ xce_ctx_capture_redirect_hint(NULL, "10.40.0.5", 9300); printf("test_capture_redirect_hint_null_ctx_safe: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 2.3 — ReplyWaitWake RED→GREEN test. * * On an endpoint error/peer reset, the existing disconnect path injects a * transport error into ctx->wbuf so a blocked cbuf_get returns instead of * hanging. This test proves the wake mechanism end-to-end on a real wbuf: a * thread blocks in cbuf_get, the "disconnect" injects -ECONNRESET, and the * blocked waiter wakes with exactly that transport error (Req 4.4). * ════════════════════════════════════════════════════════════════════════ */ typedef struct { cbuf_t wbuf; int woke_status; /* what the blocked cbuf_get observed */ int got; /* cbuf_get return code */ } fo_wake_ctx_t; static void fo_wake_waiter(void *p, int idx) { fo_wake_ctx_t *w = (fo_wake_ctx_t *)p; if (idx == 0) { /* waiter: block until the disconnect injects an error */ w->got = cbuf_get(&w->wbuf, &w->woke_status); } else { /* injector: mirror handle_disconnect's wbuf error injection. A short * spin lets the waiter reach cbuf_get first so we exercise the wake, * not a pre-filled buffer. */ for (volatile int i = 0; i < 2000000; i++) { } int err = -ECONNRESET; cbuf_tryput(&w->wbuf, &err); } } /* Req 4.4: a pending cbuf_get is released with the injected transport error so * an in-flight xce_* call returns instead of hanging across an endpoint reset. */ static void test_reply_wait_wake_releases_blocked_get(void) { fo_wake_ctx_t w; memset(&w, 0, sizeof(w)); TEST_ASSERT_EQUAL_INT(0, cbuf_init(&w.wbuf, 1, sizeof(int))); int rc = fo_run_threads(2, fo_wake_waiter, &w); cbuf_fini(&w.wbuf); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_TRUE(w.got >= 0); /* cbuf_get returned, did not hang */ TEST_ASSERT_EQUAL_INT(-ECONNRESET, w.woke_status); printf("test_reply_wait_wake_releases_blocked_get: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 3.1 — decision loop (xcecli_retry_decide) RED→GREEN tests. * * xcecli_retry_decide is the PURE core of the failover decision loop: classify * → apply SUCCESS/DUP/TERMINAL to *rc → advance attempt → compute backoff → * handle budget exhaustion → report the chosen action. It does NOT sleep, open * sockets, rotate, or log. The cursor-advancing side (xce_failover_next_target) * is driven separately to prove the become-primary path advances the cursor. * * Requirements: 1.2, 3.1, 3.2 (budget), 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2. * ════════════════════════════════════════════════════════════════════════ */ /* A small, deterministic retry config for the decide tests: a low max_retries * so budget exhaustion is reached quickly, a tiny initial delay, a modest cap. */ static xce_retry_config_t fo_test_retry_cfg(void) { xce_retry_config_t cfg = XCE_RETRY_DEFAULTS; cfg.max_retries = 3; cfg.initial_retry_ms = 10; cfg.max_retry_ms = 100; cfg.backoff_multiplier = 2.0; return cfg; } /* Req 3.3, 3.4: a controller that keeps reporting no-primary (NOMASTER / * NOT_READY) is retried with backoff until the budget is exhausted; decide then * returns false with *rc == -E_XCE_NOT_READY — the distinct cluster-not-ready * code, never an opaque -EINVAL. */ static void test_decide_no_primary_exhausts_to_not_ready(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); int rc; xce_fo_action_t action; bool again; int retries = 0; do { rc = E_XCE_NOMASTER; /* every attempt: no primary known */ again = xcecli_retry_decide(&ac, &rc, /*is_write=*/false, &cfg, &st, &action); if (again) { TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action); retries++; } } while (again && retries < 100); /* hard cap so a bug can't spin forever */ TEST_ASSERT_FALSE(again); TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc); /* exactly max_retries retries happened before exhaustion */ TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries); printf("test_decide_no_primary_exhausts_to_not_ready: PASS\n"); } /* Req 3.2 / become-primary: no-primary then a served reply (status 0) → the * loop stops with success and the master cursor advanced across the retries * (the rotation side runs in the wrapper; here we advance it explicitly to * mirror that and assert the sticky cursor moved). */ static void test_decide_no_primary_then_becomes_primary(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); int rc; xce_fo_action_t action; /* attempt 0: no primary → retry (decide says backoff). */ rc = E_XCE_NOMASTER; bool a0 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); TEST_ASSERT_TRUE(a0); TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action); /* mirror the wrapper: rotate advances the cursor (no hint set) */ char host[XCE_MAXHOSTNAME_LEN]; uint16_t port; xce_failover_next_target(&ac, host, sizeof(host), &port); TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); /* attempt 1: served by the new primary → success, loop stops. */ rc = 0; bool a1 = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); TEST_ASSERT_FALSE(a1); TEST_ASSERT_EQUAL_INT(0, rc); /* cursor stayed advanced (sticky primary = where success landed) */ TEST_ASSERT_EQUAL_INT(1, ac.failover.cur_master); printf("test_decide_no_primary_then_becomes_primary: PASS\n"); } /* Req 4.2, 4.4: redirect / standby / transport causes are RETRY_NOW with NO * backoff delay applied (st->delay_ms == 0), so the wrapper rotates promptly. */ static void test_decide_retry_now_has_no_backoff_delay(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); const int now_codes[] = { E_XCE_REDIRECT, E_XCE_STANDBY, -ECONNRESET, -ECONNREFUSED, -ETIMEDOUT }; for (size_t i = 0; i < sizeof(now_codes) / sizeof(now_codes[0]); i++) { xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); int rc = now_codes[i]; xce_fo_action_t action; bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); TEST_ASSERT_TRUE(again); TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action); TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* no backoff for RETRY_NOW */ } printf("test_decide_retry_now_has_no_backoff_delay: PASS\n"); } /* Req 4.1, 6.3: no-primary causes get a jittered exponential backoff — the * delay grows across retries and never exceeds max_retry_ms. */ static void test_decide_backoff_grows_and_is_bounded(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); xce_retry_config_t cfg = XCE_RETRY_DEFAULTS; cfg.max_retries = 12; cfg.initial_retry_ms = 10; cfg.max_retry_ms = 200; cfg.backoff_multiplier = 2.0; xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); int rc; xce_fo_action_t action; int prev_base = 0; for (int i = 0; i < 8; i++) { rc = E_XCE_NOT_READY; bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); TEST_ASSERT_TRUE(again); TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action); /* delay is positive and never exceeds the cap */ TEST_ASSERT_TRUE(st.delay_ms > 0); TEST_ASSERT_TRUE(st.delay_ms <= cfg.max_retry_ms); /* the un-jittered base (initial * mult^attempt, capped) is monotone * non-decreasing; once we hit the cap it stays there. The delay floor * grows: by attempt 5+ it must exceed the very first delay. */ if (i >= 5) TEST_ASSERT_TRUE(st.delay_ms >= prev_base); if (i == 0) prev_base = st.delay_ms; } printf("test_decide_backoff_grows_and_is_bounded: PASS\n"); } /* Req 4.5: a terminal (non-retriable) status stops the loop immediately with * *rc unchanged — no retry, no -E_XCE_NOT_READY rewrite. */ static void test_decide_terminal_surfaces_unchanged(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); const int term_codes[] = { -EACCES, -EPERM, -EINVAL, -ENOENT }; for (size_t i = 0; i < sizeof(term_codes) / sizeof(term_codes[0]); i++) { xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); int rc = term_codes[i]; xce_fo_action_t action; bool again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); TEST_ASSERT_FALSE(again); TEST_ASSERT_EQUAL_INT(term_codes[i], rc); /* surfaced unchanged */ } printf("test_decide_terminal_surfaces_unchanged: PASS\n"); } /* Req 5.2: DUP on a re-sent write (is_write && attempt>0) → success (*rc = 0), * loop stops. The controller's client-jobid replay guard means the write was * already applied, so the retry is a success not a duplicate failure. */ static void test_decide_dup_on_resent_write_is_success(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); st.attempt = 1; /* a re-sent write */ int rc = -E_XCE_DUP_JOB; xce_fo_action_t action; bool again = xcecli_retry_decide(&ac, &rc, /*is_write=*/true, &cfg, &st, &action); TEST_ASSERT_FALSE(again); TEST_ASSERT_EQUAL_INT(0, rc); /* replay → success */ /* contrast: DUP on the FIRST send (attempt 0) is a genuine duplicate → * terminal, *rc surfaced unchanged. */ xcecli_retry_t st0 = XCECLI_RETRY_INIT(&ac); int rc0 = -E_XCE_DUP_JOB; bool a0 = xcecli_retry_decide(&ac, &rc0, true, &cfg, &st0, &action); TEST_ASSERT_FALSE(a0); TEST_ASSERT_EQUAL_INT(-E_XCE_DUP_JOB, rc0); printf("test_decide_dup_on_resent_write_is_success: PASS\n"); } /* Req 3.4 (D6): a transport-only cause that outlasts the budget keeps the LAST * transport rc (not -E_XCE_NOT_READY — that mapping is reserved for no-primary * exhaustion). Proves the two exhaustion causes are distinguished. */ static void test_decide_transport_exhaustion_keeps_last_rc(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); int rc; xce_fo_action_t action; bool again; int retries = 0; do { rc = -ECONNREFUSED; /* every attempt: transport failure */ again = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); if (again) { TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action); retries++; } } while (again && retries < 100); TEST_ASSERT_FALSE(again); TEST_ASSERT_EQUAL_INT(cfg.max_retries, retries); /* transport cause → last transport rc preserved, NOT rewritten */ TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc); printf("test_decide_transport_exhaustion_keeps_last_rc: PASS\n"); } /* * Regression for the live-E2E bug: after the connected primary dies, * handle_disconnect leaves ctx.state == XCE_CTX_ERR, so a FRESH xce_* call * returns -EINVAL from the per-op xceapi guard BEFORE reaching a primary. * decide() must treat the known-dead ctx as a prompt reconnect cause * (RETRY_NOW, no backoff) rather than classifying -EINVAL as TERMINAL, and on * budget exhaustion surface the distinct cluster-not-ready code, never -EINVAL. */ static void test_decide_known_dead_ctx_retries_not_terminal(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); xce_retry_config_t cfg = fo_test_retry_cfg(); xcecli_retry_t st = XCECLI_RETRY_INIT(&ac); xce_fo_action_t action; /* Known-dead ctx + the guard's -EINVAL must RETRY (reconnect), not surface. */ ac.ctx.state = XCE_CTX_ERR; int rc = -EINVAL; bool a = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); TEST_ASSERT_TRUE(a); TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_NOW, action); /* prompt rotate, no backoff */ TEST_ASSERT_EQUAL_INT(0, st.delay_ms); /* ctx stays ERR (no real reconnect in this unit) → keep retrying until the * budget is exhausted, then surface -E_XCE_NOT_READY (never -EINVAL). */ int guard = 0; do { rc = -EINVAL; a = xcecli_retry_decide(&ac, &rc, false, &cfg, &st, &action); } while (a && ++guard < 100); TEST_ASSERT_FALSE(a); TEST_ASSERT_EQUAL_INT(-E_XCE_NOT_READY, rc); printf("test_decide_known_dead_ctx_retries_not_terminal: PASS\n"); } /* Req 6.3: when MANY clients fail over simultaneously they must NOT retry in * lockstep — randomized (jittered) backoff staggers their delays. Each client * carries its OWN retry carrier (seeded from its own api_ctx pointer, exactly as * XCECLI_RETRY_INIT does in production), so N independent carriers driven through * the SAME no-primary backoff attempt must produce delays that VARY across * clients (not a single shared value) while every delay stays within the * [base/2, max_retry_ms] envelope. This drives the pure xcecli_retry_decide path * — socket-free — for many concurrent carriers and proves the anti-thundering- * herd property the single-carrier growth/bounds test does not. */ #define FO_JITTER_NCLIENTS 64 typedef struct { api_ctx_t *ac; /* shared cfg source; seeds are per-carrier */ const xce_retry_config_t *cfg; int delays[FO_JITTER_NCLIENTS]; /* one client's computed delay */ pthread_mutex_t lock; } fo_jitter_ctx_t; /* Each "client" runs one no-primary attempt through a FRESH retry carrier seeded * from a distinct value (its thread index), mirroring how distinct api_ctx * pointers seed distinct carriers. We record the jittered RETRY_BACKOFF delay it * computed for the first no-primary retry. */ static void fo_jitter_worker(void *ctx, int idx) { fo_jitter_ctx_t *c = (fo_jitter_ctx_t *)ctx; /* a distinct carrier per client: same init shape as XCECLI_RETRY_INIT but a * distinct seed per client (production seeds from the distinct ctx pointer; * here each thread stands in for a distinct client). */ xcecli_retry_t st = XCECLI_RETRY_INIT(c->ac); st.rand_seed = (unsigned int)(idx + 1) * 2654435761u; /* distinct/client */ int rc = E_XCE_NOMASTER; /* no primary → RETRY_BACKOFF */ xce_fo_action_t action; bool again = xcecli_retry_decide(c->ac, &rc, /*is_write=*/false, c->cfg, &st, &action); /* every client agrees on the action and stays within budget on attempt 1 */ TEST_ASSERT_TRUE(again); TEST_ASSERT_EQUAL_INT(XCE_FO_RETRY_BACKOFF, action); TEST_ASSERT_TRUE(st.delay_ms > 0); TEST_ASSERT_TRUE(st.delay_ms <= c->cfg->max_retry_ms); pthread_mutex_lock(&c->lock); c->delays[idx] = st.delay_ms; pthread_mutex_unlock(&c->lock); } static void test_decide_jitter_staggers_many_clients(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); /* a wide backoff window so jitter has room to spread: base after the first * retry is initial_retry_ms, jitter ∈ [base/2, base] = [100, 200]. */ xce_retry_config_t cfg = XCE_RETRY_DEFAULTS; cfg.max_retries = 20; cfg.initial_retry_ms = 200; cfg.max_retry_ms = 5000; cfg.backoff_multiplier = 2.0; fo_jitter_ctx_t jc; memset(&jc, 0, sizeof(jc)); jc.ac = ∾ jc.cfg = &cfg; TEST_ASSERT_EQUAL_INT(0, pthread_mutex_init(&jc.lock, NULL)); int rc = fo_run_threads(FO_JITTER_NCLIENTS, fo_jitter_worker, &jc); pthread_mutex_destroy(&jc.lock); TEST_ASSERT_EQUAL_INT(0, rc); /* Count distinct delay values: lock-step retry would yield a SINGLE value * across all clients. Jitter must spread them across many distinct delays. */ int distinct = 0; int min_d = jc.delays[0]; int max_d = jc.delays[0]; for (int i = 0; i < FO_JITTER_NCLIENTS; i++) { bool seen = false; for (int j = 0; j < i; j++) { if (jc.delays[j] == jc.delays[i]) { seen = true; break; } } if (!seen) distinct++; if (jc.delays[i] < min_d) min_d = jc.delays[i]; if (jc.delays[i] > max_d) max_d = jc.delays[i]; /* every client's delay stays within the jittered envelope [base/2, base] * for the first retry: base == initial_retry_ms == 200 here. */ TEST_ASSERT_TRUE(jc.delays[i] >= cfg.initial_retry_ms / 2); TEST_ASSERT_TRUE(jc.delays[i] <= cfg.initial_retry_ms); } /* the decisive anti-thundering-herd assertion: the clients did NOT retry in * lockstep — their delays span many distinct values, not one shared delay. */ TEST_ASSERT_TRUE(distinct > 1); /* with 64 clients over a ~100ms jitter span the spread must be substantial */ TEST_ASSERT_TRUE(distinct >= FO_JITTER_NCLIENTS / 4); TEST_ASSERT_TRUE(max_d > min_d); /* a real spread, not a constant */ printf("test_decide_jitter_staggers_many_clients: PASS " "(%d distinct delays across %d clients, span [%d,%d]ms)\n", distinct, FO_JITTER_NCLIENTS, min_d, max_d); } /* ════════════════════════════════════════════════════════════════════════ * Task 3.2 — Reconnect recovery (xcecli_ctx_recover) RED→GREEN tests. * * After a transport failure the disconnect path leaves ctx->state == XCE_CTX_ERR * (xcectx.c:174) and may have injected a stale transport status into the single- * slot ctx->wbuf (xcectx.c:180). xcecli_ctx_recover repairs the ctx after a * successful reconnect, with NO socket I/O so it is unit-testable: * - CI2 state reset: ERR → RUN (only ERR; RUN/DOWN/INIT left as-is) so the * next xce_* op passes the xceapi XCE_CTX_RUN guard instead of -EINVAL. * - CI1 stale-reply drain: any entry already sitting in wbuf is drained with a * non-blocking get, so the next attempt's cbuf_get cannot consume a prior- * attempt reply. * * Requirements: 1.2 (complete next request against new primary, no caller * change), 4.4 (transport → reconnect/rotate and resume). * ════════════════════════════════════════════════════════════════════════ */ /* CI2: an ERR-state ctx is restored to RUN (the reported -EINVAL bug fix). */ static void test_recover_resets_err_state_to_run(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_ERR; /* wbuf left un-init: recover must tolerate a recover that has nothing to * drain (init it so the drain path is also exercised safely). */ TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int))); int rc = xcecli_ctx_recover(&ac); cbuf_fini(&ac.ctx.wbuf); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state); printf("test_recover_resets_err_state_to_run: PASS\n"); } /* CI2: a ctx already in RUN is left in RUN; only ERR→RUN is performed. */ static void test_recover_leaves_run_state_untouched(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_RUN; TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int))); int rc = xcecli_ctx_recover(&ac); cbuf_fini(&ac.ctx.wbuf); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state); printf("test_recover_leaves_run_state_untouched: PASS\n"); } /* CI2: a DOWN ctx is NOT flipped to RUN — only an ERR transport state is * repaired; a deliberately-stopped ctx stays down. */ static void test_recover_leaves_down_state_untouched(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_DOWN; TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int))); int rc = xcecli_ctx_recover(&ac); cbuf_fini(&ac.ctx.wbuf); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(XCE_CTX_DOWN, ac.ctx.state); printf("test_recover_leaves_down_state_untouched: PASS\n"); } /* CI1: a stale status already sitting in wbuf is drained, so a subsequent * non-blocking get finds the channel EMPTY (the prior-attempt reply was * discarded, not handed to the next attempt's cbuf_get). */ static void test_recover_drains_stale_wbuf_entry(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_ERR; TEST_ASSERT_EQUAL_INT(0, cbuf_init(&ac.ctx.wbuf, 1, sizeof(int))); /* mirror handle_disconnect: a stale transport status sits in the channel */ int stale = -ECONNRESET; TEST_ASSERT_EQUAL_INT(0, cbuf_tryput(&ac.ctx.wbuf, &stale)); int rc = xcecli_ctx_recover(&ac); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(XCE_CTX_RUN, ac.ctx.state); /* the stale entry was drained: a non-blocking get now finds nothing */ int drained = 0; int g = cbuf_tryget(&ac.ctx.wbuf, &drained); cbuf_fini(&ac.ctx.wbuf); TEST_ASSERT_EQUAL_INT(-EAGAIN, g); /* channel empty after recover */ printf("test_recover_drains_stale_wbuf_entry: PASS\n"); } /* Guard: a NULL api_ctx is rejected without crashing. */ static void test_recover_null_ctx_safe(void) { TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_ctx_recover(NULL)); printf("test_recover_null_ctx_safe: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 4.1 — ReconnectRejoinHook (xcecli_set_reconnect_handler + * xcecli_invoke_rejoin) RED→GREEN tests. * * libxce owns the hook MECHANISM + ORDERING (reconnect → hook → resume); the * callback CONTENT is the consumer's. The failover machinery invokes the * registered callback once per successful reconnect, AFTER reconnect/recovery * and BEFORE the interrupted op resumes. A hook returning <0 means the new * primary rejected the daemon's join → treat it as a FAILED attempt (do not * resume; the loop rotates/backs off again). Pure clients register NO hook * (NULL) → invoke is a no-op returning 0 and the loop resumes directly. * * xcecli_invoke_rejoin is socket-free (it only calls the stored cb), so it is * unit-testable directly here without a live server. * * Requirements: 1.2 (re-establish session against new primary before resume). * ════════════════════════════════════════════════════════════════════════ */ /* A test rejoin callback: records that it ran (and the ctx/arg it saw) and * returns a caller-controlled rc so we can drive the success and failed-join * paths. */ typedef struct { int called; /* number of times the hook fired */ xcecli_ctx_t seen_ctx; /* the ctx the hook observed */ void *seen_arg; /* the opaque arg the hook observed */ int ret; /* rc the hook returns (0 ok, <0 failed rejoin) */ } fo_rejoin_probe_t; static int fo_rejoin_cb(xcecli_ctx_t ctx, void *arg) { fo_rejoin_probe_t *p = (fo_rejoin_probe_t *)arg; p->called++; p->seen_ctx = ctx; p->seen_arg = arg; return p->ret; } /* Req 1.2: the setter stores cb+arg into the failover state; invoke then calls * the registered cb exactly once, passing the opaque ctx and arg, and returns * the cb's rc (0 on a successful rejoin). */ static void test_invoke_rejoin_calls_registered_hook(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); fo_rejoin_probe_t probe = { 0 }; probe.ret = 0; /* setter stores cb+arg on the failover state */ int src = xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, fo_rejoin_cb, &probe); TEST_ASSERT_EQUAL_INT(0, src); TEST_ASSERT_TRUE(ac.failover.rejoin_cb == fo_rejoin_cb); TEST_ASSERT_TRUE(ac.failover.rejoin_arg == &probe); /* invoke fires the hook once, returns its rc (0), passes ctx + arg */ int rc = xcecli_invoke_rejoin(&ac); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(1, probe.called); TEST_ASSERT_TRUE(probe.seen_ctx == (xcecli_ctx_t)&ac); TEST_ASSERT_TRUE(probe.seen_arg == &probe); printf("test_invoke_rejoin_calls_registered_hook: PASS\n"); } /* Req 1.2: a hook that returns <0 (the new primary rejected the join) makes * invoke return that <0 — the wrapper treats it as a failed attempt and does * NOT resume the interrupted op. */ static void test_invoke_rejoin_propagates_failure(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); fo_rejoin_probe_t probe = { 0 }; probe.ret = -ECONNREFUSED; /* join rejected by new primary */ TEST_ASSERT_EQUAL_INT( 0, xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, fo_rejoin_cb, &probe)); int rc = xcecli_invoke_rejoin(&ac); TEST_ASSERT_EQUAL_INT(-ECONNREFUSED, rc); /* failure surfaced to the loop */ TEST_ASSERT_EQUAL_INT(1, probe.called); printf("test_invoke_rejoin_propagates_failure: PASS\n"); } /* Req 1.2: a pure client registers NO hook (NULL) → invoke is a no-op that * returns 0 and calls nothing, so the loop resumes directly. */ static void test_invoke_rejoin_no_hook_is_noop(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); /* leaves rejoin_cb == NULL */ TEST_ASSERT_NULL(ac.failover.rejoin_cb); int rc = xcecli_invoke_rejoin(&ac); TEST_ASSERT_EQUAL_INT(0, rc); /* no hook → no-op success */ printf("test_invoke_rejoin_no_hook_is_noop: PASS\n"); } /* Req 1.2: the setter is NULL-safe on a NULL ctx, and can clear a previously * registered hook by passing NULL (back to the pure-client no-op path). */ static void test_set_reconnect_handler_null_safe_and_clears(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); /* NULL ctx rejected without crashing */ TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_set_reconnect_handler(NULL, fo_rejoin_cb, NULL)); fo_rejoin_probe_t probe = { 0 }; TEST_ASSERT_EQUAL_INT( 0, xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, fo_rejoin_cb, &probe)); TEST_ASSERT_TRUE(ac.failover.rejoin_cb == fo_rejoin_cb); /* clearing with a NULL cb restores the no-hook no-op path */ TEST_ASSERT_EQUAL_INT( 0, xcecli_set_reconnect_handler((xcecli_ctx_t)&ac, NULL, NULL)); TEST_ASSERT_NULL(ac.failover.rejoin_cb); TEST_ASSERT_EQUAL_INT(0, xcecli_invoke_rejoin(&ac)); TEST_ASSERT_EQUAL_INT(0, probe.called); /* cleared hook never fired */ printf("test_set_reconnect_handler_null_safe_and_clears: PASS\n"); } /* Guard: invoke on a NULL api_ctx is rejected without crashing. */ static void test_invoke_rejoin_null_ctx_safe(void) { TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_invoke_rejoin(NULL)); printf("test_invoke_rejoin_null_ctx_safe: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 5.1 — XcecliRetryLoop wrapping + per-ctx lock (CI2 concurrency). * * Each reply-bearing xcecli_* op wraps its one-shot xce_* call in the failover * do/while loop AND holds ac->failover.lock for the loop's full duration so * concurrent ops on ONE shared ctx are serialised (the single-slot wbuf already * implies one in-flight request per ctx; the lock makes that explicit and covers * the reconnect/rotate window). Full failover behaviour against a live primary * is validated by E2E task 6.2; here we prove the serialisation contract. * * The XCECLI_FAILOVER_CALL macro (client_internal.h) is the shared wrapper that * every op body uses: lock → init retry carrier → do/while(xcecli_retry_next) → * unlock → return rc. We drive that exact macro here with a trivial EXPR that * mutates shared state under the held lock, so N threads contending on one ctx's * failover.lock must not lose updates (mutual exclusion), and a non-failing EXPR * runs exactly once (the success path returns immediately). * * Requirements: 1.1, 1.3, 2.4, 5.1, 5.3, and design "FailoverState → Concurrency * (CI2)". * ════════════════════════════════════════════════════════════════════════ */ #define FO_LOCK_NTHREADS 32 #define FO_LOCK_ITERS 2000 typedef struct { api_ctx_t *ac; /* the ONE shared ctx all threads contend on */ long counter; /* mutated under failover.lock via the wrapper macro */ int max_inflight;/* peak concurrent in-section (must stay 1) */ int inflight; /* current in-section count */ } fo_lock_ctx_t; /* A trivial reply-bearing EXPR that always "succeeds" (rc==0) but, as a side * effect, performs a non-atomic read-modify-write on the shared counter and * tracks the peak in-section concurrency. If the wrapper macro did NOT hold * ac->failover.lock for the section, concurrent threads would interleave the * RMW (lost updates) and/or observe inflight > 1. */ static int fo_lock_body(fo_lock_ctx_t *c) { c->inflight++; if (c->inflight > c->max_inflight) c->max_inflight = c->inflight; /* non-atomic RMW: a data race here would lose increments */ long v = c->counter; for (volatile int s = 0; s < 50; s++) { } /* widen the race window */ c->counter = v + 1; c->inflight--; return 0; /* success → wrapper returns immediately, no retry */ } /* The per-thread worker drives FO_LOCK_ITERS wrapped "ops" on the SHARED ctx. * Each op uses the real XCECLI_FAILOVER_CALL wrapper, so it acquires/releases * ac->failover.lock exactly as a production xcecli_* op does. */ static void fo_lock_worker(void *ctx, int idx) { (void)idx; fo_lock_ctx_t *c = (fo_lock_ctx_t *)ctx; api_ctx_t *ac = c->ac; for (int i = 0; i < FO_LOCK_ITERS; i++) { int rc = XCECLI_FAILOVER_CALL(ac, /*is_write=*/false, fo_lock_body(c)); TEST_ASSERT_EQUAL_INT(0, rc); } } /* Req 5.1, 2.4 (CI2): N threads sharing ONE ctx are serialised by the per-ctx * failover lock the wrapper holds — no lost updates and peak in-section * concurrency stays 1. The total counter equals N * ITERS exactly. */ static void test_failover_call_serialises_shared_ctx(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); /* the wrapper takes ac->failover.lock; init it (xcecli_start would in prod) */ TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock)); ac.failover.retry = XCE_RETRY_DEFAULTS; fo_lock_ctx_t lc; memset(&lc, 0, sizeof(lc)); lc.ac = ∾ int rc = fo_run_threads(FO_LOCK_NTHREADS, fo_lock_worker, &lc); mylock_fini(&ac.failover.lock); TEST_ASSERT_EQUAL_INT(0, rc); /* mutual exclusion held: every increment landed, none lost */ TEST_ASSERT_EQUAL_INT((long)FO_LOCK_NTHREADS * FO_LOCK_ITERS, lc.counter); /* the critical section was never entered by two threads at once */ TEST_ASSERT_EQUAL_INT(1, lc.max_inflight); printf("test_failover_call_serialises_shared_ctx: PASS\n"); } /* Req 1.1, 5.3: the wrapper returns the op's success rc on the first try (no * spurious retry) for a status-0 EXPR, and the per-op retry carrier starts at * attempt 0 — i.e. a success path does not consume the retry budget. */ static void test_failover_call_success_runs_once(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock)); ac.failover.retry = XCE_RETRY_DEFAULTS; fo_lock_ctx_t lc; memset(&lc, 0, sizeof(lc)); lc.ac = ∾ int rc = XCECLI_FAILOVER_CALL(&ac, /*is_write=*/false, fo_lock_body(&lc)); mylock_fini(&ac.failover.lock); TEST_ASSERT_EQUAL_INT(0, rc); /* success surfaced verbatim */ TEST_ASSERT_EQUAL_INT(1, lc.counter); /* body ran exactly once, no replay */ printf("test_failover_call_success_runs_once: PASS\n"); } /* ════════════════════════════════════════════════════════════════════════ * Task 5.2 — Best-effort reconnect for fire-and-forget ops. * * Fire-and-forget ops (notify_*, heartbeat, nop) send a *_req only — there is * NO reply to classify, so they do NOT use the full failover loop (no status * classification, no no-primary backoff, no retry budget). Putting a heartbeat * through the retry loop would let it block on the no-primary backoff budget, * which is explicitly forbidden. Instead the FF path makes AT MOST ONE * best-effort transport reconnect when the connection is KNOWN-DEAD, then does * a single-shot send (which may be lost — acceptable, redelivery is deferred to * worker-rejoin-resync). * * The "known-dead" signal is ctx->state == XCE_CTX_ERR (set by the disconnect * path after a detected transport failure). xcecli_ff_needs_reconnect is the * pure predicate factored out so this decision is unit-testable socket-free: * ERR → true (reconnect once, best-effort) * RUN → false (connection live, send directly) * DOWN → false (deliberately stopped — do not resurrect) * INIT → false (not yet started) * * Requirements: 1.1 (all ops go through failover machinery — FF via the * best-effort transport-reconnect variant), 5.3 (no silent drop on a path that * CAN reconnect; a known-dead connection is repaired before the single shot). * Full live reconnect-then-send is E2E task 6.2. * ════════════════════════════════════════════════════════════════════════ */ /* Req 1.1/5.3: a known-dead (ERR) connection needs a best-effort reconnect. */ static void test_ff_needs_reconnect_on_err(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_ERR; TEST_ASSERT_TRUE(xcecli_ff_needs_reconnect(&ac)); printf("test_ff_needs_reconnect_on_err: PASS\n"); } /* Req 1.1: a live (RUN) connection needs NO reconnect — send directly. */ static void test_ff_no_reconnect_on_run(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_RUN; TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(&ac)); printf("test_ff_no_reconnect_on_run: PASS\n"); } /* Req 1.1: a DOWN (deliberately stopped) or INIT (not yet started) ctx is NOT * resurrected by a fire-and-forget op — only a transport-ERR connection is. */ static void test_ff_no_reconnect_on_down_or_init(void) { api_ctx_t ac; memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_DOWN; TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(&ac)); memset(&ac, 0, sizeof(ac)); ac.ctx.state = XCE_CTX_INIT; TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(&ac)); printf("test_ff_no_reconnect_on_down_or_init: PASS\n"); } /* Guard: a NULL api_ctx is rejected as "no reconnect" without crashing. */ static void test_ff_needs_reconnect_null_safe(void) { TEST_ASSERT_FALSE(xcecli_ff_needs_reconnect(NULL)); printf("test_ff_needs_reconnect_null_safe: PASS\n"); } /* Req 5.3 (structural): the fire-and-forget path is single-shot — it makes AT * MOST ONE reconnect attempt and NEVER consumes the no-primary retry budget / * sleeps. We prove this structurally: the FF macro drives the body exactly once * regardless of the rc it returns (a "lost" send rc is surfaced verbatim, NOT * retried), and on a known-dead ctx the single reconnect attempt fires exactly * once before that single body invocation. */ typedef struct { api_ctx_t *ac; int body_calls; /* number of times the FF body ran */ int body_rc; /* rc the body returns each call */ } fo_ff_probe_t; static int fo_ff_body(fo_ff_probe_t *p) { p->body_calls++; return p->body_rc; } static void test_ff_call_is_single_shot_no_budget(void) { xce_client_config_t conf; api_ctx_t ac; fo_make_client_masters(&conf); fo_make_api_ctx(&ac, &conf); TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock)); ac.failover.retry = XCE_RETRY_DEFAULTS; /* a live RUN ctx: no reconnect, just the single-shot send */ ac.ctx.state = XCE_CTX_RUN; fo_ff_probe_t p; memset(&p, 0, sizeof(p)); p.ac = ∾ /* a "lost" fire-and-forget send (e.g. -ECONNRESET) is surfaced verbatim and * is NOT retried — the body must run exactly once. */ p.body_rc = -ECONNRESET; int rc = XCECLI_FF_CALL(&ac, fo_ff_body(&p)); mylock_fini(&ac.failover.lock); TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc); /* best-effort: surfaced as-is */ TEST_ASSERT_EQUAL_INT(1, p.body_calls); /* single shot, no retry loop */ printf("test_ff_call_is_single_shot_no_budget: PASS\n"); } /* ── Suite entry point ──────────────────────────────────────────────── */ void test_libxce_retry(void) { UNITY_BEGIN(); RUN_TEST(test_retry_defaults_are_reachable); RUN_TEST(test_fake_masters_hold_three_candidates); RUN_TEST(test_master_cursor_advances_and_wraps); RUN_TEST(test_status_table_is_complete_and_resolves); RUN_TEST(test_status_table_redirect_hint_flag); RUN_TEST(test_thread_harness_runs_all_threads); RUN_TEST(test_thread_harness_rejects_bad_args); RUN_TEST(test_classify_matches_taxonomy_table); RUN_TEST(test_classify_control_codes_never_success); RUN_TEST(test_classify_dup_replay_rule); RUN_TEST(test_classify_zero_is_success); RUN_TEST(test_classify_unknown_is_terminal); RUN_TEST(test_next_target_hint_first_then_consumed); RUN_TEST(test_next_target_rotates_and_wraps); RUN_TEST(test_next_target_hint_then_fallback_rotation); RUN_TEST(test_next_target_rejects_bad_args); RUN_TEST(test_capture_redirect_hint_present); RUN_TEST(test_capture_redirect_hint_absent_stays_empty); RUN_TEST(test_capture_redirect_hint_truncates_long_host); RUN_TEST(test_capture_redirect_hint_null_ctx_safe); RUN_TEST(test_reply_wait_wake_releases_blocked_get); RUN_TEST(test_decide_no_primary_exhausts_to_not_ready); RUN_TEST(test_decide_no_primary_then_becomes_primary); RUN_TEST(test_decide_retry_now_has_no_backoff_delay); RUN_TEST(test_decide_backoff_grows_and_is_bounded); RUN_TEST(test_decide_terminal_surfaces_unchanged); RUN_TEST(test_decide_dup_on_resent_write_is_success); RUN_TEST(test_decide_transport_exhaustion_keeps_last_rc); RUN_TEST(test_decide_known_dead_ctx_retries_not_terminal); RUN_TEST(test_decide_jitter_staggers_many_clients); RUN_TEST(test_recover_resets_err_state_to_run); RUN_TEST(test_recover_leaves_run_state_untouched); RUN_TEST(test_recover_leaves_down_state_untouched); RUN_TEST(test_recover_drains_stale_wbuf_entry); RUN_TEST(test_recover_null_ctx_safe); RUN_TEST(test_invoke_rejoin_calls_registered_hook); RUN_TEST(test_invoke_rejoin_propagates_failure); RUN_TEST(test_invoke_rejoin_no_hook_is_noop); RUN_TEST(test_set_reconnect_handler_null_safe_and_clears); RUN_TEST(test_invoke_rejoin_null_ctx_safe); RUN_TEST(test_failover_call_serialises_shared_ctx); RUN_TEST(test_failover_call_success_runs_once); RUN_TEST(test_ff_needs_reconnect_on_err); RUN_TEST(test_ff_no_reconnect_on_run); RUN_TEST(test_ff_no_reconnect_on_down_or_init); RUN_TEST(test_ff_needs_reconnect_null_safe); RUN_TEST(test_ff_call_is_single_shot_no_budget); UNITY_END(); }