/* * client_failover.c — StatusClassifier (task 2.1) + EndpointRotator (task 2.2) * for the libxce-failover spec. * * StatusClassifier: pure mapping from a reply status / transport rc to a * failover action. No I/O, no globals (requirements 2.5, 4.1-4.3, 4.5, 5.2). * * EndpointRotator: choose and connect the next failover target. The target * SELECTION (xce_failover_next_target) is a pure, socket-free helper so it is * unit-testable; xce_failover_rotate() is a thin wrapper that closes the stale * connection and reconnects the same endpoint object (requirements 2.1, 2.2, * 2.3, 4.4, 8.4). See design.md "Components and Interfaces → EndpointRotator". */ #define _GNU_SOURCE #include "xce/client_failover.h" #include #include #include #include #include #include "client_internal.h" /* api_ctx_t + EndpointRotator + retry decls */ #include "config.h" /* XCE_MAXHOSTNAME_LEN */ #include "netio.h" /* endpoint_init / endpoint_connect / endpoint_fini */ #include "utils.h" /* INFO / WARN logging macros */ #include "xce/errnox.h" /* * Controller status codes (E_XCE_*) are positive enum values, but the failover * loop may observe them as negative -errno-style rc. Accept either sign by * comparing against the magnitude. */ static int fo_abs(int v) { return v < 0 ? -v : v; } xce_fo_action_t xce_failover_classify(int status, bool is_write, int attempt) { /* Success: served by the primary. */ if (status == 0) return XCE_FO_SUCCESS; int code = fo_abs(status); /* Idempotent-replay rule (5.2): a duplicate on a re-sent write means the * controller's client-jobid replay guard already applied our write — treat * it as success. On the first send (or a non-write), it is a genuine * duplicate and is surfaced unchanged. */ if (code == E_XCE_DUP_JOB || code == EEXIST) { if (is_write && attempt > 0) return XCE_FO_SUCCESS; return XCE_FO_TERMINAL; } /* Redirect / standby: a primary exists elsewhere, or this op is * primary-only — rotate promptly, no backoff (4.2, 4.3). */ if (code == E_XCE_REDIRECT || code == E_XCE_STANDBY) return XCE_FO_RETRY_NOW; /* No primary known yet / controller initializing or electing — rotate after * jittered backoff (4.1). */ if (code == E_XCE_NOMASTER || code == E_XCE_NOT_READY) return XCE_FO_RETRY_BACKOFF; /* Transport-level failures — reconnect/rotate and retry now (4.4). */ if (code == ECONNREFUSED || code == ETIMEDOUT || code == ECONNRESET) return XCE_FO_RETRY_NOW; /* Everything else — auth/permission/invalid/not-found and any unrecognized * status — is non-retriable and surfaced to the caller (4.5). */ return XCE_FO_TERMINAL; } /* ════════════════════════════════════════════════════════════════════════ * EndpointRotator — target selection (pure) + connect wrapper (thin) * ════════════════════════════════════════════════════════════════════════ */ /* True if a redirect-address hint is currently set on the ctx. The hint is a * NUL-terminated host; an empty host ("") means "no hint" (xcectx.h). */ static bool fo_has_hint(const api_ctx_t *ac) { return ac->ctx.redirect_hint_host[0] != '\0'; } /* Clear (consume) the redirect-address hint on the ctx. One-shot + advisory. */ static void fo_clear_hint(api_ctx_t *ac) { ac->ctx.redirect_hint_host[0] = '\0'; ac->ctx.redirect_hint_port = 0; } int xce_failover_next_target(api_ctx_t *ac, char *host_out, size_t host_sz, uint16_t *port_out) { if (ac == NULL || host_out == NULL || host_sz == 0 || port_out == NULL) return -EINVAL; /* Hint-first (2.2, 8.3): if the last reply carried a redirect address, try * it first and CONSUME it. The cursor is intentionally left untouched so a * failed hint falls through to normal rotation on the next call (4.4) — * correctness never depends on the hint. */ if (fo_has_hint(ac)) { strncpy(host_out, ac->ctx.redirect_hint_host, host_sz - 1); host_out[host_sz - 1] = '\0'; *port_out = ac->ctx.redirect_hint_port; fo_clear_hint(ac); return 0; } /* No hint: advance the sticky master cursor to the NEXT configured master, * wrapping around (2.1, 8.4). */ if (ac->conf == NULL || ac->conf->nmasters <= 0) return -ENOENT; int next = (ac->failover.cur_master + 1) % ac->conf->nmasters; ac->failover.cur_master = next; const char *host = ac->conf->masters[next]; if (host == NULL) return -ENOENT; strncpy(host_out, host, host_sz - 1); host_out[host_sz - 1] = '\0'; *port_out = (uint16_t)ac->conf->master_ports[next]; return 0; } int xce_failover_rotate(api_ctx_t *ac) { if (ac == NULL) return -EINVAL; char host[XCE_MAXHOSTNAME_LEN] = { 0 }; uint16_t port = 0; int rc = xce_failover_next_target(ac, host, sizeof(host), &port); if (rc < 0) return rc; /* Re-establish the SAME endpoint object on a fresh connection to the chosen * target. xce_ctx_reconnect() (xcectx.c) owns the full connection setup that * endpoint_fini drops — close old fd/SSL, recreate the socket, RE-BIND the * iomgr (endpoint_connect needs ep->iomgr->ssl_ctx — a missing re-bind here * previously crashed in endpoint_connect), drive the SSL handshake, and * re-arm the endpoint in the running iomgr with its owner + msg/err handlers * and re-monitor the new fd so replies route to ctx->wbuf and RDHUP failover * re-arms. It does not restart iomgr/vtaskmgr or touch ctx->state (the loop's * xcecli_ctx_recover owns the ERR→RUN reset). The live socket/handshake path * is exercised by the E2E task (6.2). */ return xce_ctx_reconnect(&ac->ctx, host, (int)port); } /* ════════════════════════════════════════════════════════════════════════ * Reconnect recovery — context-state reset (CI2) + stale-reply drain (CI1) * ════════════════════════════════════════════════════════════════════════ */ int xcecli_ctx_recover(api_ctx_t* ac) { if (ac == NULL) return -EINVAL; /* CI2 — state reset: the transport-error path (handle_disconnect) left * ctx->state = XCE_CTX_ERR, which the per-op XCE_CTX_RUN guard in xceapi.c * rejects as -EINVAL — the reported bug. After a successful reconnect, * restore RUN so the next xce_* attempt passes. ONLY flip ERR→RUN: a * deliberately-stopped (DOWN) or not-yet-started (INIT/BUSY) ctx is left * untouched. */ if (ac->ctx.state == XCE_CTX_ERR) ac->ctx.state = XCE_CTX_RUN; /* CI1 — stale-reply drain: handle_disconnect may have injected a stale * transport status into the single-slot wbuf. Drain any pending entry with * a NON-BLOCKING get so the next attempt's cbuf_get cannot consume a * prior-attempt reply. cbuf_tryget returns -EAGAIN when the channel is * already empty — the normal, no-op case. The reconnect closed the old fd * first, so no further replies arrive from the dead connection. */ int stale = 0; cbuf_tryget(&ac->ctx.wbuf, &stale); return 0; } /* ════════════════════════════════════════════════════════════════════════ * Fire-and-forget best-effort reconnect predicate (task 5.2) * ════════════════════════════════════════════════════════════════════════ */ bool xcecli_ff_needs_reconnect(const api_ctx_t* ac) { if (ac == NULL) return false; /* Only a transport-ERR connection (set by the disconnect path) is treated * as known-dead and reconnected best-effort. A live RUN ctx sends directly; * a deliberately-stopped DOWN or not-yet-started INIT ctx is never * resurrected by a fire-and-forget op. */ return ac->ctx.state == XCE_CTX_ERR; } /* ════════════════════════════════════════════════════════════════════════ * ReconnectRejoinHook — invoke the consumer rejoin callback (1.2) * ════════════════════════════════════════════════════════════════════════ */ int xcecli_invoke_rejoin(api_ctx_t* ac) { if (ac == NULL) return -EINVAL; /* Pure client: no hook registered → nothing to do, resume directly. */ if (ac->failover.rejoin_cb == NULL) return 0; /* Stateful consumer: run its rejoin/state-exchange on the calling thread. * Its rc is propagated verbatim — 0 to proceed, <0 to signal the join was * rejected so the wrapper treats this attempt as failed (no resume). */ return ac->failover.rejoin_cb((xcecli_ctx_t)ac, ac->failover.rejoin_arg); } /* ════════════════════════════════════════════════════════════════════════ * XcecliRetryLoop — decision core (pure) + side-effecting wrapper * ════════════════════════════════════════════════════════════════════════ */ /* * Compute the next no-primary backoff delay (ms) for retry number `attempt` * (1-based: the delay applied BEFORE the attempt'th re-send). Base is a capped * exponential (initial_retry_ms * multiplier^(attempt-1), clamped to * max_retry_ms); jitter spreads the actual delay within [base/2, base] using * rand_r on the per-op seed so concurrent clients do not retry in lockstep * (6.3). The base is monotone non-decreasing and never exceeds max_retry_ms. */ static int fo_backoff_delay_ms(const xce_retry_config_t* cfg, int attempt, unsigned int* seed) { double base = (double)cfg->initial_retry_ms; for (int i = 1; i < attempt; i++) { base *= cfg->backoff_multiplier; if (base >= (double)cfg->max_retry_ms) { base = (double)cfg->max_retry_ms; break; } } if (base > (double)cfg->max_retry_ms) base = (double)cfg->max_retry_ms; if (base < 1.0) base = 1.0; int ibase = (int)base; /* jitter within the lower half: delay ∈ [ibase/2, ibase]. */ int half = ibase / 2; int jitter = (half > 0) ? (int)(rand_r(seed) % (unsigned int)(half + 1)) : 0; int delay = ibase - jitter; if (delay < 1) delay = 1; if (delay > cfg->max_retry_ms) delay = cfg->max_retry_ms; return delay; } bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write, const xce_retry_config_t* cfg, xcecli_retry_t* st, xce_fo_action_t* action_out) { if (ac == NULL || rc == NULL || cfg == NULL || st == NULL) return false; /* Known-dead connection (the reported bug). After the primary dies, * handle_disconnect leaves ctx.state == XCE_CTX_ERR, so a FRESH xce_* call * hits the per-op xceapi guard and returns -EINVAL *before* the op reaches a * primary — and -EINVAL would otherwise classify as TERMINAL and surface the * opaque "Invalid argument". Detect the dead-ctx state directly and treat it * as a prompt transport-style failover cause (reconnect via rotate+recover), * regardless of *rc. After the next successful reconnect the ctx returns to * RUN and subsequent attempts classify normally (success / no-primary / * transport). On budget exhaustion we never reached a primary → surface the * distinct cluster-not-ready code, never -EINVAL. */ if (ac->ctx.state == XCE_CTX_ERR) { if (action_out != NULL) *action_out = XCE_FO_RETRY_NOW; st->attempt++; if (st->attempt > cfg->max_retries) { *rc = -E_XCE_NOT_READY; return false; } st->delay_ms = 0; return true; } xce_fo_action_t action = xce_failover_classify(*rc, is_write, st->attempt); if (action_out != NULL) *action_out = action; switch (action) { case XCE_FO_SUCCESS: /* DUP-replay success surfaces as a clean 0; a real status-0 is already * 0. Either way the op succeeded — normalise and stop. */ *rc = 0; return false; case XCE_FO_TERMINAL: /* Non-retriable — surface *rc unchanged. */ return false; case XCE_FO_RETRY_NOW: case XCE_FO_RETRY_BACKOFF: break; } /* A retriable outcome consumes one retry from the budget. */ st->attempt++; if (st->attempt > cfg->max_retries) { /* Budget exhausted. Distinguish the cause (design D6): a no-primary * window maps to the distinct cluster-not-ready code; a transport-only * outage keeps the last transport rc so the caller still sees it. */ if (action == XCE_FO_RETRY_BACKOFF) *rc = -E_XCE_NOT_READY; return false; } /* Compute the pre-attempt delay: none for prompt rotation (RETRY_NOW), * a jittered exponential backoff for the no-primary case (RETRY_BACKOFF). */ st->delay_ms = (action == XCE_FO_RETRY_BACKOFF) ? fo_backoff_delay_ms(cfg, st->attempt, &st->rand_seed) : 0; return true; } /* Sleep for `ms` milliseconds, tolerating EINTR. Isolated here so the pure * decision core stays sleep-free and unit-testable. */ static void fo_sleep_ms(int ms) { if (ms <= 0) return; struct timespec ts = { .tv_sec = ms / 1000, .tv_nsec = (long)(ms % 1000) * 1000000L }; while (nanosleep(&ts, &ts) != 0 && errno == EINTR) ; /* finish the remaining interval */ } bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write, xcecli_retry_t* st) { if (ac == NULL || rc == NULL || st == NULL) return false; xce_fo_action_t action; bool retry = xcecli_retry_decide(ac, rc, is_write, &ac->failover.retry, st, &action); if (!retry) { /* Stopped. If the budget was exhausted on a no-primary window, decide * has already set *rc to -E_XCE_NOT_READY (7.2 observable). */ if (*rc == -E_XCE_NOT_READY) WARN("xcecli failover: cluster not ready after %d attempts " "(last status=%d)", st->attempt, *rc); return false; } /* RETRY_BACKOFF: honour the jittered pre-attempt delay (4.1, 6.3). */ if (action == XCE_FO_RETRY_BACKOFF) fo_sleep_ms(st->delay_ms); /* Re-point the endpoint to the redirect hint or the next master and * reconnect (2.x, 4.2, 4.4). Log the rotation target (7.1). A rotate * failure is not fatal here: the next xce_* attempt observes the dead * connection and the loop reclassifies it as a transport error. */ int rrc = xce_failover_rotate(ac); if (rrc < 0) { WARN("xcecli failover: rotate failed (rc=%d); retrying anyway", rrc); return true; } /* Reconnect succeeded: repair the ctx before the next xce_* attempt * (CI2 ERR→RUN state reset, CI1 stale-reply drain) so the per-op guard * passes and no prior-attempt reply is consumed (1.2, 4.4). */ xcecli_ctx_recover(ac); /* Log the rotation target (7.1). xce_failover_rotate consumes the redirect * hint when present (so the post-rotate target is not always the sticky * master); report the sticky master cursor as a best-effort endpoint label, * which is exact for the configured-master rotation. */ int cur = ac->failover.cur_master; const char* host = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters) ? ac->conf->masters[cur] : "(unknown)"; int port = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters) ? ac->conf->master_ports[cur] : 0; INFO("xcecli failover: attempt %d → %s:%d", st->attempt, host ? host : "(null)", port); /* ReconnectRejoinHook (1.2): a stateful consumer re-establishes its session * on the freshly-connected primary AFTER reconnect/recovery and BEFORE the * interrupted op resumes. A pure client has no hook → no-op, resume now. */ int hrc = xcecli_invoke_rejoin(ac); if (hrc < 0) { /* The new primary rejected the daemon's join: treat this attempt as * FAILED — do NOT resume into a primary that doesn't know the daemon. * Reclassify the rejoin failure as a transport-style failover cause so * the loop rotates/backs off again on the next iteration (subject to the * same retry budget). When the budget is exhausted decide() returns * false and *rc carries the last cause. */ WARN("xcecli failover: rejoin hook failed (rc=%d); rotating again", hrc); *rc = hrc; return xcecli_retry_next(ac, rc, is_write, st); } return true; }