Rongsong Shen
ffc2c590f9
xce_failover_rotate reconnected with endpoint_fini+endpoint_init+endpoint_connect but never re-bound ep->iomgr (reset by endpoint_init) nor re-armed the owner/ msg/err handlers + iomgr_monitor. endpoint_connect then dereferenced a NULL iomgr (->ssl_ctx) → SEGV at netio.c:672 during live master failover. Added xce_ctx_reconnect() in xcectx.c that mirrors xcecli_start's establishment (without restarting iomgr/vtaskmgr or touching ctx->state); rotate delegates to it. Found via live 3-node E2E. bug-610.
412 lines
18 KiB
C
412 lines
18 KiB
C
/*
|
|
* client_failover.c — StatusClassifier (task 2.1) + EndpointRotator (task 2.2)
|
|
* for the libxce-failover spec.
|
|
*
|
|
* StatusClassifier: pure mapping from a reply status / transport rc to a
|
|
* failover action. No I/O, no globals (requirements 2.5, 4.1-4.3, 4.5, 5.2).
|
|
*
|
|
* EndpointRotator: choose and connect the next failover target. The target
|
|
* SELECTION (xce_failover_next_target) is a pure, socket-free helper so it is
|
|
* unit-testable; xce_failover_rotate() is a thin wrapper that closes the stale
|
|
* connection and reconnects the same endpoint object (requirements 2.1, 2.2,
|
|
* 2.3, 4.4, 8.4). See design.md "Components and Interfaces → EndpointRotator".
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include "xce/client_failover.h"
|
|
|
|
#include <errno.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/socket.h>
|
|
#include <time.h>
|
|
|
|
#include "client_internal.h" /* api_ctx_t + EndpointRotator + retry decls */
|
|
#include "config.h" /* XCE_MAXHOSTNAME_LEN */
|
|
#include "netio.h" /* endpoint_init / endpoint_connect / endpoint_fini */
|
|
#include "utils.h" /* INFO / WARN logging macros */
|
|
#include "xce/errnox.h"
|
|
|
|
/*
|
|
* Controller status codes (E_XCE_*) are positive enum values, but the failover
|
|
* loop may observe them as negative -errno-style rc. Accept either sign by
|
|
* comparing against the magnitude.
|
|
*/
|
|
static int fo_abs(int v)
|
|
{
|
|
return v < 0 ? -v : v;
|
|
}
|
|
|
|
xce_fo_action_t xce_failover_classify(int status, bool is_write, int attempt)
|
|
{
|
|
/* Success: served by the primary. */
|
|
if (status == 0)
|
|
return XCE_FO_SUCCESS;
|
|
|
|
int code = fo_abs(status);
|
|
|
|
/* Idempotent-replay rule (5.2): a duplicate on a re-sent write means the
|
|
* controller's client-jobid replay guard already applied our write — treat
|
|
* it as success. On the first send (or a non-write), it is a genuine
|
|
* duplicate and is surfaced unchanged. */
|
|
if (code == E_XCE_DUP_JOB || code == EEXIST) {
|
|
if (is_write && attempt > 0)
|
|
return XCE_FO_SUCCESS;
|
|
return XCE_FO_TERMINAL;
|
|
}
|
|
|
|
/* Redirect / standby: a primary exists elsewhere, or this op is
|
|
* primary-only — rotate promptly, no backoff (4.2, 4.3). */
|
|
if (code == E_XCE_REDIRECT || code == E_XCE_STANDBY)
|
|
return XCE_FO_RETRY_NOW;
|
|
|
|
/* No primary known yet / controller initializing or electing — rotate after
|
|
* jittered backoff (4.1). */
|
|
if (code == E_XCE_NOMASTER || code == E_XCE_NOT_READY)
|
|
return XCE_FO_RETRY_BACKOFF;
|
|
|
|
/* Transport-level failures — reconnect/rotate and retry now (4.4). */
|
|
if (code == ECONNREFUSED || code == ETIMEDOUT || code == ECONNRESET)
|
|
return XCE_FO_RETRY_NOW;
|
|
|
|
/* Everything else — auth/permission/invalid/not-found and any unrecognized
|
|
* status — is non-retriable and surfaced to the caller (4.5). */
|
|
return XCE_FO_TERMINAL;
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* EndpointRotator — target selection (pure) + connect wrapper (thin)
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* True if a redirect-address hint is currently set on the ctx. The hint is a
|
|
* NUL-terminated host; an empty host ("") means "no hint" (xcectx.h). */
|
|
static bool fo_has_hint(const api_ctx_t *ac)
|
|
{
|
|
return ac->ctx.redirect_hint_host[0] != '\0';
|
|
}
|
|
|
|
/* Clear (consume) the redirect-address hint on the ctx. One-shot + advisory. */
|
|
static void fo_clear_hint(api_ctx_t *ac)
|
|
{
|
|
ac->ctx.redirect_hint_host[0] = '\0';
|
|
ac->ctx.redirect_hint_port = 0;
|
|
}
|
|
|
|
int xce_failover_next_target(api_ctx_t *ac, char *host_out, size_t host_sz,
|
|
uint16_t *port_out)
|
|
{
|
|
if (ac == NULL || host_out == NULL || host_sz == 0 || port_out == NULL)
|
|
return -EINVAL;
|
|
|
|
/* Hint-first (2.2, 8.3): if the last reply carried a redirect address, try
|
|
* it first and CONSUME it. The cursor is intentionally left untouched so a
|
|
* failed hint falls through to normal rotation on the next call (4.4) —
|
|
* correctness never depends on the hint. */
|
|
if (fo_has_hint(ac)) {
|
|
strncpy(host_out, ac->ctx.redirect_hint_host, host_sz - 1);
|
|
host_out[host_sz - 1] = '\0';
|
|
*port_out = ac->ctx.redirect_hint_port;
|
|
fo_clear_hint(ac);
|
|
return 0;
|
|
}
|
|
|
|
/* No hint: advance the sticky master cursor to the NEXT configured master,
|
|
* wrapping around (2.1, 8.4). */
|
|
if (ac->conf == NULL || ac->conf->nmasters <= 0)
|
|
return -ENOENT;
|
|
|
|
int next = (ac->failover.cur_master + 1) % ac->conf->nmasters;
|
|
ac->failover.cur_master = next;
|
|
|
|
const char *host = ac->conf->masters[next];
|
|
if (host == NULL)
|
|
return -ENOENT;
|
|
|
|
strncpy(host_out, host, host_sz - 1);
|
|
host_out[host_sz - 1] = '\0';
|
|
*port_out = (uint16_t)ac->conf->master_ports[next];
|
|
return 0;
|
|
}
|
|
|
|
int xce_failover_rotate(api_ctx_t *ac)
|
|
{
|
|
if (ac == NULL)
|
|
return -EINVAL;
|
|
|
|
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
|
|
uint16_t port = 0;
|
|
|
|
int rc = xce_failover_next_target(ac, host, sizeof(host), &port);
|
|
if (rc < 0)
|
|
return rc;
|
|
|
|
/* Re-establish the SAME endpoint object on a fresh connection to the chosen
|
|
* target. xce_ctx_reconnect() (xcectx.c) owns the full connection setup that
|
|
* endpoint_fini drops — close old fd/SSL, recreate the socket, RE-BIND the
|
|
* iomgr (endpoint_connect needs ep->iomgr->ssl_ctx — a missing re-bind here
|
|
* previously crashed in endpoint_connect), drive the SSL handshake, and
|
|
* re-arm the endpoint in the running iomgr with its owner + msg/err handlers
|
|
* and re-monitor the new fd so replies route to ctx->wbuf and RDHUP failover
|
|
* re-arms. It does not restart iomgr/vtaskmgr or touch ctx->state (the loop's
|
|
* xcecli_ctx_recover owns the ERR→RUN reset). The live socket/handshake path
|
|
* is exercised by the E2E task (6.2). */
|
|
return xce_ctx_reconnect(&ac->ctx, host, (int)port);
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Reconnect recovery — context-state reset (CI2) + stale-reply drain (CI1)
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
int xcecli_ctx_recover(api_ctx_t* ac)
|
|
{
|
|
if (ac == NULL)
|
|
return -EINVAL;
|
|
|
|
/* CI2 — state reset: the transport-error path (handle_disconnect) left
|
|
* ctx->state = XCE_CTX_ERR, which the per-op XCE_CTX_RUN guard in xceapi.c
|
|
* rejects as -EINVAL — the reported bug. After a successful reconnect,
|
|
* restore RUN so the next xce_* attempt passes. ONLY flip ERR→RUN: a
|
|
* deliberately-stopped (DOWN) or not-yet-started (INIT/BUSY) ctx is left
|
|
* untouched. */
|
|
if (ac->ctx.state == XCE_CTX_ERR)
|
|
ac->ctx.state = XCE_CTX_RUN;
|
|
|
|
/* CI1 — stale-reply drain: handle_disconnect may have injected a stale
|
|
* transport status into the single-slot wbuf. Drain any pending entry with
|
|
* a NON-BLOCKING get so the next attempt's cbuf_get cannot consume a
|
|
* prior-attempt reply. cbuf_tryget returns -EAGAIN when the channel is
|
|
* already empty — the normal, no-op case. The reconnect closed the old fd
|
|
* first, so no further replies arrive from the dead connection. */
|
|
int stale = 0;
|
|
cbuf_tryget(&ac->ctx.wbuf, &stale);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* Fire-and-forget best-effort reconnect predicate (task 5.2)
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
bool xcecli_ff_needs_reconnect(const api_ctx_t* ac)
|
|
{
|
|
if (ac == NULL)
|
|
return false;
|
|
|
|
/* Only a transport-ERR connection (set by the disconnect path) is treated
|
|
* as known-dead and reconnected best-effort. A live RUN ctx sends directly;
|
|
* a deliberately-stopped DOWN or not-yet-started INIT ctx is never
|
|
* resurrected by a fire-and-forget op. */
|
|
return ac->ctx.state == XCE_CTX_ERR;
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* ReconnectRejoinHook — invoke the consumer rejoin callback (1.2)
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
int xcecli_invoke_rejoin(api_ctx_t* ac)
|
|
{
|
|
if (ac == NULL)
|
|
return -EINVAL;
|
|
|
|
/* Pure client: no hook registered → nothing to do, resume directly. */
|
|
if (ac->failover.rejoin_cb == NULL)
|
|
return 0;
|
|
|
|
/* Stateful consumer: run its rejoin/state-exchange on the calling thread.
|
|
* Its rc is propagated verbatim — 0 to proceed, <0 to signal the join was
|
|
* rejected so the wrapper treats this attempt as failed (no resume). */
|
|
return ac->failover.rejoin_cb((xcecli_ctx_t)ac, ac->failover.rejoin_arg);
|
|
}
|
|
|
|
/* ════════════════════════════════════════════════════════════════════════
|
|
* XcecliRetryLoop — decision core (pure) + side-effecting wrapper
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/*
|
|
* Compute the next no-primary backoff delay (ms) for retry number `attempt`
|
|
* (1-based: the delay applied BEFORE the attempt'th re-send). Base is a capped
|
|
* exponential (initial_retry_ms * multiplier^(attempt-1), clamped to
|
|
* max_retry_ms); jitter spreads the actual delay within [base/2, base] using
|
|
* rand_r on the per-op seed so concurrent clients do not retry in lockstep
|
|
* (6.3). The base is monotone non-decreasing and never exceeds max_retry_ms.
|
|
*/
|
|
static int fo_backoff_delay_ms(const xce_retry_config_t* cfg, int attempt,
|
|
unsigned int* seed)
|
|
{
|
|
double base = (double)cfg->initial_retry_ms;
|
|
for (int i = 1; i < attempt; i++) {
|
|
base *= cfg->backoff_multiplier;
|
|
if (base >= (double)cfg->max_retry_ms) {
|
|
base = (double)cfg->max_retry_ms;
|
|
break;
|
|
}
|
|
}
|
|
if (base > (double)cfg->max_retry_ms)
|
|
base = (double)cfg->max_retry_ms;
|
|
if (base < 1.0)
|
|
base = 1.0;
|
|
|
|
int ibase = (int)base;
|
|
/* jitter within the lower half: delay ∈ [ibase/2, ibase]. */
|
|
int half = ibase / 2;
|
|
int jitter = (half > 0) ? (int)(rand_r(seed) % (unsigned int)(half + 1)) : 0;
|
|
int delay = ibase - jitter;
|
|
if (delay < 1)
|
|
delay = 1;
|
|
if (delay > cfg->max_retry_ms)
|
|
delay = cfg->max_retry_ms;
|
|
return delay;
|
|
}
|
|
|
|
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
|
|
const xce_retry_config_t* cfg, xcecli_retry_t* st,
|
|
xce_fo_action_t* action_out)
|
|
{
|
|
if (ac == NULL || rc == NULL || cfg == NULL || st == NULL)
|
|
return false;
|
|
|
|
/* Known-dead connection (the reported bug). After the primary dies,
|
|
* handle_disconnect leaves ctx.state == XCE_CTX_ERR, so a FRESH xce_* call
|
|
* hits the per-op xceapi guard and returns -EINVAL *before* the op reaches a
|
|
* primary — and -EINVAL would otherwise classify as TERMINAL and surface the
|
|
* opaque "Invalid argument". Detect the dead-ctx state directly and treat it
|
|
* as a prompt transport-style failover cause (reconnect via rotate+recover),
|
|
* regardless of *rc. After the next successful reconnect the ctx returns to
|
|
* RUN and subsequent attempts classify normally (success / no-primary /
|
|
* transport). On budget exhaustion we never reached a primary → surface the
|
|
* distinct cluster-not-ready code, never -EINVAL. */
|
|
if (ac->ctx.state == XCE_CTX_ERR) {
|
|
if (action_out != NULL)
|
|
*action_out = XCE_FO_RETRY_NOW;
|
|
st->attempt++;
|
|
if (st->attempt > cfg->max_retries) {
|
|
*rc = -E_XCE_NOT_READY;
|
|
return false;
|
|
}
|
|
st->delay_ms = 0;
|
|
return true;
|
|
}
|
|
|
|
xce_fo_action_t action =
|
|
xce_failover_classify(*rc, is_write, st->attempt);
|
|
if (action_out != NULL)
|
|
*action_out = action;
|
|
|
|
switch (action) {
|
|
case XCE_FO_SUCCESS:
|
|
/* DUP-replay success surfaces as a clean 0; a real status-0 is already
|
|
* 0. Either way the op succeeded — normalise and stop. */
|
|
*rc = 0;
|
|
return false;
|
|
|
|
case XCE_FO_TERMINAL:
|
|
/* Non-retriable — surface *rc unchanged. */
|
|
return false;
|
|
|
|
case XCE_FO_RETRY_NOW:
|
|
case XCE_FO_RETRY_BACKOFF:
|
|
break;
|
|
}
|
|
|
|
/* A retriable outcome consumes one retry from the budget. */
|
|
st->attempt++;
|
|
if (st->attempt > cfg->max_retries) {
|
|
/* Budget exhausted. Distinguish the cause (design D6): a no-primary
|
|
* window maps to the distinct cluster-not-ready code; a transport-only
|
|
* outage keeps the last transport rc so the caller still sees it. */
|
|
if (action == XCE_FO_RETRY_BACKOFF)
|
|
*rc = -E_XCE_NOT_READY;
|
|
return false;
|
|
}
|
|
|
|
/* Compute the pre-attempt delay: none for prompt rotation (RETRY_NOW),
|
|
* a jittered exponential backoff for the no-primary case (RETRY_BACKOFF). */
|
|
st->delay_ms = (action == XCE_FO_RETRY_BACKOFF)
|
|
? fo_backoff_delay_ms(cfg, st->attempt, &st->rand_seed)
|
|
: 0;
|
|
return true;
|
|
}
|
|
|
|
/* Sleep for `ms` milliseconds, tolerating EINTR. Isolated here so the pure
|
|
* decision core stays sleep-free and unit-testable. */
|
|
static void fo_sleep_ms(int ms)
|
|
{
|
|
if (ms <= 0)
|
|
return;
|
|
struct timespec ts = { .tv_sec = ms / 1000,
|
|
.tv_nsec = (long)(ms % 1000) * 1000000L };
|
|
while (nanosleep(&ts, &ts) != 0 && errno == EINTR)
|
|
; /* finish the remaining interval */
|
|
}
|
|
|
|
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
|
|
xcecli_retry_t* st)
|
|
{
|
|
if (ac == NULL || rc == NULL || st == NULL)
|
|
return false;
|
|
|
|
xce_fo_action_t action;
|
|
bool retry = xcecli_retry_decide(ac, rc, is_write, &ac->failover.retry, st,
|
|
&action);
|
|
if (!retry) {
|
|
/* Stopped. If the budget was exhausted on a no-primary window, decide
|
|
* has already set *rc to -E_XCE_NOT_READY (7.2 observable). */
|
|
if (*rc == -E_XCE_NOT_READY)
|
|
WARN("xcecli failover: cluster not ready after %d attempts "
|
|
"(last status=%d)",
|
|
st->attempt, *rc);
|
|
return false;
|
|
}
|
|
|
|
/* RETRY_BACKOFF: honour the jittered pre-attempt delay (4.1, 6.3). */
|
|
if (action == XCE_FO_RETRY_BACKOFF)
|
|
fo_sleep_ms(st->delay_ms);
|
|
|
|
/* Re-point the endpoint to the redirect hint or the next master and
|
|
* reconnect (2.x, 4.2, 4.4). Log the rotation target (7.1). A rotate
|
|
* failure is not fatal here: the next xce_* attempt observes the dead
|
|
* connection and the loop reclassifies it as a transport error. */
|
|
int rrc = xce_failover_rotate(ac);
|
|
if (rrc < 0) {
|
|
WARN("xcecli failover: rotate failed (rc=%d); retrying anyway", rrc);
|
|
return true;
|
|
}
|
|
|
|
/* Reconnect succeeded: repair the ctx before the next xce_* attempt
|
|
* (CI2 ERR→RUN state reset, CI1 stale-reply drain) so the per-op guard
|
|
* passes and no prior-attempt reply is consumed (1.2, 4.4). */
|
|
xcecli_ctx_recover(ac);
|
|
|
|
/* Log the rotation target (7.1). xce_failover_rotate consumes the redirect
|
|
* hint when present (so the post-rotate target is not always the sticky
|
|
* master); report the sticky master cursor as a best-effort endpoint label,
|
|
* which is exact for the configured-master rotation. */
|
|
int cur = ac->failover.cur_master;
|
|
const char* host =
|
|
(ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
|
|
? ac->conf->masters[cur]
|
|
: "(unknown)";
|
|
int port = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
|
|
? ac->conf->master_ports[cur]
|
|
: 0;
|
|
INFO("xcecli failover: attempt %d → %s:%d", st->attempt,
|
|
host ? host : "(null)", port);
|
|
|
|
/* ReconnectRejoinHook (1.2): a stateful consumer re-establishes its session
|
|
* on the freshly-connected primary AFTER reconnect/recovery and BEFORE the
|
|
* interrupted op resumes. A pure client has no hook → no-op, resume now. */
|
|
int hrc = xcecli_invoke_rejoin(ac);
|
|
if (hrc < 0) {
|
|
/* The new primary rejected the daemon's join: treat this attempt as
|
|
* FAILED — do NOT resume into a primary that doesn't know the daemon.
|
|
* Reclassify the rejoin failure as a transport-style failover cause so
|
|
* the loop rotates/backs off again on the next iteration (subject to the
|
|
* same retry budget). When the budget is exhausted decide() returns
|
|
* false and *rc carries the last cause. */
|
|
WARN("xcecli failover: rejoin hook failed (rc=%d); rotating again",
|
|
hrc);
|
|
*rc = hrc;
|
|
return xcecli_retry_next(ac, rc, is_write, st);
|
|
}
|
|
|
|
return true;
|
|
}
|