feat(libxce-failover): retry decision loop — pure decide (budget/backoff/jitter, D6 exhaustion) + thin next wrapper (task 3.1)

src/lib/client_failover.c

@@ -15,12 +15,15 @@
 #include "xce/client_failover.h"
 
 #include <errno.h>
+#include <stdlib.h>
 #include <string.h>
 #include <sys/socket.h>
+#include <time.h>
 
-#include "client_internal.h" /* api_ctx_t + EndpointRotator declarations */
+#include "client_internal.h" /* api_ctx_t + EndpointRotator + retry decls */
 #include "config.h"          /* XCE_MAXHOSTNAME_LEN */
 #include "netio.h"           /* endpoint_init / endpoint_connect / endpoint_fini */
+#include "utils.h"           /* INFO / WARN logging macros */
 #include "xce/errnox.h"
 
 /*
@@ -155,3 +158,152 @@ int xce_failover_rotate(api_ctx_t *ac)
     rc = endpoint_connect(&ac->ep, host, (int)port);
     return rc < 0 ? rc : 0;
 }
+
+/* ════════════════════════════════════════════════════════════════════════
+ * XcecliRetryLoop — decision core (pure) + side-effecting wrapper
+ * ════════════════════════════════════════════════════════════════════════ */
+
+/*
+ * Compute the next no-primary backoff delay (ms) for retry number `attempt`
+ * (1-based: the delay applied BEFORE the attempt'th re-send). Base is a capped
+ * exponential (initial_retry_ms * multiplier^(attempt-1), clamped to
+ * max_retry_ms); jitter spreads the actual delay within [base/2, base] using
+ * rand_r on the per-op seed so concurrent clients do not retry in lockstep
+ * (6.3). The base is monotone non-decreasing and never exceeds max_retry_ms.
+ */
+static int fo_backoff_delay_ms(const xce_retry_config_t* cfg, int attempt,
+                               unsigned int* seed)
+{
+    double base = (double)cfg->initial_retry_ms;
+    for (int i = 1; i < attempt; i++) {
+        base *= cfg->backoff_multiplier;
+        if (base >= (double)cfg->max_retry_ms) {
+            base = (double)cfg->max_retry_ms;
+            break;
+        }
+    }
+    if (base > (double)cfg->max_retry_ms)
+        base = (double)cfg->max_retry_ms;
+    if (base < 1.0)
+        base = 1.0;
+
+    int ibase = (int)base;
+    /* jitter within the lower half: delay ∈ [ibase/2, ibase]. */
+    int half   = ibase / 2;
+    int jitter = (half > 0) ? (int)(rand_r(seed) % (unsigned int)(half + 1)) : 0;
+    int delay  = ibase - jitter;
+    if (delay < 1)
+        delay = 1;
+    if (delay > cfg->max_retry_ms)
+        delay = cfg->max_retry_ms;
+    return delay;
+}
+
+bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
+                         const xce_retry_config_t* cfg, xcecli_retry_t* st,
+                         xce_fo_action_t* action_out)
+{
+    if (ac == NULL || rc == NULL || cfg == NULL || st == NULL)
+        return false;
+
+    xce_fo_action_t action =
+        xce_failover_classify(*rc, is_write, st->attempt);
+    if (action_out != NULL)
+        *action_out = action;
+
+    switch (action) {
+    case XCE_FO_SUCCESS:
+        /* DUP-replay success surfaces as a clean 0; a real status-0 is already
+         * 0. Either way the op succeeded — normalise and stop. */
+        *rc = 0;
+        return false;
+
+    case XCE_FO_TERMINAL:
+        /* Non-retriable — surface *rc unchanged. */
+        return false;
+
+    case XCE_FO_RETRY_NOW:
+    case XCE_FO_RETRY_BACKOFF:
+        break;
+    }
+
+    /* A retriable outcome consumes one retry from the budget. */
+    st->attempt++;
+    if (st->attempt > cfg->max_retries) {
+        /* Budget exhausted. Distinguish the cause (design D6): a no-primary
+         * window maps to the distinct cluster-not-ready code; a transport-only
+         * outage keeps the last transport rc so the caller still sees it. */
+        if (action == XCE_FO_RETRY_BACKOFF)
+            *rc = -E_XCE_NOT_READY;
+        return false;
+    }
+
+    /* Compute the pre-attempt delay: none for prompt rotation (RETRY_NOW),
+     * a jittered exponential backoff for the no-primary case (RETRY_BACKOFF). */
+    st->delay_ms = (action == XCE_FO_RETRY_BACKOFF)
+                       ? fo_backoff_delay_ms(cfg, st->attempt, &st->rand_seed)
+                       : 0;
+    return true;
+}
+
+/* Sleep for `ms` milliseconds, tolerating EINTR. Isolated here so the pure
+ * decision core stays sleep-free and unit-testable. */
+static void fo_sleep_ms(int ms)
+{
+    if (ms <= 0)
+        return;
+    struct timespec ts = { .tv_sec  = ms / 1000,
+                           .tv_nsec = (long)(ms % 1000) * 1000000L };
+    while (nanosleep(&ts, &ts) != 0 && errno == EINTR)
+        ; /* finish the remaining interval */
+}
+
+bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
+                       xcecli_retry_t* st)
+{
+    if (ac == NULL || rc == NULL || st == NULL)
+        return false;
+
+    xce_fo_action_t action;
+    bool retry = xcecli_retry_decide(ac, rc, is_write, &ac->failover.retry, st,
+                                     &action);
+    if (!retry) {
+        /* Stopped. If the budget was exhausted on a no-primary window, decide
+         * has already set *rc to -E_XCE_NOT_READY (7.2 observable). */
+        if (*rc == -E_XCE_NOT_READY)
+            WARN("xcecli failover: cluster not ready after %d attempts "
+                 "(last status=%d)",
+                 st->attempt, *rc);
+        return false;
+    }
+
+    /* RETRY_BACKOFF: honour the jittered pre-attempt delay (4.1, 6.3). */
+    if (action == XCE_FO_RETRY_BACKOFF)
+        fo_sleep_ms(st->delay_ms);
+
+    /* Re-point the endpoint to the redirect hint or the next master and
+     * reconnect (2.x, 4.2, 4.4). Log the rotation target (7.1). A rotate
+     * failure is not fatal here: the next xce_* attempt observes the dead
+     * connection and the loop reclassifies it as a transport error. */
+    int rrc = xce_failover_rotate(ac);
+    if (rrc < 0) {
+        WARN("xcecli failover: rotate failed (rc=%d); retrying anyway", rrc);
+    } else {
+        /* Log the rotation target (7.1). xce_failover_rotate consumes the
+         * redirect hint when present (so the post-rotate target is not always
+         * the sticky master); report the sticky master cursor as a best-effort
+         * endpoint label, which is exact for the configured-master rotation. */
+        int cur = ac->failover.cur_master;
+        const char* host =
+            (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
+                ? ac->conf->masters[cur]
+                : "(unknown)";
+        int port = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
+                       ? ac->conf->master_ports[cur]
+                       : 0;
+        INFO("xcecli failover: attempt %d → %s:%d", st->attempt,
+             host ? host : "(null)", port);
+    }
+
+    return true;
+}