Rongsong Shen
8a6c339a55
xcecli_jobsubmit() could return rc>=0 (success) while leaving its *jobid output parameter at 0, reproducible under concurrent client load immediately after a fresh single-node election (~1-3/9 clients per round in the original repro). Root cause: XCECLI_FAILOVER_CALL's idempotent-replay rule (client_failover.c's xce_failover_classify) normalizes a retried resend that hits the server's EEXIST/inflight-collision rejection into overall success -- but that rejection reply carries no payload (handle_jobsubmit's EEXIST branches reply with RequestReply_data_nodata, job.c), so xce_jobsubmit() never populates *jobid on the winning attempt, nor did it on the earlier failing attempt (raw status was already negative). Adds jobsubmit_finalize_jobid() (client_internal.h/client.c), called from xcecli_jobsubmit() right after XCECLI_FAILOVER_CALL returns. Sets *jobid = submit->jobid (the client-generated id already on the wire -- the controller commits/echoes this exact value on every genuine success reply, bug-786/job.c:1439) whenever the overall call succeeded, otherwise leaves it untouched. Placed at the jobsubmit-specific wrapper layer, not the generic classify/decide layer, which stays fully generic. 3 new socket-free unit tests in test_libxce_retry.c (following this codebase's direct-state-simulation style for I/O-adjacent logic). Live-validated: full ASAN rebuild clean, libxce test suite 1/1 OK with all 3 new tests passing, and a 5-round/45-client concurrent-submit repro against a live single-node cluster showed 0 "no clean ack" failures (vs. 1-3/9 per round pre-fix). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
360 lines
19 KiB
C
360 lines
19 KiB
C
#if !defined(_CLIENT_INTERNAL_H_)
|
|
|
|
#define _CLIENT_INTERNAL_H_ 1
|
|
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "client.h"
|
|
#include "mylock.h"
|
|
#include "netio.h"
|
|
#include "request.h"
|
|
#include "xce/client_failover.h" /* xce_fo_action_t */
|
|
#include "xcectx.h"
|
|
|
|
/*
|
|
* client_internal.h — private libxce client state shared between client.c and
|
|
* the forthcoming failover helper (client_failover.c, task 1.x).
|
|
*
|
|
* api_ctx_t was previously file-local to client.c. It is moved here so the
|
|
* failover helper can operate on it without exposing it through the public
|
|
* API in client.h.
|
|
*/
|
|
|
|
typedef enum {
|
|
API_CTX_INIT = 0,
|
|
API_CTX_RUN,
|
|
API_CTX_DOWN,
|
|
API_CTX_ERR
|
|
} api_ctx_state_t;
|
|
|
|
/*
|
|
* xcecli_reconnect_cb (consumer rejoin hook) and its public setter
|
|
* xcecli_set_reconnect_handler() are declared in the public client.h (included
|
|
* above) as of task 4.1. The failover state below carries the registered hook.
|
|
*/
|
|
|
|
/*
|
|
* xce_failover_state_t: per-context failover state. Persists across calls so a
|
|
* sticky primary and the retry tunables survive between requests.
|
|
*
|
|
* lock — serializes xcecli_* per ctx (shared-ctx safety)
|
|
* cur_master — index into conf->masters[] (sticky primary)
|
|
* retry — defaults from XCE_RETRY_DEFAULTS unless overridden
|
|
* rejoin_cb — consumer rejoin hook; NULL for pure clients
|
|
* rejoin_arg — opaque argument passed to rejoin_cb
|
|
*/
|
|
typedef struct {
|
|
mylock_t lock;
|
|
int cur_master;
|
|
xce_retry_config_t retry;
|
|
xcecli_reconnect_cb rejoin_cb;
|
|
void* rejoin_arg;
|
|
} xce_failover_state_t;
|
|
|
|
typedef struct {
|
|
xce_client_config_t* conf;
|
|
endpoint_t ep;
|
|
xce_ctx_t ctx;
|
|
xce_failover_state_t failover;
|
|
} api_ctx_t;
|
|
|
|
/*
|
|
* EndpointRotator (task 2.2) — these operate on the private api_ctx_t, so they
|
|
* live in this internal header rather than the public xce/client_failover.h.
|
|
*
|
|
* xce_failover_next_target(): PURE target-selection (no I/O). If a redirect-
|
|
* address hint is set on ac->ctx, write it out and CONSUME it (one-shot,
|
|
* advisory). Otherwise advance the sticky master cursor (ac->failover.cur_master)
|
|
* to the next configured master in ac->conf, wrapping around. Returns 0,
|
|
* -EINVAL on a NULL/zero argument, or -ENOENT if no masters and no hint.
|
|
* Correctness never depends on the hint.
|
|
*
|
|
* xce_failover_rotate(): close the stale connection on ac->ep and reconnect the
|
|
* SAME endpoint object to the chosen target. Thin wrapper over the selector +
|
|
* endpoint_connect(); the live socket path is exercised by the E2E task.
|
|
*/
|
|
int xce_failover_next_target(api_ctx_t* ac, char* host_out, size_t host_sz,
|
|
uint16_t* port_out);
|
|
int xce_failover_rotate(api_ctx_t* ac);
|
|
|
|
/*
|
|
* XcecliRetryLoop / decision helper (task 3.1) — see design.md
|
|
* "Components and Interfaces → xcecli_retry_next".
|
|
*
|
|
* xcecli_retry_t: per-op retry carrier. One instance lives on the caller's
|
|
* stack for the duration of a single xcecli_* op's do/while loop.
|
|
* attempt — number of re-sends so far (0 on the first send).
|
|
* delay_ms — backoff to apply BEFORE the next attempt (0 for RETRY_NOW).
|
|
* rand_seed — per-op rand_r() state for jittered backoff (6.3).
|
|
*/
|
|
typedef struct {
|
|
int attempt;
|
|
int delay_ms;
|
|
unsigned int rand_seed;
|
|
} xcecli_retry_t;
|
|
|
|
/*
|
|
* XCECLI_RETRY_INIT(ac): initialise a retry carrier from an api_ctx_t. The seed
|
|
* mixes the ctx pointer so concurrent ops on distinct ctxs jitter independently.
|
|
*/
|
|
#define XCECLI_RETRY_INIT(ac) \
|
|
((xcecli_retry_t){ .attempt = 0, \
|
|
.delay_ms = 0, \
|
|
.rand_seed = (unsigned int)(uintptr_t)(ac) })
|
|
|
|
/*
|
|
* xcecli_retry_decide(): the PURE decision core (no I/O, no sleep, no socket,
|
|
* no logging) so it is unit-testable. Given the rc returned by the last xce_*
|
|
* attempt, it:
|
|
* - classifies *rc via xce_failover_classify(*rc, is_write, st->attempt);
|
|
* - on SUCCESS applies the DUP-replay rule (sets *rc = 0) and returns false;
|
|
* - on TERMINAL returns false leaving *rc as-is;
|
|
* - on a retriable action advances st->attempt, computes st->delay_ms
|
|
* (0 for RETRY_NOW; a jittered exponential backoff bounded by
|
|
* cfg->max_retry_ms for RETRY_BACKOFF), reports the action in *action_out,
|
|
* and returns true — UNLESS the retry budget (cfg->max_retries) is now
|
|
* exhausted, in which case it returns false and, for a no-primary cause,
|
|
* sets *rc = -E_XCE_NOT_READY (a transport-only cause keeps the last rc).
|
|
*
|
|
* It does NOT reset ctx state, drain wbuf, sleep, rotate, or invoke the rejoin
|
|
* hook — those are the side-effecting wrapper's job (xcecli_retry_next) and
|
|
* later tasks (3.2 state-reset/drain, 4.1 rejoin hook).
|
|
*/
|
|
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
|
|
const xce_retry_config_t* cfg, xcecli_retry_t* st,
|
|
xce_fo_action_t* action_out);
|
|
|
|
/*
|
|
* xcecli_retry_next(): the side-effecting wrapper the xcecli_* do/while loops
|
|
* call. = xcecli_retry_decide() + (RETRY_BACKOFF: a real st->delay_ms sleep) +
|
|
* (any retry: xce_failover_rotate(ac) + INFO log of the rotation target, 7.1) +
|
|
* (budget exhaustion: WARN log of the attempt count, 7.2). Returns true → the
|
|
* caller re-invokes its xce_* op; false → stop, *rc is final.
|
|
*/
|
|
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
|
|
xcecli_retry_t* st);
|
|
|
|
/*
|
|
* XCECLI_FAILOVER_CALL(ac, is_write, EXPR) — the shared XcecliRetryLoop wrapper
|
|
* (task 5.1). Turns a one-shot reply-bearing xce_* call into a transparent
|
|
* failover loop, holding the per-ctx lock (ac->failover.lock) for the FULL loop
|
|
* duration so concurrent ops on a shared ctx are serialised (CI2): the
|
|
* single-slot wbuf already implies one in-flight request per ctx; the lock makes
|
|
* that explicit and covers the reconnect/rotate window.
|
|
*
|
|
* ac — api_ctx_t* (already NULL-checked by the caller's guard)
|
|
* is_write — true for mutating ops (DUP-on-replay → success, 5.2), false reads
|
|
* EXPR — the unchanged one-shot xce_* call, evaluated each iteration; its
|
|
* rc is what the loop classifies and ultimately returns
|
|
*
|
|
* Evaluates to the final rc (>=0 success; -E_XCE_NOT_READY on no-primary budget
|
|
* exhaustion; terminal errors surfaced as-is). Used as the RHS of `return`.
|
|
*
|
|
* Implemented as a GCC/clang statement-expression so it yields a value and keeps
|
|
* each public xcecli_* signature/return convention intact (R1.3). The lock is
|
|
* released on every exit path because the do/while always falls through to the
|
|
* unlock before the result is produced.
|
|
*/
|
|
#define XCECLI_FAILOVER_CALL(ac, is_write, EXPR) \
|
|
__extension__({ \
|
|
api_ctx_t* _fo_ac = (ac); \
|
|
xcecli_retry_t _fo_st = XCECLI_RETRY_INIT(_fo_ac); \
|
|
int _fo_rc; \
|
|
mylock_lock(&_fo_ac->failover.lock); \
|
|
do { \
|
|
_fo_rc = (EXPR); \
|
|
} while ( \
|
|
xcecli_retry_next(_fo_ac, &_fo_rc, (is_write), &_fo_st)); \
|
|
mylock_unlock(&_fo_ac->failover.lock); \
|
|
_fo_rc; \
|
|
})
|
|
|
|
/*
|
|
* XCECLI_FAILOVER_CALL_COUNT(ac, EXPR) — sibling of XCECLI_FAILOVER_CALL for
|
|
* xce_get-family calls (bug-1436).
|
|
*
|
|
* xce_get_with_condition (request.c) — and every xcecli_* wrapper built on it
|
|
* (joblist, reslist, workerlist, resval, jobinfo) — returns a NON-NEGATIVE ROW
|
|
* COUNT on success (`rc = getreply.nrows`), not a fixed 0/status code; a
|
|
* negative return is the ONLY failure signal. XCECLI_FAILOVER_CALL assumes
|
|
* every wrapped EXPR follows the "0 = success" status-code convention and
|
|
* unconditionally feeds the raw return value through xce_failover_classify(),
|
|
* which recognizes retriable/terminal outcomes by comparing MAGNITUDE against
|
|
* fixed constants (E_XCE_* >= 10001, and POSIX errno values it deliberately
|
|
* accepts in either sign — ECONNRESET=104, ETIMEDOUT=110, ECONNREFUSED=111,
|
|
* client_failover.c fo_abs()). A joblist/reslist/workerlist/resval reply whose
|
|
* row count happens to equal one of those constants (e.g. exactly 111 jobs ==
|
|
* ECONNREFUSED) is silently misclassified as a transport failure on EVERY
|
|
* call with that exact count — forcing a spurious xce_failover_rotate() to the
|
|
* NEXT configured master even though the read fully succeeded. Since
|
|
* get-class ops are primary-only, landing on a standby correctly (but
|
|
* pointlessly) fails with E_XCE_STANDBY and rotates again; the cycle repeats
|
|
* for every subsequent same-sized call in the same batch, burning the finite
|
|
* max_retries budget (20) on self-inflicted churn until it coincides with a
|
|
* genuine transient (e.g. a peer mid-restart after a just-completed election)
|
|
* and exhausts the budget while rotating, surfacing a real
|
|
* ECONNRESET/ECONNREFUSED to the caller — bug-1436 ("xce_joblist: Connection
|
|
* reset by peer" immediately after a successful wait-primary; reproduced live
|
|
* at exactly 111 tracked jobs, the first count in the /xce-test-ha soak's
|
|
* arithmetic progression — 3 + 3*round — that lands on 111 == ECONNREFUSED).
|
|
*
|
|
* Fix: only enter the retry/classify loop when EXPR's raw result is NEGATIVE.
|
|
* A non-negative result (0 or a genuine row count, of ANY magnitude) is
|
|
* unconditionally success and returned as-is — it can never be misread as a
|
|
* status code. A dead ctx (XCE_CTX_ERR) is still caught correctly: the per-op
|
|
* XCE_CTX_RUN guard in xceapi.c makes EXPR itself return a negative -EINVAL in
|
|
* that state, which enters the loop and is reclassified by
|
|
* xcecli_retry_decide's dead-connection branch (checked before magnitude
|
|
* classification, regardless of the stale *rc it was invoked with).
|
|
*
|
|
* is_write is hardcoded false: every xce_get-family op is a read.
|
|
*/
|
|
#define XCECLI_FAILOVER_CALL_COUNT(ac, EXPR) \
|
|
__extension__({ \
|
|
api_ctx_t* _fc_ac = (ac); \
|
|
xcecli_retry_t _fc_st = XCECLI_RETRY_INIT(_fc_ac); \
|
|
int _fc_rc; \
|
|
mylock_lock(&_fc_ac->failover.lock); \
|
|
do { \
|
|
_fc_rc = (EXPR); \
|
|
} while ((_fc_rc < 0) && \
|
|
xcecli_retry_next(_fc_ac, &_fc_rc, /*is_write=*/false, \
|
|
&_fc_st)); \
|
|
mylock_unlock(&_fc_ac->failover.lock); \
|
|
_fc_rc; \
|
|
})
|
|
|
|
/*
|
|
* xcecli_ctx_recover (task 3.2) — repair the inner xce_ctx_t after a successful
|
|
* reconnect so the next xce_* attempt can proceed. Socket-free (no I/O, no
|
|
* sleep) so it is unit-testable; xcecli_retry_next invokes it after a successful
|
|
* xce_failover_rotate(), before re-sending the interrupted op.
|
|
*
|
|
* CI2 (state reset): the transport-error path left ctx->state = XCE_CTX_ERR
|
|
* (xcectx.c:174), which the per-op XCE_CTX_RUN guard rejects as -EINVAL
|
|
* (xceapi.c) — the reported bug. recover restores XCE_CTX_RUN, but ONLY from
|
|
* ERR: a DOWN/INIT ctx (deliberately stopped / not yet started) is left
|
|
* as-is, and a RUN ctx is already fine.
|
|
* CI1 (stale-reply drain): handle_disconnect may have injected a stale status
|
|
* into the single-slot ctx->wbuf (xcectx.c:180). recover drains any pending
|
|
* entry with a non-blocking cbuf_tryget so the next attempt's cbuf_get
|
|
* cannot consume a prior-attempt reply. Reconnect closes the old fd first,
|
|
* so the dead connection yields no further replies — this drain is enough.
|
|
*
|
|
* Returns 0 on success, -EINVAL on a NULL ac.
|
|
*/
|
|
int xcecli_ctx_recover(api_ctx_t* ac);
|
|
|
|
/*
|
|
* xcecli_ff_needs_reconnect (task 5.2) — PURE predicate for the fire-and-forget
|
|
* best-effort reconnect decision. Fire-and-forget ops (notify_*, heartbeat, nop)
|
|
* send a *_req only — there is NO reply to classify, so they must NOT go through
|
|
* the full failover retry loop (no status classification, no no-primary backoff,
|
|
* no retry budget; a heartbeat must never block on the backoff budget). Instead
|
|
* the FF path makes AT MOST ONE best-effort transport reconnect when the
|
|
* connection is KNOWN-DEAD, then does a single-shot send (which may be lost —
|
|
* acceptable; redelivery is deferred to worker-rejoin-resync).
|
|
*
|
|
* The "known-dead" signal is ctx->state == XCE_CTX_ERR (set by the disconnect
|
|
* path after a detected transport failure). Returns true ONLY for ERR so that a
|
|
* deliberately-stopped (DOWN) or not-yet-started (INIT) ctx is never resurrected
|
|
* by a fire-and-forget op. NULL-safe: returns false. Socket-free + side-effect
|
|
* free so it is unit-testable.
|
|
*/
|
|
bool xcecli_ff_needs_reconnect(const api_ctx_t* ac);
|
|
|
|
/*
|
|
* XCECLI_FF_CALL(ac, EXPR) — the fire-and-forget best-effort wrapper (task 5.2).
|
|
* Mirrors XCECLI_FAILOVER_CALL's per-ctx locking (CI2 — shared-ctx safety) but
|
|
* is single-shot, NOT a retry loop: there is no reply to classify for an FF op,
|
|
* so the full failover machinery (classification, no-primary backoff, retry
|
|
* budget) is deliberately NOT used. Putting a heartbeat through the retry loop
|
|
* would let it block on the no-primary backoff budget — explicitly forbidden.
|
|
*
|
|
* Behaviour, holding ac->failover.lock for the whole sequence:
|
|
* 1. if the connection is KNOWN-DEAD (xcecli_ff_needs_reconnect), attempt ONE
|
|
* best-effort reconnect: xce_failover_rotate(ac) + xcecli_ctx_recover(ac).
|
|
* A failed reconnect is non-fatal — the send below is simply best-effort.
|
|
* 2. evaluate EXPR exactly ONCE (the unchanged one-shot xce_* fire-and-forget
|
|
* send) and yield its rc verbatim. A lost send rc is surfaced as-is, never
|
|
* retried.
|
|
*
|
|
* ac — api_ctx_t* (already NULL-checked by the caller's guard)
|
|
* EXPR — the unchanged one-shot xce_* *_req-only call
|
|
*
|
|
* Implemented as a GCC/clang statement-expression so it yields the send rc and
|
|
* keeps each public xcecli_* signature/return convention intact (R1.3).
|
|
*/
|
|
#define XCECLI_FF_CALL(ac, EXPR) \
|
|
__extension__({ \
|
|
api_ctx_t* _ff_ac = (ac); \
|
|
int _ff_rc; \
|
|
mylock_lock(&_ff_ac->failover.lock); \
|
|
if (xcecli_ff_needs_reconnect(_ff_ac)) { \
|
|
(void)xce_failover_rotate(_ff_ac); \
|
|
xcecli_ctx_recover(_ff_ac); \
|
|
} \
|
|
_ff_rc = (EXPR); \
|
|
mylock_unlock(&_ff_ac->failover.lock); \
|
|
_ff_rc; \
|
|
})
|
|
|
|
/*
|
|
* jobsubmit_finalize_jobid (bug-1425) — guarantee xcecli_jobsubmit()'s *jobid
|
|
* output param is populated on every overall success, including one the
|
|
* XCecliRetryLoop's idempotent-replay rule (5.2, xce_failover_classify)
|
|
* produced by normalizing an EEXIST/E_XCE_DUP_JOB reply from a RETRIED
|
|
* resend into rc == 0.
|
|
*
|
|
* The gap: that normalization happens in the generic, type-erased
|
|
* xcecli_retry_decide() layer, one level above xce_jobsubmit() (xceapi.c) —
|
|
* which only ever copies the reply's jobid into *jobid on ITS OWN raw
|
|
* non-negative status. The dup-rejection reply itself carries NO payload
|
|
* (handle_jobsubmit's EEXIST/inflight-collision branches reply with
|
|
* RequestReply_data_nodata, job.c), so *jobid is never written on the
|
|
* winning (dup) attempt, and was never written on the earlier failing
|
|
* attempt either (whose raw status was already negative). Caller-visible
|
|
* result: xcecli_jobsubmit() returns 0 (success) with *jobid left at
|
|
* whatever the caller initialized it to — reproduced live via a concurrent
|
|
* 9-client submit burst immediately after a fresh single-node election
|
|
* (~10.0-10.1s: one full 10s cbuf_timedget deadline on the failing first
|
|
* attempt, then a fast dup-rejection reply on the resend).
|
|
*
|
|
* The job a successful submission belongs to is unambiguously the one THIS
|
|
* client generated and put on the wire in the first place (submit->jobid —
|
|
* job submission ids are client-generated; the controller commits/echoes
|
|
* this exact value verbatim on every genuine success reply, bug-786,
|
|
* job.c:1439's `jobid = submit->jobid`). So on ANY overall success this is
|
|
* always the correct id to report, regardless of which attempt in the retry
|
|
* sequence actually produced the winning rc. No socket/reply-decode
|
|
* dependency — pure, and exposed non-static so it is unit-testable without
|
|
* a live connection (test_libxce_retry.c).
|
|
*
|
|
* Called by xcecli_jobsubmit() (client.c) immediately after
|
|
* XCECLI_FAILOVER_CALL returns. A no-op (jobid left untouched) when
|
|
* overall_rc < 0 (the call genuinely failed) or jobid is NULL.
|
|
*/
|
|
void jobsubmit_finalize_jobid(uint64_t* jobid, uint64_t submitted_jobid,
|
|
int overall_rc);
|
|
|
|
/*
|
|
* xcecli_invoke_rejoin (task 4.1) — invoke the consumer rejoin hook once after a
|
|
* successful reconnect/recovery and BEFORE the interrupted op resumes. Socket-
|
|
* free (it only calls the stored callback), so it is unit-testable. The
|
|
* xcecli_retry_next wrapper calls it AFTER xce_failover_rotate +
|
|
* xcecli_ctx_recover and BEFORE re-sending the op.
|
|
*
|
|
* If no hook is registered (rejoin_cb == NULL, a pure client) it is a no-op and
|
|
* returns 0 so the loop resumes directly. Otherwise it returns the callback's
|
|
* rc: 0 to proceed, <0 to treat the rejoin as FAILED (the new primary rejected
|
|
* the daemon's join) — the wrapper then does NOT resume but rotates/backs off on
|
|
* the next iteration.
|
|
*
|
|
* Returns 0 on a successful (or absent) rejoin, the hook's <0 rc on a failed
|
|
* rejoin, or -EINVAL on a NULL ac.
|
|
*/
|
|
int xcecli_invoke_rejoin(api_ctx_t* ac);
|
|
|
|
#endif
|