feat(libxce-failover): wrap 18 reply-bearing xcecli_* ops in failover loop under per-ctx lock; correct nop OpCategory (task 5.1)

src/lib/testsuite/test_libxce_retry.c

@@ -32,6 +32,7 @@
 
 #include "client_internal.h"  /* api_ctx_t, xce_failover_state_t */
 #include "failover_harness.h"
+#include "mylock.h"
 #include "request.h"
 #include "xce/client_failover.h"
 #include "xce/errnox.h"
@@ -1272,6 +1273,128 @@ static void test_invoke_rejoin_null_ctx_safe(void)
     printf("test_invoke_rejoin_null_ctx_safe: PASS\n");
 }
 
+/* ════════════════════════════════════════════════════════════════════════
+ * Task 5.1 — XcecliRetryLoop wrapping + per-ctx lock (CI2 concurrency).
+ *
+ * Each reply-bearing xcecli_* op wraps its one-shot xce_* call in the failover
+ * do/while loop AND holds ac->failover.lock for the loop's full duration so
+ * concurrent ops on ONE shared ctx are serialised (the single-slot wbuf already
+ * implies one in-flight request per ctx; the lock makes that explicit and covers
+ * the reconnect/rotate window). Full failover behaviour against a live primary
+ * is validated by E2E task 6.2; here we prove the serialisation contract.
+ *
+ * The XCECLI_FAILOVER_CALL macro (client_internal.h) is the shared wrapper that
+ * every op body uses: lock → init retry carrier → do/while(xcecli_retry_next) →
+ * unlock → return rc. We drive that exact macro here with a trivial EXPR that
+ * mutates shared state under the held lock, so N threads contending on one ctx's
+ * failover.lock must not lose updates (mutual exclusion), and a non-failing EXPR
+ * runs exactly once (the success path returns immediately).
+ *
+ * Requirements: 1.1, 1.3, 2.4, 5.1, 5.3, and design "FailoverState → Concurrency
+ * (CI2)".
+ * ════════════════════════════════════════════════════════════════════════ */
+
+#define FO_LOCK_NTHREADS 32
+#define FO_LOCK_ITERS    2000
+
+typedef struct {
+    api_ctx_t *ac;          /* the ONE shared ctx all threads contend on        */
+    long       counter;     /* mutated under failover.lock via the wrapper macro */
+    int        max_inflight;/* peak concurrent in-section (must stay 1)          */
+    int        inflight;    /* current in-section count                          */
+} fo_lock_ctx_t;
+
+/* A trivial reply-bearing EXPR that always "succeeds" (rc==0) but, as a side
+ * effect, performs a non-atomic read-modify-write on the shared counter and
+ * tracks the peak in-section concurrency. If the wrapper macro did NOT hold
+ * ac->failover.lock for the section, concurrent threads would interleave the
+ * RMW (lost updates) and/or observe inflight > 1. */
+static int fo_lock_body(fo_lock_ctx_t *c)
+{
+    c->inflight++;
+    if (c->inflight > c->max_inflight)
+        c->max_inflight = c->inflight;
+
+    /* non-atomic RMW: a data race here would lose increments */
+    long v = c->counter;
+    for (volatile int s = 0; s < 50; s++) {
+    } /* widen the race window */
+    c->counter = v + 1;
+
+    c->inflight--;
+    return 0; /* success → wrapper returns immediately, no retry */
+}
+
+/* The per-thread worker drives FO_LOCK_ITERS wrapped "ops" on the SHARED ctx.
+ * Each op uses the real XCECLI_FAILOVER_CALL wrapper, so it acquires/releases
+ * ac->failover.lock exactly as a production xcecli_* op does. */
+static void fo_lock_worker(void *ctx, int idx)
+{
+    (void)idx;
+    fo_lock_ctx_t *c  = (fo_lock_ctx_t *)ctx;
+    api_ctx_t     *ac = c->ac;
+
+    for (int i = 0; i < FO_LOCK_ITERS; i++) {
+        int rc = XCECLI_FAILOVER_CALL(ac, /*is_write=*/false, fo_lock_body(c));
+        TEST_ASSERT_EQUAL_INT(0, rc);
+    }
+}
+
+/* Req 5.1, 2.4 (CI2): N threads sharing ONE ctx are serialised by the per-ctx
+ * failover lock the wrapper holds — no lost updates and peak in-section
+ * concurrency stays 1. The total counter equals N * ITERS exactly. */
+static void test_failover_call_serialises_shared_ctx(void)
+{
+    xce_client_config_t conf;
+    api_ctx_t           ac;
+    fo_make_client_masters(&conf);
+    fo_make_api_ctx(&ac, &conf);
+
+    /* the wrapper takes ac->failover.lock; init it (xcecli_start would in prod) */
+    TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock));
+    ac.failover.retry = XCE_RETRY_DEFAULTS;
+
+    fo_lock_ctx_t lc;
+    memset(&lc, 0, sizeof(lc));
+    lc.ac = &ac;
+
+    int rc = fo_run_threads(FO_LOCK_NTHREADS, fo_lock_worker, &lc);
+    mylock_fini(&ac.failover.lock);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    /* mutual exclusion held: every increment landed, none lost */
+    TEST_ASSERT_EQUAL_INT((long)FO_LOCK_NTHREADS * FO_LOCK_ITERS, lc.counter);
+    /* the critical section was never entered by two threads at once */
+    TEST_ASSERT_EQUAL_INT(1, lc.max_inflight);
+
+    printf("test_failover_call_serialises_shared_ctx: PASS\n");
+}
+
+/* Req 1.1, 5.3: the wrapper returns the op's success rc on the first try (no
+ * spurious retry) for a status-0 EXPR, and the per-op retry carrier starts at
+ * attempt 0 — i.e. a success path does not consume the retry budget. */
+static void test_failover_call_success_runs_once(void)
+{
+    xce_client_config_t conf;
+    api_ctx_t           ac;
+    fo_make_client_masters(&conf);
+    fo_make_api_ctx(&ac, &conf);
+    TEST_ASSERT_EQUAL_INT(0, mylock_init(&ac.failover.lock));
+    ac.failover.retry = XCE_RETRY_DEFAULTS;
+
+    fo_lock_ctx_t lc;
+    memset(&lc, 0, sizeof(lc));
+    lc.ac = &ac;
+
+    int rc = XCECLI_FAILOVER_CALL(&ac, /*is_write=*/false, fo_lock_body(&lc));
+    mylock_fini(&ac.failover.lock);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);          /* success surfaced verbatim         */
+    TEST_ASSERT_EQUAL_INT(1, lc.counter);  /* body ran exactly once, no replay  */
+
+    printf("test_failover_call_success_runs_once: PASS\n");
+}
+
 /* ── Suite entry point ──────────────────────────────────────────────── */
 
 void test_libxce_retry(void)
@@ -1315,5 +1438,7 @@ void test_libxce_retry(void)
     RUN_TEST(test_invoke_rejoin_no_hook_is_noop);
     RUN_TEST(test_set_reconnect_handler_null_safe_and_clears);
     RUN_TEST(test_invoke_rejoin_null_ctx_safe);
+    RUN_TEST(test_failover_call_serialises_shared_ctx);
+    RUN_TEST(test_failover_call_success_runs_once);
     UNITY_END();
 }