From 387ccfa8dc83e2d8a5141e6e21ffaa23e41693e4 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Wed, 17 Jun 2026 00:05:41 +0000 Subject: [PATCH] =?UTF-8?q?feat(libxce-failover):=20foundation=20=E2=80=94?= =?UTF-8?q?=20per-ctx=20failover=20state,=20redirect-hint=20fields,=20remo?= =?UTF-8?q?ve=20orphaned=20retry=20primitive=20(task=201.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .kiro/specs/libxce-failover/brief.md | 150 ++++ .kiro/specs/libxce-failover/design.md | 540 ++++++++++++++ .kiro/specs/libxce-failover/requirements.md | 179 +++++ .kiro/specs/libxce-failover/research.md | 127 ++++ .kiro/specs/libxce-failover/spec.json | 22 + .kiro/specs/libxce-failover/tasks.md | 103 +++ .wolf/anatomy.md | 26 +- .wolf/cron-state.json | 2 +- .wolf/hooks/_session.json | 176 ++--- .wolf/memory.md | 1 + .wolf/token-ledger.json | 783 +++++++++++++++++++- src/include/request.h | 33 +- src/include/xcectx.h | 10 + src/lib/client.c | 29 +- src/lib/client_internal.h | 63 ++ src/lib/request.c | 104 +-- src/lib/testsuite/test_libxce_retry.c | 336 +-------- 17 files changed, 2122 insertions(+), 562 deletions(-) create mode 100644 .kiro/specs/libxce-failover/brief.md create mode 100644 .kiro/specs/libxce-failover/design.md create mode 100644 .kiro/specs/libxce-failover/requirements.md create mode 100644 .kiro/specs/libxce-failover/research.md create mode 100644 .kiro/specs/libxce-failover/spec.json create mode 100644 .kiro/specs/libxce-failover/tasks.md create mode 100644 src/lib/client_internal.h diff --git a/.kiro/specs/libxce-failover/brief.md b/.kiro/specs/libxce-failover/brief.md new file mode 100644 index 0000000..807d8c4 --- /dev/null +++ b/.kiro/specs/libxce-failover/brief.md @@ -0,0 +1,150 @@ +# Brief: libxce-failover + +## Problem + +Client applications using libxce do **not** survive controller primary failover. When the +primary controller is killed and a new one is elected, in-flight and subsequent libxce calls +fail with an opaque `Invalid argument` (EINVAL) instead of either (a) retrying transparently +against the new primary or (b) reporting a clear "cluster is not ready" during the election +window. + +Reproduction (3-node HA cluster, `set masters localhost:9991,localhost:9992,localhost:9993`): +1. Primary elected; `jobsub` / `joblist` work. +2. Kill the primary (`ctrl-1`). +3. Before a new primary is elected, `joblist` should report **cluster not ready**. +4. After a new primary is elected, all commands should transparently route to the new + primary with no error. + +Observed instead: `joblist` keeps returning stale cached output, then `xce_joblist: Invalid +argument` indefinitely — the client never fails over. + +## Current State + +- The requirement already exists: **`ha-config-bootstrap` Requirement 7 — "Client-Side + Transparent Retry for Transient Controller Errors"** (acceptance criteria 7.1–7.6), and its + tasks 10 / 10.1 / 10.2 are marked complete. +- Only the **standalone primitive** was delivered: + - `xce_request_with_retry()` is implemented at `src/lib/request.c:1956` and unit-tested in + `src/lib/testsuite/test_libxce_retry.c` (NOT_READY backoff, STANDBY+redirect reconnect, + STANDBY-without-redirect fallback, non-retriable surface-immediately, max-retries, + 50-client concurrency). `xce_retry_config_t` defaults: `max_retries=20`, + `initial_retry_ms=50`, `max_retry_ms=5000`, `backoff_multiplier=2.0`. +- **Gap 1 — wiring:** every production API path bypasses the wrapper. `xce_joblist` → + `xce_get_req` (`src/lib/request.c:1373`) → plain `xce_request()` (`request.c:1385`). The + same is true for ~18 other `xce_*_req` wrappers — none call `xce_request_with_retry()`. +- **Gap 2 — multi-master discovery:** the retry primitive operates on a single `endpoint_t` + and only follows an explicit `redirect_host/redirect_port`. A *killed* primary cannot + redirect, so the client must **rotate through its configured master list** (9991/9992/9993) + to discover the new primary. That orchestration does not exist; the client connects to + `masters[0]` once in `xcecli_start()` (`src/lib/client.c:122`) and never re-selects. +- Connections are long-lived and cached in `api_ctx_t->ep`; there is no dead-primary + detection or automatic reconnect at the API level. + +## Desired Outcome + +Failover is transparent to client applications: +- Short failovers are hidden: libxce retries across the configured masters with backoff and + succeeds against the new primary once elected. +- If the election outlasts the retry budget, the call returns a clear `E_XCE_NOT_READY` + ("cluster not ready") rather than `EINVAL`. +- Both read and write requests fail over (writes rely on existing server-side commit-barrier + + idempotency guards to dedup re-sends). +- No per-daemon ad-hoc retry loops — the behavior lives entirely in libxce. + +## Approach + +**Central chokepoint (Approach A).** Make failover the default behavior at the single point +all wrappers already funnel through: + +1. Give the request layer access to the cluster/master list + a "current primary" index + (design choice: a back-reference on `endpoint_t`, or a small `xce_cluster_t` the endpoint + references — resolved in design phase). +2. Upgrade `xce_request_with_retry()` to, in addition to its existing redirect/backoff cases: + - rotate through the master list on connection failure (`ECONNREFUSED`/`ETIMEDOUT`/RDHUP) or + `STANDBY`-without-redirect, reconnecting to each candidate to find the new primary; + - apply the bounded backoff budget and, on expiry, surface `E_XCE_NOT_READY`. +3. Route the production `xce_*_req` wrappers through the retry/failover path (e.g. make + `xce_request()` the failover wrapper over an inner `xce_request_raw()`), wiring every API + at once. + +Chosen because it is the only approach that covers the entire client surface at one +integration point, reuses the already-tested primitive, treats reads and writes uniformly, and +keeps the requirement out of caller code — matching "transparent to the client application." + +## Scope + +- **In:** + - Wire all production libxce request paths through transparent retry/failover. + - Multi-master rotation to discover the new primary after the connected one dies. + - Bounded-budget semantics: transparent retry, then `E_XCE_NOT_READY` ("cluster not ready"). + - Failover for both read and write requests (rely on server idempotency for writes). + - Tests: extend `test_libxce_retry.c` for master-list rotation + budget→NOT_READY; a 3-node + live failover integration test (kill primary, assert NOT_READY window then transparent + recovery). +- **Out:** + - Server-side election, WAL, commit barrier, idempotency guards (already owned by the HA + server specs; this spec consumes them). + - Changing the `xce_test` REPL behavior beyond what surfaces the new error semantics. + - Per-request idempotency/retry opt-in flags (writes retry by default per decision). + +## Boundary Candidates + +- Master-list + current-primary selection state and rotation policy. +- The retry/failover wrapper (backoff budget, error classification, NOT_READY surfacing). +- Integration seam: how production `xce_*_req` wrappers reach the failover path. +- Error-surface mapping: connection/EINVAL vs. `E_XCE_NOT_READY` vs. `E_XCE_STANDBY` vs. + non-retriable (auth/perm/invalid/noent). + +## Out of Boundary + +- Any controller/server-side change (election timing, redirect content, idempotency). +- New transport/probe opcodes unless design shows rotation cannot work with existing replies. + +## Upstream / Downstream + +- **Upstream:** `ha-config-bootstrap` (Req 7 retry primitive + `xce_retry_config_t` + error + codes `E_XCE_NOT_READY` / `E_XCE_STANDBY` + `redirect_host/port` reply fields); + `ha-commit-barrier-rollout` (server idempotency guards that make write retries safe); + `ha-single-leader-role` / election specs (define when a standby/primary answers). +- **Downstream:** worker, scheduler, and apiserver clients — all gain transparent failover and + can drop any ad-hoc retry handling; the `/xce-deploy-ha` live test harness. + +## Existing Spec Touchpoints + +- **Extends:** consumes and completes the intent of `ha-config-bootstrap` Req 7 (the primitive + was delivered but never integrated); this spec owns the integration + multi-master failover. +- **Adjacent:** `ha-repl-hardening`, `ha-convergence-recovery` (server-side failover behavior); + `worker-rejoin-resync` (worker-side rejoin, distinct from client-library failover). + +## Protocol Correction (2026-06-16, requirements phase) + +The discovery notes above assumed the controller signals non-primary via `E_XCE_STANDBY` with +`redirect_host/redirect_port` fields. Code review found the real taxonomy (`errnox.h:22-25`): +`E_XCE_REDIRECT` (standby knows a primary exists; `join.c:260-268`), `E_XCE_NOMASTER` (standby, +no primary known yet; `join.c:270-278`), `E_XCE_NOT_READY` (INIT/ELECTING), `E_XCE_STANDBY` +(primary-only op on a standby). The controller's redirect path sends `E_XCE_REDIRECT` with +`data_which = nodata` — **status only, no address** — so the existing `xce_request_with_retry()` +condition (`E_XCE_STANDBY` + populated redirect fields, `request.c:2010`) never matches a real +controller. Decisions (confirmed): classify on the actual statuses — `E_XCE_REDIRECT` → +fast-rotate the master list to find the existing primary (use an address if the reply carries +one, else rotate); `E_XCE_NOMASTER`/`E_XCE_NOT_READY` → backoff → cluster-not-ready; +`E_XCE_STANDBY` → rotate. The existing retry unit tests (keyed on the non-emitted shape) are +corrected to the real statuses. Tracked as bug-608. + +**Scope expansion (confirmed):** the redirect address is absent today only because `join.c` +never fills the reply's `redirect_host/redirect_port` fields — those fields already exist in the +wire schema (`controller.capnp:568-569`) and the reply struct (`typemap.h:225-226`), and the +controller already knows the address (`join.c:263`). One small controller change to populate them +is now **in scope** (Requirement 8), making redirect-based failover deterministic (direct connect +to the primary, O(1)); client-side master-list rotation remains as a defensive fallback for an +older controller that omits the address. + +## Constraints + +- C library (libxce); follow project error-handling (`errnox`, early-return) and async-I/O + rules — `xce_request`/`xce_reply` non-blocking; never block the iomgr thread. +- TDD mandatory: failing tests in `src/lib/testsuite/` before implementation. +- Touches HA/failover behavior → full Kiro workflow (requirements → design → tasks), not the + quick path, per CLAUDE.md. +- Reuse the existing `xce_request_with_retry()` primitive and its config struct; do not fork a + second retry implementation. diff --git a/.kiro/specs/libxce-failover/design.md b/.kiro/specs/libxce-failover/design.md new file mode 100644 index 0000000..d4ccb52 --- /dev/null +++ b/.kiro/specs/libxce-failover/design.md @@ -0,0 +1,540 @@ +# Design Document — libxce-failover + +## Overview + +**Purpose**: This feature delivers transparent controller-primary failover to every client +application built on libxce (worker, scheduler, apiserver via cgo, and the `xce_test` harness), +so that a controller failover is hidden when short and reported clearly when not. + +**Users**: libxce callers issue normal requests (`xce_joblist`, `xce_jobsub`, …). When the +connected primary dies, libxce locates the newly elected primary and completes the request there, +or — if no primary exists yet — returns a distinct cluster-not-ready error instead of an opaque +`EINVAL`. No caller code changes. + +**Impact**: Today the client connects to one master (`client.c:122`), routes every request through +the fire-and-forget `xce_request()`, and has no reply-status-driven retry on the production path +(`xce_request_with_retry()` only retries in test-hook mode). This design places failover at the +**`client.c` (`xcecli_*`) layer** — the universal client surface that every consumer goes through +(worker, scheduler, apiserver-via-cgo, `xce_test`) and the only layer whose context (`api_ctx_t`) +co-locates the master list (`conf`), the endpoint (`ep`), and the reply context (`ctx`). Each +`xcecli_*` op already calls its internal `xce_*` op, which **returns the controller status**; the +failover loop wraps that call, classifies the status, and reconnects/rotates over `api_ctx.conf`. +It adds master-list rotation, redirect-address following, and a small controller change so redirect +replies carry the primary address. `xceapi.c` ops are unchanged. + +### Goals +- Route all libxce request/reply round trips through one transparent failover path (1.1, 1.3). +- Discover the new primary via redirect address, falling back to master-list rotation (2.x, 8.x). +- Hide short failovers via bounded backoff; surface a distinct cluster-not-ready on exhaustion (3.x). +- Fail over reads and writes uniformly, relying on controller idempotency for writes (5.x). +- Reuse the existing retry config/defaults; stagger retries with jitter (6.x). + +### Non-Goals +- Controller-side election, WAL, commit barrier, write-idempotency logic (consumed, not changed). +- Per-daemon adoption (master-list wiring, liveness-safe blocking) — `component-failover-adoption`. +- New transport opcodes or new reply fields (redirect fields already exist). + +## Boundary Commitments + +### This Spec Owns +- The client-side failover loop: classify status → reconnect/rotate/backoff → resume, including the + cluster-not-ready terminal mapping, for **reply-bearing** `xcecli_*` ops. +- Client-side master-list cursor and current-primary stickiness within a context. +- Capturing the redirect address from a reply as a best-effort hint (no correctness dependency). +- A **reconnect/rejoin hook**: a consumer-registered callback the failover machinery invokes after + it reconnects to a new primary and *before* it resumes the interrupted op. libxce owns the hook + mechanism, its ordering, and invoking it; it does **not** own the callback's content. +- A per-ctx concurrency lock so `xcecli_*` failover is safe when one ctx is shared across threads. +- The single controller change that populates `redirect_host/port` on `E_XCE_REDIRECT` replies. +- Realignment of the libxce retry unit tests to the real controller status taxonomy. + +### Out of Boundary +- How the controller decides primary/standby/redirect/no-primary (election, `ha_should_redirect`). +- The **content** of the rejoin callback: the join/state-exchange a daemon performs on reconnect + (owned by `component-failover-adoption`) and the worker's running-jobs re-sync (owned by + `worker-rejoin-resync`). This spec provides and invokes the hook; the daemons fill it in. +- Redelivery of fire-and-forget notifications lost across a failover (also `worker-rejoin-resync`). +- Worker/scheduler/apiserver adoption and their threading/liveness handling of bounded blocking. +- Any reply payload beyond the already-defined `redirect_host/redirect_port`. + +### Allowed Dependencies +- Existing controller status codes (`errnox.h:22-25`) and redirect reply fields (`typemap.h:225`). +- `endpoint_connect()` / endpoint I/O (`netio.*`), request encode/enqueue (`request.c`), and the + context wait channel (`cbuf`, `xcectx.c`). +- `xce_retry_config_t` + `XCE_RETRY_DEFAULTS` (request.h) and the cluster/peer config (`cluster.*`). + +### Revalidation Triggers +- Change to the reply `status`/`redirect_*` contract → re-check client classification + controller. +- Change to the async reply-dispatch model (`handle_reply` → `cbuf` signalling) → re-check round trip. +- Change to `api_ctx_t`/endpoint ownership → re-check reconnect-reuses-same-endpoint assumption. +- New non-primary status code added to the controller → extend client classification. +- Change to the reconnect/rejoin hook signature or its invocation ordering → re-check daemon rejoin + registration (`component-failover-adoption`) and worker resync (`worker-rejoin-resync`). + +## Architecture + +### Existing Architecture Analysis +- **Layering — `xcecli_*` is the universal surface.** Every consumer calls `client.c`'s `xcecli_*` + ops, never `xceapi.c`'s `xce_*` directly: worker (`controller_connector.c:184,272`), scheduler + (`scheduler.c:457,463`), apiserver via cgo (`clib.go:34,36`), `xce_test`. Each `xcecli_*` is a thin + wrapper that calls its `xce_*` counterpart, which **returns the controller status** as its rc. +- **Only `api_ctx_t` reaches the master list.** `api_ctx_t = { conf (masters), ep, ctx }` + (client.c:23). The inner `xce_ctx_t` (xceapi) holds `ep`/`wbuf`/`state` but **no** `conf` + (xcectx.h) — so failover belongs at the client layer, where the master list is already in scope. +- **`xce_*` fail fast; `xcecli_*` retries.** The async send/wait stays inside `xce_*` unchanged: a + single attempt that returns the controller status (or transport error). The `xcecli_*` wrapper + classifies that rc and, on a retriable status, reconnects/rotates `api_ctx.ep` over `api_ctx.conf` + and re-invokes `xce_*`. No async internals, thunks, or master-list plumbing are introduced. +- **Reconnect reuses the endpoint.** `endpoint_connect(ep,host,port)` re-points the same + `endpoint_t`, preserving the `msg_handler` wiring that routes replies to `ctx->wbuf`. +- **Stateful consumers need a rejoin on reconnect.** `xce_test` is a pure client — reconnect + + resend suffices. But the apiserver, worker, and scheduler hold a *session* with the primary + established via `xcecli_join` (role + attrs); a freshly elected primary has no live session for a + reconnecting daemon until it re-joins, and a worker must additionally re-sync its running-jobs + state. So failover for these consumers is **reconnect → rejoin/state-exchange → resume**, not just + reconnect → resend. libxce models this with a consumer-registered reconnect hook; the hook content + lives in `component-failover-adoption` (join) and `worker-rejoin-resync` (job state). +- **Two op shapes.** Reply-bearing ops (`xce_joblist`/`xce_jobsubmit`/`xce_hello`/…) do + `xce_*_req` + `cbuf_get` → return a controller status the loop can classify. Fire-and-forget ops + (`xce_notify*`, heartbeat) do `xce_*_req` only (no `cbuf_get`, xceapi.c:282) → return the enqueue + result, never a controller status. Failover classification applies only to the former. +- **Patterns preserved**: `xce_*` ops, `xce_*_req` enqueue wrappers, and `xce_request`/`xce_reply` + are all unchanged; failover is additive in `xcecli_*`. Trade-off: ~33 `xcecli_*` wrappers each gain + a small retry loop (more lines), but the policy lives in one shared decision helper (fewer moving + parts) — an explicit simplicity-over-LOC choice. + +### Architecture Pattern & Boundary Map + +```mermaid +graph LR + Caller[consumer worker scheduler apiserver xce_test] --> Cli[xcecli op in client.c] + Cli --> Loop[do while retry loop] + Loop --> Decide[xcecli_retry_next decision helper] + Loop --> Xce[xce op in xceapi one shot] + Xce --> Ctrl[controller] + Ctrl --> Reply[handle_reply] + Reply --> Capture[capture redirect hint into ctx] + Decide --> Rotate[reconnect or rotate over api_ctx conf and ep] + Decide --> Backoff[jittered backoff] +``` + +- **Selected pattern**: a thin retry loop in each `xcecli_*` wrapper, driven by one shared decision + helper (`xcecli_retry_next`). Rationale: `client.c` is the universal surface AND the only layer + whose `api_ctx_t` co-locates master list + endpoint + ctx; `xce_*` already returns the status. +- **Dependency direction**: `types → cluster/config → endpoint(netio) → xceapi(xce_*) → client(xcecli_*)`. + Each layer imports only leftward. Failover logic is added at the `client` layer (rightmost), + depending on `xce_*` and `cluster/config`; `xceapi` does not depend on it. + +### Technology Stack + +| Layer | Choice / Version | Role in Feature | Notes | +|-------|------------------|-----------------|-------| +| Client wrappers | C (gnu17), libxce `client.c` | `xcecli_*` retry loop + decision helper + rotate | Reuses `xce_*`, `endpoint_connect`, `cluster` | +| Client API | C (gnu17), libxce `xceapi.c` | **Unchanged** — one-shot `xce_*` returns status | No edits | +| Controller | C (gnu17) | Populate redirect address on `E_XCE_REDIRECT` | ~2 lines in `join.c`; fields already in schema | +| Tests | GoogleTest (testsuite) | Realign retry tests to real status taxonomy | Test the `xcecli_*` retry path | + +## File Structure Plan + +### New Files +``` +src/lib/client_failover.c # xcecli_retry_next decision helper, StatusClassifier, rotate/reconnect over api_ctx +src/lib/client_internal.h # private: api_ctx_t + xce_failover_state_t shared by client.c and client_failover.c +``` +> `api_ctx_t` is currently file-local to `client.c`; move its definition to `client_internal.h` so the +> failover helper can operate on it without exposing it in the public API. + +### Modified Files +- `src/lib/client.c` — wrap each **reply-bearing** `xcecli_*` op body in a `do { rc = + xce_(...); } while (xcecli_retry_next(ac,&rc,is_write,&st));` loop (per OpCategory); + fire-and-forget ops get the transport-reconnect-only variant; add `xce_failover_state_t` to + `api_ctx_t` (lock + cursor + retry config + rejoin hook); add `xcecli_set_reconnect_handler()`; + init + take `failover.lock` per op; init in `xcecli_start()`; keep initial connect to `masters[0]`. +- `src/include/client.h` — declare `xcecli_set_reconnect_handler()` + `xcecli_reconnect_cb`. +- `src/lib/xcectx.c` — `handle_reply()` copies the redirect-hint address (`reply.redirect_host/port`, + when present) into `xce_ctx_t` before the reply is freed; the existing error path (`XCE_CTX_ERR` + + `wbuf` error inject, xcectx.c:137-154) is the wake hook reused on reconnect. +- `src/include/xcectx.h` — add the fixed-size redirect-hint `host/port` fields to `xce_ctx_t`. +- `src/include/request.h` / `src/lib/request.c` — keep `xce_retry_config_t` + `XCE_RETRY_DEFAULTS`; + **remove** the orphaned no-op `xce_request_with_retry()` (superseded by the `xcecli_*` loop). +- `src/lib/xceapi.c` — **unchanged** (one-shot `xce_*` ops already return the controller status). +- `src/controller/join.c` — populate `reply.redirect_host = c->ha.primary_host` / + `reply.redirect_port = c->ha.primary_port` on the `E_XCE_REDIRECT` path (audit for other sites). +- `src/lib/testsuite/test_libxce_retry.c` — retarget from the retired primitive to the `xcecli_*` + retry path + `StatusClassifier`; cover `E_XCE_REDIRECT`/`NOMASTER`/`NOT_READY`/`STANDBY`, + master rotation, and budget→not-ready. + +## System Flows + +### Failover loop (per `xcecli_*` op) + +```mermaid +sequenceDiagram + participant C as Caller + participant F as xcecli op do while loop + participant E as Endpoint + participant K as Controller + C->>F: xcecli op + loop until success, terminal, or budget exhausted + F->>E: xce op one shot + E->>K: request + K-->>E: reply status plus optional redirect addr + E-->>F: xce op returns status + alt status success + F-->>C: ok + else redirect or standby or transport error + F->>E: rotate (hint address first if set, else next master) then invoke rejoin hook + else no-primary or not-ready + F->>F: backoff jittered then rotate + else non-retriable + F-->>C: surface error + end + end + F-->>C: cluster-not-ready on budget exhaustion +``` + +After any reconnect to a new endpoint, the failover machinery invokes the consumer-registered +reconnect/rejoin hook (if set) **before** re-sending the interrupted op, so a stateful daemon +re-establishes its session (and a worker re-syncs job state) against the new primary. Pure clients +(`xce_test`) register no hook and skip straight to re-send. + +Key decisions: redirect/standby/transport rotate **without** the no-primary backoff delay (4.2); +only no-primary/not-ready consume the backoff budget (3.1, 4.1); on success the served master +becomes the sticky current primary (2.4). + +### Status classification + +```mermaid +stateDiagram-v2 + [*] --> Classify + Classify --> Success: status==0 or DUP on re-sent write + Classify --> RetryNow: REDIRECT or STANDBY or transport error + Classify --> RetryBackoff: NOMASTER or NOT_READY + Classify --> Terminal: AUTH PERM INVALID NOENT + RetryNow --> Rotate + RetryBackoff --> Rotate + Rotate --> Classify: hint first else next master + Success --> [*] + Terminal --> [*] +``` + +## Requirements Traceability + +| Requirement | Summary | Components | Flows | +|-------------|---------|------------|-------| +| 1.1, 1.3 | All ops through failover; preserve API | XcecliRetryLoop (all `xcecli_*`) | Failover loop | +| 1.2 | Complete next request against new primary | XcecliRetryLoop, EndpointRotator | Failover loop | +| 2.1 | Unreachable → try other masters | EndpointRotator | Classification | +| 2.2, 8.3, 8.4 | Redirect addr if present else rotate | RedirectCapture, EndpointRotator | Classification | +| 2.3, 4.3 | Standby → rotate | StatusClassifier, EndpointRotator | Classification | +| 2.4 | Sticky current primary | FailoverState | Failover loop | +| 2.5 | Redirect/standby/no-primary ≠ success | StatusClassifier | Classification | +| 3.1, 4.1 | No-primary → backoff to budget | XcecliRetryLoop, FailoverState | Failover loop | +| 3.2 | Success before budget | XcecliRetryLoop | Failover loop | +| 3.3, 3.4 | Budget exhausted → distinct cluster-not-ready | XcecliRetryLoop | Failover loop | +| 4.2 | Redirect → prompt switch, no backoff | EndpointRotator | Classification | +| 4.4 | Transport → reconnect/rotate | EndpointRotator, ReplyWaitWake | Classification | +| 4.5 | Non-retriable → surface | StatusClassifier | Classification | +| 1.2 | Re-establish session against new primary | ReconnectRejoinHook | Failover loop | +| 5.1, 5.3 | Reads+writes; no silent drop (reply-bearing) | XcecliRetryLoop, OpCategory | Failover loop | +| 5.2 | Write retry not double-applied | StatusClassifier (DUP→success) + controller idempotency | — | +| 6.1, 6.2, 6.3 | Configurable params, defaults, jitter | FailoverState, XcecliRetryLoop | Failover loop | +| 7.1, 7.2 | Log failover target + exhaustion | XcecliRetryLoop | — | +| 8.1, 8.2 | Controller populates redirect addr | ControllerRedirectAddress | — | + +## Components and Interfaces + +| Component | Layer | Intent | Req Coverage | Key Dependencies | Contracts | +|-----------|-------|--------|--------------|------------------|-----------| +| XcecliRetryLoop | client (`client.c`) | `do/while` wrapper around each reply-bearing `xce_*` call | 1.1,1.2,3.x,5.x,7.x | xcecli_retry_next(P0) | Service | +| xcecli_retry_next | client (`client_failover.c`) | Decision helper: classify, rotate, rejoin, backoff, budget | 1.2,3.x,4.x,7.x | EndpointRotator(P0), StatusClassifier(P0), ReconnectRejoinHook(P0), FailoverState(P0) | Service | +| ReconnectRejoinHook | client (`client.c`) | Invoke consumer callback after reconnect, before resume | 1.2 | api_ctx_t(P0) | Service, State | +| OpCategory | client (`client.c`) | Per-op shape: reply-bearing / write / fire-and-forget | 1.1,5.1,5.2 | — | State | +| EndpointRotator | client (`client_failover.c`) | Reconnect to addr or next master over api_ctx | 2.1,2.2,2.3,4.2,4.4 | endpoint_connect(P0), FailoverState(P0) | Service | +| StatusClassifier | client (`client_failover.c`) | Map status → action | 2.5,4.1,4.2,4.3,4.5,5.2 | errnox codes(P0) | Service | +| FailoverState | client (`api_ctx_t`) | Master cursor, retry cfg, sticky primary | 2.4,6.1 | xce_client_config_t(P0) | State | +| RedirectCapture | ctx (`xcectx.c`) | Copy redirect-hint address into ctx (if present) | 2.2,8.3 | xce_reply_t(P0) | State | +| ReplyWaitWake | ctx (`xcectx.c`) | Release cbuf_get on endpoint error | 4.4 | endpoint err_handler(P0) | Service | +| ControllerRedirectAddress | controller | Fill redirect_host/port | 8.1,8.2 | ha_state primary_host/port(P0) | State | + +### client (libxce) + +#### XcecliRetryLoop — the wrapper pattern in each `xcecli_*` + +| Field | Detail | +|-------|--------| +| Intent | Turn each one-shot `xce_*` call into a transparent failover loop with no change to `xce_*` | +| Requirements | 1.1, 1.2, 3.1, 3.2, 3.3, 3.4, 5.1, 5.3, 7.1, 7.2 | + +**Responsibilities & Constraints** +- Each `xcecli_*` op runs its existing `xce_*` call inside a `do/while` driven by `xcecli_retry_next`. +- `xce_*` ops are unchanged: each is a single attempt that returns the controller status (fail fast). +- Never drops a request silently (5.3): the loop returns the op's success value or a distinct error. + +**Contracts**: Service [x] +```c +/* Pattern applied to every xcecli_* op (joblist shown; is_write=false for reads). */ +int xcecli_joblist(xcecli_ctx_t c, xce_job_t ***out) { + api_ctx_t *ac = (api_ctx_t *)c; + xcecli_retry_t st = XCECLI_RETRY_INIT(ac); /* attempt=0, delay/seed from ac->failover.retry */ + int rc; + do { + rc = xce_joblist(ac->ctx, out); /* unchanged one-shot xceapi op */ + } while (xcecli_retry_next(ac, &rc, /*is_write=*/false, &st)); + return rc; /* >=0 ok; -E_XCE_NOT_READY = cluster not ready */ +} +``` +- Postconditions: `>=0` success (sticky primary updated, 2.4); `-E_XCE_NOT_READY` on no-primary + budget exhaustion (3.3); terminal errors surfaced as-is (4.5). For writes, pass `is_write=true`. + +#### xcecli_retry_next — decision helper + +| Field | Detail | +|-------|--------| +| Intent | Decide whether to retry after an `xce_*` returned `rc`; perform reconnect/rotate/backoff | +| Requirements | 1.2, 3.1, 3.3, 3.4, 4.1, 4.2, 4.3, 4.4, 5.2, 7.1, 7.2 | + +**Responsibilities & Constraints** +- Owns the retry budget, backoff progression, sticky-primary update, and cluster-not-ready mapping. +- After a transport error the existing path left `ctx->state = XCE_CTX_ERR`; on a successful reconnect + this helper restores `XCE_CTX_RUN` so the next `xce_*` passes its guard (CI2). +- Drains any stale entry from `ctx->wbuf` after reconnect, before the next attempt (CI1); reconnect + closes the old fd first, so the dead connection yields no further replies. + +**Contracts**: Service [x] +```c +typedef struct { int attempt; int delay_ms; unsigned int rand_seed; } xcecli_retry_t; + +/* Returns true → caller re-invokes its xce_* op; false → stop, *rc is final. + * On DUP write replay it sets *rc = 0 (success) and returns false (5.2). */ +bool xcecli_retry_next(api_ctx_t *ac, int *rc, bool is_write, xcecli_retry_t *st); +``` +- Decisions: SUCCESS/TERMINAL → return false; RETRY_NOW → `EndpointRotator` (hint-first) then true, + no backoff (4.2); RETRY_BACKOFF → jittered sleep then `EndpointRotator` then true (4.1); attempts + beyond `retry.max_retries` while no-primary → set `*rc = -E_XCE_NOT_READY`, return false (3.3, 3.4). +- Validation: INFO log target on rotation (7.1); WARN log attempt count on exhaustion (7.2). + +#### EndpointRotator + +| Field | Detail | +|-------|--------| +| Intent | Re-point `ac->ep` to a redirect address or the next master in `ac->conf` | +| Requirements | 2.1, 2.2, 2.3, 4.2, 4.4, 8.4 | + +**Responsibilities & Constraints** +- On retry, if a redirect-address **hint** was captured, try it first (then clear it); otherwise + advance the master cursor and reconnect to the next configured master (8.3, 8.4). A failed hint + just falls through to rotation on the next retry — correctness never depends on it. +- Reuse the same `endpoint_t` (reply routing to `ctx->wbuf` preserved); close stale fd/ssl first. + +**Contracts**: Service [x] +```c +/* Reconnect ac->ep to the redirect-hint address if one is set (consumes it), else advance to the + * next configured master in ac->conf and reconnect. Returns 0 or -errno. */ +int xce_failover_rotate(api_ctx_t *ac); +``` + +#### StatusClassifier + +| Field | Detail | +|-------|--------| +| Intent | Map a reply status / transport rc to a failover action | +| Requirements | 2.5, 4.1, 4.2, 4.3, 4.5, 5.2 | + +**Contracts**: Service [x] +```c +typedef enum { + XCE_FO_SUCCESS, /* status == 0, or idempotent-replay success (DUP on a re-sent write) */ + XCE_FO_RETRY_NOW, /* E_XCE_REDIRECT / E_XCE_STANDBY / transport error — rotate, no backoff */ + XCE_FO_RETRY_BACKOFF, /* E_XCE_NOMASTER / E_XCE_NOT_READY — rotate after jittered backoff */ + XCE_FO_TERMINAL /* AUTH/PERM/INVALID/NOENT and other non-retriable — surface */ +} xce_fo_action_t; + +/* attempt == 0 for the first send; >0 for a re-send after failover. + * is_write distinguishes mutating ops (job submit) from reads. + * The redirect ADDRESS is NOT a classifier concern — it is a hint consumed by EndpointRotator. */ +xce_fo_action_t xce_failover_classify(int status, bool is_write, int attempt); +``` +- Invariants: `E_XCE_REDIRECT`/`STANDBY`/`NOMASTER`/`NOT_READY` are never reported as success (2.5). +- **Idempotent-replay rule (5.2):** `-E_XCE_DUP_JOB` / `-EEXIST` on a **re-sent write** (`is_write && + attempt > 0`) maps to `XCE_FO_SUCCESS` — the controller's client-supplied-`jobid` replay guard + (`job.c:56,95`) means "already applied" = our write succeeded. On `attempt == 0` the same status + is a genuine duplicate and maps to `XCE_FO_TERMINAL` (surfaced unchanged). + +#### OpCategory — per-op shape (CI1) + +| Field | Detail | +|-------|--------| +| Intent | Classify each `xcecli_*` op so its wrapper applies the right failover behavior | +| Requirements | 1.1, 5.1, 5.2 | + +Three categories drive the wrapper generated for each of the ~33 ops: + +| Category | Ops (examples) | `xce_*` shape | Failover behavior | +|----------|----------------|---------------|-------------------| +| Reply-read | `joblist`, `reslist`, `workerlist`, `clusterconf`, `resval`, `hello`, `nop` | `*_req` + `cbuf_get` → status | Full failover; `is_write=false` | +| Reply-write | `jobsubmit`, `jobctrl`, `queuectrl`, `hostctrl`, `update_resvals`, `runtask`, `join` | `*_req` + `cbuf_get` → status | Full failover; `is_write=true` (DUP-replay→success) | +| Fire-and-forget | `notify_*` (jobstart/finish/fail/stop/chkpt), `heartbeat`, `wakeup` | `*_req` only, **no status** | Best-effort: reconnect on a known-dead connection only; **no** standby/redirect detection, **no** no-primary backoff. Redelivery deferred to `worker-rejoin-resync` | + +- The exact per-op assignment is enumerated during task generation; `join` is reply-write and is the + op that triggers `E_XCE_REDIRECT` at reconnect time (it carries the daemon's role/attrs). + +#### ReconnectRejoinHook + +| Field | Detail | +|-------|--------| +| Intent | Let a stateful consumer re-establish its session on the new primary after a reconnect, before the interrupted op resumes | +| Requirements | 1.2 | + +**Responsibilities & Constraints** +- libxce owns the hook *mechanism* and *ordering* (reconnect → hook → resume); the callback *content* + (join handshake, worker job re-sync) belongs to `component-failover-adoption` / `worker-rejoin-resync`. +- Invoked once per successful reconnect to a *new* endpoint, before the next `xce_*` attempt. If the + hook fails (e.g. the new primary rejects the join), the failover loop treats it like a failed + attempt (rotate/backoff per the returned status). Pure clients register no hook (no-op path). + +**Contracts**: Service [x] / State [x] +```c +/* Registered via the client API; cb runs on the calling thread after reconnect. + * Return 0 to proceed with the interrupted op; <0 to treat the rejoin as failed. */ +typedef int (*xcecli_reconnect_cb)(xcecli_ctx_t ctx, void *arg); +int xcecli_set_reconnect_handler(xcecli_ctx_t ctx, xcecli_reconnect_cb cb, void *arg); +``` + +### client / ctx (libxce) + +#### FailoverState (on `api_ctx_t`) + +**Contracts**: State [x] +```c +/* On api_ctx_t — persists across calls (sticky primary + tunables). */ +typedef struct { + mylock_t lock; /* serializes xcecli_* per ctx (CI2 — shared-ctx safety) */ + int cur_master; /* index into conf->masters[] (sticky primary) */ + xce_retry_config_t retry; /* defaults from XCE_RETRY_DEFAULTS unless overridden */ + xcecli_reconnect_cb rejoin_cb; /* consumer rejoin hook; NULL for pure clients */ + void *rejoin_arg; +} xce_failover_state_t; + +/* On xce_ctx_t — written by handle_reply (which only sees xce_ctx_t), read by the rotator via ac->ctx. */ +/* char redirect_hint_host[XCE_MAXHOSTNAME_LEN]; hint from last redirect reply; "" if none */ +/* uint16_t redirect_hint_port; */ +``` +- State model: `cur_master` updated on a successful served-as-primary reply (2.4); retry config from + `XCE_RETRY_DEFAULTS` unless overridden (6.1, 6.2). The redirect hint is read by the rotator on the + next retry and cleared on consumption; it is purely advisory. +- **Concurrency (CI2):** the apiserver shares one ctx across goroutines (a heartbeat goroutine plus + request handlers, `clib.go:182`). Each `xcecli_*` op takes `failover.lock` for the duration of its + `do/while` loop, so reconnect/rotate (which closes `ac->ep` and drains `wbuf`) cannot race another + in-flight call on the same ctx. The single-slot `wbuf` already implies one in-flight request per + ctx; the lock makes that explicit and covers the reconnect window. + +#### RedirectCapture (in `handle_reply`) + +**Contracts**: State [x] +- On a redirect reply that carries an address, copy `reply.redirect_host` (capnp-owned, freed with + the reply — see request.c:243-250 hazard) + `redirect_port` into the fixed-size `xce_ctx_t` hint + fields before the reply is freed; leave the hint empty otherwise. Because the hint is advisory, the + copy needs no sequence correlation or lifetime guarantees beyond "copy now, use-or-discard next + retry." The status continues through the existing `cbuf_tryput(&ctx->wbuf,&status)` path + (xcectx.c:303), so `xceapi.c` is unchanged. + +#### ReplyWaitWake (endpoint error path) + +**Contracts**: Service [x] +- On endpoint error/RDHUP for `ctx->ep`, release a pending `cbuf_get` with a transport error + (`-ECONNRESET`/`-ETIMEDOUT`) so the `xce_*` call returns and the loop rotates instead of hanging + (4.4, F7). The existing error path already injects an error into `wbuf` and sets `ctx->state = + XCE_CTX_ERR` (xcectx.c:148-154) — this component reuses it. +- **State reset (CI2):** that error path leaves `ctx->state = XCE_CTX_ERR`, which the per-op guard + rejects as `-EINVAL` (xceapi.c:14,38,…) — the reported bug. `xcecli_retry_next` restores + `ctx->state = XCE_CTX_RUN` after a successful reconnect, before the next `xce_*` attempt, so the + guard (re-checked per attempt) passes. **CI1 drain:** the helper also drains any stale `wbuf` entry + after reconnect so a late reply from the dead connection is never consumed by the next attempt. + +### controller + +#### ControllerRedirectAddress (`join.c`) + +**Contracts**: State [x] +- On the `E_XCE_REDIRECT` path, set `reply.redirect_host = c->ha.primary_host` and + `reply.redirect_port = c->ha.primary_port` (8.1). When the primary is unknown, keep returning + `E_XCE_NOMASTER` rather than an addressless redirect (8.2 — already the case at join.c:270). +- Audit during implementation: confirm `join.c` is the sole `E_XCE_REDIRECT` emitter; apply the + same populate to any other site found. + +## Error Handling + +### Error Strategy +Classify every round-trip outcome (StatusClassifier) into success / rotate-now / backoff / terminal, +and bound the loop by the retry budget. Repair the context on transport failure rather than leaving +it in a non-RUN state (F7) so failover is repeatable. + +**Context state model (CI2):** the existing transport-error path sets `ctx->state = XCE_CTX_ERR` +(xcectx.c:148); the per-op `XCE_CTX_RUN` guard then rejects with `-EINVAL` (xceapi.c:14,…) — the +reported bug. `xcecli_retry_next` owns the transition back: after a successful reconnect it sets +`ctx->state = XCE_CTX_RUN` before the next `xce_*` attempt, so the per-op guard (re-checked each +attempt) passes. The guard otherwise stays exactly as today — `xceapi.c` is unchanged. + +**Stale-reply avoidance (CI1):** `ctx->wbuf` is a single-slot, status-only channel +(`cbuf_init(...,1,sizeof(int))`, xcectx.c:185). Reconnect closes the old fd **first**, so the dead +connection produces no further replies; `xcecli_retry_next` then drains any entry already sitting in +`wbuf` before the next attempt. This drain-after-close is sufficient — no per-reply sequence +correlation is needed (a deliberate simplification). + +### Error Categories and Responses +- **Transport (refused/timeout/reset)**: reconnect/rotate, retry; wake any pending wait (4.4). +- **No primary (NOMASTER/NOT_READY)**: jittered backoff; on budget exhaustion return the distinct + `-E_XCE_NOT_READY` cluster-not-ready code (3.3, 3.4) — never `-EINVAL`. +- **Redirect/standby**: rotate (address-first) without backoff (4.2, 4.3). +- **Terminal (AUTH/PERM/INVALID/NOENT)**: surface immediately, unchanged (4.5). + +### Monitoring +INFO log on each rotation with the target endpoint (7.1); WARN log on budget exhaustion with the +attempt count and last status (7.2). Reuse existing libxce logging. + +## Performance & Scalability + +Rejoin-on-failover cost is **accepted as-is** (decision 2026-06-16): failover is a rare, exceptional +event, so the cost is not on any hot path, and *some* handshake is unavoidable — a freshly elected +primary has no live connection to a reconnecting daemon until it re-joins. Characteristics: + +- Scheduler/apiserver join carry trivial payloads (`NULL` attrs); worker join carries a small + `hostattrs_t`. All reuse the daemons' existing startup join path — no new heavy machinery. +- On rejoin, apiserver/scheduler receive **full** state (e.g. the full job list), not a delta; for a + large cluster with many workers this is sizable but infrequent. This is simpler and correct, and + still cheaper than a full-history replace with no snapshot (cf. IBM LSF). +- **Herd control**: the jittered backoff (6.3) staggers simultaneous reconnects after a failover. + +**Deferred optimization (YAGNI):** a lightweight "re-attach" (session-token hello letting the primary +skip re-pushing already-replicated state, or delta-only state transfer) is intentionally **not** +designed now — delta reconciliation is non-trivial and full-state-on-rejoin is acceptable. Revisit +only if failover-rejoin latency is *measured* as a problem at scale. Recorded in `roadmap.md` backlog. + +## Testing Strategy + +### Unit Tests (`test_libxce_retry.c`, realigned) +- `xce_failover_classify`: REDIRECT+addr→ROTATE_ADDR; REDIRECT/STANDBY no-addr→ROTATE; + NOMASTER/NOT_READY→BACKOFF; AUTH/PERM/INVALID/NOENT→TERMINAL; 0→SUCCESS (2.5, 4.1-4.5). +- `xcecli_retry_next` loop: NOMASTER until budget → returns `-E_XCE_NOT_READY` (3.3); becomes primary + mid-loop → success, sticky cursor advanced (2.4, 3.2); non-retriable → immediate surface (4.5). +- Stale-reply avoidance (CI1): after a reconnect (old fd closed first), a stale entry left in `wbuf` + is drained before the next attempt, so no prior-attempt status is consumed; no sequence matching (1.2, 5.3). +- Context state (CI2): after a transport error drives `XCE_CTX_ERR`, a successful reconnect restores + `XCE_CTX_RUN` and the retry completes — never surfaces `-EINVAL` (1.2, 3.2). +- Idempotent write (CI3): `DUP_JOB` on a re-sent `jobsub` (attempt>0) → success; on attempt 0 → + surfaced as duplicate (5.2). +- EndpointRotator: redirect address used first (8.3); absent → next master (8.4); rotation reuses + same endpoint object. +- RetryConfig: NULL cfg → defaults (6.2); jitter staggers concurrent retries (6.3, reuse 50-client). + +### Integration / E2E (3-node, `/xce-deploy-ha` + `xce_test`) +- Submit jobs, kill primary, `joblist`: returns cluster-not-ready during the election window, then + transparently succeeds against the new primary with no error (1.2, 3.2, 3.3) — the bug reproduction. +- Redirect path: contact a standby directly → client lands on primary in one hop via the populated + redirect address (8.1, 8.3). +- Write failover: `jobsub` across a failover → exactly one job created (5.1, 5.2 via controller idempotency). + +### Controller Unit Test +- `handle_join` redirect path populates `redirect_host/port` from `ha.primary_host/port` (8.1); + unknown primary still yields `E_XCE_NOMASTER` (8.2). diff --git a/.kiro/specs/libxce-failover/requirements.md b/.kiro/specs/libxce-failover/requirements.md new file mode 100644 index 0000000..f59ba6f --- /dev/null +++ b/.kiro/specs/libxce-failover/requirements.md @@ -0,0 +1,179 @@ +# Requirements Document + +## Introduction + +This feature makes XCE controller primary failover transparent to client applications built on +libxce (the XCE client library), including the worker, scheduler, and API server daemons and +the `xce_test` REPL. + +Today, when the connected controller primary is killed in an HA cluster, libxce keeps using the +dead connection and returns an opaque invalid-argument error indefinitely: it never fails over +to the newly elected primary and never reports a clear "cluster not ready" during the election +window. The upstream requirement (`ha-config-bootstrap` Requirement 7) delivered only a +standalone retry primitive that no production request path invokes and that cannot discover a +new primary after the connected one dies. This spec completes that intent on the client side: +it routes all client requests through transparent retry/failover, rotates the configured master +list to locate the new primary, and surfaces a distinct cluster-not-ready error only when an +election outlasts the retry budget. + +## Boundary Context + +- **In scope**: client-side transparent failover for all libxce request types; discovery of the + new primary by rotating the configured master list, using the redirect address (when present) as + a best-effort hint to try first; bounded transparent retry that surfaces a distinct + cluster-not-ready error on budget exhaustion; failover for both read and write requests; + configurable retry/backoff parameters with safe defaults; operator-visible logging of failover + events; one small controller change so that redirect responses carry the primary's host and port + (the wire schema and reply struct already define these fields); end-to-end verification with the + `xce_test` client (the controller test harness, and the original reproduction case for this bug). +- **Out of scope**: controller-side election, log replication, commit barrier, and write + idempotency (consumed, not changed); introducing new probe or transport message types or new + reply fields (the redirect address fields already exist); per-request retry opt-in flags (writes + fail over by default); **adoption by the worker, scheduler, and apiserver daemons** — their + master-list configuration and liveness-safe handling of bounded blocking are owned by the + follow-on spec `component-failover-adoption` (depends on this spec). +- **Adjacent expectations**: libxce classifies controller responses by the distinct statuses the + controller actually emits — a *redirect* status (a primary exists but this node is a standby), a + *no-primary* status (this standby knows no primary yet), a *not-ready* status (the controller is + still initializing or electing), and a *standby* status (the operation is primary-only). With + the in-scope controller change, a redirect response carries the primary's host and port; libxce + treats this as a **best-effort hint** — it tries the hinted address first, and rotates its + configured master list when the hint is absent, stale, or unreachable. Correctness never depends + on the hint. libxce relies on controllers to deduplicate re-sent + writes through their commit-barrier idempotency guards, and on the configured master list (e.g. + `set masters host:port,host:port,host:port`) being the full set of controller candidates. +- **Stateful consumers**: a pure client (`xce_test`) needs only reconnect + resend. Stateful daemons + (apiserver, worker, scheduler) hold a session established via join; after a failover they must + re-establish it on the new primary (and a worker re-syncs its running jobs). libxce provides a + reconnect/rejoin hook for this; the hook *content* is owned by `component-failover-adoption` and + `worker-rejoin-resync`. Fire-and-forget notifications have no reply, so libxce cannot detect a + standby for them or guarantee delivery across a failover — redelivery is `worker-rejoin-resync`. + +## Requirements + +### Requirement 1: Transparent Failover for All Client Requests + +**Objective:** As a developer of a libxce-based daemon, I want every controller request to fail +over automatically inside the library, so that I do not write ad-hoc retry or reconnect loops in +caller code. + +#### Acceptance Criteria +1. The libxce client library shall route every controller request, both read/query and + mutating/write requests, through the transparent retry-and-failover path. +2. When controller failover occurs and a new primary becomes available, the libxce client + library shall complete the next request against the new primary without requiring any change + to caller code. +3. While failover handling is active, the libxce client library shall preserve the existing + public API call signatures and success/error return conventions so that existing callers + continue to compile and run unchanged. + +### Requirement 2: New-Primary Discovery Across the Configured Master List + +**Objective:** As a client application, I want libxce to find the new primary after the connected +one dies, so that my requests continue to reach an authoritative controller. + +#### Acceptance Criteria +1. If the connected controller becomes unreachable through connection refused, connection + timeout, or peer reset, the libxce client library shall attempt the other controllers in its + configured master list to locate the current primary. +2. When a contacted controller responds with a redirect status indicating that a primary exists + elsewhere, the libxce client library shall locate the primary by rotating its configured master + list, trying the redirect address (when the response carries one) first as a best-effort hint. +3. When a contacted controller responds that the requested operation is primary-only (a standby + status), the libxce client library shall treat it as a redirect and attempt the other + controllers in its configured master list. +4. When a controller accepts and serves the request as primary, the libxce client library shall + treat it as the current primary and direct subsequent requests to it. +5. The libxce client library shall not consider a request successful when it was answered by a + controller reporting a redirect, standby, no-primary, or not-ready status. + +### Requirement 3: Election-Window Behavior and Cluster-Not-Ready Reporting + +**Objective:** As a client application, I want short failovers hidden and longer outages reported +clearly, so that I can distinguish "retry later" from a real failure. + +#### Acceptance Criteria +1. While every reachable controller in the configured master list reports no primary exists (a + no-primary or not-ready status), the libxce client library shall retry across the configured + controllers with exponential backoff until the retry budget is exhausted. +2. When a primary becomes available before the retry budget is exhausted, the libxce client + library shall complete the request successfully and return no error to the caller. +3. If the retry budget is exhausted while no controller has become primary, the libxce client + library shall return a distinct cluster-not-ready error to the caller. +4. The cluster-not-ready error shall be distinguishable by the caller from transport connection + errors and from request-validation errors, so that a no-primary window is never reported as an + opaque invalid-argument error. + +### Requirement 4: Error Classification — Retriable vs. Terminal + +**Objective:** As a client application, I want transient controller errors retried and permanent +errors surfaced immediately, so that I neither hang on unrecoverable errors nor fail on recoverable ones. + +#### Acceptance Criteria +1. If a controller returns a no-primary or not-ready status, the libxce client library shall retry + the request with exponential backoff. +2. If a controller returns a redirect status (a primary exists elsewhere), the libxce client + library shall locate and switch to the primary per Requirement 2 and retry promptly, without + waiting for the full backoff delay used for the no-primary case. +3. If a controller returns a standby status (the operation is primary-only), the libxce client + library shall switch to another configured controller per Requirement 2 and retry. +4. If a transport-level connection refused, timeout, or peer reset occurs, the libxce client + library shall reconnect or rotate to another configured controller and retry. +5. If a controller returns a non-retriable error, specifically authentication failure, permission + denied, invalid request, or not found, the libxce client library shall surface that error to + the caller immediately without retry. + +### Requirement 5: Read and Write Failover Semantics + +**Objective:** As a client application, I want both queries and mutating operations to survive +failover without duplicating side effects, so that failover is safe as well as transparent. + +#### Acceptance Criteria +1. The libxce client library shall apply transparent failover to both read/query requests and + mutating/write requests, such as job submission. +2. When a write request is retried after a failover, the libxce client library shall not cause the + operation to be applied more than once, relying on the controller's idempotency handling for + re-sent writes. +3. While failing over, the libxce client library shall not silently drop a request; it shall + either complete the request against a primary or surface a distinct error to the caller. + +### Requirement 6: Configurable Retry Parameters and Thundering-Herd Avoidance + +**Objective:** As an operator, I want retry behavior tunable with safe defaults and protection +against synchronized retries, so that failover does not overwhelm a freshly elected primary. + +#### Acceptance Criteria +1. The libxce client library shall provide configurable retry parameters: maximum retries, + initial delay, maximum delay, and backoff multiplier. +2. Where no retry configuration is supplied, the libxce client library shall apply safe + production defaults. +3. When many clients fail over simultaneously, the libxce client library shall stagger their + retry attempts through randomized backoff so that they do not retry in lockstep against the new + primary. + +### Requirement 7: Failover Observability + +**Objective:** As an operator, I want failover activity visible in logs, so that I can confirm +and diagnose client failover during a controller outage. + +#### Acceptance Criteria +1. When the libxce client library fails over to a different controller, it shall log the failover + event including the target controller endpoint. +2. When the libxce client library returns a cluster-not-ready error after exhausting its retry + budget, it shall log the exhaustion including the number of attempts made. + +### Requirement 8: Redirect Responses Carry the Primary Address (used as a hint) + +**Objective:** As a client application, I want a standby's redirect to *hint* where the primary is, +so libxce can usually reach it in one hop while still working correctly without the hint. + +#### Acceptance Criteria +1. When a standby controller redirects a client because it knows a primary exists elsewhere, the + controller shall include the primary's host and port in the redirect response. +2. While a standby controller does not know the primary's address, the controller shall return a + no-primary status rather than a redirect that omits the address. +3. When the libxce client library receives a redirect response that includes a primary address, it + shall try that address first as a best-effort hint. +4. If the redirect carries no address, or the hinted address is unreachable or not the primary, the + libxce client library shall fall back to rotating its configured master list; correctness shall + not depend on the hint. diff --git a/.kiro/specs/libxce-failover/research.md b/.kiro/specs/libxce-failover/research.md new file mode 100644 index 0000000..bd5747d --- /dev/null +++ b/.kiro/specs/libxce-failover/research.md @@ -0,0 +1,127 @@ +# Research Log — libxce-failover + +## Discovery Scope + +Light, integration-focused discovery (extension of existing libxce). Goal: find the exact +seams to wire transparent failover into the client request/reply path. Verified by reading the +endpoint, client-config, request-core, reply, and controller-redirect code paths. + +## Key Findings + +### F1 — The async request/reply model determines the chokepoint (DECISIVE) +`xce_request(ep, req)` (request.c:317) only **encodes + enqueues** a packet — fire-and-forget. +The controller's reply (and its `status`) is delivered asynchronously: the endpoint's +`msg_handler` runs `handle_reply()` (xcectx.c:30) which signals the caller's wait channel, and +public ops block on `cbuf_get(&ctx->wbuf, &status)` (xceapi.c:116). Therefore the controller +status (`E_XCE_REDIRECT`/`NOMASTER`/`NOT_READY`/`STANDBY`) is observable only at the +**send-and-wait round trip**, not at `xce_request`. Failover orchestration must wrap that round +trip, where the `api_ctx_t` is in scope. + +### F2 — `xce_request_with_retry()` production branch is a no-op stub +request.c:1972-1981: with no test hook installed, it does `xce_request(ep,req); return 0;`. The +entire retry/classify/backoff loop executes only under `xce_retry_send_hook` (tests). So the +"done" primitive cannot fail over against a real controller. Its hook contract +(`int hook(ep, req, reply_out)`) fakes a synchronous reply that production never produces. + +### F3 — Redirect address is plumbed in the wire/struct but never populated or surfaced +- Wire schema has the fields: `controller.capnp:568-569` (`redirectHost`/`redirectPort`). +- Reply struct has them: `typemap.h:225-226` (`char *redirect_host; uint16_t redirect_port;`). +- Controller never sets them: every redirect site (`join.c:264`) sends `E_XCE_REDIRECT` with + `data_which = nodata`; grep finds **zero** production assignments to `reply.redirect_*`. +- Client never surfaces them: `handle_reply()` (xcectx.c:34) returns only `reply.status`; the + redirect fields are dropped before any caller sees them. + +### F4 — Context already holds everything failover needs +`api_ctx_t` (client.c:23): `{ xce_client_config_t *conf; endpoint_t ep; xce_ctx_t ctx; }`. +`xce_client_config_t` (client.h:16): `masters[N_MAX_MASTERS]` + `master_ports[]` + `nmasters`. +So the master list, the endpoint to reconnect, and the reply channel are all reachable from one +context object. Missing: a current-master cursor, the retry config, and captured redirect target. + +### F5 — Endpoint reconnect is straightforward and reusable +`endpoint_connect(ep, host, port)` (netio.c:659) does socket connect + SSL handshake on the +**same** `endpoint_t`. Reconnecting to a different master reuses the same endpoint object, so the +`msg_handler`/`owner` wiring that routes replies to `ctx->wbuf` stays intact. + +### F6 — `xcecli_start()` only ever connects `masters[0]` (client.c:122) +No primary discovery at startup; single endpoint, never re-selected. Confirms the gap. + +### F7 — Original EINVAL source +`xce_get_with_condition()` returns `-EINVAL` when `ctx->state != XCE_CTX_RUN` (xceapi.c:104). +After the primary dies the connection breaks and the ctx is left non-RUN, so every subsequent +call short-circuits to `-EINVAL` — the opaque error in the bug report. Failover must repair the +endpoint/ctx state instead of leaving it dead. + +## Decisions (synthesis) + +- **D1 (final — client-layer, supersedes the earlier xceapi round-trip idea):** failover lives in + the **`client.c` `xcecli_*` layer**, not at `xce_request` and not at `xceapi.c`. Evidence: every + consumer (worker `controller_connector.c`, scheduler `scheduler.c`, apiserver `clib.go`, xce_test) + calls `xcecli_*`, never `xce_*` directly; and only `api_ctx_t` (client.c) holds the master list — + `xce_ctx_t` (xceapi) does not. So `client.c` is both the universal surface and the only layer with + conf+ep+ctx co-located. **`xce_*` stay one-shot/fail-fast and unchanged**; each `xcecli_*` wraps + its `xce_*` call in a `do/while` driven by a shared `xcecli_retry_next` helper + (`src/lib/client_failover.c` + `client_internal.h`). Trade-off accepted (user): ~33 wrappers gain + a small loop (more LOC) but the policy is one helper and no async/thunk/plumbing changes — simpler + and lower-risk. The orphaned `xce_request_with_retry()` is removed, not wired. +- **D2:** retire the no-op production branch of `xce_request_with_retry()`. Keep `xce_retry_config_t` + + `XCE_RETRY_DEFAULTS`. Realign the unit tests (and hook contract) to the real status taxonomy + and the round-trip shape (allowed by Req 8 / requirements decision; tracked in bug-608). +- **D3:** `handle_reply()` captures `reply.redirect_host/port` + status into the ctx (deep-copy the + host string before the decoded reply is freed) so the failover loop can prefer the address. +- **D4:** on endpoint error/RDHUP, the err path must release the pending `cbuf_get` with a + transport error so the loop reconnects instead of hanging. +- **D5:** controller populates `reply.redirect_host/port` from `ha.primary_host/port` at every + `E_XCE_REDIRECT` site (join.c confirmed sole site; audit during implementation). +- **D6:** budget-exhaustion mapping — no-primary cause → return distinct `-E_XCE_NOT_READY` + ("cluster not ready"); transport-only cause → return last transport error. Never `-EINVAL`. + +### Decisions added after design review (validate-design pass) +- **D7 (CI1 — reply correlation):** `ctx->wbuf` is single-slot, status-only + (`cbuf_init(...,1,sizeof(int))`, xcectx.c:185). `handle_reply` records `reply.seq` into the ctx; + `xcecli_retry_next` drains any stale `wbuf` entry after reconnect (and reconnect closes the old fd + first → no further old replies), so the next `xce_*` attempt cannot consume a prior-attempt reply. +- **D8 (CI2 — context state reset):** the existing transport-error path sets `ctx->state = + XCE_CTX_ERR` (xcectx.c:148), which the per-op guard rejects as `-EINVAL` (xceapi.c:14,…) — the + reported bug. `xcecli_retry_next` restores `XCE_CTX_RUN` after a successful reconnect, before the + next `xce_*` attempt, so the unchanged per-op guard passes. `xceapi.c` is not modified. +### Simplification pass (goal: keep libxce easy to use) +- **D13 (redirect = hint, not contract):** the controller still populates the redirect address + (forward-compatible, kept), but the client treats it as a **best-effort hint** — try it first if + present, else rotate the master list; correctness never depends on it. This removes the heavy + machinery the address-as-contract demanded: **drop the `reply_seq` correlation** (CI1 becomes + drain-after-close only — reconnect closes the old fd first, then drain `wbuf`), and **collapse the + classifier from 5 actions to 4** (`SUCCESS` / `RETRY_NOW` / `RETRY_BACKOFF` / `TERMINAL`) — the + hint is consumed by `EndpointRotator`, not a separate classifier branch. Net: fewer fields, no + lifetime-hazard dependency, simpler classification, same user-visible behavior. The rejoin hook + and per-ctx lock are kept (real needs). Rationale: with 3 masters and rare failover, rotation is + cheap; the hint is a latency nicety, not a correctness mechanism. + +### Decisions added after 2nd design review (validate-design pass 2) +- **D10 (CI1 — op categories):** the ~33 `xcecli_*` ops split into reply-read (full failover, + `is_write=false`), reply-write (full failover, `is_write=true`, DUP-replay→success), and + fire-and-forget (`notify_*`/`heartbeat`/`wakeup` — `xce_*_req` only, no `cbuf_get`/status). The + failover classifier applies only to reply-bearing ops; fire-and-forget gets transport-reconnect + best-effort, and redelivery of lost notifies is deferred to `worker-rejoin-resync`. +- **D11 (CI2 — concurrency):** apiserver shares one ctx across goroutines (`clib.go:182`); add a + `failover.lock` taken per `xcecli_*` op so reconnect/rotate cannot race another in-flight call. +- **D12 (rejoin hook — user requirement):** stateful daemons (apiserver/worker/scheduler) need a + rejoin/state-exchange on reconnect; a worker also re-syncs its running-jobs state. libxce adds + `xcecli_set_reconnect_handler()`; the failover loop invokes it after reconnect, before resume. The + hook *content* is owned by `component-failover-adoption` (join) and `worker-rejoin-resync` (job + state). Pure clients (`xce_test`) register none. + +- **D9 (CI3 — idempotent-write replay):** job submit carries a **client-generated** `jobid` + (`job.c:56`) with a server `-EEXIST`/`E_XCE_DUP_JOB` replay guard (`job.c:95`). The classifier maps + `DUP_JOB` on a re-sent write (`is_write && attempt>0`) to success; on attempt 0 it is a genuine + duplicate (terminal). This is what makes 5.2 actually satisfiable rather than assumed. + +## Risks + +- **R-async-blocking:** the round trip blocks longer under retry. Acceptable for CLI/apiserver + (already blocking); for async C daemons it's the `component-failover-adoption` spec's concern + (out of boundary here). +- **R-reply-routing on reconnect:** re-sending after reconnect must correlate the reply to the + same ctx wait channel; mitigated by reusing the same `endpoint_t` (F5) and matching on `seq`. +- **R-redirect-lifetime:** `redirect_host` is a decoded capnp string freed with the reply; must be + copied into a ctx-owned buffer (D3) to avoid use-after-free (cf. the `init_reply_by_req` + memset hazard already documented at request.c:243-250). diff --git a/.kiro/specs/libxce-failover/spec.json b/.kiro/specs/libxce-failover/spec.json new file mode 100644 index 0000000..12421f5 --- /dev/null +++ b/.kiro/specs/libxce-failover/spec.json @@ -0,0 +1,22 @@ +{ + "feature_name": "libxce-failover", + "created_at": "2026-06-16T10:30:55Z", + "updated_at": "2026-06-16T10:52:00Z", + "language": "en", + "phase": "tasks-generated", + "approvals": { + "requirements": { + "generated": true, + "approved": true + }, + "design": { + "generated": true, + "approved": true + }, + "tasks": { + "generated": true, + "approved": true + } + }, + "ready_for_implementation": false +} diff --git a/.kiro/specs/libxce-failover/tasks.md b/.kiro/specs/libxce-failover/tasks.md new file mode 100644 index 0000000..0373eee --- /dev/null +++ b/.kiro/specs/libxce-failover/tasks.md @@ -0,0 +1,103 @@ +# Implementation Plan + +- [ ] 1. Foundation: failover state, captured-reply fields, and cleanup +- [x] 1.1 Establish shared failover state and reply-capture fields + - Expose the client context to a new client-internal header so the failover helper can operate on it + - Add per-context failover state: a lock, the current-master cursor, retry parameters (with safe defaults), and a reconnect/rejoin callback slot + - Add fixed-size redirect-hint host/port fields to the client reply context (advisory hint; no sequence correlation) + - Initialize all new state when the client starts; keep the retry-config defaults; remove the orphaned, never-wired retry primitive + - Done: libxce builds with the new per-context state present and the dead primitive removed; retry defaults are reachable + - _Requirements: 6.1, 6.2_ +- [ ] 1.2 Realign the retry test harness to the new model (RED) + - Retarget the existing libxce retry tests from the removed primitive to the new classifier and client-wrapper retry path + - Add fixtures simulating each controller status (redirect with and without address, no-primary, not-ready, standby, duplicate-job) and transport failures, plus a fake multi-master list and a shared-context concurrency harness + - Done: the test file compiles against the new symbols and fails (RED) because the implementation does not yet exist + - _Requirements: 2.5, 4.1, 4.2, 4.3, 4.4, 4.5, 5.2, 8.3, 8.4_ + +- [ ] 2. Core: status classification, endpoint rotation, reply-path capture +- [ ] 2.1 Implement status classification + - Map each controller status and transport result to one of four actions: success, retry-now (redirect/standby/transport — rotate without backoff), retry-after-backoff (no-primary/not-ready), terminal + - Never classify redirect, standby, no-primary, or not-ready as success + - Treat a duplicate-job/exists status on a re-sent write as success (idempotent replay); on a first attempt treat it as a genuine duplicate surfaced to the caller + - Done: classification unit tests pass for every status and for both duplicate-replay cases + - _Requirements: 2.5, 4.1, 4.2, 4.3, 4.5, 5.2_ + - _Boundary: StatusClassifier_ +- [ ] 2.2 Implement endpoint reconnect and master rotation + - On rotate, try the captured redirect-hint address first when one is set (then clear it); otherwise advance the master cursor to the next configured master and reconnect + - Close the stale connection first and reuse the same endpoint object so reply routing stays intact; a failed hint simply falls through to the next master + - Done: rotation unit tests show hint-first then next-master fallback, reusing a single endpoint object, with correctness independent of the hint + - _Requirements: 2.1, 2.2, 2.3, 4.4, 8.4_ + - _Boundary: EndpointRotator_ + - _Depends: 1.1_ +- [ ] 2.3 (P) Capture redirect on the reply path and wake the wait on endpoint error + - In the reply handler, copy the redirect-hint host/port into the client reply context when a redirect reply carries one, before the reply is freed; leave empty otherwise (advisory hint, no sequence correlation) + - On endpoint error or peer reset, release any pending reply wait with a transport error so the in-flight call returns instead of hanging + - Keep the status flowing through the existing wait-channel path so the public API layer stays unchanged + - Done: after a redirect reply the captured hint address is readable from the context; a simulated endpoint error releases a pending wait with a transport error (no hang); the API layer is untouched + - _Requirements: 2.2, 4.4, 8.3_ + - _Boundary: RedirectCapture, ReplyWaitWake_ + - _Depends: 1.1_ + +- [ ] 3. Core: the retry/failover decision helper +- [ ] 3.1 Implement the decision loop (classify, rotate, backoff, budget) + - Combine classification, rotation, and the captured redirect into one decision: success and terminal stop the loop; redirect, standby, and transport results rotate without backoff; no-primary and not-ready apply jittered backoff + - Track the retry budget; on exhaustion with no primary, return the distinct cluster-not-ready code + - Update the sticky current-master on a successful served-as-primary reply; log the target on each rotation and the attempt count on budget exhaustion + - Done: helper-level tests show no-primary-until-budget returns cluster-not-ready, become-primary returns success with the cursor advanced, and redirect/transport rotate without the no-primary backoff + - _Requirements: 1.2, 3.1, 3.2, 3.3, 3.4, 4.1, 4.2, 4.4, 6.3, 7.1, 7.2_ + - _Depends: 2.1, 2.2, 2.3_ + - _Boundary: xcecli_retry_next_ +- [ ] 3.2 Implement reconnect recovery: context-state reset and stale-reply drain + - After a successful reconnect, restore the context from its error state to its running state before the next attempt, so a transport error never leaves the context returning an invalid-argument error + - Drain any stale reply left in the wait channel after reconnect so the next attempt cannot consume a prior-attempt reply + - Done: a transport error followed by reconnect no longer surfaces an invalid-argument error and the context is reusable; a unit test injecting a stale reply confirms it is drained, not returned + - _Requirements: 1.2, 4.4_ + - _Depends: 3.1_ + - _Boundary: xcecli_retry_next_ + +- [ ] 4. Core: reconnect/rejoin hook +- [ ] 4.1 Add the reconnect/rejoin hook mechanism + - Provide a client API to register a reconnect callback and argument, stored in the failover state + - Invoke the callback after each successful reconnect to a new primary and before resuming the interrupted op; treat a failing callback as a failed attempt that rotates or backs off per its result + - Done: with a registered hook a simulated failover invokes it before resume; with no hook the loop resumes directly + - _Requirements: 1.2_ + - _Depends: 3.1, 3.2_ + +- [ ] 5. Integration: wrap the client ops; controller redirect address +- [ ] 5.1 Wrap reply-bearing client ops in the failover loop under the per-context lock + - Wrap each reply-bearing read and write op so it runs its single-shot call inside the decision loop, holding the per-context lock for the loop's duration + - Tag write ops so the idempotent-replay rule applies; preserve every public signature and success/error convention + - Done: existing callers compile unchanged; a reply-bearing op transparently completes against a new primary after a simulated failover; concurrent use of one context is serialized + - _Requirements: 1.1, 1.3, 2.4, 5.1, 5.3_ + - _Depends: 3.1, 3.2, 4.1_ + - _Boundary: XcecliRetryLoop, FailoverState_ +- [ ] 5.2 Apply best-effort reconnect to fire-and-forget ops + - Give the notify, heartbeat, and wakeup ops a transport-reconnect-only variant: reconnect on a known-dead connection, with no status classification and no no-primary backoff + - Done: a fire-and-forget op on a dead connection triggers a reconnect attempt and never blocks on the no-primary backoff budget; lost-notification redelivery remains out of scope + - _Requirements: 1.1, 5.3_ + - _Depends: 3.1_ + - _Boundary: XcecliRetryLoop_ +- [ ] 5.3 (P) Populate the primary address in controller redirect replies + - On the standby redirect path, fill the reply's redirect host and port from the known primary address; keep returning the no-primary status when the primary is unknown + - Audit for any other redirect-emitting sites and apply the same population + - Add a controller unit test: the redirect path populates the address, and an unknown primary still yields the no-primary status + - Done: the controller test passes and a standby redirect reply carries the primary address + - _Requirements: 8.1, 8.2_ + - _Boundary: ControllerRedirectAddress_ + +- [ ] 6. Validation: unit suite and live failover +- [ ] 6.1 Bring the realigned unit suite green + - Make the retry tests pass for classification, budget-to-cluster-not-ready, sticky cursor, hint-first then rotate, duplicate-replay-to-success, jitter and many-client staggering, stale-reply drain, context-state recovery, and concurrent shared-context use + - Done: the full libxce retry unit suite passes under the project's test and ASAN build + - _Requirements: 2.4, 2.5, 3.2, 3.3, 4.1, 4.2, 4.3, 4.4, 4.5, 5.2, 6.2, 6.3, 8.3, 8.4_ + - _Depends: 5.1, 5.2_ +- [ ] 6.2 Live 3-node failover end-to-end + - On a 3-node cluster, submit jobs, kill the primary, and verify joblist reports cluster-not-ready during the election window then transparently succeeds against the new primary + - Verify a request sent to a standby lands on the primary — in one hop when the redirect hint is present, otherwise via master-list rotation + - Verify a job submission spanning a failover creates exactly one job + - Done: the original reproduction passes — no invalid-argument error, transparent recovery, and a single job created + - _Requirements: 1.2, 3.2, 3.3, 5.1, 5.2, 8.1, 8.3_ + - _Depends: 5.1, 5.3_ + +## Implementation Notes +- 1.1: design said `XCE_MAXHOST_LEN`; real constant is `XCE_MAXHOSTNAME_LEN` (config.h, =256) — design corrected. `api_ctx_t` now in private `client_internal.h`; `xcecli_reconnect_cb` typedef lives there until task 4.1 adds the public setter. Build: `cd src && nix develop --command bash tools/xce-build.sh lib`. diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index 8f227d5..913575b 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,23 +1,25 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-06-16T07:17:57.828Z -> Files: 507 tracked | Anatomy hits: 0 | Misses: 0 +> Auto-maintained by OpenWolf. Last scanned: 2026-06-16T08:19:04.353Z +> Files: 516 tracked | Anatomy hits: 0 | Misses: 0 ## ./ - `.clang-format` (~226 tok) - `.gitignore` — Git ignore rules (~88 tok) +- `.mcp.json` (~7 tok) - `.opencode.json` (~55 tok) - `CLAUDE.md` — OpenWolf (~1400 tok) - `package-lock.json` — npm lock file (~1039 tok) - `package.json` — Node.js package manifest (~1 tok) - `README.md` — Project documentation (~612 tok) +- `uninstall-code-review-graph.sh` — Fully uninstall `code-review-graph` (the Python/uv MCP tool). (~627 tok) ## .claude/ - `settings.json` (~497 tok) - `settings.json.bak` (~329 tok) -- `settings.local.json` (~187 tok) +- `settings.local.json` (~186 tok) - `ucai-formatter-cache.local.json` (~17 tok) ## .claude/rules/ @@ -296,6 +298,11 @@ - `spec.json` (~142 tok) - `tasks.md` — Implementation Plan: ha-convergence-recovery (~5679 tok) +## .kiro/specs/ha-elect-ctx-lifecycle/ + +- `brief.md` — Discovery Brief: ha-elect-ctx-lifecycle (~774 tok) +- `spec.json` (~130 tok) + ## .kiro/specs/ha-heartbeat-liveness/ - `design.md` — Design Document: ha-heartbeat-liveness (~5409 tok) @@ -319,7 +326,7 @@ - `requirements.md` — Requirements Document (~3604 tok) - `research.md` — Gap Analysis: ha-prevote (~6077 tok) - `spec.json` (~126 tok) -- `tasks.md` — Implementation Plan (~3129 tok) +- `tasks.md` — Implementation Plan (~3197 tok) ## .kiro/specs/ha-repl-hardening/ @@ -710,6 +717,17 @@ - `3c576da7137df22cc5fbd982fd231d9d40d374a77c7d618081ba7018f5eec36c.json` (~4654 tok) - `3ce0572164f9daaf2eee6a108f57bb947fb3d812c5578c7451eed78062e38b68.json` (~8024 tok) +## src/controller/ + +- `cluster.c` — include "config.h" (~15511 tok) +- `ha.c` — define _GNU_SOURCE (~52565 tok) +- `ha.h` — ifndef HA_H (~14911 tok) +- `persist.c` — define _GNU_SOURCE (~18295 tok) + +## src/include/ + +- `persist.h` — pragma once (~2668 tok) + ## src/lib/testsuite/ - `test_raft.c` — include (~36211 tok) diff --git a/.wolf/cron-state.json b/.wolf/cron-state.json index 3419f5b..b2a76b6 100644 --- a/.wolf/cron-state.json +++ b/.wolf/cron-state.json @@ -1,5 +1,5 @@ { - "last_heartbeat": "2026-06-16T07:25:23.733Z", + "last_heartbeat": "2026-06-16T08:25:23.734Z", "engine_status": "running", "execution_log": [ { diff --git a/.wolf/hooks/_session.json b/.wolf/hooks/_session.json index 716c7e0..4b0f9ec 100644 --- a/.wolf/hooks/_session.json +++ b/.wolf/hooks/_session.json @@ -1,143 +1,131 @@ { - "session_id": "session-2026-06-16-2344", - "started": "2026-06-16T06:44:27.944Z", + "session_id": "session-2026-06-16-0057", + "started": "2026-06-16T07:57:38.410Z", "files_read": { - "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/requirements.md": { - "count": 6, - "tokens": 3604, - "first_read": "2026-06-16T06:45:04.651Z" + "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json": { + "count": 2, + "tokens": 7, + "first_read": "2026-06-16T07:58:57.528Z" }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c": { - "count": 9, - "tokens": 0, - "first_read": "2026-06-16T06:45:17.576Z" - }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_connection_reuse.c": { + "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/commit.c": { "count": 1, "tokens": 0, - "first_read": "2026-06-16T06:45:26.432Z" + "first_read": "2026-06-16T07:58:58.420Z" }, - "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md": { + "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/worker.c": { + "count": 1, + "tokens": 0, + "first_read": "2026-06-16T07:59:27.185Z" + }, + "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/join.c": { + "count": 1, + "tokens": 0, + "first_read": "2026-06-16T07:59:27.920Z" + }, + "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c": { "count": 4, - "tokens": 3129, - "first_read": "2026-06-16T06:48:13.339Z" + "tokens": 0, + "first_read": "2026-06-16T07:59:43.058Z" }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c": { - "count": 2, - "tokens": 7618, - "first_read": "2026-06-16T06:48:14.326Z" - }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_failover_prevote.c": { - "count": 2, - "tokens": 3896, - "first_read": "2026-06-16T06:48:24.387Z" - }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_vote_gate.c": { + "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/meson.build": { "count": 1, - "tokens": 9075, - "first_read": "2026-06-16T06:48:24.556Z" + "tokens": 0, + "first_read": "2026-06-16T08:00:30.531Z" }, "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/controller_test.c": { "count": 1, - "tokens": 1469, - "first_read": "2026-06-16T06:48:31.170Z" - }, - "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/design.md": { - "count": 4, - "tokens": 6905, - "first_read": "2026-06-16T06:48:40.155Z" - }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/raft/raft.c": { - "count": 3, "tokens": 0, - "first_read": "2026-06-16T06:52:15.949Z" + "first_read": "2026-06-16T08:00:40.931Z" }, - "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/research.md": { - "count": 1, - "tokens": 5295, - "first_read": "2026-06-16T07:10:16.063Z" - }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c": { + "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json": { "count": 2, - "tokens": 0, - "first_read": "2026-06-16T07:10:25.883Z" + "tokens": 497, + "first_read": "2026-06-16T08:06:45.661Z" }, - "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/raft/raft.h": { + "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/persist.h": { "count": 1, "tokens": 0, - "first_read": "2026-06-16T07:10:26.237Z" - }, - "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-persistent-ha/bffce5f2-ffc9-4dc1-b956-0d3d7223fc0d/tasks/bei9rto5x.output": { - "count": 1, - "tokens": 0, - "first_read": "2026-06-16T07:20:19.102Z" + "first_read": "2026-06-16T08:17:36.150Z" } }, "files_written": [ { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", - "action": "edit", - "tokens": 20, - "at": "2026-06-16T06:46:55.771Z" + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "action": "create", + "tokens": 7, + "at": "2026-06-16T07:59:01.938Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", "action": "edit", - "tokens": 3353, - "at": "2026-06-16T06:50:23.587Z" + "tokens": 11, + "at": "2026-06-16T08:07:10.456Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", "action": "edit", - "tokens": 14, - "at": "2026-06-16T06:54:20.104Z" + "tokens": 10, + "at": "2026-06-16T08:07:18.236Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", - "action": "edit", - "tokens": 3780, - "at": "2026-06-16T06:58:28.913Z" + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "action": "create", + "tokens": 7, + "at": "2026-06-16T08:09:05.843Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", - "action": "edit", - "tokens": 16, - "at": "2026-06-16T07:08:51.022Z" + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "action": "create", + "tokens": 497, + "at": "2026-06-16T08:09:28.949Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", - "action": "edit", - "tokens": 4104, - "at": "2026-06-16T07:12:02.527Z" + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/uninstall-code-review-graph.sh", + "action": "create", + "tokens": 672, + "at": "2026-06-16T08:09:44.867Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/persist.h", "action": "edit", - "tokens": 142, - "at": "2026-06-16T07:12:14.641Z" + "tokens": 74, + "at": "2026-06-16T08:18:22.177Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/research.md", + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", "action": "edit", - "tokens": 1012, - "at": "2026-06-16T07:13:49.151Z" + "tokens": 56, + "at": "2026-06-16T08:18:33.020Z" }, { - "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", "action": "edit", - "tokens": 17, - "at": "2026-06-16T07:17:57.840Z" + "tokens": 411, + "at": "2026-06-16T08:18:48.563Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "action": "edit", + "tokens": 166, + "at": "2026-06-16T08:18:57.385Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "action": "edit", + "tokens": 63, + "at": "2026-06-16T08:19:04.362Z" } ], "edit_counts": { - ".kiro/specs/ha-prevote/tasks.md": 4, - "src/controller/testsuite/test_ha_prevote.c": 2, - "src/lib/testsuite/test_raft.c": 2, - ".kiro/specs/ha-prevote/research.md": 1 + ".mcp.json": 2, + ".claude/settings.json": 3, + "uninstall-code-review-graph.sh": 1, + "src/include/persist.h": 1, + "src/controller/persist.c": 4 }, - "anatomy_hits": 10, - "anatomy_misses": 4, - "repeated_reads_warned": 24, + "anatomy_hits": 1, + "anatomy_misses": 8, + "repeated_reads_warned": 5, "cerebrum_warnings": 0, - "stop_count": 1 + "stop_count": 3 } \ No newline at end of file diff --git a/.wolf/memory.md b/.wolf/memory.md index 76c70b7..dcc0116 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -1798,3 +1798,4 @@ Implemented sub-spec 3/4 of ha-consistency-invariants autonomously (per-task imp | 01:19 | Edited src/controller/persist.c | 4→6 lines | ~59 | | 15:55 | Fixed controller test timeout: interruptible fsync_worker sleep (condvar) so persist_stop joins promptly | persist.c, persist.h | suite 300s+ TIMEOUT → 10s, 0 failures | ~25k | | 01:23 | Session end: 11 writes across 5 files (.mcp.json, settings.json, uninstall-code-review-graph.sh, persist.h, persist.c) | 9 reads | ~2478 tok | +| 01:24 | Session end: 11 writes across 5 files (.mcp.json, settings.json, uninstall-code-review-graph.sh, persist.h, persist.c) | 9 reads | ~2478 tok | diff --git a/.wolf/token-ledger.json b/.wolf/token-ledger.json index b1a00df..207f4f2 100644 --- a/.wolf/token-ledger.json +++ b/.wolf/token-ledger.json @@ -2,14 +2,14 @@ "version": 1, "created_at": "2026-05-21T04:35:43.577Z", "lifetime": { - "total_tokens_estimated": 42216062, - "total_reads": 10106, - "total_writes": 15352, - "total_sessions": 113, - "anatomy_hits": 6986, - "anatomy_misses": 2948, - "repeated_reads_blocked": 31561, - "estimated_savings_vs_bare_cli": 384256039 + "total_tokens_estimated": 42334085, + "total_reads": 10168, + "total_writes": 15414, + "total_sessions": 114, + "anatomy_hits": 7012, + "anatomy_misses": 2984, + "repeated_reads_blocked": 31627, + "estimated_savings_vs_bare_cli": 384382023 }, "sessions": [ { @@ -143860,6 +143860,773 @@ "repeated_reads_blocked": 24, "anatomy_lookups": 10 } + }, + { + "id": "session-2026-06-16-2344", + "started": "2026-06-16T06:44:27.944Z", + "ended": "2026-06-16T07:49:34.801Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/requirements.md", + "tokens_estimated": 3604, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_connection_reuse.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 3129, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "tokens_estimated": 7618, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_failover_prevote.c", + "tokens_estimated": 3896, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_vote_gate.c", + "tokens_estimated": 9075, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/controller_test.c", + "tokens_estimated": 1469, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/design.md", + "tokens_estimated": 6905, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/raft/raft.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/research.md", + "tokens_estimated": 5295, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/raft/raft.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-persistent-ha/bffce5f2-ffc9-4dc1-b956-0d3d7223fc0d/tasks/bei9rto5x.output", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/cluster.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 20, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "tokens_estimated": 3353, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 14, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "tokens_estimated": 3780, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 16, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "tokens_estimated": 4104, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "tokens_estimated": 142, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/research.md", + "tokens_estimated": 1012, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 17, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c", + "tokens_estimated": 32, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c", + "tokens_estimated": 13, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.h", + "tokens_estimated": 86, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/cluster.c", + "tokens_estimated": 449, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 40991, + "output_tokens_estimated": 13038, + "reads_count": 16, + "writes_count": 13, + "repeated_reads_blocked": 26, + "anatomy_lookups": 10 + } + }, + { + "id": "session-2026-06-16-2344", + "started": "2026-06-16T06:44:27.944Z", + "ended": "2026-06-16T07:56:37.342Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/requirements.md", + "tokens_estimated": 3604, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_connection_reuse.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 3129, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "tokens_estimated": 7618, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_failover_prevote.c", + "tokens_estimated": 3896, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_vote_gate.c", + "tokens_estimated": 9075, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/controller_test.c", + "tokens_estimated": 1469, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/design.md", + "tokens_estimated": 6905, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/raft/raft.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/research.md", + "tokens_estimated": 5295, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/raft/raft.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/tmp/claude-1000/-home-rshen-projects-xce-worktrees-persistent-ha/bffce5f2-ffc9-4dc1-b956-0d3d7223fc0d/tasks/bei9rto5x.output", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/cluster.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.local.json", + "tokens_estimated": 187, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/CLAUDE.md", + "tokens_estimated": 1400, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/explore-codebase/skill.md", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 20, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "tokens_estimated": 3353, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 14, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test_ha_prevote.c", + "tokens_estimated": 3780, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 16, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "tokens_estimated": 4104, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/lib/testsuite/test_raft.c", + "tokens_estimated": 142, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/research.md", + "tokens_estimated": 1012, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 17, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c", + "tokens_estimated": 32, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.c", + "tokens_estimated": 13, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/ha.h", + "tokens_estimated": 86, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/cluster.c", + "tokens_estimated": 449, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 11, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-elect-ctx-lifecycle/spec.json", + "tokens_estimated": 130, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 10, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.local.json", + "tokens_estimated": 20, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-elect-ctx-lifecycle/brief.md", + "tokens_estimated": 884, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 64, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-prevote/tasks.md", + "tokens_estimated": 73, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/CLAUDE.md", + "tokens_estimated": 25, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 43075, + "output_tokens_estimated": 14255, + "reads_count": 20, + "writes_count": 21, + "repeated_reads_blocked": 26, + "anatomy_lookups": 13 + } + }, + { + "id": "session-2026-06-16-0057", + "started": "2026-06-16T07:57:38.410Z", + "ended": "2026-06-16T08:10:11.254Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/commit.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/worker.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/join.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/meson.build", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/controller_test.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "was_repeated": true, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 11, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 10, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/uninstall-code-review-graph.sh", + "tokens_estimated": 672, + "action": "create" + } + ], + "totals": { + "input_tokens_estimated": 504, + "output_tokens_estimated": 1204, + "reads_count": 8, + "writes_count": 6, + "repeated_reads_blocked": 4, + "anatomy_lookups": 1 + } + }, + { + "id": "session-2026-06-16-0057", + "started": "2026-06-16T07:57:38.410Z", + "ended": "2026-06-16T08:23:24.362Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/commit.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/worker.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/join.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/meson.build", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/controller_test.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/persist.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 11, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 10, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/uninstall-code-review-graph.sh", + "tokens_estimated": 672, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/persist.h", + "tokens_estimated": 74, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 56, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 411, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 166, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 63, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 504, + "output_tokens_estimated": 1974, + "reads_count": 9, + "writes_count": 11, + "repeated_reads_blocked": 5, + "anatomy_lookups": 1 + } + }, + { + "id": "session-2026-06-16-0057", + "started": "2026-06-16T07:57:38.410Z", + "ended": "2026-06-16T08:24:49.631Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/commit.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/worker.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/join.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/meson.build", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/controller_test.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/persist.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 11, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 10, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.mcp.json", + "tokens_estimated": 7, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/settings.json", + "tokens_estimated": 497, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/uninstall-code-review-graph.sh", + "tokens_estimated": 672, + "action": "create" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/persist.h", + "tokens_estimated": 74, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 56, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 411, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 166, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/persist.c", + "tokens_estimated": 63, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 504, + "output_tokens_estimated": 1974, + "reads_count": 9, + "writes_count": 11, + "repeated_reads_blocked": 5, + "anatomy_lookups": 1 + } } ], "daemon_usage": [], diff --git a/src/include/request.h b/src/include/request.h index 91c71ea..7427cc7 100644 --- a/src/include/request.h +++ b/src/include/request.h @@ -176,10 +176,12 @@ extern int xce_upgrade_config_req_extract_raw(const uint8_t *pkt_data, size_t *out_len, uint8_t **out_sha256); -/* ── Task 10.2: Client-Side Transparent Retry (Req 7.1–7.6) ───────────────── +/* ── Client-Side Transparent Retry tunables (Req 6.1, 6.2) ────────────────── * - * xce_retry_config_t: configurable retry parameters with safe production defaults. - * Pass NULL to xce_request_with_retry() to use defaults. + * xce_retry_config_t: configurable retry parameters with safe production + * defaults, consumed by the libxce client failover loop (client.c / + * client_failover.c). XCE_RETRY_DEFAULTS supplies the defaults applied when no + * override is configured. */ typedef struct { int max_retries; /* default 20 */ @@ -190,29 +192,4 @@ typedef struct { extern const xce_retry_config_t XCE_RETRY_DEFAULTS; -/* - * xce_request_with_retry: transparent retry wrapper for transient controller errors. - * - * - XCE_ERR_NOT_READY → backoff, retry same endpoint (Req 7.1) - * - XCE_ERR_STANDBY with redirect_host/port → reconnect, retry (Req 7.2) - * - XCE_ERR_STANDBY without redirect → backoff, retry (Req 7.2 fallback) - * - ECONNREFUSED / ETIMEDOUT → reconnect, retry (Req 7.3) - * - EPERM / EINVAL / ENOENT / EACCES → surface immediately (Req 7.4) - * - Exhausted max_retries → surface last error (Req 7.5) - * - Backoff jitter via rand_r() to prevent thundering herd (Req 7.6) - * - * NULL cfg → uses XCE_RETRY_DEFAULTS. - */ -extern int xce_request_with_retry(endpoint_t *ep, - xce_request_t *req, - const xce_retry_config_t *cfg); - -/* - * xce_retry_send_hook: test-only injection point for the send+receive round-trip. - * When non-NULL, xce_request_with_retry() calls this instead of the real network path. - * Returns 0 on success (reply_out populated), -errno on transport failure. - */ -extern int (*xce_retry_send_hook)(endpoint_t *ep, xce_request_t *req, - xce_reply_t *reply_out); - #endif diff --git a/src/include/xcectx.h b/src/include/xcectx.h index a80d7f7..a82d268 100644 --- a/src/include/xcectx.h +++ b/src/include/xcectx.h @@ -2,6 +2,9 @@ #define _XCECTX_H_ 1 +#include + +#include "config.h" /* XCE_MAXHOSTNAME_LEN */ #include "netio.h" #include "request.h" #include "vtaskmgr.h" @@ -28,6 +31,13 @@ typedef struct { cbuf_t wbuf; dataset_t* dataset; + /* Redirect-hint address captured from the last redirect reply by + * handle_reply() (which only sees xce_ctx_t) and read by the failover + * rotator on the next retry. Empty host ("") means no hint. Advisory only; + * correctness never depends on it. */ + char redirect_hint_host[XCE_MAXHOSTNAME_LEN]; + uint16_t redirect_hint_port; + struct { union { char* debugmsg; diff --git a/src/lib/client.c b/src/lib/client.c index fc94b86..81dd81a 100644 --- a/src/lib/client.c +++ b/src/lib/client.c @@ -4,6 +4,7 @@ #include #include "client.h" +#include "client_internal.h" #include "controller.capnp.h" #include "mylock.h" @@ -13,19 +14,6 @@ #include "xcectx.h" #include "xlist.h" -typedef enum { - API_CTX_INIT = 0, - API_CTX_RUN, - API_CTX_DOWN, - API_CTX_ERR -} api_ctx_state_t; - -typedef struct { - xce_client_config_t* conf; - endpoint_t ep; - xce_ctx_t ctx; -} api_ctx_t; - int xcecli_ctx_init(xcecli_ctx_t* c, xce_client_config_t* cfg) { api_ctx_t* ctx; @@ -126,6 +114,20 @@ int xcecli_start(xcecli_ctx_t c) return rc; } + /* Initialize per-context failover state: sticky primary starts at + * masters[0] (the initial connect target above), retry tunables default + * to XCE_RETRY_DEFAULTS, and no rejoin hook is installed for pure clients. + */ + rc = mylock_init(&(ctx->failover.lock)); + if (rc < 0) { + endpoint_fini(&(ctx->ep)); + return rc; + } + ctx->failover.cur_master = 0; + ctx->failover.retry = XCE_RETRY_DEFAULTS; + ctx->failover.rejoin_cb = NULL; + ctx->failover.rejoin_arg = NULL; + rc = xce_ctx_start(&(ctx->ctx)); return rc; @@ -141,6 +143,7 @@ int xcecli_stop(xcecli_ctx_t c) xce_ctx_stop(&(ctx->ctx)); endpoint_fini(&(ctx->ep)); + mylock_fini(&(ctx->failover.lock)); return 0; } diff --git a/src/lib/client_internal.h b/src/lib/client_internal.h new file mode 100644 index 0000000..70bd38e --- /dev/null +++ b/src/lib/client_internal.h @@ -0,0 +1,63 @@ +#if !defined(_CLIENT_INTERNAL_H_) + +#define _CLIENT_INTERNAL_H_ 1 + +#include + +#include "client.h" +#include "mylock.h" +#include "netio.h" +#include "request.h" +#include "xcectx.h" + +/* + * client_internal.h — private libxce client state shared between client.c and + * the forthcoming failover helper (client_failover.c, task 1.x). + * + * api_ctx_t was previously file-local to client.c. It is moved here so the + * failover helper can operate on it without exposing it through the public + * API in client.h. + */ + +typedef enum { + API_CTX_INIT = 0, + API_CTX_RUN, + API_CTX_DOWN, + API_CTX_ERR +} api_ctx_state_t; + +/* + * xcecli_reconnect_cb: consumer rejoin hook invoked after a transparent + * reconnect/failover and before the in-flight request resumes. NULL for pure + * clients (e.g. xce_test). The public declaration and the + * xcecli_set_reconnect_handler() setter are introduced by a later task; this + * private typedef lets the failover state carry the hook now. + */ +typedef int (*xcecli_reconnect_cb)(xcecli_ctx_t c, void* arg); + +/* + * xce_failover_state_t: per-context failover state. Persists across calls so a + * sticky primary and the retry tunables survive between requests. + * + * lock — serializes xcecli_* per ctx (shared-ctx safety) + * cur_master — index into conf->masters[] (sticky primary) + * retry — defaults from XCE_RETRY_DEFAULTS unless overridden + * rejoin_cb — consumer rejoin hook; NULL for pure clients + * rejoin_arg — opaque argument passed to rejoin_cb + */ +typedef struct { + mylock_t lock; + int cur_master; + xce_retry_config_t retry; + xcecli_reconnect_cb rejoin_cb; + void* rejoin_arg; +} xce_failover_state_t; + +typedef struct { + xce_client_config_t* conf; + endpoint_t ep; + xce_ctx_t ctx; + xce_failover_state_t failover; +} api_ctx_t; + +#endif diff --git a/src/lib/request.c b/src/lib/request.c index 888ca02..0d3924b 100644 --- a/src/lib/request.c +++ b/src/lib/request.c @@ -1920,109 +1920,17 @@ int xce_clear_reason_req(endpoint_t* ep, xce_clear_reason_t* reason) return xce_request(ep, &request); } -/* ── Task 10.2: Client-Side Transparent Retry (Req 7.1–7.6) ──────────────── +/* ── Client-Side Transparent Retry tunables (Req 6.1, 6.2) ───────────────── * - * xce_retry_send_hook: test-injectable send+receive round-trip. - * NULL in production; set by unit tests to return controlled replies. + * XCE_RETRY_DEFAULTS provides safe production defaults for the failover retry + * loop that lives in the libxce client layer (client.c / client_failover.c). + * The orphaned no-op xce_request_with_retry() primitive and its + * xce_retry_send_hook test seam were removed; they are superseded by the + * xcecli_* failover loop. */ -int (*xce_retry_send_hook)(endpoint_t *ep, xce_request_t *req, - xce_reply_t *reply_out) = NULL; - const xce_retry_config_t XCE_RETRY_DEFAULTS = { .max_retries = 20, .initial_retry_ms = 50, .max_retry_ms = 5000, .backoff_multiplier = 2.0, }; - -/* is_non_retriable: errors that should surface immediately without retry */ -static int is_non_retriable(int status) -{ - return (status == -EPERM || status == -EACCES || - status == -EINVAL || status == -ENOENT); -} - -/* ms_sleep_jittered: sleep delay_ms + random jitter via rand_r (Req 7.6) */ -static void ms_sleep_jittered(int delay_ms, unsigned int *seed) -{ - if (delay_ms <= 0) return; - int jitter_ms = (int)((rand_r(seed) % (delay_ms / 4 + 1))); - long total_ns = ((long)delay_ms + jitter_ms) * 1000000L; - struct timespec ts = { .tv_sec = total_ns / 1000000000L, - .tv_nsec = total_ns % 1000000000L }; - nanosleep(&ts, NULL); -} - -int xce_request_with_retry(endpoint_t *ep, - xce_request_t *req, - const xce_retry_config_t *cfg) -{ - if (cfg == NULL) - cfg = &XCE_RETRY_DEFAULTS; - - unsigned int rand_seed = (unsigned int)(uintptr_t)ep ^ (unsigned int)time(NULL); - int delay_ms = cfg->initial_retry_ms; - int last_rc = -ETIMEDOUT; - - for (int attempt = 0; attempt <= cfg->max_retries; attempt++) { - xce_reply_t reply; - memset(&reply, 0, sizeof(reply)); - - int rc; - if (xce_retry_send_hook) { - rc = xce_retry_send_hook(ep, req, &reply); - } else { - /* Production path: fire-and-forget send; reply handling is caller's - * responsibility via the event loop. Return send error directly. */ - rc = xce_request(ep, req); - if (rc != 0) - return rc; - return 0; /* async: caller reads reply via event loop */ - } - - /* Transport-level failure (ECONNREFUSED, ETIMEDOUT) → retry (Req 7.3) */ - if (rc == -ECONNREFUSED || rc == -ETIMEDOUT) { - last_rc = rc; - if (attempt < cfg->max_retries) - ms_sleep_jittered(delay_ms, &rand_seed); - goto next_backoff; - } - - /* Any other transport error → surface immediately */ - if (rc != 0) - return rc; - - /* Application-layer status from reply */ - int status = (int)reply.status; - - if (status == 0) - return 0; /* success */ - - last_rc = status; - - /* Non-retriable errors: surface immediately (Req 7.4) */ - if (is_non_retriable(status)) - return status; - - /* XCE_ERR_STANDBY with redirect → reconnect (Req 7.2). - * In hook mode, the hook already updated ep/redirect state. - * Just retry; redirect handling is at the transport layer. */ - if (status == -E_XCE_STANDBY && reply.redirect_host != NULL - && reply.redirect_port != 0) { - /* no extra sleep for redirect — just retry immediately */ - continue; - } - - /* XCE_ERR_NOT_READY or XCE_ERR_STANDBY without redirect → backoff */ - if (attempt < cfg->max_retries) - ms_sleep_jittered(delay_ms, &rand_seed); - -next_backoff: - { - int next = (int)((double)delay_ms * cfg->backoff_multiplier); - delay_ms = next > cfg->max_retry_ms ? cfg->max_retry_ms : next; - } - } - - return last_rc; -} diff --git a/src/lib/testsuite/test_libxce_retry.c b/src/lib/testsuite/test_libxce_retry.c index e84431a..ede17e0 100644 --- a/src/lib/testsuite/test_libxce_retry.c +++ b/src/lib/testsuite/test_libxce_retry.c @@ -1,324 +1,35 @@ /* - * test_libxce_retry.c — RED-phase tests for xce_request_with_retry() (tasks 10.1/10.2). + * test_libxce_retry.c — placeholder for the libxce failover retry tests. * - * Uses xce_retry_send_hook to inject mock round-trip outcomes without real - * network connections. Each mock call returns a configured status/redirect. + * The orphaned xce_request_with_retry() primitive and its xce_retry_send_hook + * test seam were removed (superseded by the xcecli_* failover loop). Task 1.2 + * rewrites this suite against the new failover model (StatusClassifier, master + * rotation, redirect-hint, budget → cluster-not-ready). * - * Requirements: 7.1, 7.2, 7.3, 7.4, 7.5, 7.6 + * For now this is a minimal compiling placeholder that asserts the retained + * retry-tunables scaffolding (xce_retry_config_t + XCE_RETRY_DEFAULTS) is + * reachable, keeping the lib test suite GREEN between tasks 1.1 and 1.2. + * + * Requirements: 6.1, 6.2 */ #define _GNU_SOURCE -#include -#include -#include -#include #include -#include -#include -#include -#include #include #include "request.h" -#include "xce/errnox.h" -/* ── Forward declarations for functions-under-test ───────────────── */ -/* xce_request_with_retry is not yet implemented — linker will fail (RED). */ -extern int xce_request_with_retry(endpoint_t *ep, - xce_request_t *req, - const xce_retry_config_t *cfg); - -/* - * xce_retry_send_hook: test seam for synchronous request-reply. - * When set, xce_request_with_retry calls this instead of the real network path. - * The hook fills in *reply_out and returns 0 on success, -errno on failure. - */ -extern int (*xce_retry_send_hook)(endpoint_t *ep, xce_request_t *req, - xce_reply_t *reply_out); - -/* ── Default retry config for tests (fast backoff) ──────────────── */ -static const xce_retry_config_t g_fast_cfg = { - .max_retries = 5, - .initial_retry_ms = 1, - .max_retry_ms = 10, - .backoff_multiplier = 2.0, -}; - -/* ── Mock state ─────────────────────────────────────────────────── */ - -#define MAX_MOCK_REPLIES 32 - -struct mock_reply { - int32_t status; /* xce_reply_t.status to return */ - int send_rc; /* return code from the hook (0 or -errno) */ - char *redirect_host; /* non-NULL if status == -E_XCE_STANDBY */ - uint16_t redirect_port; -}; - -static struct mock_reply g_replies[MAX_MOCK_REPLIES]; -static int g_reply_count = 0; -static int g_reply_idx = 0; -static int g_call_count = 0; - -static void reset_mock(void) +/* Req 6.1, 6.2: the retry-tunables struct + safe production defaults are + * retained and reachable for the failover loop. */ +static void test_retry_defaults_are_reachable(void) { - for (int i = 0; i < g_reply_count; i++) { - free(g_replies[i].redirect_host); - g_replies[i].redirect_host = NULL; - } - g_reply_count = 0; - g_reply_idx = 0; - g_call_count = 0; - memset(g_replies, 0, sizeof(g_replies)); -} + xce_retry_config_t cfg = XCE_RETRY_DEFAULTS; -static void add_reply(int32_t status, int send_rc, - const char *rhost, uint16_t rport) -{ - if (g_reply_count >= MAX_MOCK_REPLIES) return; - int i = g_reply_count++; - g_replies[i].status = status; - g_replies[i].send_rc = send_rc; - g_replies[i].redirect_host = rhost ? strdup(rhost) : NULL; - g_replies[i].redirect_port = rport; -} + TEST_ASSERT_EQUAL_INT(20, cfg.max_retries); + TEST_ASSERT_EQUAL_INT(50, cfg.initial_retry_ms); + TEST_ASSERT_EQUAL_INT(5000, cfg.max_retry_ms); + TEST_ASSERT_TRUE(cfg.backoff_multiplier > 1.0); -static int mock_send(endpoint_t *ep, xce_request_t *req, xce_reply_t *reply_out) -{ - (void)ep; (void)req; - g_call_count++; - - if (g_reply_idx >= g_reply_count) { - /* Default: return success after mocks exhausted */ - memset(reply_out, 0, sizeof(*reply_out)); - reply_out->status = 0; - return 0; - } - - struct mock_reply *m = &g_replies[g_reply_idx++]; - if (m->send_rc != 0) - return m->send_rc; - - memset(reply_out, 0, sizeof(*reply_out)); - reply_out->status = m->status; - reply_out->redirect_host = m->redirect_host; - reply_out->redirect_port = m->redirect_port; - return 0; -} - -static void install_mock(void) -{ - xce_retry_send_hook = mock_send; -} - -/* ── Tests ──────────────────────────────────────────────────────── */ - -/* Req 7.1: E_XCE_NOT_READY → retry with backoff, eventually succeed */ -static void test_not_ready_retries_then_succeeds(void) -{ - install_mock(); - reset_mock(); - - /* 3 NOT_READY responses, then success */ - add_reply(-E_XCE_NOT_READY, 0, NULL, 0); - add_reply(-E_XCE_NOT_READY, 0, NULL, 0); - add_reply(-E_XCE_NOT_READY, 0, NULL, 0); - add_reply(0, 0, NULL, 0); /* success */ - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - TEST_ASSERT_EQUAL_INT(0, rc); - TEST_ASSERT_EQUAL_INT(4, g_call_count); - - printf("test_not_ready_retries_then_succeeds: PASS\n"); -} - -/* Req 7.2: E_XCE_STANDBY with redirect → reconnect to redirect, retry */ -static void test_standby_with_redirect_reconnects_and_retries(void) -{ - install_mock(); - reset_mock(); - - /* Standby response with redirect */ - add_reply(-E_XCE_STANDBY, 0, "primary.host", 9990); - add_reply(0, 0, NULL, 0); /* success on redirect target */ - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - TEST_ASSERT_EQUAL_INT(0, rc); - TEST_ASSERT_EQUAL_INT(2, g_call_count); - - printf("test_standby_with_redirect_reconnects_and_retries: PASS\n"); -} - -/* Req 7.2 fallback: E_XCE_STANDBY without redirect → backoff, retry same */ -static void test_standby_without_redirect_falls_back_to_backoff(void) -{ - install_mock(); - reset_mock(); - - /* STANDBY without redirect, then success */ - add_reply(-E_XCE_STANDBY, 0, NULL, 0); - add_reply(0, 0, NULL, 0); - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - TEST_ASSERT_EQUAL_INT(0, rc); - TEST_ASSERT_EQUAL_INT(2, g_call_count); - - printf("test_standby_without_redirect_falls_back_to_backoff: PASS\n"); -} - -/* Req 7.4: non-retriable error (EPERM) → surface immediately, no retry */ -static void test_non_retriable_error_surfaces_immediately(void) -{ - install_mock(); - reset_mock(); - - /* Auth/perm failure — must NOT retry */ - add_reply(-EPERM, 0, NULL, 0); - /* Add a success after — it should never be reached */ - add_reply(0, 0, NULL, 0); - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - TEST_ASSERT_EQUAL_INT(-EPERM, rc); - TEST_ASSERT_EQUAL_INT(1, g_call_count); /* only 1 attempt */ - - printf("test_non_retriable_error_surfaces_immediately: PASS\n"); -} - -/* Req 7.5: max_retries exhausted → surface last error */ -static void test_max_retries_exhausted_returns_last_error(void) -{ - install_mock(); - reset_mock(); - - /* Fill with NOT_READY beyond max_retries */ - for (int i = 0; i <= g_fast_cfg.max_retries + 1; i++) - add_reply(-E_XCE_NOT_READY, 0, NULL, 0); - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - /* Must return the last error, not 0 */ - TEST_ASSERT_NOT_EQUAL(0, rc); - /* Must have tried max_retries+1 times total */ - TEST_ASSERT_EQUAL_INT(g_fast_cfg.max_retries + 1, g_call_count); - - printf("test_max_retries_exhausted_returns_last_error: PASS\n"); -} - -/* Req 7.3: ECONNREFUSED → reconnect, retry */ -static void test_conn_refused_retries(void) -{ - install_mock(); - reset_mock(); - - add_reply(0, -ECONNREFUSED, NULL, 0); - add_reply(0, 0, NULL, 0); - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - TEST_ASSERT_EQUAL_INT(0, rc); - TEST_ASSERT_EQUAL_INT(2, g_call_count); - - printf("test_conn_refused_retries: PASS\n"); -} - -/* NULL cfg → uses defaults (max_retries=20, etc.) */ -static void test_null_cfg_uses_defaults(void) -{ - install_mock(); - reset_mock(); - - add_reply(0, 0, NULL, 0); /* immediate success */ - - xce_request_t req; - memset(&req, 0, sizeof(req)); - - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, NULL); - TEST_ASSERT_EQUAL_INT(0, rc); - - printf("test_null_cfg_uses_defaults: PASS\n"); -} - -/* Req 7.6: 50 concurrent simulated clients all succeed (backoff prevents starvation) */ - -struct concurrent_client_args { - int id; - int rc; - atomic_int *success_count; -}; - -static void *concurrent_client_fn(void *arg) -{ - struct concurrent_client_args *a = arg; - xce_request_t req; - memset(&req, 0, sizeof(req)); - - /* Each client sees NOT_READY twice, then success */ - int rc = xce_request_with_retry((endpoint_t*)0x1, &req, &g_fast_cfg); - a->rc = rc; - if (rc == 0) - atomic_fetch_add(a->success_count, 1); - return NULL; -} - -static int concurrent_mock_send(endpoint_t *ep, xce_request_t *req, - xce_reply_t *reply_out) -{ - (void)ep; (void)req; - static atomic_int s_call_count = 0; - int n = atomic_fetch_add(&s_call_count, 1); - - memset(reply_out, 0, sizeof(*reply_out)); - - /* First 3 calls per "client" return NOT_READY, then succeed. - * Since we can't track per-client state here, use a simple global - * threshold: first 100 calls NOT_READY, rest succeed. */ - if (n < 100) - reply_out->status = -E_XCE_NOT_READY; - else - reply_out->status = 0; - return 0; -} - -static void test_50_concurrent_clients_all_succeed(void) -{ - xce_retry_send_hook = concurrent_mock_send; - - const int N = 50; - pthread_t threads[50]; - struct concurrent_client_args args[50]; - atomic_int success_count; - atomic_init(&success_count, 0); - - for (int i = 0; i < N; i++) { - args[i].id = i; - args[i].rc = -1; - args[i].success_count = &success_count; - pthread_create(&threads[i], NULL, concurrent_client_fn, &args[i]); - } - - /* Wait for all clients (with 30s overall deadline from Unity timeout) */ - for (int i = 0; i < N; i++) - pthread_join(threads[i], NULL); - - TEST_ASSERT_EQUAL_INT(N, atomic_load(&success_count)); - - printf("test_50_concurrent_clients_all_succeed: PASS (%d/%d)\n", - atomic_load(&success_count), N); + printf("test_retry_defaults_are_reachable: PASS\n"); } /* ── Suite entry point ──────────────────────────────────────────── */ @@ -326,13 +37,6 @@ static void test_50_concurrent_clients_all_succeed(void) void test_libxce_retry(void) { UNITY_BEGIN(); - RUN_TEST(test_not_ready_retries_then_succeeds); - RUN_TEST(test_standby_with_redirect_reconnects_and_retries); - RUN_TEST(test_standby_without_redirect_falls_back_to_backoff); - RUN_TEST(test_non_retriable_error_surfaces_immediately); - RUN_TEST(test_max_retries_exhausted_returns_last_error); - RUN_TEST(test_conn_refused_retries); - RUN_TEST(test_null_cfg_uses_defaults); - RUN_TEST(test_50_concurrent_clients_all_succeed); + RUN_TEST(test_retry_defaults_are_reachable); UNITY_END(); }