lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'working' into job_peek_lchen

Browse source 
job_peek_lchen: Add fetchfile feature (job output/error peek proxy)
working: HA/Raft infrastructure (consensus, replication, config mgmt)

Conflict resolution summary:
- controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30
- cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK
- job.c: kept full fetchfile pending-request subsystem (~183 lines)
- request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS
- helper.c: added missing preVoteRequest/preVoteReply mappings
- All 15 new test registrations (test_cbuf.c + 14 HA tests)
This commit is contained in:
Lei Chen 2026-07-22 13:57:25 +08:00
commit 3a2094a461
529 changed files with 345174 additions and 5048 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/lib/CLAUDE.md
View file

@ -12,12 +12,12 @@ meson build && ninja -C build
## Architecture
- Produces `libxce.a` (static) and/or `libxce.so` (shared)
- Linked by all C/C++ system components: controller, worker, scheduler
- Linked by all C system components: controller, worker, scheduler
- Public headers exposed via `src/include/xce/`
- Test suite in `src/lib/testsuite/`
## Conventions
- C standard: C99 or later
- C standard: gnu17
- Format with `clang-format` (config at repo root if present)
- All public API headers go in `src/include/xce/` — not in `src/lib/`
- Implementation files stay in `src/lib/`
@ -27,4 +27,4 @@ meson build && ninja -C build
- `src/lib/testsuite/` — Library unit tests
## See Also
- Root guidelines: `/workspace/CLAUDE.md`
- Root guidelines: `CLAUDE.md` (repo root)

318
src/lib/client.c
View file

@ -1,31 +1,21 @@
#include <errno.h>
#include <pthread.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include "client.h"
#include "client_internal.h"
#include "controller.capnp.h"
#include "mylock.h"
#include "request.h"
#include "xce/errnox.h" /* E_XCE_NOT_READY */
#include "xce/types.h"
#include "xceapi.h"
#include "xcectx.h"
#include "xlist.h"
typedef enum {
API_CTX_INIT = 0,
API_CTX_RUN,
API_CTX_DOWN,
API_CTX_ERR
} api_ctx_state_t;
typedef struct {
xce_client_config_t* conf;
endpoint_t ep;
xce_ctx_t ctx;
} api_ctx_t;
int xcecli_ctx_init(xcecli_ctx_t* c, xce_client_config_t* cfg)
{
api_ctx_t* ctx;
@ -88,6 +78,27 @@ int xcecli_set_handler(xcecli_ctx_t c, xce_request_handler_t handler,
return xce_ctx_set_handler(&(ctx->ctx), handler, args);
}
/*
* xcecli_set_reconnect_handler (task 4.1) — register the consumer rejoin hook
* the failover loop invokes after a transparent reconnect, before resuming the
* interrupted op. A NULL cb clears any previously registered hook (back to the
* pure-client no-op path). Stores cb+arg on the per-ctx failover state.
*/
int xcecli_set_reconnect_handler(xcecli_ctx_t c, xcecli_reconnect_cb cb,
void* arg)
{
api_ctx_t* ctx = (api_ctx_t*)c;
if (c == NULL) {
return -EINVAL;
}
ctx->failover.rejoin_cb = cb;
ctx->failover.rejoin_arg = arg;
return 0;
}
int xcecli_wakeup(xcecli_ctx_t c, int status)
{
api_ctx_t* ctx = (api_ctx_t*)c;
@ -112,19 +123,62 @@ int xcecli_try_wakeup(xcecli_ctx_t c, int status)
int xcecli_start(xcecli_ctx_t c)
{
int rc;
int rc = -ECONNREFUSED;
api_ctx_t* ctx = (api_ctx_t*)c;
int i;
int connected_idx = -1;
if (c == NULL) {
return -EINVAL;
}
rc = endpoint_connect(&(ctx->ep), ctx->conf->masters[0],
ctx->conf->master_ports[0]);
/* Initial connect rotates through ALL configured masters, not just
* masters[0] (bug-854 dead-first-master): if the first master is down
* (connection-refused after a failover that killed it), a hardcoded
* masters[0] connect fails the whole `client` session before the failover
* loop — which only runs on an ALREADY-connected ctx — ever gets a chance
* to rotate. Try each master in order; the first that connects becomes the
* sticky cursor start. The subsequent ctrlstatus/op failover loop then
* rotates onward to the authoritative primary as needed.
*
* ctx->ep was endpoint_init'd + bound to the iomgr in xcecli_init. A failed
* endpoint_connect leaves the socket fd unusable, so before retrying the
* NEXT master we must rebuild the endpoint exactly as xce_ctx_reconnect
* does: endpoint_fini → endpoint_init (fresh fd) → re-set peer + re-bind the
* iomgr (endpoint_connect needs ep->iomgr->ssl_ctx). */
for (i = 0; i < ctx->conf->nmasters; i++) {
if (i > 0) {
endpoint_fini(&(ctx->ep));
if (endpoint_init(&(ctx->ep), AF_INET, SOCK_STREAM, 0, 0) < 0)
continue;
endpoint_set_peer(&(ctx->ep), &(ctx->ctx.myinfo));
ctx->ep.iomgr = &(ctx->ctx.iomgr);
}
rc = endpoint_connect(&(ctx->ep), ctx->conf->masters[i],
ctx->conf->master_ports[i]);
if (rc >= 0) {
connected_idx = i;
break;
}
}
if (connected_idx < 0) {
/* No master reachable at all — surface the last transport error. */
endpoint_fini(&(ctx->ep));
return rc;
}
/* Initialize per-context failover state: sticky primary starts at the master
* we actually connected to (not always index 0), retry tunables default to
* XCE_RETRY_DEFAULTS, and no rejoin hook is installed for pure clients. */
rc = mylock_init(&(ctx->failover.lock));
if (rc < 0) {
endpoint_fini(&(ctx->ep));
return rc;
}
ctx->failover.cur_master = connected_idx;
ctx->failover.retry = XCE_RETRY_DEFAULTS;
ctx->failover.rejoin_cb = NULL;
ctx->failover.rejoin_arg = NULL;
rc = xce_ctx_start(&(ctx->ctx));
@ -141,6 +195,7 @@ int xcecli_stop(xcecli_ctx_t c)
xce_ctx_stop(&(ctx->ctx));
endpoint_fini(&(ctx->ep));
mylock_fini(&(ctx->failover.lock));
return 0;
}
@ -175,7 +230,8 @@ int xcecli_hello(xcecli_ctx_t c, char* msg)
return -EINVAL;
}
return xce_hello(&(ctx->ctx), msg);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_hello(&(ctx->ctx), msg));
}
int xcecli_nop(xcecli_ctx_t c)
@ -186,7 +242,10 @@ int xcecli_nop(xcecli_ctx_t c)
return -EINVAL;
}
return xce_nop(&(ctx->ctx));
/* Fire-and-forget: best-effort transport reconnect on a known-dead
* connection, then a single-shot send. No status classification, no
* no-primary backoff, no retry budget (task 5.2). */
return XCECLI_FF_CALL(ctx, xce_nop(&(ctx->ctx)));
}
int xcecli_heartbeat(xcecli_ctx_t c, int hb_interval)
@ -197,7 +256,10 @@ int xcecli_heartbeat(xcecli_ctx_t c, int hb_interval)
return -EINVAL;
}
return xce_heartbeat(&(ctx->ctx), hb_interval);
/* Fire-and-forget: best-effort transport reconnect on a known-dead
* connection, then a single-shot send (task 5.2). A heartbeat must NEVER go
* through the no-primary backoff retry budget. */
return XCECLI_FF_CALL(ctx, xce_heartbeat(&(ctx->ctx), hb_interval));
}
int xcecli_shutdown(xcecli_ctx_t c)
@ -219,7 +281,8 @@ int xcecli_reslist(xcecli_ctx_t c, xce_resource_t*** res)
return -EINVAL;
}
return xce_reslist(&(ctx->ctx), res);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_reslist(&(ctx->ctx), res));
}
int xcecli_join(xcecli_ctx_t c, int role, hostattrs_t* attrs)
@ -229,7 +292,8 @@ int xcecli_join(xcecli_ctx_t c, int role, hostattrs_t* attrs)
return -EINVAL;
}
return xce_join(&(ctx->ctx), role, attrs);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_join(&(ctx->ctx), role, attrs));
}
int xcecli_workerlist(xcecli_ctx_t c, xce_worker_t*** workers)
@ -239,7 +303,8 @@ int xcecli_workerlist(xcecli_ctx_t c, xce_worker_t*** workers)
return -EINVAL;
}
return xce_workerlist(&(ctx->ctx), workers);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_workerlist(&(ctx->ctx), workers));
}
int xcecli_jobdispatch(xcecli_ctx_t c, xce_job_dispatch_t* info)
@ -250,7 +315,8 @@ int xcecli_jobdispatch(xcecli_ctx_t c, xce_job_dispatch_t* info)
return -EINVAL;
}
return xce_jobdispatch(&(ctx->ctx), info);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_jobdispatch(&(ctx->ctx), info));
}
int xcecli_jobsubmit(xcecli_ctx_t c, xce_job_submit_t* submit, uint64_t* jobid)
@ -261,7 +327,8 @@ int xcecli_jobsubmit(xcecli_ctx_t c, xce_job_submit_t* submit, uint64_t* jobid)
return -EINVAL;
}
return xce_jobsubmit(&(ctx->ctx), submit, jobid);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_jobsubmit(&(ctx->ctx), submit, jobid));
}
int xcecli_joblist(xcecli_ctx_t c, xce_job_t*** jobs)
@ -272,7 +339,8 @@ int xcecli_joblist(xcecli_ctx_t c, xce_job_t*** jobs)
return -EINVAL;
}
return xce_joblist(&(ctx->ctx), jobs);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_joblist(&(ctx->ctx), jobs));
}
int xcecli_clusterconf(xcecli_ctx_t c, xce_clusterconf_t** conf)
@ -283,7 +351,165 @@ int xcecli_clusterconf(xcecli_ctx_t c, xce_clusterconf_t** conf)
return -EINVAL;
}
return xce_clusterconf(&(ctx->ctx), conf);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_clusterconf(&(ctx->ctx), conf));
}
int xcecli_ctrlstatus(xcecli_ctx_t c, xce_ctrlstatus_t** status)
{
api_ctx_t* ctx = (api_ctx_t*)c;
if (c == NULL) {
return -EINVAL;
}
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_ctrlstatus(&(ctx->ctx), status));
}
/*
* The primary's responder role on the wire is ctrl_role_t PRIMARY. That enum
* lives in the controller-private ha.h (CTRL_PRIMARY == 0) and is intentionally
* not exposed to libxce; the wire ordinal is part of the ctrlstatus reply
* contract (controller/ctrlstatus.c stamps responderRole = ha->role). We pin it
* here as a named constant so the comparison reads intentionally rather than as
* a bare 0. Any reorder of ctrl_role_t is a Revalidation Trigger for this spec.
*/
#define XCECLI_CTRLSTATUS_ROLE_PRIMARY ((uint8_t)0)
bool xcecli_ctrlstatus_is_authoritative(const xce_ctrlstatus_t* status)
{
if (status == NULL) {
return false;
}
return status->responderRole == XCECLI_CTRLSTATUS_ROLE_PRIMARY;
}
int xcecli_ctrlstatus_primary_addr(const xce_ctrlstatus_t* status,
char* host_out, size_t host_sz,
uint16_t* port_out)
{
int i;
if ((status == NULL) || (host_out == NULL) || (host_sz == 0)
|| (port_out == NULL)) {
return -EINVAL;
}
/* No known primary (empty/NULL primaryName) ⇒ nothing to target; the caller
* falls back to plain master rotation (Req 2.3 unknown-primary). */
if ((status->primaryName == NULL) || (status->primaryName[0] == '\0')) {
return -ENOENT;
}
/* Map the reported primaryName to its configured address via the reply's
* peers list. The list is a small bounded set (configured controllers). */
for (i = 0; i < status->peers.count; i++) {
xce_ctrlpeer_t* p = status->peers.list[i];
if ((p == NULL) || (p->name == NULL) || (p->host == NULL)) {
continue;
}
if (strcmp(p->name, status->primaryName) != 0) {
continue;
}
if ((p->host[0] == '\0') || (p->port == 0)) {
/* named primary has no usable address — treat as absent */
break;
}
strncpy(host_out, p->host, host_sz - 1);
host_out[host_sz - 1] = '\0';
*port_out = (uint16_t)p->port;
return 0;
}
/* primaryName not present in peers (or no usable address): fall back to
* rotation rather than targeting a bogus address. */
return -ENOENT;
}
int xcecli_ctrlstatus_primary(xcecli_ctx_t c, xce_ctrlstatus_t** status)
{
api_ctx_t* ctx = (api_ctx_t*)c;
xce_ctrlstatus_t* hint;
int rc;
int attempt;
int max_attempts;
if ((c == NULL) || (status == NULL)) {
return -EINVAL;
}
/* Bound the redirect loop by the configured failover retry budget so we
* never spin unbounded chasing a never-appearing primary (each inner
* xcecli_ctrlstatus already runs its own full reach-some-node failover).
* Floor it at the master-ring size so a standby-first / dead-first master
* list is always traversable in full before giving up (bug-854): one extra
* attempt lets the rotation that lands on the primary actually query it. */
max_attempts = ctx->failover.retry.max_retries;
if (max_attempts < 1) {
max_attempts = 1;
}
if (ctx->conf != NULL && ctx->conf->nmasters + 1 > max_attempts) {
max_attempts = ctx->conf->nmasters + 1;
}
for (attempt = 0; attempt < max_attempts; attempt++) {
hint = NULL;
rc = xcecli_ctrlstatus(c, &hint);
if (rc < 0) {
/* no reachable controller within the inner budget ⇒ typed failure;
* *status untouched (never a hint reply as the result, Req 9.2). */
return rc;
}
if (xcecli_ctrlstatus_is_authoritative(hint)) {
/* the primary itself answered — authoritative result */
*status = hint;
return 0;
}
/* Non-authoritative reply (standby/electing/init). The inner
* xcecli_ctrlstatus SUCCEEDS against ANY reachable node — ctrlstatus is
* answerable in every role — so it classifies the standby's status-0
* reply as success and NEVER rotates off it on its own. We must advance
* the endpoint here, or the loop re-queries the same non-primary every
* attempt and exhausts the budget without ever reaching the primary
* (bug-854). Capture a redirect hint when the responder named the primary
* (1-hop direct redirect), then FORCE rotation: xce_failover_rotate
* consumes the hint when set, else advances to the next configured
* master; recover repairs the ctx so the next attempt's send proceeds.
* Mirror xcecli_retry_next: only recover when the rotate succeeded — a
* failed rotate leaves the ctx for the next inner call's own transport-
* error failover to heal. All under failover.lock (the inner call takes
* and releases it itself, so it is not held here). */
{
char phost[XCE_MAXHOSTNAME_LEN];
uint16_t pport = 0;
mylock_lock(&ctx->failover.lock);
if (xcecli_ctrlstatus_primary_addr(hint, phost, sizeof(phost),
&pport)
== 0) {
xce_ctx_capture_redirect_hint(&ctx->ctx, phost, pport);
}
if (xce_failover_rotate(ctx) == 0) {
xcecli_ctx_recover(ctx);
}
mylock_unlock(&ctx->failover.lock);
}
/* Free the non-authoritative interim reply before the next attempt — no
* leak, and we never return a hint as the authoritative result. */
xcecli_free_ctrlstatus(hint);
}
/* Budget exhausted without the primary answering: typed failure, *status
* untouched (Req 9.2 — never return a non-primary reply as authoritative).
*/
return -E_XCE_NOT_READY;
}
int xcecli_notify_jobfinish(xcecli_ctx_t c, xce_jobfinish_t* jfinish)
@ -318,7 +544,8 @@ int xcecli_update_resvals(xcecli_ctx_t c, int nres, xce_resval_t** vals)
return -EINVAL;
}
rc = xce_update_resvals(&(ctx->ctx), nres, vals);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_update_resvals(&(ctx->ctx), nres, vals));
return rc;
}
@ -334,8 +561,10 @@ int xcecli_update_jusage(xcecli_ctx_t c, uint64_t jobid, int npids, int* pids,
return -EINVAL;
}
rc = xce_update_jusage(&(ctx->ctx), jobid, npids, pids, npgids, pgids,
nusage, usages);
rc = XCECLI_FAILOVER_CALL(
ctx, /*is_write=*/true,
xce_update_jusage(&(ctx->ctx), jobid, npids, pids, npgids, pgids, nusage,
usages));
return rc;
}
@ -352,7 +581,8 @@ int xcecli_resval(xcecli_ctx_t c, char* host, char* res, xce_resval_t*** val)
return -EINVAL;
}
return xce_resval(&(ctx->ctx), host, res, val);
return XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_resval(&(ctx->ctx), host, res, val));
}
int xcecli_notify(xcecli_ctx_t c, xce_notify_data_t* data)
@ -367,7 +597,12 @@ int xcecli_notify(xcecli_ctx_t c, xce_notify_data_t* data)
return -EINVAL;
}
return xce_notify(&(ctx->ctx), data);
/* Fire-and-forget chokepoint: every thin notify_* wrapper delegates
* here, so applying the best-effort reconnect at this single point covers
* them all without double-wrapping/double-locking. Best-effort transport
* reconnect on a known-dead connection, then a single-shot send (task
* 5.2). */
return XCECLI_FF_CALL(ctx, xce_notify(&(ctx->ctx), data));
}
int xcecli_notify_jobstart(xcecli_ctx_t c, job_pidinfo_t* pidinfo)
@ -417,7 +652,8 @@ int xcecli_jobctrl(xcecli_ctx_t c, xce_jobctrl_t* ctrlinfo)
return -EINVAL;
}
rc = xce_jobctrl(&(ctx->ctx), ctrlinfo);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_jobctrl(&(ctx->ctx), ctrlinfo));
return rc;
}
@ -511,7 +747,8 @@ int xcecli_queuectrl(xcecli_ctx_t c, xce_queuectrl_t* ctrlinfo)
return -EINVAL;
}
rc = xce_queuectrl(&(ctx->ctx), ctrlinfo);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_queuectrl(&(ctx->ctx), ctrlinfo));
return rc;
}
@ -525,7 +762,8 @@ int xcecli_hostctrl(xcecli_ctx_t c, xce_hostctrl_t* ctrlinfo)
return -EINVAL;
}
rc = xce_hostctrl(&(ctx->ctx), ctrlinfo);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_hostctrl(&(ctx->ctx), ctrlinfo));
return rc;
}
@ -542,7 +780,8 @@ int xcecli_runtask(xcecli_ctx_t c, xce_taskrun_t* info)
* controller to trace back
* client
*/
rc = xce_runtask(&(ctx->ctx), info);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_runtask(&(ctx->ctx), info));
return rc;
}
@ -634,7 +873,8 @@ int xcecli_set_pend_reason(xcecli_ctx_t c, xce_preason_set_t* reasons)
return -EINVAL;
}
rc = xce_set_pend_reason(&(ctx->ctx), reasons);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_set_pend_reason(&(ctx->ctx), reasons));
return rc;
}
@ -657,7 +897,8 @@ int xcecli_clear_reason(xcecli_ctx_t c, int njobs, uint64_t* jobs)
return 0;
}
rc = xce_clear_reason(&(ctx->ctx), &reason);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_clear_reason(&(ctx->ctx), &reason));
return rc;
}
@ -670,7 +911,8 @@ int xcecli_jobinfo(xcecli_ctx_t c, uint64_t jobid, xce_job_t** job)
return -EINVAL;
}
rc = xce_jobinfo(&(ctx->ctx), jobid, job);
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,
xce_jobinfo(&(ctx->ctx), jobid, job));
return rc;
}

412
src/lib/client_failover.c Normal file
View file

@ -0,0 +1,412 @@
/*
* client_failover.c — StatusClassifier (task 2.1) + EndpointRotator (task 2.2)
* for the libxce-failover spec.
*
* StatusClassifier: pure mapping from a reply status / transport rc to a
* failover action. No I/O, no globals (requirements 2.5, 4.1-4.3, 4.5, 5.2).
*
* EndpointRotator: choose and connect the next failover target. The target
* SELECTION (xce_failover_next_target) is a pure, socket-free helper so it is
* unit-testable; xce_failover_rotate() is a thin wrapper that closes the stale
* connection and reconnects the same endpoint object (requirements 2.1, 2.2,
* 2.3, 4.4, 8.4). See design.md "Components and Interfaces → EndpointRotator".
*/
#define _GNU_SOURCE
#include "xce/client_failover.h"
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <time.h>
#include "client_internal.h" /* api_ctx_t + EndpointRotator + retry decls */
#include "config.h" /* XCE_MAXHOSTNAME_LEN */
#include "netio.h" /* endpoint_init / endpoint_connect / endpoint_fini */
#include "utils.h" /* INFO / WARN logging macros */
#include "xce/errnox.h"
/*
* Controller status codes (E_XCE_*) are positive enum values, but the failover
* loop may observe them as negative -errno-style rc. Accept either sign by
* comparing against the magnitude.
*/
static int fo_abs(int v)
{
return v < 0 ? -v : v;
}
xce_fo_action_t xce_failover_classify(int status, bool is_write, int attempt)
{
/* Success: served by the primary. */
if (status == 0)
return XCE_FO_SUCCESS;
int code = fo_abs(status);
/* Idempotent-replay rule (5.2): a duplicate on a re-sent write means the
* controller's client-jobid replay guard already applied our write — treat
* it as success. On the first send (or a non-write), it is a genuine
* duplicate and is surfaced unchanged. */
if (code == E_XCE_DUP_JOB || code == EEXIST) {
if (is_write && attempt > 0)
return XCE_FO_SUCCESS;
return XCE_FO_TERMINAL;
}
/* Redirect / standby: a primary exists elsewhere, or this op is
* primary-only — rotate promptly, no backoff (4.2, 4.3). */
if (code == E_XCE_REDIRECT || code == E_XCE_STANDBY)
return XCE_FO_RETRY_NOW;
/* No primary known yet / controller initializing or electing — rotate after
* jittered backoff (4.1). */
if (code == E_XCE_NOMASTER || code == E_XCE_NOT_READY)
return XCE_FO_RETRY_BACKOFF;
/* Transport-level failures — reconnect/rotate and retry now (4.4). */
if (code == ECONNREFUSED || code == ETIMEDOUT || code == ECONNRESET)
return XCE_FO_RETRY_NOW;
/* Everything else — auth/permission/invalid/not-found and any unrecognized
* status — is non-retriable and surfaced to the caller (4.5). */
return XCE_FO_TERMINAL;
}
/* ════════════════════════════════════════════════════════════════════════
* EndpointRotator — target selection (pure) + connect wrapper (thin)
* ════════════════════════════════════════════════════════════════════════ */
/* True if a redirect-address hint is currently set on the ctx. The hint is a
* NUL-terminated host; an empty host ("") means "no hint" (xcectx.h). */
static bool fo_has_hint(const api_ctx_t *ac)
{
return ac->ctx.redirect_hint_host[0] != '\0';
}
/* Clear (consume) the redirect-address hint on the ctx. One-shot + advisory. */
static void fo_clear_hint(api_ctx_t *ac)
{
ac->ctx.redirect_hint_host[0] = '\0';
ac->ctx.redirect_hint_port = 0;
}
int xce_failover_next_target(api_ctx_t *ac, char *host_out, size_t host_sz,
uint16_t *port_out)
{
if (ac == NULL || host_out == NULL || host_sz == 0 || port_out == NULL)
return -EINVAL;
/* Hint-first (2.2, 8.3): if the last reply carried a redirect address, try
* it first and CONSUME it. The cursor is intentionally left untouched so a
* failed hint falls through to normal rotation on the next call (4.4) —
* correctness never depends on the hint. */
if (fo_has_hint(ac)) {
strncpy(host_out, ac->ctx.redirect_hint_host, host_sz - 1);
host_out[host_sz - 1] = '\0';
*port_out = ac->ctx.redirect_hint_port;
fo_clear_hint(ac);
return 0;
}
/* No hint: advance the sticky master cursor to the NEXT configured master,
* wrapping around (2.1, 8.4). */
if (ac->conf == NULL || ac->conf->nmasters <= 0)
return -ENOENT;
int next = (ac->failover.cur_master + 1) % ac->conf->nmasters;
ac->failover.cur_master = next;
const char *host = ac->conf->masters[next];
if (host == NULL)
return -ENOENT;
strncpy(host_out, host, host_sz - 1);
host_out[host_sz - 1] = '\0';
*port_out = (uint16_t)ac->conf->master_ports[next];
return 0;
}
int xce_failover_rotate(api_ctx_t *ac)
{
if (ac == NULL)
return -EINVAL;
char host[XCE_MAXHOSTNAME_LEN] = { 0 };
uint16_t port = 0;
int rc = xce_failover_next_target(ac, host, sizeof(host), &port);
if (rc < 0)
return rc;
/* Re-establish the SAME endpoint object on a fresh connection to the chosen
* target. xce_ctx_reconnect() (xcectx.c) owns the full connection setup that
* endpoint_fini drops — close old fd/SSL, recreate the socket, RE-BIND the
* iomgr (endpoint_connect needs ep->iomgr->ssl_ctx — a missing re-bind here
* previously crashed in endpoint_connect), drive the SSL handshake, and
* re-arm the endpoint in the running iomgr with its owner + msg/err handlers
* and re-monitor the new fd so replies route to ctx->wbuf and RDHUP failover
* re-arms. It does not restart iomgr/vtaskmgr or touch ctx->state (the loop's
* xcecli_ctx_recover owns the ERR→RUN reset). The live socket/handshake path
* is exercised by the E2E task (6.2). */
return xce_ctx_reconnect(&ac->ctx, host, (int)port);
}
/* ════════════════════════════════════════════════════════════════════════
* Reconnect recovery — context-state reset (CI2) + stale-reply drain (CI1)
* ════════════════════════════════════════════════════════════════════════ */
int xcecli_ctx_recover(api_ctx_t* ac)
{
if (ac == NULL)
return -EINVAL;
/* CI2 — state reset: the transport-error path (handle_disconnect) left
* ctx->state = XCE_CTX_ERR, which the per-op XCE_CTX_RUN guard in xceapi.c
* rejects as -EINVAL — the reported bug. After a successful reconnect,
* restore RUN so the next xce_* attempt passes. ONLY flip ERR→RUN: a
* deliberately-stopped (DOWN) or not-yet-started (INIT/BUSY) ctx is left
* untouched. */
if (ac->ctx.state == XCE_CTX_ERR)
ac->ctx.state = XCE_CTX_RUN;
/* CI1 — stale-reply drain: handle_disconnect may have injected a stale
* transport status into the single-slot wbuf. Drain any pending entry with
* a NON-BLOCKING get so the next attempt's cbuf_get cannot consume a
* prior-attempt reply. cbuf_tryget returns -EAGAIN when the channel is
* already empty — the normal, no-op case. The reconnect closed the old fd
* first, so no further replies arrive from the dead connection. */
int stale = 0;
cbuf_tryget(&ac->ctx.wbuf, &stale);
return 0;
}
/* ════════════════════════════════════════════════════════════════════════
* Fire-and-forget best-effort reconnect predicate (task 5.2)
* ════════════════════════════════════════════════════════════════════════ */
bool xcecli_ff_needs_reconnect(const api_ctx_t* ac)
{
if (ac == NULL)
return false;
/* Only a transport-ERR connection (set by the disconnect path) is treated
* as known-dead and reconnected best-effort. A live RUN ctx sends directly;
* a deliberately-stopped DOWN or not-yet-started INIT ctx is never
* resurrected by a fire-and-forget op. */
return ac->ctx.state == XCE_CTX_ERR;
}
/* ════════════════════════════════════════════════════════════════════════
* ReconnectRejoinHook — invoke the consumer rejoin callback (1.2)
* ════════════════════════════════════════════════════════════════════════ */
int xcecli_invoke_rejoin(api_ctx_t* ac)
{
if (ac == NULL)
return -EINVAL;
/* Pure client: no hook registered → nothing to do, resume directly. */
if (ac->failover.rejoin_cb == NULL)
return 0;
/* Stateful consumer: run its rejoin/state-exchange on the calling thread.
* Its rc is propagated verbatim — 0 to proceed, <0 to signal the join was
* rejected so the wrapper treats this attempt as failed (no resume). */
return ac->failover.rejoin_cb((xcecli_ctx_t)ac, ac->failover.rejoin_arg);
}
/* ════════════════════════════════════════════════════════════════════════
* XcecliRetryLoop — decision core (pure) + side-effecting wrapper
* ════════════════════════════════════════════════════════════════════════ */
/*
* Compute the next no-primary backoff delay (ms) for retry number `attempt`
* (1-based: the delay applied BEFORE the attempt'th re-send). Base is a capped
* exponential (initial_retry_ms * multiplier^(attempt-1), clamped to
* max_retry_ms); jitter spreads the actual delay within [base/2, base] using
* rand_r on the per-op seed so concurrent clients do not retry in lockstep
* (6.3). The base is monotone non-decreasing and never exceeds max_retry_ms.
*/
static int fo_backoff_delay_ms(const xce_retry_config_t* cfg, int attempt,
unsigned int* seed)
{
double base = (double)cfg->initial_retry_ms;
for (int i = 1; i < attempt; i++) {
base *= cfg->backoff_multiplier;
if (base >= (double)cfg->max_retry_ms) {
base = (double)cfg->max_retry_ms;
break;
}
}
if (base > (double)cfg->max_retry_ms)
base = (double)cfg->max_retry_ms;
if (base < 1.0)
base = 1.0;
int ibase = (int)base;
/* jitter within the lower half: delay ∈ [ibase/2, ibase]. */
int half = ibase / 2;
int jitter = (half > 0) ? (int)(rand_r(seed) % (unsigned int)(half + 1)) : 0;
int delay = ibase - jitter;
if (delay < 1)
delay = 1;
if (delay > cfg->max_retry_ms)
delay = cfg->max_retry_ms;
return delay;
}
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
const xce_retry_config_t* cfg, xcecli_retry_t* st,
xce_fo_action_t* action_out)
{
if (ac == NULL || rc == NULL || cfg == NULL || st == NULL)
return false;
/* Known-dead connection (the reported bug). After the primary dies,
* handle_disconnect leaves ctx.state == XCE_CTX_ERR, so a FRESH xce_* call
* hits the per-op xceapi guard and returns -EINVAL *before* the op reaches a
* primary — and -EINVAL would otherwise classify as TERMINAL and surface the
* opaque "Invalid argument". Detect the dead-ctx state directly and treat it
* as a prompt transport-style failover cause (reconnect via rotate+recover),
* regardless of *rc. After the next successful reconnect the ctx returns to
* RUN and subsequent attempts classify normally (success / no-primary /
* transport). On budget exhaustion we never reached a primary → surface the
* distinct cluster-not-ready code, never -EINVAL. */
if (ac->ctx.state == XCE_CTX_ERR) {
if (action_out != NULL)
*action_out = XCE_FO_RETRY_NOW;
st->attempt++;
if (st->attempt > cfg->max_retries) {
*rc = -E_XCE_NOT_READY;
return false;
}
st->delay_ms = 0;
return true;
}
xce_fo_action_t action =
xce_failover_classify(*rc, is_write, st->attempt);
if (action_out != NULL)
*action_out = action;
switch (action) {
case XCE_FO_SUCCESS:
/* DUP-replay success surfaces as a clean 0; a real status-0 is already
* 0. Either way the op succeeded — normalise and stop. */
*rc = 0;
return false;
case XCE_FO_TERMINAL:
/* Non-retriable — surface *rc unchanged. */
return false;
case XCE_FO_RETRY_NOW:
case XCE_FO_RETRY_BACKOFF:
break;
}
/* A retriable outcome consumes one retry from the budget. */
st->attempt++;
if (st->attempt > cfg->max_retries) {
/* Budget exhausted. Distinguish the cause (design D6): a no-primary
* window maps to the distinct cluster-not-ready code; a transport-only
* outage keeps the last transport rc so the caller still sees it. */
if (action == XCE_FO_RETRY_BACKOFF)
*rc = -E_XCE_NOT_READY;
return false;
}
/* Compute the pre-attempt delay: none for prompt rotation (RETRY_NOW),
* a jittered exponential backoff for the no-primary case (RETRY_BACKOFF). */
st->delay_ms = (action == XCE_FO_RETRY_BACKOFF)
? fo_backoff_delay_ms(cfg, st->attempt, &st->rand_seed)
: 0;
return true;
}
/* Sleep for `ms` milliseconds, tolerating EINTR. Isolated here so the pure
* decision core stays sleep-free and unit-testable. */
static void fo_sleep_ms(int ms)
{
if (ms <= 0)
return;
struct timespec ts = { .tv_sec = ms / 1000,
.tv_nsec = (long)(ms % 1000) * 1000000L };
while (nanosleep(&ts, &ts) != 0 && errno == EINTR)
; /* finish the remaining interval */
}
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
xcecli_retry_t* st)
{
if (ac == NULL || rc == NULL || st == NULL)
return false;
xce_fo_action_t action;
bool retry = xcecli_retry_decide(ac, rc, is_write, &ac->failover.retry, st,
&action);
if (!retry) {
/* Stopped. If the budget was exhausted on a no-primary window, decide
* has already set *rc to -E_XCE_NOT_READY (7.2 observable). */
if (*rc == -E_XCE_NOT_READY)
WARN("xcecli failover: cluster not ready after %d attempts "
"(last status=%d)",
st->attempt, *rc);
return false;
}
/* RETRY_BACKOFF: honour the jittered pre-attempt delay (4.1, 6.3). */
if (action == XCE_FO_RETRY_BACKOFF)
fo_sleep_ms(st->delay_ms);
/* Re-point the endpoint to the redirect hint or the next master and
* reconnect (2.x, 4.2, 4.4). Log the rotation target (7.1). A rotate
* failure is not fatal here: the next xce_* attempt observes the dead
* connection and the loop reclassifies it as a transport error. */
int rrc = xce_failover_rotate(ac);
if (rrc < 0) {
WARN("xcecli failover: rotate failed (rc=%d); retrying anyway", rrc);
return true;
}
/* Reconnect succeeded: repair the ctx before the next xce_* attempt
* (CI2 ERR→RUN state reset, CI1 stale-reply drain) so the per-op guard
* passes and no prior-attempt reply is consumed (1.2, 4.4). */
xcecli_ctx_recover(ac);
/* Log the rotation target (7.1). xce_failover_rotate consumes the redirect
* hint when present (so the post-rotate target is not always the sticky
* master); report the sticky master cursor as a best-effort endpoint label,
* which is exact for the configured-master rotation. */
int cur = ac->failover.cur_master;
const char* host =
(ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
? ac->conf->masters[cur]
: "(unknown)";
int port = (ac->conf != NULL && cur >= 0 && cur < ac->conf->nmasters)
? ac->conf->master_ports[cur]
: 0;
INFO("xcecli failover: attempt %d → %s:%d", st->attempt,
host ? host : "(null)", port);
/* ReconnectRejoinHook (1.2): a stateful consumer re-establishes its session
* on the freshly-connected primary AFTER reconnect/recovery and BEFORE the
* interrupted op resumes. A pure client has no hook → no-op, resume now. */
int hrc = xcecli_invoke_rejoin(ac);
if (hrc < 0) {
/* The new primary rejected the daemon's join: treat this attempt as
* FAILED — do NOT resume into a primary that doesn't know the daemon.
* Reclassify the rejoin failure as a transport-style failover cause so
* the loop rotates/backs off again on the next iteration (subject to the
* same retry budget). When the budget is exhausted decide() returns
* false and *rc carries the last cause. */
WARN("xcecli failover: rejoin hook failed (rc=%d); rotating again",
hrc);
*rc = hrc;
return xcecli_retry_next(ac, rc, is_write, st);
}
return true;
}

267
src/lib/client_internal.h Normal file
View file

@ -0,0 +1,267 @@
#if !defined(_CLIENT_INTERNAL_H_)
#define _CLIENT_INTERNAL_H_ 1
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include "client.h"
#include "mylock.h"
#include "netio.h"
#include "request.h"
#include "xce/client_failover.h" /* xce_fo_action_t */
#include "xcectx.h"
/*
* client_internal.h — private libxce client state shared between client.c and
* the forthcoming failover helper (client_failover.c, task 1.x).
*
* api_ctx_t was previously file-local to client.c. It is moved here so the
* failover helper can operate on it without exposing it through the public
* API in client.h.
*/
typedef enum {
API_CTX_INIT = 0,
API_CTX_RUN,
API_CTX_DOWN,
API_CTX_ERR
} api_ctx_state_t;
/*
* xcecli_reconnect_cb (consumer rejoin hook) and its public setter
* xcecli_set_reconnect_handler() are declared in the public client.h (included
* above) as of task 4.1. The failover state below carries the registered hook.
*/
/*
* xce_failover_state_t: per-context failover state. Persists across calls so a
* sticky primary and the retry tunables survive between requests.
*
* lock — serializes xcecli_* per ctx (shared-ctx safety)
* cur_master — index into conf->masters[] (sticky primary)
* retry — defaults from XCE_RETRY_DEFAULTS unless overridden
* rejoin_cb — consumer rejoin hook; NULL for pure clients
* rejoin_arg — opaque argument passed to rejoin_cb
*/
typedef struct {
mylock_t lock;
int cur_master;
xce_retry_config_t retry;
xcecli_reconnect_cb rejoin_cb;
void* rejoin_arg;
} xce_failover_state_t;
typedef struct {
xce_client_config_t* conf;
endpoint_t ep;
xce_ctx_t ctx;
xce_failover_state_t failover;
} api_ctx_t;
/*
* EndpointRotator (task 2.2) — these operate on the private api_ctx_t, so they
* live in this internal header rather than the public xce/client_failover.h.
*
* xce_failover_next_target(): PURE target-selection (no I/O). If a redirect-
* address hint is set on ac->ctx, write it out and CONSUME it (one-shot,
* advisory). Otherwise advance the sticky master cursor (ac->failover.cur_master)
* to the next configured master in ac->conf, wrapping around. Returns 0,
* -EINVAL on a NULL/zero argument, or -ENOENT if no masters and no hint.
* Correctness never depends on the hint.
*
* xce_failover_rotate(): close the stale connection on ac->ep and reconnect the
* SAME endpoint object to the chosen target. Thin wrapper over the selector +
* endpoint_connect(); the live socket path is exercised by the E2E task.
*/
int xce_failover_next_target(api_ctx_t* ac, char* host_out, size_t host_sz,
uint16_t* port_out);
int xce_failover_rotate(api_ctx_t* ac);
/*
* XcecliRetryLoop / decision helper (task 3.1) — see design.md
* "Components and Interfaces → xcecli_retry_next".
*
* xcecli_retry_t: per-op retry carrier. One instance lives on the caller's
* stack for the duration of a single xcecli_* op's do/while loop.
* attempt — number of re-sends so far (0 on the first send).
* delay_ms — backoff to apply BEFORE the next attempt (0 for RETRY_NOW).
* rand_seed — per-op rand_r() state for jittered backoff (6.3).
*/
typedef struct {
int attempt;
int delay_ms;
unsigned int rand_seed;
} xcecli_retry_t;
/*
* XCECLI_RETRY_INIT(ac): initialise a retry carrier from an api_ctx_t. The seed
* mixes the ctx pointer so concurrent ops on distinct ctxs jitter independently.
*/
#define XCECLI_RETRY_INIT(ac) \
((xcecli_retry_t){ .attempt = 0, \
.delay_ms = 0, \
.rand_seed = (unsigned int)(uintptr_t)(ac) })
/*
* xcecli_retry_decide(): the PURE decision core (no I/O, no sleep, no socket,
* no logging) so it is unit-testable. Given the rc returned by the last xce_*
* attempt, it:
* - classifies *rc via xce_failover_classify(*rc, is_write, st->attempt);
* - on SUCCESS applies the DUP-replay rule (sets *rc = 0) and returns false;
* - on TERMINAL returns false leaving *rc as-is;
* - on a retriable action advances st->attempt, computes st->delay_ms
* (0 for RETRY_NOW; a jittered exponential backoff bounded by
* cfg->max_retry_ms for RETRY_BACKOFF), reports the action in *action_out,
* and returns true — UNLESS the retry budget (cfg->max_retries) is now
* exhausted, in which case it returns false and, for a no-primary cause,
* sets *rc = -E_XCE_NOT_READY (a transport-only cause keeps the last rc).
*
* It does NOT reset ctx state, drain wbuf, sleep, rotate, or invoke the rejoin
* hook — those are the side-effecting wrapper's job (xcecli_retry_next) and
* later tasks (3.2 state-reset/drain, 4.1 rejoin hook).
*/
bool xcecli_retry_decide(api_ctx_t* ac, int* rc, bool is_write,
const xce_retry_config_t* cfg, xcecli_retry_t* st,
xce_fo_action_t* action_out);
/*
* xcecli_retry_next(): the side-effecting wrapper the xcecli_* do/while loops
* call. = xcecli_retry_decide() + (RETRY_BACKOFF: a real st->delay_ms sleep) +
* (any retry: xce_failover_rotate(ac) + INFO log of the rotation target, 7.1) +
* (budget exhaustion: WARN log of the attempt count, 7.2). Returns true → the
* caller re-invokes its xce_* op; false → stop, *rc is final.
*/
bool xcecli_retry_next(api_ctx_t* ac, int* rc, bool is_write,
xcecli_retry_t* st);
/*
* XCECLI_FAILOVER_CALL(ac, is_write, EXPR) — the shared XcecliRetryLoop wrapper
* (task 5.1). Turns a one-shot reply-bearing xce_* call into a transparent
* failover loop, holding the per-ctx lock (ac->failover.lock) for the FULL loop
* duration so concurrent ops on a shared ctx are serialised (CI2): the
* single-slot wbuf already implies one in-flight request per ctx; the lock makes
* that explicit and covers the reconnect/rotate window.
*
* ac — api_ctx_t* (already NULL-checked by the caller's guard)
* is_write — true for mutating ops (DUP-on-replay → success, 5.2), false reads
* EXPR — the unchanged one-shot xce_* call, evaluated each iteration; its
* rc is what the loop classifies and ultimately returns
*
* Evaluates to the final rc (>=0 success; -E_XCE_NOT_READY on no-primary budget
* exhaustion; terminal errors surfaced as-is). Used as the RHS of `return`.
*
* Implemented as a GCC/clang statement-expression so it yields a value and keeps
* each public xcecli_* signature/return convention intact (R1.3). The lock is
* released on every exit path because the do/while always falls through to the
* unlock before the result is produced.
*/
#define XCECLI_FAILOVER_CALL(ac, is_write, EXPR) \
__extension__({ \
api_ctx_t* _fo_ac = (ac); \
xcecli_retry_t _fo_st = XCECLI_RETRY_INIT(_fo_ac); \
int _fo_rc; \
mylock_lock(&_fo_ac->failover.lock); \
do { \
_fo_rc = (EXPR); \
} while ( \
xcecli_retry_next(_fo_ac, &_fo_rc, (is_write), &_fo_st)); \
mylock_unlock(&_fo_ac->failover.lock); \
_fo_rc; \
})
/*
* xcecli_ctx_recover (task 3.2) — repair the inner xce_ctx_t after a successful
* reconnect so the next xce_* attempt can proceed. Socket-free (no I/O, no
* sleep) so it is unit-testable; xcecli_retry_next invokes it after a successful
* xce_failover_rotate(), before re-sending the interrupted op.
*
* CI2 (state reset): the transport-error path left ctx->state = XCE_CTX_ERR
* (xcectx.c:174), which the per-op XCE_CTX_RUN guard rejects as -EINVAL
* (xceapi.c) — the reported bug. recover restores XCE_CTX_RUN, but ONLY from
* ERR: a DOWN/INIT ctx (deliberately stopped / not yet started) is left
* as-is, and a RUN ctx is already fine.
* CI1 (stale-reply drain): handle_disconnect may have injected a stale status
* into the single-slot ctx->wbuf (xcectx.c:180). recover drains any pending
* entry with a non-blocking cbuf_tryget so the next attempt's cbuf_get
* cannot consume a prior-attempt reply. Reconnect closes the old fd first,
* so the dead connection yields no further replies — this drain is enough.
*
* Returns 0 on success, -EINVAL on a NULL ac.
*/
int xcecli_ctx_recover(api_ctx_t* ac);
/*
* xcecli_ff_needs_reconnect (task 5.2) — PURE predicate for the fire-and-forget
* best-effort reconnect decision. Fire-and-forget ops (notify_*, heartbeat, nop)
* send a *_req only — there is NO reply to classify, so they must NOT go through
* the full failover retry loop (no status classification, no no-primary backoff,
* no retry budget; a heartbeat must never block on the backoff budget). Instead
* the FF path makes AT MOST ONE best-effort transport reconnect when the
* connection is KNOWN-DEAD, then does a single-shot send (which may be lost —
* acceptable; redelivery is deferred to worker-rejoin-resync).
*
* The "known-dead" signal is ctx->state == XCE_CTX_ERR (set by the disconnect
* path after a detected transport failure). Returns true ONLY for ERR so that a
* deliberately-stopped (DOWN) or not-yet-started (INIT) ctx is never resurrected
* by a fire-and-forget op. NULL-safe: returns false. Socket-free + side-effect
* free so it is unit-testable.
*/
bool xcecli_ff_needs_reconnect(const api_ctx_t* ac);
/*
* XCECLI_FF_CALL(ac, EXPR) — the fire-and-forget best-effort wrapper (task 5.2).
* Mirrors XCECLI_FAILOVER_CALL's per-ctx locking (CI2 — shared-ctx safety) but
* is single-shot, NOT a retry loop: there is no reply to classify for an FF op,
* so the full failover machinery (classification, no-primary backoff, retry
* budget) is deliberately NOT used. Putting a heartbeat through the retry loop
* would let it block on the no-primary backoff budget — explicitly forbidden.
*
* Behaviour, holding ac->failover.lock for the whole sequence:
* 1. if the connection is KNOWN-DEAD (xcecli_ff_needs_reconnect), attempt ONE
* best-effort reconnect: xce_failover_rotate(ac) + xcecli_ctx_recover(ac).
* A failed reconnect is non-fatal — the send below is simply best-effort.
* 2. evaluate EXPR exactly ONCE (the unchanged one-shot xce_* fire-and-forget
* send) and yield its rc verbatim. A lost send rc is surfaced as-is, never
* retried.
*
* ac — api_ctx_t* (already NULL-checked by the caller's guard)
* EXPR — the unchanged one-shot xce_* *_req-only call
*
* Implemented as a GCC/clang statement-expression so it yields the send rc and
* keeps each public xcecli_* signature/return convention intact (R1.3).
*/
#define XCECLI_FF_CALL(ac, EXPR) \
__extension__({ \
api_ctx_t* _ff_ac = (ac); \
int _ff_rc; \
mylock_lock(&_ff_ac->failover.lock); \
if (xcecli_ff_needs_reconnect(_ff_ac)) { \
(void)xce_failover_rotate(_ff_ac); \
xcecli_ctx_recover(_ff_ac); \
} \
_ff_rc = (EXPR); \
mylock_unlock(&_ff_ac->failover.lock); \
_ff_rc; \
})
/*
* xcecli_invoke_rejoin (task 4.1) — invoke the consumer rejoin hook once after a
* successful reconnect/recovery and BEFORE the interrupted op resumes. Socket-
* free (it only calls the stored callback), so it is unit-testable. The
* xcecli_retry_next wrapper calls it AFTER xce_failover_rotate +
* xcecli_ctx_recover and BEFORE re-sending the op.
*
* If no hook is registered (rejoin_cb == NULL, a pure client) it is a no-op and
* returns 0 so the loop resumes directly. Otherwise it returns the callback's
* rc: 0 to proceed, <0 to treat the rejoin as FAILED (the new primary rejected
* the daemon's join) — the wrapper then does NOT resume but rotates/backs off on
* the next iteration.
*
* Returns 0 on a successful (or absent) rejoin, the hook's <0 rc on a failed
* rejoin, or -EINVAL on a NULL ac.
*/
int xcecli_invoke_rejoin(api_ctx_t* ac);
#endif

191
src/lib/cluster.c Normal file
View file

@ -0,0 +1,191 @@
/* src/lib/cluster.c — cluster discovery and reconnect (P2.5-2/3)
*
* xce_connect_cluster() iterates the candidate list and returns the first
* successful endpoint_t* (cast to void*). In production the real SSL
* connect is performed via do_connect(); unit tests override s_connect_fn.
*
* xce_reconnect_step() drives a non-blocking state machine:
* IDLE → CONNECTING → CONNECTED (happy path)
* IDLE → CONNECTING → BACKOFF → CONNECTING → … (retrying)
*
* Backoff sleep is skipped when s_connect_fn is set (i.e., in unit tests)
* so tests do not need to wait for real timeouts.
*/
#include <errno.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "xce/cluster.h"
/* ── Peer list parsing ─────────────────────────────────────────── */
int xce_parse_peers(const char* peers_str, xce_cluster_config_t* cfg)
{
cfg->candidates_count = 0;
if (!peers_str || peers_str[0] == '\0')
return -1;
char buf[2048];
strncpy(buf, peers_str, sizeof(buf) - 1);
buf[sizeof(buf) - 1] = '\0';
char* saveptr;
char* tok = strtok_r(buf, ",", &saveptr);
while (tok && cfg->candidates_count < MAX_HA_CANDIDATES) {
/* Format: name=host:port OR host:port */
char* eq = strchr(tok, '=');
char* addr = eq ? eq + 1 : tok;
char* colon = strrchr(addr, ':');
if (!colon)
return -1;
*colon = '\0';
xce_candidate_t* cand = &cfg->candidates[cfg->candidates_count];
if (eq) {
*eq = '\0';
strncpy(cand->name, tok, XCE_MAXNAME_LEN - 1);
cand->name[XCE_MAXNAME_LEN - 1] = '\0';
} else {
cand->name[0] = '\0';
}
strncpy(cand->hostname, addr, XCE_MAXHOSTNAME_LEN - 1);
cand->hostname[XCE_MAXHOSTNAME_LEN - 1] = '\0';
cand->port = atoi(colon + 1);
cfg->candidates_count++;
tok = strtok_r(NULL, ",", &saveptr);
}
return cfg->candidates_count > 0 ? 0 : -1;
}
/* ── Test seam ─────────────────────────────────────────────────── */
static xce_connect_fn_t s_connect_fn = NULL;
void xce_cluster_set_connect_fn(xce_connect_fn_t fn)
{
s_connect_fn = fn;
}
/* do_connect: calls the mock when set, otherwise returns NULL (stub for
* the real SSL connect via netio — to be wired in a later task). */
static void* do_connect(const char* host, int port, const char* ca_cert)
{
if (s_connect_fn)
return s_connect_fn(host, port, ca_cert);
/* Production path: real SSL connect via netio — stub for now */
(void)host;
(void)port;
(void)ca_cert;
return NULL;
}
/* ── Validation ────────────────────────────────────────────────── */
int xce_cluster_config_validate(const xce_cluster_config_t* cfg)
{
if (!cfg)
return -EINVAL;
if (cfg->candidates_count < 1 || cfg->candidates_count > MAX_HA_CANDIDATES)
return -EINVAL;
for (int i = 0; i < cfg->candidates_count; i++) {
if (cfg->candidates[i].hostname[0] == '\0')
return -EINVAL;
if (cfg->candidates[i].port <= 0)
return -EINVAL;
}
if (cfg->ca_cert_path[0] == '\0')
return -EINVAL;
if (cfg->identity[0] == '\0')
return -EINVAL;
if (cfg->candidates_count > 3)
fprintf(stderr,
"WARN: HA cluster has %d candidates; validated for <=3 only\n",
cfg->candidates_count);
return 0;
}
/* ── Connect with candidate fallback ───────────────────────────── */
void* xce_connect_cluster(xce_cluster_config_t* cfg)
{
if (xce_cluster_config_validate(cfg) != 0)
return NULL;
for (int i = 0; i < cfg->candidates_count; i++) {
void* ep = do_connect(cfg->candidates[i].hostname,
cfg->candidates[i].port,
cfg->ca_cert_path);
if (ep)
return ep;
}
return NULL;
}
/* ── Reconnect state machine ───────────────────────────────────── */
int xce_reconnect_init(xce_reconnect_ctx_t* rctx, xce_cluster_config_t* cfg)
{
if (!rctx || !cfg)
return -EINVAL;
memset(rctx, 0, sizeof(*rctx));
rctx->config = cfg;
rctx->state = XCE_RECONNECT_IDLE;
rctx->backoff_ms = XCE_RECONNECT_INITIAL_BACKOFF_MS;
return 0;
}
int xce_reconnect_step(xce_reconnect_ctx_t* rctx)
{
if (!rctx)
return -EINVAL;
switch (rctx->state) {
case XCE_RECONNECT_IDLE:
rctx->state = XCE_RECONNECT_CONNECTING;
return 0; /* next step does the actual connect */
case XCE_RECONNECT_CONNECTING: {
void* ep = xce_connect_cluster(rctx->config);
if (ep) {
rctx->endpoint = ep;
rctx->state = XCE_RECONNECT_CONNECTED;
rctx->attempt = 0;
rctx->backoff_ms = XCE_RECONNECT_INITIAL_BACKOFF_MS;
} else {
rctx->attempt++;
rctx->state = XCE_RECONNECT_BACKOFF;
}
return 0;
}
case XCE_RECONNECT_BACKOFF: {
/* Skip sleep in test context (mock connect set) to keep tests fast */
if (!s_connect_fn) {
struct timespec ts = {
.tv_sec = rctx->backoff_ms / 1000,
.tv_nsec = (long)(rctx->backoff_ms % 1000) * 1000000L,
};
nanosleep(&ts, NULL);
}
/* Exponential backoff with cap */
rctx->backoff_ms *= XCE_RECONNECT_BACKOFF_MULTIPLIER;
if (rctx->backoff_ms > XCE_RECONNECT_MAX_BACKOFF_MS)
rctx->backoff_ms = XCE_RECONNECT_MAX_BACKOFF_MS;
rctx->state = XCE_RECONNECT_CONNECTING;
return 0;
}
case XCE_RECONNECT_CONNECTED:
case XCE_RECONNECT_FAILED:
return -1;
}
return 0;
}
void xce_reconnect_fini(xce_reconnect_ctx_t* rctx)
{
if (rctx)
memset(rctx, 0, sizeof(*rctx));
}

120
src/lib/delay.c
View file

@ -1,12 +1,28 @@
#include <errno.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "delay.h"
#include "utils.h"
/* Return 1 if a ms-task's deadline has passed, 0 otherwise. */
static int ms_task_due(const delay_task_t* t)
{
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
if (now.tv_sec > t->nextrun_ts.tv_sec) {
return 1;
}
if (now.tv_sec == t->nextrun_ts.tv_sec
&& now.tv_nsec >= t->nextrun_ts.tv_nsec) {
return 1;
}
return 0;
}
static void* delay_worker(void* args)
{
delaymgr_t* mgr = (delaymgr_t*)args;
@ -15,19 +31,35 @@ static void* delay_worker(void* args)
ordq_item_t* it;
delay_task_t* t;
time_t now;
int is_ms_task;
pthread_testcancel();
sleep(mgr->precision);
{
long ms = (long)mgr->precision_ms;
struct timespec ts = {
.tv_sec = ms / 1000,
.tv_nsec = (ms % 1000) * 1000000L,
};
nanosleep(&ts, NULL);
}
it = ordq_peek(&(mgr->tasks), NULL);
if (it == NULL) {
continue;
}
now = time(NULL);
t = (delay_task_t*)it->data;
if (t->nextrun > now) {
continue;
t = (delay_task_t*)it->data;
is_ms_task = (t->nextrun_ts.tv_nsec != 0 || t->nextrun_ts.tv_sec != 0);
if (is_ms_task) {
if (!ms_task_due(t)) {
continue;
}
} else {
now = time(NULL);
if (t->nextrun > now) {
continue;
}
}
it = ordq_get(&(mgr->tasks), NULL);
@ -41,10 +73,17 @@ static void* delay_worker(void* args)
if (t->runcount == 0) {
FREEUP(t);
} else {
time_t nextrun = now + t->run_period;
if (is_ms_task) {
/* advance deadline by run_period ms */
long add_ns = (long)t->run_period * 1000000L;
/* put back to taskq as it need to be run again */
t->nextrun = nextrun;
t->nextrun_ts.tv_nsec += add_ns;
t->nextrun_ts.tv_sec += t->nextrun_ts.tv_nsec / 1000000000L;
t->nextrun_ts.tv_nsec %= 1000000000L;
} else {
/* put back to taskq as it need to be run again */
t->nextrun = now + t->run_period;
}
ordq_put(&(mgr->tasks), t);
}
@ -54,21 +93,30 @@ static void* delay_worker(void* args)
return NULL;
}
static int task_cmp(void* arg1, void* arg2)
static int task_cmp(void* ctx, void* arg1, void* arg2)
{
(void)ctx;
delay_task_t* t1 = (delay_task_t*)arg1;
delay_task_t* t2 = (delay_task_t*)arg2;
time_t k1 = t1->nextrun;
time_t k2 = t2->nextrun;
if (k1 > k2) {
return 1;
}
/* ms-tasks: compare by nextrun_ts (nanosecond-precision deadline).
* Both delay_task_ms and ordq_put use nextrun==0 as the ms-task marker. */
int t1_is_ms = (t1->nextrun_ts.tv_nsec != 0 || t1->nextrun_ts.tv_sec != 0);
int t2_is_ms = (t2->nextrun_ts.tv_nsec != 0 || t2->nextrun_ts.tv_sec != 0);
if (k1 == k2) {
if (t1_is_ms && t2_is_ms) {
if (t1->nextrun_ts.tv_sec > t2->nextrun_ts.tv_sec) return 1;
if (t1->nextrun_ts.tv_sec < t2->nextrun_ts.tv_sec) return -1;
if (t1->nextrun_ts.tv_nsec > t2->nextrun_ts.tv_nsec) return 1;
if (t1->nextrun_ts.tv_nsec < t2->nextrun_ts.tv_nsec) return -1;
return 0;
}
/* Mixed or seconds-based: fall back to nextrun comparison. */
time_t k1 = t1->nextrun;
time_t k2 = t2->nextrun;
if (k1 > k2) return 1;
if (k1 == k2) return 0;
return -1;
}
@ -100,7 +148,7 @@ int delaymgr_init(delaymgr_t* mgr, int precision, task_run_fn_t runner_fn)
runner_fn = default_runner_fn;
}
mgr->precision = precision;
mgr->precision_ms = precision;
mgr->running = 0;
mgr->runner_fn = runner_fn;
mgr->worker = 0;
@ -198,3 +246,41 @@ int delay_task(delaymgr_t* mgr, int delay, int runcount,
return ordq_put(&(mgr->tasks), t);
}
int delay_task_ms(delaymgr_t* mgr, int delay_ms, int runcount,
delay_task_entry_fn_t entry, void* args)
{
delay_task_t* t;
struct timespec now;
long add_ns;
if ((mgr == NULL) || (delay_ms <= 0) || (entry == NULL)) {
return -EINVAL;
}
t = (delay_task_t*)calloc(1, sizeof(*t));
if (t == NULL) {
return -ENOMEM;
}
clock_gettime(CLOCK_MONOTONIC, &now);
add_ns = (long)delay_ms * 1000000L;
t->nextrun_ts.tv_sec = now.tv_sec + (now.tv_nsec + add_ns) / 1000000000L;
t->nextrun_ts.tv_nsec = (now.tv_nsec + add_ns) % 1000000000L;
/* Ensure nextrun_ts is never all-zero so the ms-task flag is unambiguous.
* tv_nsec == 0 can only arise when add_ns is an exact multiple of 1s and
* now.tv_nsec == 0; in that edge case bump tv_nsec by 1 ns. */
if (t->nextrun_ts.tv_nsec == 0) {
t->nextrun_ts.tv_nsec = 1;
}
t->nextrun = 0; /* unused for ms-tasks */
t->runcount = runcount;
t->run_period = delay_ms;
t->entry = entry;
t->args = args;
return ordq_put(&(mgr->tasks), t);
}

91
src/lib/helper.c
View file

@ -1,4 +1,5 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include "controller.capnp.h"
@ -147,6 +148,44 @@ char* xce_request_op_name(xce_request_t* req)
return "setpreason";
case OPCode_clrpreason:
return "clrpreason";
case OPCode_heartbeat:
return "heartbeat";
case OPCode_redirect:
return "redirect";
case OPCode_steppingDown:
return "steppingDown";
case OPCode_replEntry:
return "replEntry";
case OPCode_replAck:
return "replAck";
case OPCode_replSnapshot:
return "replSnapshot";
case OPCode_replCatchUpDone:
return "replCatchUpDone";
case OPCode_replConfig:
return "replConfig";
case OPCode_replConfigPush:
return "replConfigPush";
case OPCode_upgradeConfig:
return "upgradeConfig";
case OPCode_upgradeConfigAck:
return "upgradeConfigAck";
case OPCode_requestVote:
return "requestVote";
case OPCode_probeConfigTag:
return "probeConfigTag";
case OPCode_walAck:
return "walAck";
case OPCode_walCatchupEntry:
return "walCatchupEntry";
case OPCode_snapshotTransfer:
return "snapshotTransfer";
case OPCode_preVoteRequest:
return "preVoteRequest";
case OPCode_preVoteReply:
return "preVoteReply";
case OPCode_ctrlstatus:
return "ctrlstatus";
case OPCode_fetchfile:
return "fetchfile";
default:
@ -332,3 +371,55 @@ char* xce_dataset_table(int table)
return "unknown";
}
char* xce_notify_event_name(int event)
{
static char buf[32];
switch (event) {
case NotifyData_data_jobPidInfo:
return "jobPidInfo";
case NotifyData_data_jobFailInfo:
return "jobFailInfo";
case NotifyData_data_jobDispatchFail:
return "jobDispatchFail";
case NotifyData_data_jobFinishInfo:
return "jobFinishInfo";
case NotifyData_data_schedStartJob:
return "schedStartJob";
case NotifyData_data_schedExitJob:
return "schedExitJob";
case NotifyData_data_schedFailJob:
return "schedFailJob";
case NotifyData_data_schedUpdateJob:
return "schedUpdateJob";
case NotifyData_data_schedUpdateResvals:
return "schedUpdateResvals";
case NotifyData_data_stoppedJob:
return "stoppedJob";
case NotifyData_data_resumedJob:
return "resumedJob";
case NotifyData_data_ckptinfo:
return "ckptinfo";
case NotifyData_data_schedUpdateQueue:
return "schedUpdateQueue";
case NotifyData_data_taskPidInfo:
return "taskPidInfo";
case NotifyData_data_taskFailInfo:
return "taskFailInfo";
case NotifyData_data_taskFinish:
return "taskFinish";
case NotifyData_data_postDoneJob:
return "postDoneJob";
case NotifyData_data_schedJobRusages:
return "schedJobRusages";
case NotifyData_data_updateJob:
return "updateJob";
default:
break;
}
snprintf(buf, sizeof(buf), "UNKNOWN(%d)", event);
return buf;
}

67
src/lib/hook.c
View file

@ -1,4 +1,5 @@
#include <errno.h>
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
@ -41,19 +42,75 @@ int xce_add_hook(hooksys_t* s, int typ, hook_fn_t fn, void* data)
return -EINVAL;
}
/* Single scan of the type's slots:
* - if (fn,data) is ALREADY registered and active, adding again is a
* no-op (idempotent). xce_disable_hook removes only ONE matching slot
* and xce_run_hooks fires ALL slots, so a duplicate (fn,data) would
* fire twice and survive a single disable — the bug-845 double-append
* (persist_start + ha_register_persist_hooks both register the same
* persist_job_hook_fun closure). Dedup is keyed on (fn,data): the same
* fn with a DISTINCT closure stays a legitimate second registration.
* - otherwise remember the first disabled (NULL-fn) slot to reuse. */
{
xlist_t* l = &s->hooks[typ];
hook_item_t* reuse = NULL;
int i;
for (i = 0; i < l->count; i++) {
hook_item_t* slot = (hook_item_t*)l->data[i];
if (!slot) continue;
hook_fn_t cur =
atomic_load_explicit(&slot->fn, memory_order_acquire);
if (cur == NULL) {
if (!reuse) reuse = slot;
continue;
}
if (cur == fn && slot->data == data)
return 0; /* already registered — no duplicate */
}
if (reuse != NULL) {
reuse->data = data;
atomic_store_explicit(&reuse->fn, fn, memory_order_release);
return 0;
}
}
/* No reusable slot — allocate new */
it = (hook_item_t*)calloc(1, sizeof(*it));
if (it == NULL) {
return -ENOMEM;
}
it->fn = fn;
it->data = data;
atomic_store_explicit(&it->fn, fn, memory_order_release);
xlist_store(&(s->hooks[typ]), it);
return 0;
}
int xce_disable_hook(hooksys_t* s, int typ, hook_fn_t fn)
{
xlist_t* l;
int i;
if (!s || !fn || typ < 0 || typ >= s->maxtypes) {
return -EINVAL;
}
l = &s->hooks[typ];
for (i = 0; i < l->count; i++) {
hook_item_t* it = (hook_item_t*)l->data[i];
if (!it) continue;
hook_fn_t cur = atomic_load_explicit(&it->fn, memory_order_acquire);
if (cur == fn) {
atomic_store_explicit(&it->fn, (hook_fn_t)NULL,
memory_order_release);
return 0;
}
}
return -ENOENT;
}
/* HOOK_BEFORE_EVENT returns
* negative value will cause
* fail
@ -76,14 +133,20 @@ int xce_run_hooks(hooksys_t* s, int type, int situation, int detail, ...)
for (i = 0; i < l->count; i++) {
hook_item_t* it = l->data[i];
hook_fn_t local_fn;
va_list ap;
if (it == NULL) {
continue;
}
local_fn = atomic_load_explicit(&it->fn, memory_order_acquire);
if (!local_fn) {
continue; /* disabled slot */
}
va_start(ap, detail);
rc = (*(it->fn))(type, situation, detail, it->data, ap);
rc = local_fn(type, situation, detail, it->data, ap);
va_end(ap);
if ((situation == HOOK_BEFORE_EVENT) && (rc < 0)) {

29
src/lib/kv.c
View file

@ -248,6 +248,35 @@ static int kvtab_find(meta_obj_t* obj, void* key, void** value)
return 1;
}
/*
* kvtab_foreach — walk every live entry and invoke cb(key, index, arg).
* index is recovered by casting the stored void* value back to int.
* Caller must hold any necessary external lock before calling.
*/
int kvtab_foreach(var kvtab_var, kvtab_foreach_fn_t cb, void* arg)
{
if (kvtab_var == NULL || cb == NULL) {
return -EINVAL;
}
meta_obj_t* obj = RTTI_DATA_TO_OBJ(kvtab_var);
kvtab_t* t = RTTI_OBJ_TO_DATA(kvtab_t, obj);
int count = 0;
uint32_t i;
for (i = 0; i < t->nslots; i++) {
kvent_t* e = t->slots[i];
while (e != NULL) {
int idx = (int)(long)e->value;
cb(e->key, idx, arg);
count++;
e = e->next;
}
}
return count;
}
typeinfo_t kvtab_type
= { .name = "kvtab",
.privateSize = 0,

112
src/lib/kvmgr.c
View file

@ -1,12 +1,15 @@
#include <errno.h>
#include <inttypes.h>
#include <rtti.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "htab.h"
#include "kv.h"
#include "kvmgr.h"
#include "mylock.h"
#include "persist.h"
#include "utils.h"
#include "xce/types.h"
#include "zarray.h"
@ -290,7 +293,9 @@ typeinfo_t xkvmgr_type = {
.vget_vec = (vget_vec_fn_t)xkvmgr_get_vec,
.vput = (vput_fn_t)xkvmgr_put,
.vput_vec = (vput_vec_fn_t)xkvmgr_put_vec,
.vclear = (vclear_fn_t)NULL },
.vclear = (vclear_fn_t)NULL,
.vfreeze = NULL, /* wired in P0-2 */
.vthaw = NULL /* wired in P0-2 */ },
.iter
= &(intf_iterator_t) { .make = (iter_make_fn_t)xkvmgr_make_iter,
.next = (iter_next_fn_t)xkvmgr_iter_next,
@ -300,3 +305,108 @@ typeinfo_t xkvmgr_type = {
.lock = &(intf_lock_t) { .lock = (lock_fn_t)xkvmgr_lock,
.unlock = (unlock_fn_t)xkvmgr_unlock } }
};
/* ------------------------------------------------------------------ */
/* kvmgr_capture_keys — P0-14a */
/* ------------------------------------------------------------------ */
typedef struct {
slot_key_t* keys;
size_t capacity;
size_t filled;
kvmgr_key_type_t key_type;
int oom;
} capture_ctx_t;
static void capture_one(void* raw_key, int index, void* arg)
{
capture_ctx_t* ctx = (capture_ctx_t*)arg;
if (ctx->oom || ctx->filled >= ctx->capacity) {
return;
}
slot_key_t* sk = &ctx->keys[ctx->filled];
sk->index = index;
if (ctx->key_type == KVMGR_KEY_STRING) {
const char* src = (const char*)raw_key;
strncpy(sk->key, src, XCE_MAXNAME_LEN - 1);
sk->key[XCE_MAXNAME_LEN - 1] = '\0';
sk->key_len = strlen(sk->key) + 1;
} else {
/* Integer key — format as decimal string */
int64_t intval = (int64_t)(long)raw_key;
int n = snprintf(sk->key, XCE_MAXNAME_LEN,
"%" PRId64, intval);
if (n < 0 || n >= XCE_MAXNAME_LEN) {
/* Truncated — mark as oom so caller can detect */
ctx->oom = 1;
return;
}
sk->key_len = (size_t)n + 1;
}
ctx->filled++;
}
/*
* Count live entries by iterating; kvtab_foreach returns the count.
* We use a NULL callback to count first, then allocate, then fill.
*/
static void count_one(void* key, int index, void* arg)
{
(void)key;
(void)index;
size_t* n = (size_t*)arg;
(*n)++;
}
int kvmgr_capture_keys(var kvmgr_var,
slot_key_t** keys_out,
size_t* n_keys_out)
{
*keys_out = NULL;
*n_keys_out = 0;
if (kvmgr_var == NULL) {
return -EINVAL;
}
/* The public API pointer (var) points to the data region;
* convert to the meta_obj_t header, then extract xkvmgr_t. */
meta_obj_t* obj = RTTI_DATA_TO_OBJ(kvmgr_var);
xkvmgr_t* kvm = RTTI_OBJ_TO_DATA(xkvmgr_t, obj);
/* Count live entries */
size_t n = 0;
kvtab_foreach(kvm->kvtab, count_one, &n);
if (n == 0) {
return 0;
}
slot_key_t* keys = calloc(n, sizeof(slot_key_t));
if (keys == NULL) {
return -ENOMEM;
}
capture_ctx_t ctx = {
.keys = keys,
.capacity = n,
.filled = 0,
.key_type = kvm->key_type,
.oom = 0,
};
kvtab_foreach(kvm->kvtab, capture_one, &ctx);
if (ctx.oom) {
free(keys);
return -ENOMEM;
}
*keys_out = keys;
*n_keys_out = ctx.filled;
return 0;
}

18
src/lib/meson.build
View file

@ -68,6 +68,7 @@ libxce_src = [ gen_srcs[0],
xce_resreq_parser[0],
xce_select_parser[0],
'client.c',
'client_failover.c',
'dataq.c',
'hook.c',
'htab.c',
@ -104,6 +105,8 @@ libxce_src = [ gen_srcs[0],
'resobj.c',
'wobj.c',
'configobj.c',
'cluster.c',
'raft/raft.c',
]
incdir = include_directories('../include')
@ -164,6 +167,20 @@ libxce_test_src = ['testsuite/test.c',
'testsuite/test_qobj.c',
'testsuite/test_request.c',
'testsuite/test_cbuf.c',
'testsuite/test_zarray_cow.c',
'testsuite/test_kvmgr_capture.c',
'testsuite/test_hooks.c',
'testsuite/test_cluster.c',
'testsuite/test_cluster_parse.c',
'testsuite/test_libxce_retry.c',
'testsuite/test_netio_on_connected.c',
'testsuite/test_netio_arm_failure.c',
'testsuite/test_netio_close_propagation.c',
'testsuite/test_request_send_propagation.c',
'testsuite/test_raft.c',
'testsuite/test_wal_wire.c',
'testsuite/test_ctrlstatus.c',
'raft/raft_sim.c',
]
libxce_test_deps = [libxce_deps, librtti_dep, unity_dep, libxce_dep]
@ -171,6 +188,7 @@ libxce_test = executable('xcelib_test',
libxce_test_src,
include_directories: [incdir, build_inc],
dependencies: libxce_test_deps,
c_args: ['-DXCE_TESTING'],
install: true)
test('libxce test', libxce_test, timeout: 120)

150
src/lib/netio.c
View file

@ -4,6 +4,7 @@
#include <netdb.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -68,6 +69,41 @@ int io_fsm_start(io_fsm_t* fsm, io_packet_t* pkt)
return 0;
}
/*
* io_classify_io_error: map a failed SSL_read/SSL_write (io_rc <= 0) to an FSM
* step result, given SSL_get_error()'s value and the errno captured right after
* the SSL call.
*
* returns 0 → retriable would-block (WANT_READ/WANT_WRITE, or an
* EAGAIN/EWOULDBLOCK syscall): the FSM stays put and retries on
* the next readiness event.
* returns <0 → the peer closed or reset the connection: the caller MUST move
* the FSM to FSM_STATE_ERROR and propagate this so a blocked
* reply-waiter is woken. A clean EOF reports errno==0, and the
* old code returned `-errno == 0` here — making the close look
* like success, so handle_input never reached handle_error and a
* blocked cbuf_get() hung forever (bug-648). We therefore return
* -ECONNRESET whenever errno is 0.
*
* Pure (no SSL object, no endpoint state) so it is unit-testable.
*/
int io_classify_io_error(int ssl_err, int ecode, int io_rc)
{
switch (ssl_err) {
case SSL_ERROR_WANT_READ:
case SSL_ERROR_WANT_WRITE:
return 0;
case SSL_ERROR_SYSCALL:
if ((io_rc < 0) && ((ecode == EAGAIN) || (ecode == EWOULDBLOCK)))
return 0;
return (ecode != 0) ? -ecode : -ECONNRESET;
default:
/* SSL_ERROR_ZERO_RETURN (clean TLS close_notify) and any other SSL
* error: the session is closed — never report success. */
return (ecode != 0) ? -ecode : -ECONNRESET;
}
}
static int io_fsm_do(io_fsm_t* fsm, char* data, size_t len)
{
int rc = 0;
@ -86,32 +122,15 @@ static int io_fsm_do(io_fsm_t* fsm, char* data, size_t len)
}
if (rc <= 0) {
int err = 0;
int ecode = 0;
int ecode = errno;
int err = SSL_get_error(ep->ssl, rc);
ecode = errno;
err = SSL_get_error(ep->ssl, rc);
switch (err) {
case SSL_ERROR_WANT_READ:
case SSL_ERROR_WANT_WRITE:
rc = 0;
break;
case SSL_ERROR_SYSCALL:
if ((rc < 0) && ((ecode == EAGAIN) || (ecode == EWOULDBLOCK))) {
rc = 0;
} else {
fsm->state = FSM_STATE_ERROR;
rc = -errno;
}
if (rc == 0) {
/* we meet EOF */
fsm->state = FSM_STATE_ERROR;
}
break;
default:
/* Classify the failure. A negative result means the connection is
* closed/errored — move to ERROR and propagate (never return 0 on a
* close, or a blocked reply-waiter is never woken: bug-648). */
rc = io_classify_io_error(err, ecode, rc);
if (rc < 0) {
fsm->state = FSM_STATE_ERROR;
rc = -errno;
break;
}
return rc;
@ -410,7 +429,13 @@ static int handle_output(endpoint_t* ep)
ep->fsm_out.pkt = NULL;
endpoint_update_epoll(ep);
/* Re-arm (drop EPOLLOUT now the queue is drained). A failure here means
* the fd is no longer registered — surface it so io_handler runs the
* error path rather than leaving a silently-dead endpoint (bug-646). */
rc = endpoint_update_epoll(ep);
if (rc < 0) {
return rc;
}
}
return 0;
@ -438,6 +463,10 @@ static int handle_ssl_handshake(endpoint_t* ep)
rc = SSL_do_handshake(ep->ssl);
if (rc == 1) {
ep->ssl_handshake_done = 1;
if (ep->on_connected) {
ep->on_connected(ep);
ep->on_connected = NULL;
}
return 1;
}
@ -449,8 +478,11 @@ static int handle_ssl_handshake(endpoint_t* ep)
rc = 0;
break;
default:
errno = ecode;
perror("SSL_do_handshake");
/* Only log if it's a genuine error (not errno=0 after success) */
if (ecode != 0) {
errno = ecode;
perror("SSL_do_handshake");
}
rc = handle_error(ep);
break;
}
@ -550,27 +582,41 @@ static int endpoint_calc_flags(endpoint_t* ep)
static int endpoint_update_epoll(endpoint_t* ep)
{
int flags;
int rc = 0;
mylock_lock(&(ep->lock));
flags = endpoint_calc_flags(ep);
if (flags != ep->flags) {
if (ep->iomgr != NULL) {
int rc1;
int saved_errno;
mylock_lock(&(ep->iomgr->lock));
rc1 = iomgr_update_poll(ep->iomgr, ep, flags);
rc1 = iomgr_update_poll(ep->iomgr, ep, flags);
saved_errno = errno;
mylock_unlock(&(ep->iomgr->lock));
if (rc1 >= 0) {
ep->flags = flags;
} else {
WARN("rc1=%d < 0", rc1);
/* Do NOT swallow this: an unarmed poll interest silently
* strands all I/O on the endpoint — outbound bytes never flush
* and inbound replies never wake a blocked cbuf_get(), so a
* synchronous caller hangs forever while io_worker sleeps in
* epoll_wait(). EPOLL_CTL_MOD fails with ENOENT when the fd is
* no longer registered (e.g. after a failed reconnect).
* Surface the failure so the caller tears the endpoint down and
* the failover loop rotates instead of hanging (bug-646). */
rc = (saved_errno != 0) ? -saved_errno : -EIO;
WARN("endpoint_update_epoll: iomgr_update_poll failed "
"(rc1=%d errno=%d) — endpoint marked dead",
rc1, saved_errno);
}
}
}
mylock_unlock(&(ep->lock));
return 0;
return rc;
}
static void outq_item_killer(data_item_t* it)
@ -658,12 +704,15 @@ int endpoint_connect(endpoint_t* ep, char* host, int port)
return rc;
}
ep->ssl = SSL_new(iomgr->ssl_ctx);
/* If ep->ssl is already set (e.g. pre-created with client context for
* outgoing connections), use it; otherwise create from iomgr->ssl_ctx. */
if (ep->ssl == NULL) {
ep->ssl = SSL_new(iomgr->ssl_ctx);
SSL_set_connect_state(ep->ssl);
}
SSL_set_fd(ep->ssl, ep->fd);
SSL_set_connect_state(ep->ssl);
set_fd_nonblocking(ep->fd);
ep->flags = 0;
@ -769,6 +818,19 @@ int endpoint_set_err_handler(endpoint_t* ep, err_handler_t h)
return 1;
}
int endpoint_set_on_connected(endpoint_t* ep, void (*cb)(endpoint_t*))
{
if (ep == NULL) {
return -EINVAL;
}
mylock_lock(&(ep->lock));
ep->on_connected = cb;
mylock_unlock(&(ep->lock));
return 0;
}
int endpoint_enq(endpoint_t* ep, data_item_t* it)
{
int rc = 0;
@ -783,7 +845,22 @@ int endpoint_enq(endpoint_t* ep, data_item_t* it)
rc = dataq_enq(&(ep->outq), it);
mylock_unlock(&(ep->lock));
endpoint_update_epoll(ep);
/* Arm the output interest. If the poll registration cannot be updated
* (e.g. EPOLL_CTL_MOD on an fd no longer registered after a failed
* reconnect), the enqueued bytes will never flush and the endpoint is dead.
* Surface a RETRIABLE transport error so the caller propagates it (the
* request.c senders return it; the client failover loop classifies
* -ECONNRESET as retriable and rotates — bug-650).
*
* Do NOT invoke handle_error()/the err_handler here. endpoint_enq() runs on
* the *caller's* thread, which may already hold a lock the err_handler needs
* — e.g. the controller's primary_heartbeat_fn holds ha->lock and
* vote_ep_err_handler re-acquires it, self-deadlocking the controller
* (bug-652). The real teardown stays on the io_worker thread, which drives
* handle_error() off the next epoll error event holding no caller locks. */
if (endpoint_update_epoll(ep) < 0) {
return -ECONNRESET;
}
return rc;
}
@ -926,6 +1003,11 @@ SSL_CTX* create_ssl_context(char* cert_file, char* key_file)
int iomgr_init(iomgr_t* iomgr)
{
/* Ignore SIGPIPE — writing to a closed peer socket returns EPIPE
* instead of killing the daemon. All daemons call iomgr_init before
* any socket I/O, so this protects controller, worker, and scheduler. */
signal(SIGPIPE, SIG_IGN);
mylock_init(&(iomgr->lock));
iomgr->epoll_fd = epoll_create1(0);
if (iomgr->epoll_fd < 0) {

853
src/lib/raft/raft.c Normal file
View file

@ -0,0 +1,853 @@
#include "raft.h"
void raft_snapshot(const raft_state_t *s, raft_snapshot_t *snap)
{
snap->term = s->term;
snap->role = s->role;
snap->voted_for = s->voted_for;
snap->voted_for_term = s->voted_for_term;
snap->votes_granted = s->votes_granted;
snap->election_decided = s->election_decided;
}
int raft_init(raft_state_t *s, const raft_config_t *cfg)
{
/* Validate required callbacks and n_peers before touching *s */
if (cfg->n_peers < 1)
return -EINVAL;
/* Upper bound: next_index[]/match_index[]/pre_vote_replies[] are fixed-size
* RAFT_MAX_PEERS arrays indexed by peer in [0, n_peers). Reject oversize
* clusters so the leader-progress and pre-vote loops can never OOB-write. */
if (cfg->n_peers > RAFT_MAX_PEERS)
return -EINVAL;
/* Even cluster sizes (>1) have no fault-tolerance advantage over the next
* smaller odd size and can mislead operators into thinking split-brain is
* impossible. Reject to prevent misconfiguration. */
if (cfg->n_peers > 1 && cfg->n_peers % 2 == 0)
return -EINVAL;
if (!cfg->cbs)
return -EINVAL;
if (!cfg->cbs->on_send_vote_request)
return -EINVAL;
if (!cfg->cbs->on_become_leader)
return -EINVAL;
if (!cfg->cbs->on_become_follower)
return -EINVAL;
if (!cfg->cbs->on_persist)
return -EINVAL;
if (!cfg->cbs->on_schedule_timeout)
return -EINVAL;
if (!cfg->cbs->on_cancel_timeout)
return -EINVAL;
if (!cfg->cbs->on_start_election_after_delay)
return -EINVAL;
/* Log-replication required callbacks (raft-log-replication Req 7.1–7.4):
* all four must be non-NULL — the library owns the protocol decisions and
* delegates the I/O (persist/read-term/send/truncate) through them. */
if (!cfg->cbs->on_persist_entry)
return -EINVAL;
if (!cfg->cbs->on_get_entry_term)
return -EINVAL;
if (!cfg->cbs->on_send_append_entries)
return -EINVAL;
if (!cfg->cbs->on_truncate_from)
return -EINVAL;
/* Commit-advance trigger (raft-commit-safety Req 2.5): required — the library
* decides whether commit_index advanced; this callback triggers the
* controller's existing commit barrier. */
if (!cfg->cbs->on_commit_advanced)
return -EINVAL;
s->term = cfg->initial_term;
s->role = RAFT_FOLLOWER;
s->voted_for = cfg->initial_voted_for;
/* A restored vote belongs to the restored term: persistence writes
* (term, voted_for) as a pair. Leaving voted_for_term at 0 would let the
* duplicate-vote check miss after restart, granting a second vote in the
* same term (Election Safety violation). */
s->voted_for_term = (cfg->initial_voted_for != -1) ? cfg->initial_term : 0;
s->n_peers = cfg->n_peers;
s->self_idx = cfg->self_idx;
s->votes_granted = 0;
s->replies_received = 0;
/* Defensive: a node must never carry stale per-peer vote flags into its
* first election (mirrors pre_vote_replies). raft_start_election re-clears
* them, but zero here so a never-elected node has a clean array. */
memset(s->vote_replies, 0, sizeof(s->vote_replies));
s->election_decided = 0;
s->election_timeout_ms = cfg->election_timeout_ms ? cfg->election_timeout_ms : 2000;
s->my_priority = cfg->my_priority;
/* Log length restored from the adapter's recovered WAL length; the library
* never reads storage (same discipline as initial_term/voted_for). 0 on
* first boot. (raft-log-replication Req 1.1; design "log_len restored at
* init from initial_log_len".) */
s->log_len = cfg->initial_log_len;
/* Commit index restored from the adapter's recovered committed LSN; the
* library never reads storage (same discipline as initial_term/log_len). 0 on
* first boot. (raft-commit-safety Req 1.1.) */
s->commit_index = cfg->initial_commit_index;
s->cbs = cfg->cbs;
s->ctx = cfg->ctx;
return 0;
}
/* Initialize per-peer leader progress on becoming leader (raft.tla BecomeLeader,
* Req 3.1): next_index[j] = log_len + 1, match_index[j] = 0 for every peer j. */
static void raft_init_leader_progress(raft_state_t *s)
{
for (int j = 0; j < s->n_peers; j++) {
s->next_index[j] = s->log_len + 1;
s->match_index[j] = 0;
}
}
int raft_start_election(raft_state_t *s)
{
/* Guard: reject if already CANDIDATE or LEADER */
if (s->role == RAFT_CANDIDATE || s->role == RAFT_LEADER)
return -EALREADY;
/* Increment term and record self-vote before any RPCs (split-brain prevention) */
s->term++;
s->voted_for = s->self_idx;
s->voted_for_term = s->term;
s->votes_granted = 1; /* self-vote */
s->replies_received = 0;
/* Fresh per-peer vote-reply tally for this election (mirrors
* pre_vote_replies reset). Mark self as already-counted so a stray
* self-addressed reply cannot double-count the self-vote that
* votes_granted already includes. */
memset(s->vote_replies, 0, sizeof(s->vote_replies));
if (s->self_idx >= 0 && s->self_idx < RAFT_MAX_PEERS)
s->vote_replies[s->self_idx] = 1;
s->election_decided = 0;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "term_update", NULL);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "self_vote", NULL);
/* Persist new term and self-vote before sending any RPCs */
s->cbs->on_persist(s->ctx, s->term, s->self_idx);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "election_start", NULL);
/* Single-node shortcircuit: win immediately, no RPCs, no timeout */
if (s->n_peers == 1) {
s->role = RAFT_LEADER;
s->election_decided = 1;
raft_init_leader_progress(s);
s->cbs->on_become_leader(s->ctx, s->term);
return RAFT_ELECTED_IMMEDIATELY;
}
/* Multi-node: transition to CANDIDATE */
s->role = RAFT_CANDIDATE;
/* Send RequestVote to all peers except self */
for (int i = 0; i < s->n_peers; i++) {
if (i == s->self_idx)
continue;
s->cbs->on_send_vote_request(s->ctx, i, s->term);
}
/* Schedule election timeout watchdog */
s->cbs->on_schedule_timeout(s->ctx, s->election_timeout_ms);
return RAFT_OK;
}
/* (lastLogTerm, lastLogIndex) freshness predicate (raft.tla
* HandleRequestVoteRequest logOk, line 285, closes G6). The voter's own last-log
* tuple is (on_get_entry_term(log_len), log_len) — term 0 when the log is empty,
* matching the on_get_entry_term 0-means-absent sentinel. The library owns this
* decision (design "Option B: library owns logOk") and applies it identically on
* the binding vote and the PreVote eligibility paths. */
static int raft_log_ok(const raft_state_t *s, uint64_t cand_last_log_term,
uint64_t cand_last_log_index)
{
uint64_t local_last_log_term =
(s->log_len > 0) ? s->cbs->on_get_entry_term(s->ctx, s->log_len) : 0;
return (cand_last_log_term > local_last_log_term) ||
(cand_last_log_term == local_last_log_term &&
cand_last_log_index >= s->log_len);
}
int raft_on_vote_request(raft_state_t *s, int from_idx, uint64_t term,
uint64_t cand_last_log_term,
uint64_t cand_last_log_index,
uint64_t *reply_term)
{
/* Step 1: stale term — deny immediately, no state change */
if (term < s->term) {
*reply_term = s->term;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_stale_term", NULL);
return RAFT_DENY;
}
/* Step 2: term upgrade — update term, clear vote, step down to FOLLOWER.
* If we were LEADER, notify the HA FSM so it updates ha->role.
* Without this, the HA FSM stays in CTRL_PRIMARY after losing the term,
* never stepping down to STANDBY → split-brain. */
if (term > s->term) {
int was_leader = (s->role == RAFT_LEADER);
s->term = term;
s->voted_for = -1;
s->voted_for_term = 0;
s->role = RAFT_FOLLOWER;
if (was_leader) {
s->cbs->on_become_follower(s->ctx, s->term, from_idx);
}
}
/* reply_term reflects our (possibly updated) current term from here on */
*reply_term = s->term;
/* Step 3: role gate — same-term challenger denied by live leader.
*
* This only fires when term == s->term (the equal-term case). For
* term > s->term, Step 2 already downgraded s->role to RAFT_FOLLOWER
* before we reach here, so the check below is false and we fall through
* to grant/deny based on freshness.
*
* A RequestVote for the same term as the current leader means the sender
* started a competing candidacy in a term the leader already won — deny. */
if (s->role == RAFT_LEADER) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_role", NULL);
return RAFT_DENY;
}
/* Step 4: duplicate-vote check — already voted in this term.
* Idempotent re-grant: if this is the same candidate we already voted for
* in this term, fall through so we re-persist and re-grant (network retry
* path). This is safe: the candidate has already committed to this term
* and we already durably recorded the vote.
* Different candidate: deny — standard Raft invariant of at most one vote
* per node per term. */
if (s->voted_for != -1 && s->voted_for_term == term) {
if (s->voted_for != from_idx) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_already_voted", NULL);
return RAFT_DENY;
}
/* same candidate — fall through to re-persist and re-grant */
}
/* Step 5: (lastLogTerm, lastLogIndex) freshness gate (raft.tla
* HandleRequestVoteRequest logOk, 285, closes G6). This precedes the
* application gate: a !logOk denies regardless of on_vote_eligible (Req 3.4),
* without weakening the stale-term / one-vote-per-term / persist-before-grant
* invariants above (Req 3.5). */
if (!raft_log_ok(s, cand_last_log_term, cand_last_log_index)) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_log_stale", NULL);
return RAFT_DENY;
}
/* Step 6: application-level policy gate (freshness, log lsn, etc.) */
if (s->cbs->on_vote_eligible != NULL) {
if (s->cbs->on_vote_eligible(s->ctx, from_idx, term) == 0) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_denied_freshness", NULL);
return RAFT_DENY;
}
}
/* Step 7: persist before granting — durability guarantee */
if (s->cbs->on_persist(s->ctx, term, from_idx) != 0) {
/* Persist failed — deny without updating voted_for */
return RAFT_DENY;
}
/* Grant: record vote */
s->voted_for = from_idx;
s->voted_for_term = term;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_granted", NULL);
return RAFT_GRANT;
}
int raft_on_pre_vote_request(raft_state_t *s, int from_idx, uint64_t cand_term,
uint64_t cand_last_log_term,
uint64_t cand_last_log_index)
{
/* PreVote is a side-effect-free straw poll: it answers "would you vote for
* me in cand_term?" WITHOUT changing any persistent state. Unlike
* raft_on_vote_request, this function never adopts cand_term (even when
* higher — that adoption is the very thing PreVote exists to prevent), never
* touches term / voted_for / voted_for_term / role, and never calls
* on_persist. The library applies the SAME (lastLogTerm, lastLogIndex)
* logOk freshness gate as the real-vote path (via raft_log_ok), so a stale
* candidate is rejected here exactly as it would be in a binding vote. */
/* Step 1: stale term — deny (mirrors raft_on_vote_request Step 1). A
* candidate proposing a term we already passed could never win a real
* election, so it cannot win a PreVote either. No term adoption: even when
* cand_term > s->term we only evaluate, never adopt. */
if (cand_term < s->term) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_denied_stale_term", NULL);
return RAFT_DENY;
}
/* Step 2: (lastLogTerm, lastLogIndex) freshness gate — the IDENTICAL logOk
* as the binding vote path (raft.tla 285, closes G6 for PreVote, Req 4.1).
* The library owns this predicate (the dispatcher no longer applies freshness
* before this call). Side-effect-free: a deny mutates nothing and never
* persists (Req 4.4). */
if (!raft_log_ok(s, cand_last_log_term, cand_last_log_index)) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_denied_log_stale", NULL);
return RAFT_DENY;
}
/* Step 3: application-level eligibility gate — reuse the EXISTING
* on_vote_eligible callback (primary-deny / fresh-heartbeat stickiness /
* catch-up deny), identical to raft_on_vote_request's application gate. If
* NULL, every Raft-eligible candidate is granted. */
if (s->cbs->on_vote_eligible != NULL) {
if (s->cbs->on_vote_eligible(s->ctx, from_idx, cand_term) == 0) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_denied_eligibility", NULL);
return RAFT_DENY;
}
}
/* Grant — record NOTHING. No voted_for write (so a later real vote in this
* term is still grantable), no persist, no term/role change. */
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_granted", NULL);
return RAFT_GRANT;
}
int raft_begin_pre_vote(raft_state_t *s)
{
/* Pre-image of raft_start_election: probe peers + tally a self-grant, but
* STRIP every persistent side effect. Per the header INVARIANT (Req 1.2,
* 1.3, 6.2) this function MUST NOT touch s->term, s->voted_for,
* s->voted_for_term, or s->role, and MUST NOT call on_persist. The node
* stays FOLLOWER throughout the round; CANDIDATE happens later in the
* unchanged raft_start_election once the PreVote round is WON. */
/* Prospective term is a LOCAL value only — never assigned to s->term and
* never persisted (this is the whole point of PreVote). */
uint64_t prospective_term = s->term + 1;
/* Reset and arm the round tally on the raft state. Self always grants. */
s->pre_vote_in_progress = 1;
s->pre_votes_granted = 1; /* self-grant */
memset(s->pre_vote_replies, 0, sizeof(s->pre_vote_replies));
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_start", NULL);
/* Single-node: no peers, the self-grant is already a majority. Win
* immediately and let the caller proceed to raft_start_election. */
if (s->n_peers == 1)
return RAFT_PRE_VOTE_WON;
/* Probe every peer except self with the prospective term (NULL-checked:
* if the callback is unset, the caller owns the transport). */
if (s->cbs->on_send_pre_vote_request) {
for (int i = 0; i < s->n_peers; i++) {
if (i == s->self_idx)
continue;
s->cbs->on_send_pre_vote_request(s->ctx, i, prospective_term);
}
}
return RAFT_OK;
}
int raft_on_pre_vote_reply(raft_state_t *s, int peer_idx, int granted)
{
/* Pure straw-poll tally — mirrors raft_on_vote_reply's quorum and
* all-replied-no-quorum logic, but STRIPS every side effect. Per the
* header INVARIANT (Req 1.4, 1.5, 6.2) this function MUST NOT fire
* on_become_leader / on_become_follower, MUST NOT touch s->term /
* s->voted_for / s->voted_for_term / s->role, and MUST NOT call on_persist.
* It only writes the pre_vote_* tally fields and returns a status the
* controller (task 4.1) acts on. There is no term in a pre-vote reply, so
* no term adoption is even possible here. */
/* Bounds guard: ignore replies from an out-of-range peer (mirrors the
* per-peer dedup array bound). A self reply (peer_idx == self_idx) is
* never expected — self is granted by raft_begin_pre_vote — and would also
* be filtered by the already-set flag, but the range check alone is the
* defensive boundary. */
if (peer_idx < 0 || peer_idx >= s->n_peers || peer_idx >= RAFT_MAX_PEERS) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_reply_out_of_range", NULL);
return RAFT_PRE_VOTE_PENDING;
}
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_reply_in", NULL);
/* Per-peer idempotency: a duplicate reply from a peer we already counted is
* ignored (no double-count). Stay undecided. */
if (s->pre_vote_replies[peer_idx]) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_reply_dup", NULL);
return RAFT_PRE_VOTE_PENDING;
}
s->pre_vote_replies[peer_idx] = 1;
/* Accumulate grant. pre_votes_granted already includes the self-grant set
* by raft_begin_pre_vote. */
if (granted)
s->pre_votes_granted++;
/* Quorum mirrors raft_on_vote_reply: n_peers is the TOTAL cluster size
* including self. */
int quorum = (s->n_peers / 2) + 1;
if (s->pre_votes_granted >= quorum) {
s->pre_vote_in_progress = 0;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_won", NULL);
return RAFT_PRE_VOTE_WON;
}
/* All non-self peers have replied without reaching quorum → lost. Count
* the set per-peer flags (a self reply, if it ever arrived, would have been
* range-filtered out of n_peers - 1 expectation only if self_idx is in
* range, so compare against n_peers - 1 replied flags). */
int replied = 0;
for (int i = 0; i < s->n_peers; i++)
replied += s->pre_vote_replies[i];
if (replied == s->n_peers - 1) {
s->pre_vote_in_progress = 0;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "pre_vote_lost", NULL);
return RAFT_PRE_VOTE_LOST;
}
/* Still awaiting replies. */
return RAFT_PRE_VOTE_PENDING;
}
int raft_on_vote_reply(raft_state_t *s, int from_idx, int granted, uint64_t term)
{
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_reply_in", NULL);
/* Step 1: term upgrade — higher term means we lost the election */
if (term > s->term) {
s->term = term;
s->role = RAFT_FOLLOWER;
s->election_decided = 1;
s->cbs->on_become_follower(s->ctx, s->term, -1);
s->cbs->on_cancel_timeout(s->ctx);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "became_follower", NULL);
return RAFT_BECAME_FOLLOWER;
}
/* DropStaleResponse (raft.tla DropStaleResponse, mterm < currentTerm):
* a reply from an older term is discarded, NOT counted toward this
* election — and it is NOT a higher term, so it must not step us down. */
if (term < s->term) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_reply_stale_drop", NULL);
return RAFT_PENDING;
}
/* Step 2: no-op guard — election already decided */
if (s->election_decided) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_reply_noop", NULL);
return RAFT_PENDING;
}
/* Bounds guard: ignore a reply from an out-of-range peer (mirrors the
* pre-vote per-peer dedup bound). */
if (from_idx < 0 || from_idx >= s->n_peers || from_idx >= RAFT_MAX_PEERS) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_reply_out_of_range", NULL);
return RAFT_PENDING;
}
/* Per-peer idempotency (raft.tla votesGranted is a SET): a duplicate reply
* from a peer we already counted is ignored — a legal DuplicateMessage must
* not double-count toward quorum (Election Safety). */
if (s->vote_replies[from_idx]) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "vote_reply_dup", NULL);
return RAFT_PENDING;
}
s->vote_replies[from_idx] = 1;
/* Step 3: count this reply */
s->replies_received++;
/* Step 4: accumulate grant */
if (granted)
s->votes_granted++;
/* Step 5: quorum check */
int quorum = (s->n_peers / 2) + 1;
if (s->votes_granted >= quorum) {
s->role = RAFT_LEADER;
s->election_decided = 1;
raft_init_leader_progress(s);
s->cbs->on_become_leader(s->ctx, s->term);
s->cbs->on_cancel_timeout(s->ctx);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "quorum_reached", NULL);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "became_leader", NULL);
return RAFT_BECAME_LEADER;
}
/* Step 6: all-replied-no-quorum */
if (s->replies_received == s->n_peers - 1 && s->votes_granted < quorum) {
s->role = RAFT_FOLLOWER;
s->election_decided = 1;
s->cbs->on_become_follower(s->ctx, s->term, -1);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "became_follower", NULL);
return RAFT_BECAME_FOLLOWER;
}
/* Step 7: still waiting */
return RAFT_PENDING;
}
void raft_on_election_timeout(raft_state_t *s)
{
if (s->election_decided) {
/* Req 6.3: cancel any pending timer and be a no-op */
s->cbs->on_cancel_timeout(s->ctx);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "election_timeout_noop", NULL);
return;
}
s->role = RAFT_FOLLOWER;
s->election_decided = 1;
s->cbs->on_become_follower(s->ctx, s->term, -1);
s->cbs->on_cancel_timeout(s->ctx);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "election_timeout", NULL);
}
void raft_on_leader_lost(raft_state_t *s)
{
s->voted_for = -1;
s->role = RAFT_FOLLOWER;
s->cbs->on_start_election_after_delay(s->ctx,
(uint32_t)s->my_priority * s->election_timeout_ms);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "leader_lost", NULL);
}
int raft_check_and_start_election(raft_state_t *s)
{
if (s->voted_for != -1 && s->voted_for != s->self_idx) {
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "jitter_skip", NULL);
return RAFT_SKIP_ELECTION;
}
/* Precondition: raft_on_leader_lost sets role=FOLLOWER, so -EALREADY is unreachable here */
int rc = raft_start_election(s);
(void)rc;
return RAFT_ELECTION_STARTED;
}
int raft_log_append(raft_state_t *s, const void *entry, size_t len,
uint64_t *out_index)
{
/* Role gate (Req 2.2): only a leader may grow the log. Reject without any
* log mutation or callback (raft.tla ClientRequest is enabled only at the
* leader). */
if (s->role != RAFT_LEADER)
return -EPERM;
/* Assign the next 1-based index (== WAL LSN) and stamp the leader's current
* term, mirroring raft.tla ClientRequest's [term ↦ currentTerm, value ↦ v]
* (Req 2.1, 2.3, 2.4). */
uint64_t index = s->log_len + 1;
uint64_t term = s->term;
/* Persist-before-act (Req 2.3, 2.5): durably write the entry BEFORE
* advancing log_len. On failure, propagate the error verbatim and leave the
* log untouched (no phantom entry). */
int rc = s->cbs->on_persist_entry(s->ctx, index, term, entry, len);
if (rc != 0)
return rc;
/* Commit the append only after a successful persist. */
s->log_len = index;
if (out_index)
*out_index = index;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "log_append", NULL);
return RAFT_OK;
}
int raft_send_append_entries(raft_state_t *s, int peer_idx,
uint64_t leader_commit)
{
/* Leader-only (Req 3.5): AppendEntries is enabled only at the leader in
* raft.tla. Off the leader this is a no-op — no callback, no state change. */
if (s->role != RAFT_LEADER)
return RAFT_OK;
/* prevLogIndex = nextIndex[j] - 1; prevLogTerm = on_get_entry_term(prevIdx)
* when prevIdx > 0, else 0 (the "absent" sentinel, matching raft.tla
* prevLogTerm == 0 when prevLogIndex == 0). Do NOT query the callback at
* index 0 (Req 3.2). */
uint64_t prev_log_index = s->next_index[peer_idx] - 1;
uint64_t prev_log_term = (prev_log_index > 0)
? s->cbs->on_get_entry_term(s->ctx, prev_log_index)
: 0;
/* lastEntry = Min(log_len, next_index[j]); send SubSeq(log, next_index[j],
* lastEntry) — at most one entry. When next_index[j] > log_len there is
* nothing to send → a 0-entry heartbeat (Req 3.3). */
int n_entries = (s->next_index[peer_idx] <= s->log_len) ? 1 : 0;
/* leader_commit is passed through opaquely; this spec never interprets or
* advances commit (Req 3.4 — commit is owned by raft-commit-safety). */
int rc = s->cbs->on_send_append_entries(s->ctx, peer_idx, prev_log_index,
prev_log_term, leader_commit,
n_entries);
if (rc != 0)
return rc;
/* Trace only on a successful send (mirrors raft_log_append, which traces
* after a confirmed persist) — a failed emit must not look successful. */
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "ae_send", NULL);
return RAFT_OK;
}
/* Fill *out as a rejection (success=0, match_index=0) at the current term and
* return RAFT_OK. Every reject branch shares this shape (raft.tla reject, 333). */
static int raft_ae_reject(raft_state_t *s, raft_append_result_t *out,
const char *trace_event)
{
out->reply_term = s->term;
out->success = 0;
out->match_index = 0;
if (trace_event && s->cbs->on_trace)
s->cbs->on_trace(s->ctx, trace_event, NULL);
return RAFT_OK;
}
int raft_on_append_entries_request(raft_state_t *s, int leader_idx,
uint64_t term, uint64_t prev_log_index,
uint64_t prev_log_term, uint64_t leader_commit,
const void *entry, size_t entry_len,
uint64_t entry_term, int entries_len,
raft_append_result_t *out)
{
/* leader_commit is accepted into the contract but NOT acted on in this spec
* — commit-index advance is owned by raft-commit-safety (Req 3.4 / 5.4). */
(void)leader_commit;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "ae_recv", NULL);
/* ---- Step 1: UpdateTerm FIRST (raft.tla 406, Req 6.1–6.5 — closes G5) ----
* Applied on EVERY incoming AppendEntries/heartbeat, BEFORE logOk, so a
* strictly-higher-term RPC is never spuriously rejected for a log mismatch.
* Strictly-greater-than only: equal/lower term never alters term/voted_for. */
if (term > s->term) {
int was_leader = (s->role == RAFT_LEADER);
s->term = term;
s->voted_for = -1;
s->voted_for_term = 0;
s->role = RAFT_FOLLOWER;
/* Persist the adopted (term, voted_for) before processing the rest of
* the RPC (durability ordering, Req 6.2). */
s->cbs->on_persist(s->ctx, s->term, -1);
/* A node that was leader must notify the HA FSM so it steps down
* (Req 6.3), mirroring the vote-path higher-term step-down. */
if (was_leader)
s->cbs->on_become_follower(s->ctx, s->term, leader_idx);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "term_adopt_ae", NULL);
}
/* ---- Step 2: stale-term reject (raft.tla 333) ----
* After UpdateTerm, term < s->term can only happen for a strictly-lower
* incoming term (the higher case adopted above). */
if (term < s->term)
return raft_ae_reject(s, out, "ae_reject_stale");
/* ---- Step 3: candidate step-down at equal term (raft.tla 346) ----
* A candidate that sees an AppendEntries at its own term concedes that a
* leader already exists for this term and returns to follower. */
if (s->role == RAFT_CANDIDATE)
s->role = RAFT_FOLLOWER;
/* ---- Step 4: logOk (raft.tla 328, Req 4.1 — closes G2) ----
* logOk = prevLogIndex == 0
* OR (prevLogIndex <= log_len AND on_get_entry_term(prevLogIndex) == prevLogTerm).
* The explicit prevLogIndex <= log_len bound mirrors raft.tla:330
* (prevLogIndex <= Len(log)). The on_get_entry_term 0 sentinel already
* rejects absent entries for any non-zero prevLogTerm, but the bound also
* defends against a malformed sender presenting prevLogTerm==0 with
* prevLogIndex past the local log (0==0 would otherwise spuriously pass). */
int log_ok = (prev_log_index == 0) ||
(prev_log_index <= s->log_len &&
s->cbs->on_get_entry_term(s->ctx, prev_log_index) == prev_log_term);
/* ---- Step 5: reject if !logOk (follower path, raft.tla 333, Req 4.2) ---- */
if (!log_ok)
return raft_ae_reject(s, out, "ae_reject_logmismatch");
/* ---- Step 6: accept (raft.tla 351), index = prevLogIndex + 1 ----
* match_index reported on success is prevLogIndex + entries_len for every
* accept sub-case (already-have / conflict-truncate / append). */
uint64_t index = prev_log_index + 1;
uint64_t match_index = prev_log_index + (uint64_t)entries_len;
/* already-have: a heartbeat (no entries), or the local entry at `index`
* already carries the incoming entry's term → success, no log rewrite. */
if (entries_len == 0 ||
s->cbs->on_get_entry_term(s->ctx, index) == entry_term) {
out->reply_term = s->term;
out->success = 1;
out->match_index = match_index;
/* must not shrink the log (already-have keeps the existing suffix) */
if (s->log_len < match_index)
s->log_len = match_index;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "ae_already_have", NULL);
return RAFT_OK;
}
/* conflict-truncate (raft.tla 375, Req 4.5 — closes G3): a local entry at
* `index` exists with a DIFFERENT term. Delete the divergent suffix, then
* (intentional safe collapse) re-persist the leader's entry in the same
* handler invocation. The truncate is ALWAYS followed by the re-persist. */
if (s->cbs->on_get_entry_term(s->ctx, index) != 0) {
int rc = s->cbs->on_truncate_from(s->ctx, index);
if (rc != 0)
/* truncate failed → report a non-success so the leader retries; the
* follower's log is not corrupted (it simply did not converge). */
return raft_ae_reject(s, out, "ae_conflict_truncate_fail");
rc = s->cbs->on_persist_entry(s->ctx, index, entry_term, entry, entry_len);
if (rc != 0)
return raft_ae_reject(s, out, "ae_conflict_persist_fail");
out->reply_term = s->term;
out->success = 1;
out->match_index = match_index;
s->log_len = match_index;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "ae_conflict_truncate", NULL);
return RAFT_OK;
}
/* append (raft.tla 383, Req 4.6): the local log ends exactly at
* prevLogIndex (no entry at `index`) → append the incoming entry. */
{
int rc = s->cbs->on_persist_entry(s->ctx, index, entry_term, entry, entry_len);
if (rc != 0)
return raft_ae_reject(s, out, "ae_append_persist_fail");
out->reply_term = s->term;
out->success = 1;
out->match_index = match_index;
if (s->log_len < match_index)
s->log_len = match_index;
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "ae_append", NULL);
return RAFT_OK;
}
}
int raft_advance_commit(raft_state_t *s)
{
/* Leader-only (raft.tla AdvanceCommitIndex is enabled only at the leader).
* Off the leader: -EPERM, no change, no callback. */
if (s->role != RAFT_LEADER)
return -EPERM;
/* Compute Max(agreeIndexes): the highest index N (1..log_len) for which a
* QUORUM of the cluster has match_index >= N. Self is treated as having all
* its own entries (match_index == log_len), so it counts toward every
* candidate N <= log_len. Scan from log_len downward and stop at the first N
* that reaches quorum (that N is the maximum, since lower indexes are
* implied by every node that agrees on a higher one). (raft.tla 259/262.) */
int quorum = (s->n_peers / 2) + 1;
uint64_t agreed = 0; /* 0 = no index reached quorum */
for (uint64_t n = s->log_len; n >= 1; n--) {
int count = 1; /* self always has index n (n <= log_len) */
for (int j = 0; j < s->n_peers; j++) {
if (j == s->self_idx)
continue;
if (s->match_index[j] >= n)
count++;
}
if (count >= quorum) {
agreed = n;
break;
}
}
/* No index reached quorum → nothing to commit. */
if (agreed == 0)
return RAFT_OK;
/* Current-term Figure-8 rule (raft.tla 270): advance commit ONLY when the
* entry at the agreed index is from the CURRENT term. A leader must never
* commit a previous-term entry on replication count alone — doing so risks
* the Figure-8 safety violation where a committed entry is later
* overwritten. */
if (s->cbs->on_get_entry_term(s->ctx, agreed) != s->term)
return RAFT_OK;
/* Monotonic: never lower commit_index; only a strict advance fires the
* callback (and exactly once). */
if (agreed <= s->commit_index)
return RAFT_OK;
s->commit_index = agreed;
s->cbs->on_commit_advanced(s->ctx, s->commit_index);
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, "commit_advanced", NULL);
return RAFT_OK;
}
int raft_on_append_entries_reply(raft_state_t *s, int peer_idx, int success,
uint64_t match_index, uint64_t reply_term)
{
/* ---- Stale-term drop FIRST (raft.tla HandleAppendEntriesResponse 415, Req 5.4) ----
* The response is meaningful only if it pertains to the leader's current
* term (m.mterm == currentTerm[i]). A reply carrying any other term — from
* an old in-flight RPC or a future leader — is discarded with no state
* change. No commit index is ever advanced here (Req 5.4; owned by
* raft-commit-safety). */
if (reply_term != s->term)
return RAFT_OK;
if (success) {
/* success (raft.tla 397, Req 5.2): nextIndex'[peer] = m.mmatchIndex + 1,
* matchIndex'[peer] = m.mmatchIndex. */
s->match_index[peer_idx] = match_index;
s->next_index[peer_idx] = match_index + 1;
} else {
/* failure (raft.tla 401, Req 5.3): nextIndex'[peer] =
* Max(nextIndex[peer] - 1, 1) — back off one index, floored at 1.
* matchIndex is left unchanged. */
uint64_t next = s->next_index[peer_idx];
s->next_index[peer_idx] = (next > 1) ? (next - 1) : 1;
}
if (s->cbs->on_trace)
s->cbs->on_trace(s->ctx, success ? "ae_reply_success" : "ae_reply_fail",
NULL);
return RAFT_OK;
}

1147
src/lib/raft/raft.h Normal file
View file

File diff suppressed because it is too large Load diff

2510
src/lib/raft/raft_sim.c Normal file
View file

File diff suppressed because it is too large Load diff

1089
src/lib/request.c
View file

File diff suppressed because it is too large Load diff

107
src/lib/testsuite/failover_harness.h Normal file
View file

@ -0,0 +1,107 @@
/*
* failover_harness.h — reusable, test-only fixtures for the libxce-failover
* spec (test_libxce_retry.c and the later component tasks 2.1/2.2/2.3/3.1/3.2,
* plus the concurrency tasks 5.1/6.1).
*
* This header is HARNESS ONLY. It references NO not-yet-existing failover
* production symbols (e.g. xce_failover_classify, xcecli_retry_next,
* xce_failover_rotate) — those arrive in later tasks. Everything here compiles
* and links against the current libxce surface so the lib test suite stays
* GREEN between this task (1.2) and the component tasks.
*
* What it provides:
* 1. A fake multi-master client config (3 masters) + a master-list cursor
* that the rotation logic will later consume (advance/wrap).
* 2. A controller-status taxonomy table mapping every status the failover
* classifier must handle (REDIRECT with/without hint, NOMASTER, NOT_READY,
* STANDBY, DUP_JOB/EEXIST) and the transport errors (ECONNREFUSED,
* ETIMEDOUT, ECONNRESET) to a stable description + expected failover
* action category — so later tasks can table-drive the classifier RED/GREEN.
* 3. A generic N-pthread concurrency harness so 5.1/6.1 can spawn N threads
* against a shared ctx without re-rolling pthread plumbing.
*
* Requirements laid down for: 2.5, 4.1-4.5, 5.2, 8.3, 8.4 (taxonomy table);
* 2.1, 2.4, 8.4 (master cursor); 6.3 / concurrency reuse (thread harness).
*/
#ifndef XCE_FAILOVER_HARNESS_H
#define XCE_FAILOVER_HARNESS_H
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdbool.h>
#include <stddef.h>
#include "xce/cluster.h" /* xce_cluster_config_t, xce_candidate_t, MAX_HA_CANDIDATES */
#include "xce/errnox.h" /* E_XCE_REDIRECT / NOMASTER / NOT_READY / STANDBY / DUP_JOB */
/* ── 1. Fake multi-master config + cursor ─────────────────────────────── */
/* Number of fake masters the rotation fixtures stand up. Kept <= MAX_HA_CANDIDATES. */
#define FO_FAKE_MASTERS 3
/* A master-list cursor: mirrors the "advance to next master / wrap" behaviour
* the real EndpointRotator (task 2.2) will drive over api_ctx->conf. Pure value
* type — no I/O, no production dependency. */
typedef struct {
const xce_cluster_config_t *cfg;
int cur; /* index into cfg->candidates[] */
} fo_master_cursor_t;
/* Populate `cfg` with FO_FAKE_MASTERS distinct host:port candidates. Returns the
* candidate count written. */
int fo_make_fake_masters(xce_cluster_config_t *cfg);
/* Initialise a cursor at master 0. */
void fo_cursor_init(fo_master_cursor_t *cur, const xce_cluster_config_t *cfg);
/* Advance the cursor to the next master, wrapping at the end. Returns the new
* index. */
int fo_cursor_advance(fo_master_cursor_t *cur);
/* The candidate the cursor currently points at (NULL if cfg empty). */
const xce_candidate_t *fo_cursor_current(const fo_master_cursor_t *cur);
/* ── 2. Controller-status taxonomy table ──────────────────────────────── */
/* Expected failover action for each status — mirrors design.md StatusClassifier
* (XCE_FO_SUCCESS / RETRY_NOW / RETRY_BACKOFF / TERMINAL). This is the HARNESS'
* expectation table; the real xce_fo_action_t enum is introduced by task 2.1,
* and its tests will assert the real classifier matches these expectations. */
typedef enum {
FO_ACT_SUCCESS = 0, /* status 0, or idempotent DUP replay on a re-sent write */
FO_ACT_RETRY_NOW, /* REDIRECT / STANDBY / transport error — rotate, no backoff */
FO_ACT_RETRY_BACKOFF,/* NOMASTER / NOT_READY — rotate after jittered backoff */
FO_ACT_TERMINAL /* non-retriable — surface to caller */
} fo_expected_action_t;
typedef struct {
const char *name; /* human label, e.g. "E_XCE_REDIRECT+hint" */
int status; /* status/rc as the classifier sees it */
bool has_hint; /* redirect carried an address hint? */
bool is_transport;/* negative -errno transport error? */
fo_expected_action_t expect; /* expected first-send classification */
const char *desc; /* one-line description */
} fo_status_row_t;
/* The complete taxonomy table and its length. Defined in test_libxce_retry.c.
* Covers: REDIRECT (with + without hint), NOMASTER, NOT_READY, STANDBY,
* DUP_JOB, -EEXIST, and the three transport errors. */
extern const fo_status_row_t FO_STATUS_TABLE[];
extern const size_t FO_STATUS_TABLE_LEN;
/* Look up a row by its `status` field (first match), or NULL. */
const fo_status_row_t *fo_status_lookup(int status);
/* ── 3. Generic N-thread concurrency harness ──────────────────────────── */
/* A worker fn run on each spawned thread. `ctx` is a shared context the caller
* owns; `idx` is the thread's 0-based index. */
typedef void (*fo_thread_fn)(void *ctx, int idx);
/* Spawn `n` pthreads, each invoking `fn(ctx, idx)`, and join them all. Returns 0
* if every thread was created and joined, -errno on the first failure. Generic
* so 5.1 (shared-ctx safety) and 6.1 (50-client jitter) reuse it unchanged. */
int fo_run_threads(int n, fo_thread_fn fn, void *ctx);
#endif /* XCE_FAILOVER_HARNESS_H */

26
src/lib/testsuite/test.c
View file

@ -30,6 +30,19 @@ extern void test_qobj(void);
extern void test_wobj(void);
extern void test_request(void);
extern void test_cbuf(void);
extern void test_zarray_cow(void);
extern void test_kvmgr_capture(void);
extern void test_hooks(void);
extern void test_cluster(void);
extern void test_cluster_parse(void);
extern void test_libxce_retry(void);
extern void test_netio_on_connected(void);
extern void test_netio_arm_failure(void);
extern void test_netio_close_propagation(void);
extern void test_request_send_propagation(void);
extern void test_raft(void);
extern void test_wal_wire(void);
extern void test_ctrlstatus(void);
void setUp(void) { return; }
@ -129,6 +142,19 @@ int main(int argc, char* argv[])
TEST(test_qobj);
TEST(test_request);
TEST(test_cbuf);
TEST(test_zarray_cow);
TEST(test_kvmgr_capture);
TEST(test_hooks);
TEST(test_cluster);
TEST(test_cluster_parse);
TEST(test_libxce_retry);
TEST(test_netio_on_connected);
TEST(test_netio_arm_failure);
TEST(test_netio_close_propagation);
TEST(test_request_send_propagation);
TEST(test_raft);
TEST(test_wal_wire);
TEST(test_ctrlstatus);
return UNITY_END();
}

181
src/lib/testsuite/test_cluster.c Normal file
View file

@ -0,0 +1,181 @@
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <unity/unity.h>
#include "xce/cluster.h"
/* ── Test seam: mock connect function ───────────────────────────── */
static int s_connect_call_count = 0;
static int s_connect_fail_n = 0; /* fail first N attempts, then succeed */
static char s_last_host[256] = "";
static int s_last_port = 0;
static void* mock_connect(const char* host, int port, const char* ca_cert)
{
(void)ca_cert;
strncpy(s_last_host, host, sizeof(s_last_host) - 1);
s_last_port = port;
s_connect_call_count++;
if (s_connect_call_count <= s_connect_fail_n)
return NULL;
return (void*)0xDEADBEEF; /* non-NULL = success */
}
static xce_cluster_config_t make_valid_config(void)
{
xce_cluster_config_t cfg = {0};
strncpy(cfg.candidates[0].hostname, "10.0.1.1",
XCE_MAXHOSTNAME_LEN - 1);
cfg.candidates[0].port = 7000;
strncpy(cfg.candidates[1].hostname, "10.0.1.2",
XCE_MAXHOSTNAME_LEN - 1);
cfg.candidates[1].port = 7001;
cfg.candidates_count = 2;
strncpy(cfg.ca_cert_path, "/etc/xce/ca.pem",
sizeof(cfg.ca_cert_path) - 1);
strncpy(cfg.identity, "worker-01", sizeof(cfg.identity) - 1);
return cfg;
}
/* ── Test cases ─────────────────────────────────────────────────── */
/* Test 1: validate rejects empty config */
static void test_validate_rejects_zero_candidates(void)
{
xce_cluster_config_t cfg = {0};
cfg.candidates_count = 0;
TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg));
printf("test_validate_rejects_zero_candidates: PASS\n");
}
/* Test 2: validate rejects too many candidates */
static void test_validate_rejects_over_max_candidates(void)
{
xce_cluster_config_t cfg = {0};
cfg.candidates_count = MAX_HA_CANDIDATES + 1;
TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg));
printf("test_validate_rejects_over_max_candidates: PASS\n");
}
/* Test 3: validate rejects missing identity */
static void test_validate_rejects_empty_identity(void)
{
xce_cluster_config_t cfg = make_valid_config();
cfg.identity[0] = '\0';
TEST_ASSERT_NOT_EQUAL(0, xce_cluster_config_validate(&cfg));
printf("test_validate_rejects_empty_identity: PASS\n");
}
/* Test 4: validate accepts valid config */
static void test_validate_accepts_valid_config(void)
{
xce_cluster_config_t cfg = make_valid_config();
TEST_ASSERT_EQUAL_INT(0, xce_cluster_config_validate(&cfg));
printf("test_validate_accepts_valid_config: PASS\n");
}
/* Test 5: connect succeeds on first candidate */
static void test_connect_succeeds_first_candidate(void)
{
xce_cluster_config_t cfg = make_valid_config();
s_connect_call_count = 0;
s_connect_fail_n = 0;
xce_cluster_set_connect_fn(mock_connect);
void* ep = xce_connect_cluster(&cfg);
TEST_ASSERT_NOT_NULL(ep);
TEST_ASSERT_EQUAL_INT(1, s_connect_call_count);
TEST_ASSERT_EQUAL_STRING("10.0.1.1", s_last_host);
TEST_ASSERT_EQUAL_INT(7000, s_last_port);
xce_cluster_set_connect_fn(NULL);
printf("test_connect_succeeds_first_candidate: PASS\n");
}
/* Test 6: connect falls back to second candidate when first fails */
static void test_connect_fallback_to_second_candidate(void)
{
xce_cluster_config_t cfg = make_valid_config();
s_connect_call_count = 0;
s_connect_fail_n = 1; /* first attempt fails */
xce_cluster_set_connect_fn(mock_connect);
void* ep = xce_connect_cluster(&cfg);
TEST_ASSERT_NOT_NULL(ep);
TEST_ASSERT_EQUAL_INT(2, s_connect_call_count);
TEST_ASSERT_EQUAL_STRING("10.0.1.2", s_last_host);
xce_cluster_set_connect_fn(NULL);
printf("test_connect_fallback_to_second_candidate: PASS\n");
}
/* Test 7: connect returns NULL when all candidates fail */
static void test_connect_returns_null_all_fail(void)
{
xce_cluster_config_t cfg = make_valid_config();
s_connect_call_count = 0;
s_connect_fail_n = 999; /* all fail */
xce_cluster_set_connect_fn(mock_connect);
void* ep = xce_connect_cluster(&cfg);
TEST_ASSERT_NULL(ep);
xce_cluster_set_connect_fn(NULL);
printf("test_connect_returns_null_all_fail: PASS\n");
}
/* Test 8: reconnect state machine — IDLE → CONNECTING → CONNECTED */
static void test_reconnect_step_reaches_connected(void)
{
xce_cluster_config_t cfg = make_valid_config();
xce_reconnect_ctx_t rctx;
s_connect_call_count = 0;
s_connect_fail_n = 0;
xce_cluster_set_connect_fn(mock_connect);
xce_reconnect_init(&rctx, &cfg);
TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_IDLE, (int)rctx.state);
int steps = 0;
while (rctx.state != XCE_RECONNECT_CONNECTED &&
rctx.state != XCE_RECONNECT_FAILED && steps < 100) {
xce_reconnect_step(&rctx);
steps++;
}
TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_CONNECTED, (int)rctx.state);
TEST_ASSERT_NOT_NULL(rctx.endpoint);
xce_reconnect_fini(&rctx);
xce_cluster_set_connect_fn(NULL);
printf("test_reconnect_step_reaches_connected: PASS\n");
}
/* Test 9: reconnect state machine backs off on failure */
static void test_reconnect_step_backs_off_on_failure(void)
{
xce_cluster_config_t cfg = make_valid_config();
cfg.candidates_count = 1; /* only one candidate so it hits backoff */
xce_reconnect_ctx_t rctx;
s_connect_call_count = 0;
s_connect_fail_n = 999; /* always fail */
xce_cluster_set_connect_fn(mock_connect);
xce_reconnect_init(&rctx, &cfg);
xce_reconnect_step(&rctx); /* IDLE → CONNECTING (attempt connect) */
xce_reconnect_step(&rctx); /* CONNECTING → BACKOFF (all candidates exhausted) */
/* When mock is set, sleep is skipped — state must be BACKOFF */
TEST_ASSERT_EQUAL_INT(XCE_RECONNECT_BACKOFF, (int)rctx.state);
TEST_ASSERT_TRUE(rctx.backoff_ms > 0);
xce_reconnect_fini(&rctx);
xce_cluster_set_connect_fn(NULL);
printf("test_reconnect_step_backs_off_on_failure: PASS\n");
}
/* ── Test group entry point ─────────────────────────────────────── */
void test_cluster(void)
{
RUN_TEST(test_validate_rejects_zero_candidates);
RUN_TEST(test_validate_rejects_over_max_candidates);
RUN_TEST(test_validate_rejects_empty_identity);
RUN_TEST(test_validate_accepts_valid_config);
RUN_TEST(test_connect_succeeds_first_candidate);
RUN_TEST(test_connect_fallback_to_second_candidate);
RUN_TEST(test_connect_returns_null_all_fail);
RUN_TEST(test_reconnect_step_reaches_connected);
RUN_TEST(test_reconnect_step_backs_off_on_failure);
}

160
src/lib/testsuite/test_cluster_parse.c Normal file
View file

@ -0,0 +1,160 @@
/* test_cluster_parse.c — unit tests for the parse_peers() helper used by
* worker and scheduler main.c (P2.5-5 / P2.5-6).
*
* The parser is a static function in the respective main.c files. To keep
* the test self-contained we duplicate a simplified reference implementation
* here and test the same contract. Integration with the real binaries is
* covered by the build (both must compile).
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "xce/cluster.h"
/* ── Reference implementation of parse_peers (mirrors worker/main.c) ─── */
static int parse_peers_local(const char* peers_str, xce_cluster_config_t* cfg)
{
cfg->candidates_count = 0;
if (!peers_str || peers_str[0] == '\0')
return -1;
char buf[2048];
strncpy(buf, peers_str, sizeof(buf) - 1);
buf[sizeof(buf) - 1] = '\0';
char* saveptr;
char* tok = strtok_r(buf, ",", &saveptr);
while (tok && cfg->candidates_count < MAX_HA_CANDIDATES) {
/* Format: name=host:port OR host:port */
char* eq = strchr(tok, '=');
char* addr = eq ? eq + 1 : tok;
char* colon = strrchr(addr, ':');
if (!colon)
return -1;
*colon = '\0';
strncpy(cfg->candidates[cfg->candidates_count].hostname, addr,
XCE_MAXHOSTNAME_LEN - 1);
cfg->candidates[cfg->candidates_count].hostname[XCE_MAXHOSTNAME_LEN - 1]
= '\0';
cfg->candidates[cfg->candidates_count].port = atoi(colon + 1);
cfg->candidates_count++;
tok = strtok_r(NULL, ",", &saveptr);
}
return cfg->candidates_count > 0 ? 0 : -1;
}
/* ── Test cases ─────────────────────────────────────────────────────── */
static void test_parse_peers_single(void)
{
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local("ctrl-1=10.0.1.1:7000", &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(1, cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("10.0.1.1", cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(7000, cfg.candidates[0].port);
printf("test_parse_peers_single: PASS\n");
}
static void test_parse_peers_multiple(void)
{
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local(
"ctrl-1=10.0.1.1:7000,ctrl-2=10.0.1.2:7001,ctrl-3=10.0.1.3:7002",
&cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(3, cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("10.0.1.2", cfg.candidates[1].hostname);
TEST_ASSERT_EQUAL_INT(7001, cfg.candidates[1].port);
TEST_ASSERT_EQUAL_STRING("10.0.1.3", cfg.candidates[2].hostname);
TEST_ASSERT_EQUAL_INT(7002, cfg.candidates[2].port);
printf("test_parse_peers_multiple: PASS\n");
}
static void test_parse_peers_no_name(void)
{
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local("10.0.1.1:7000,10.0.1.2:7001", &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(2, cfg.candidates_count);
TEST_ASSERT_EQUAL_STRING("10.0.1.1", cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(7000, cfg.candidates[0].port);
TEST_ASSERT_EQUAL_STRING("10.0.1.2", cfg.candidates[1].hostname);
TEST_ASSERT_EQUAL_INT(7001, cfg.candidates[1].port);
printf("test_parse_peers_no_name: PASS\n");
}
static void test_parse_peers_empty_returns_error(void)
{
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local("", &cfg);
TEST_ASSERT_NOT_EQUAL(0, rc);
printf("test_parse_peers_empty_returns_error: PASS\n");
}
static void test_parse_peers_null_returns_error(void)
{
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local(NULL, &cfg);
TEST_ASSERT_NOT_EQUAL(0, rc);
printf("test_parse_peers_null_returns_error: PASS\n");
}
static void test_parse_peers_missing_port_returns_error(void)
{
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local("ctrl-1=10.0.1.1", &cfg); /* no colon:port */
TEST_ASSERT_NOT_EQUAL(0, rc);
printf("test_parse_peers_missing_port_returns_error: PASS\n");
}
static void test_parse_peers_first_entry_parsed_correctly(void)
{
/* Verify the name= prefix is stripped and only addr:port is stored */
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local("primary=192.168.10.1:9000", &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_STRING("192.168.10.1", cfg.candidates[0].hostname);
TEST_ASSERT_EQUAL_INT(9000, cfg.candidates[0].port);
printf("test_parse_peers_first_entry_parsed_correctly: PASS\n");
}
static void test_parse_peers_respects_max_candidates(void)
{
/* Build a string with MAX_HA_CANDIDATES + 2 entries; only MAX accepted */
char big[2048] = "";
char entry[128] = "";
int n = MAX_HA_CANDIDATES + 2;
for (int i = 0; i < n; i++) {
snprintf(entry, sizeof(entry), "10.0.1.%d:%d", i + 1, 7000 + i);
if (i > 0)
strncat(big, ",", sizeof(big) - strlen(big) - 1);
strncat(big, entry, sizeof(big) - strlen(big) - 1);
}
xce_cluster_config_t cfg = {0};
int rc = parse_peers_local(big, &cfg);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(MAX_HA_CANDIDATES, cfg.candidates_count);
printf("test_parse_peers_respects_max_candidates: PASS\n");
}
/* ── Test group entry point ─────────────────────────────────────────── */
void test_cluster_parse(void)
{
RUN_TEST(test_parse_peers_single);
RUN_TEST(test_parse_peers_multiple);
RUN_TEST(test_parse_peers_no_name);
RUN_TEST(test_parse_peers_empty_returns_error);
RUN_TEST(test_parse_peers_null_returns_error);
RUN_TEST(test_parse_peers_missing_port_returns_error);
RUN_TEST(test_parse_peers_first_entry_parsed_correctly);
RUN_TEST(test_parse_peers_respects_max_candidates);
}

22
src/lib/testsuite/test_config.c
View file

@ -27,6 +27,27 @@ static void test_array(void)
TEST_ASSERT_TRUE(MAX_ARRAY_IDX == 32767);
}
static void test_persistence_constants(void)
{
TEST_ASSERT_TRUE(MAX_STANDBYS == 3);
TEST_ASSERT_TRUE(MAX_HA_CANDIDATES == 4);
TEST_ASSERT_TRUE(WAL_FORMAT_VERSION == 1);
TEST_ASSERT_TRUE(WAL_ENTRY_TYPE_MAX == 100);
TEST_ASSERT_TRUE(MAX_WAL_PAYLOAD_BYTES == (1 << 20));
TEST_ASSERT_TRUE(MAX_SNAPSHOT_BYTES == (256 * 1024 * 1024));
TEST_ASSERT_TRUE(DEFAULT_WAL_SYNC_MS == 10);
TEST_ASSERT_TRUE(DEFAULT_SNAP_INTERVAL_S == 60);
TEST_ASSERT_TRUE(DEFAULT_SNAP_WAL_SIZE_MB == 64);
TEST_ASSERT_TRUE(DEFAULT_SNAP_KEEP_N == 2);
TEST_ASSERT_TRUE(DEFAULT_FINISHED_RETAIN_S == 3600);
TEST_ASSERT_TRUE(DEFAULT_WORKER_RETAIN_S == 86400);
TEST_ASSERT_TRUE(DEFAULT_SYNC_TIMEOUT_MS == 500);
TEST_ASSERT_TRUE(DEFAULT_HB_INTERVAL_MS == 3000);
TEST_ASSERT_TRUE(DEFAULT_TRANSIENT_JOB_TIMEOUT_S == 300);
TEST_ASSERT_TRUE(WAL_HDR_CRC_SIZE == 32);
TEST_ASSERT_TRUE(WAL_CRC_NONE == 0xFFFFFFFFu);
}
void test_config(void)
{
printf("\n>> config test\n\n");
@ -36,4 +57,5 @@ void test_config(void)
RUN_TEST(test_timestamp);
RUN_TEST(test_tag);
RUN_TEST(test_array);
RUN_TEST(test_persistence_constants);
}

405
src/lib/testsuite/test_ctrlstatus.c Normal file
View file

@ -0,0 +1,405 @@
/* test_ctrlstatus.c — Unity tests for the controller-status (ctrlstatus)
* libxce client reply path (controller-status task 3.1).
*
* Proves the reply path that xcecli_ctrlstatus relies on:
* 1. A true ENCODE→DECODE round-trip of xce_ctrlstatus_t through the
* generated Cap'n Proto codec (encode_CtrlStatus/write_CtrlStatus →
* bytes → read_CtrlStatus/decode_CtrlStatus): every status field,
* including the peers list and the configChecksum hex Text, survives
* the wire (Req 1.1 — the client decodes every field a handler emits).
* 2. xcecli_ctrlstatus(NULL, ...) returns a typed negative rather than a
* partial/empty success (Req 7.3 — no reachable controller ⇒ typed
* failure).
*
* A full live round-trip (client → running controller → handler reply)
* needs a controller and is the E2E task; here we prove the encode/decode
* round-trip + the no-context typed failure only.
*/
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "client.h"
#include "config.h" /* XCE_MAXHOSTNAME_LEN */
#include "request.h"
#include "xce/types.h"
#include "xcectx.h" /* xce_ctx_t, xce_handle_reply (bug-1077 regression) */
/* generated Cap'n Proto codec for CtrlStatus / CtrlPeer */
#include "controller.capnp.h"
/* ── encode→decode round-trip of xce_ctrlstatus_t ───────────────────────── */
static void test_ctrlstatus_roundtrip_all_fields(void)
{
/* Arrange: a fully-populated status with two configured peers */
xce_ctrlpeer_t p0 = {
.name = "ctrl-a", .host = "10.0.0.1", .port = 9991
};
xce_ctrlpeer_t p1 = {
.name = "ctrl-b", .host = "10.0.0.2", .port = 9992
};
xce_ctrlpeer_t* peerlist[2] = { &p0, &p1 };
xce_ctrlstatus_t in;
memset(&in, 0, sizeof(in));
in.clusterName = "lab-cluster";
in.primaryName = "ctrl-a";
in.responderName = "ctrl-b";
in.responderRole = 3; /* STANDBY */
in.term = 42;
in.configVersion = 0x1122334455667788ULL;
in.configChecksum = "1122334455667788aabbccddeeff0011"
"2233445566778899aabbccddeeff0011";
in.committedConfigVersion = 0x1122334455667788ULL;
in.committedLsn = 5000;
in.appliedLsn = 4999;
in.fenced = 1;
in.peers.count = 2;
in.peers.list = peerlist;
in.peerCount = 2;
/* Act: encode into a capnp segment, serialize to bytes, then decode
* a fresh struct back out — mirrors xce_encode/xce_decode. */
struct capn c;
struct capn_segment* cs;
struct CtrlStatus cap_in;
CtrlStatus_ptr sp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
memset(&cap_in, 0, sizeof(cap_in));
encode_CtrlStatus(cs, &cap_in, &in);
sp = new_CtrlStatus(cs);
write_CtrlStatus(&cap_in, sp);
capn_setp(capn_root(&c), 0, sp.p);
size_t sz = capn_size(&c);
uint8_t* buf = (uint8_t*)malloc(sz);
TEST_ASSERT_NOT_NULL(buf);
TEST_ASSERT_EQUAL_size_t(sz, capn_write_mem(&c, buf, sz, 0));
capn_free(&c);
/* decode side */
struct capn dc;
struct CtrlStatus cap_out;
CtrlStatus_ptr dp;
xce_ctrlstatus_t out;
capn_init_mem(&dc, buf, sz, 0);
dp.p = capn_getp(capn_root(&dc), 0, 1);
read_CtrlStatus(&cap_out, dp);
memset(&out, 0, sizeof(out));
decode_CtrlStatus(&out, &cap_out);
/* Assert: every scalar/text field survived */
TEST_ASSERT_NOT_NULL(out.clusterName);
TEST_ASSERT_EQUAL_STRING("lab-cluster", out.clusterName);
TEST_ASSERT_NOT_NULL(out.primaryName);
TEST_ASSERT_EQUAL_STRING("ctrl-a", out.primaryName);
TEST_ASSERT_NOT_NULL(out.responderName);
TEST_ASSERT_EQUAL_STRING("ctrl-b", out.responderName);
TEST_ASSERT_EQUAL_UINT8(3, out.responderRole);
TEST_ASSERT_EQUAL_UINT64(42, out.term);
TEST_ASSERT_EQUAL_UINT64(0x1122334455667788ULL, out.configVersion);
TEST_ASSERT_NOT_NULL(out.configChecksum);
TEST_ASSERT_EQUAL_STRING(in.configChecksum, out.configChecksum);
TEST_ASSERT_EQUAL_UINT64(0x1122334455667788ULL, out.committedConfigVersion);
TEST_ASSERT_EQUAL_UINT64(5000, out.committedLsn);
TEST_ASSERT_EQUAL_UINT64(4999, out.appliedLsn);
TEST_ASSERT_EQUAL_UINT32(1, out.fenced);
TEST_ASSERT_EQUAL_UINT32(2, out.peerCount);
/* Assert: the peers list survived with both entries */
TEST_ASSERT_EQUAL_INT(2, out.peers.count);
TEST_ASSERT_NOT_NULL(out.peers.list);
TEST_ASSERT_NOT_NULL(out.peers.list[0]);
TEST_ASSERT_EQUAL_STRING("ctrl-a", out.peers.list[0]->name);
TEST_ASSERT_EQUAL_STRING("10.0.0.1", out.peers.list[0]->host);
TEST_ASSERT_EQUAL_UINT32(9991, out.peers.list[0]->port);
TEST_ASSERT_NOT_NULL(out.peers.list[1]);
TEST_ASSERT_EQUAL_STRING("ctrl-b", out.peers.list[1]->name);
TEST_ASSERT_EQUAL_STRING("10.0.0.2", out.peers.list[1]->host);
TEST_ASSERT_EQUAL_UINT32(9992, out.peers.list[1]->port);
free_CtrlStatus(&out);
capn_free(&dc);
free(buf);
printf("test_ctrlstatus_roundtrip_all_fields: PASS\n");
}
/* An "unknown" primary (empty primaryName) must round-trip as empty/NULL,
* never garbage — the client renders it as "unknown" (Req 1.3, 2.2). */
static void test_ctrlstatus_roundtrip_unknown_primary(void)
{
xce_ctrlstatus_t in;
memset(&in, 0, sizeof(in));
in.clusterName = "lab-cluster";
in.primaryName = ""; /* unknown */
in.responderName = "ctrl-a";
in.responderRole = 1; /* ELECTING */
in.peers.count = 0;
in.peers.list = NULL;
in.peerCount = 0;
struct capn c;
struct capn_segment* cs;
struct CtrlStatus cap_in;
CtrlStatus_ptr sp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
memset(&cap_in, 0, sizeof(cap_in));
encode_CtrlStatus(cs, &cap_in, &in);
sp = new_CtrlStatus(cs);
write_CtrlStatus(&cap_in, sp);
capn_setp(capn_root(&c), 0, sp.p);
size_t sz = capn_size(&c);
uint8_t* buf = (uint8_t*)malloc(sz);
TEST_ASSERT_NOT_NULL(buf);
capn_write_mem(&c, buf, sz, 0);
capn_free(&c);
struct capn dc;
struct CtrlStatus cap_out;
CtrlStatus_ptr dp;
xce_ctrlstatus_t out;
capn_init_mem(&dc, buf, sz, 0);
dp.p = capn_getp(capn_root(&dc), 0, 1);
read_CtrlStatus(&cap_out, dp);
memset(&out, 0, sizeof(out));
decode_CtrlStatus(&out, &cap_out);
TEST_ASSERT_EQUAL_STRING("ctrl-a", out.responderName);
TEST_ASSERT_EQUAL_UINT8(1, out.responderRole);
/* empty primaryName decodes to NULL or "" — not a stale/guessed value */
if (out.primaryName != NULL) {
TEST_ASSERT_EQUAL_STRING("", out.primaryName);
}
TEST_ASSERT_EQUAL_INT(0, out.peers.count);
TEST_ASSERT_EQUAL_UINT32(0, out.peerCount);
free_CtrlStatus(&out);
capn_free(&dc);
free(buf);
printf("test_ctrlstatus_roundtrip_unknown_primary: PASS\n");
}
/* ── no reachable controller ⇒ typed failure (Req 7.3) ──────────────────── */
static void test_xcecli_ctrlstatus_null_ctx_typed_failure(void)
{
xce_ctrlstatus_t* status = (xce_ctrlstatus_t*)0xdeadbeef;
int rc = xcecli_ctrlstatus(NULL, &status);
/* typed negative, not a partial/empty success */
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
printf("test_xcecli_ctrlstatus_null_ctx_typed_failure: PASS\n");
}
/* ── authoritative (primary-only) read: task 3.2 ─────────────────────────── */
/* The authoritative call must reject a NULL/unconnected ctx with the same
* typed negative as the any-node read — it must NEVER hand back a hint reply
* as the authoritative result (Req 9.2). The `status` out-param is left
* untouched on failure (it still holds the poisoned sentinel below). */
static void test_xcecli_ctrlstatus_primary_null_ctx_typed_failure(void)
{
xce_ctrlstatus_t* status = (xce_ctrlstatus_t*)0xdeadbeef;
int rc = xcecli_ctrlstatus_primary(NULL, &status);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
/* never returned a hint reply as the authoritative result */
TEST_ASSERT_EQUAL_PTR((xce_ctrlstatus_t*)0xdeadbeef, status);
printf("test_xcecli_ctrlstatus_primary_null_ctx_typed_failure: PASS\n");
}
/* NULL status out-param is also a typed failure (boundary validation). */
static void test_xcecli_ctrlstatus_primary_null_out_typed_failure(void)
{
int rc = xcecli_ctrlstatus_primary(NULL, NULL);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
printf("test_xcecli_ctrlstatus_primary_null_out_typed_failure: PASS\n");
}
/* The pure authoritative-decision helper: a reply is authoritative iff the
* responder's role is PRIMARY (ctrl_role_t PRIMARY == 0). A standby (1),
* init (2), or electing (3) responder is a non-authoritative HINT — the call
* must keep following the redirect (Req 9.1, 2.3, 3.2). */
static void test_ctrlstatus_is_authoritative_decision(void)
{
xce_ctrlstatus_t reply;
memset(&reply, 0, sizeof(reply));
reply.responderRole = 0; /* PRIMARY */
TEST_ASSERT_TRUE(xcecli_ctrlstatus_is_authoritative(&reply));
reply.responderRole = 1; /* STANDBY */
TEST_ASSERT_FALSE(xcecli_ctrlstatus_is_authoritative(&reply));
reply.responderRole = 2; /* INIT */
TEST_ASSERT_FALSE(xcecli_ctrlstatus_is_authoritative(&reply));
reply.responderRole = 3; /* ELECTING */
TEST_ASSERT_FALSE(xcecli_ctrlstatus_is_authoritative(&reply));
/* NULL reply is never authoritative (defensive). */
TEST_ASSERT_FALSE(xcecli_ctrlstatus_is_authoritative(NULL));
printf("test_ctrlstatus_is_authoritative_decision: PASS\n");
}
/* The pure next-target helper: given a standby's hint reply, map the reported
* primaryName to its host:port via the reply's peers list so the next query
* can target the primary directly (the redirect step of Req 9.1). */
static void test_ctrlstatus_primary_addr_from_peers(void)
{
xce_ctrlpeer_t p0 = { .name = "ctrl-a", .host = "10.0.0.1", .port = 9991 };
xce_ctrlpeer_t p1 = { .name = "ctrl-b", .host = "10.0.0.2", .port = 9992 };
xce_ctrlpeer_t* peerlist[2] = { &p0, &p1 };
xce_ctrlstatus_t hint;
memset(&hint, 0, sizeof(hint));
hint.responderName = "ctrl-a";
hint.responderRole = 1; /* STANDBY */
hint.primaryName = "ctrl-b"; /* the standby knows ctrl-b is primary */
hint.peers.count = 2;
hint.peers.list = peerlist;
hint.peerCount = 2;
char host[XCE_MAXHOSTNAME_LEN];
uint16_t port = 0;
int rc = xcecli_ctrlstatus_primary_addr(&hint, host, sizeof(host), &port);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_STRING("10.0.0.2", host);
TEST_ASSERT_EQUAL_UINT16(9992, port);
printf("test_ctrlstatus_primary_addr_from_peers: PASS\n");
}
/* When the standby reports no known primary (empty primaryName) or the named
* primary is not in the peers list, the address lookup fails (-ENOENT) so the
* caller falls back to plain master rotation rather than targeting a bogus
* address (Req 2.3 unknown-primary, redirect best-effort). */
static void test_ctrlstatus_primary_addr_unknown_or_missing(void)
{
xce_ctrlpeer_t p0 = { .name = "ctrl-a", .host = "10.0.0.1", .port = 9991 };
xce_ctrlpeer_t* peerlist[1] = { &p0 };
xce_ctrlstatus_t hint;
memset(&hint, 0, sizeof(hint));
hint.responderName = "ctrl-a";
hint.responderRole = 3; /* ELECTING */
hint.peers.count = 1;
hint.peers.list = peerlist;
hint.peerCount = 1;
char host[XCE_MAXHOSTNAME_LEN];
uint16_t port = 7;
/* unknown primary (empty) */
hint.primaryName = "";
TEST_ASSERT_EQUAL_INT(-ENOENT, xcecli_ctrlstatus_primary_addr(
&hint, host, sizeof(host), &port));
/* NULL primary */
hint.primaryName = NULL;
TEST_ASSERT_EQUAL_INT(-ENOENT, xcecli_ctrlstatus_primary_addr(
&hint, host, sizeof(host), &port));
/* named primary not present in peers */
hint.primaryName = "ctrl-z";
TEST_ASSERT_EQUAL_INT(-ENOENT, xcecli_ctrlstatus_primary_addr(
&hint, host, sizeof(host), &port));
/* NULL/zero args are rejected */
hint.primaryName = "ctrl-a";
TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_ctrlstatus_primary_addr(
NULL, host, sizeof(host), &port));
TEST_ASSERT_EQUAL_INT(-EINVAL, xcecli_ctrlstatus_primary_addr(
&hint, NULL, sizeof(host), &port));
TEST_ASSERT_EQUAL_INT(
-EINVAL, xcecli_ctrlstatus_primary_addr(&hint, host, 0, &port));
TEST_ASSERT_EQUAL_INT(
-EINVAL,
xcecli_ctrlstatus_primary_addr(&hint, host, sizeof(host), NULL));
printf("test_ctrlstatus_primary_addr_unknown_or_missing: PASS\n");
}
/* ── reply-union type-confusion regression (bug-1077) ───────────────────────
*
* ctx->reply.u is a single untagged union shared by every opcode (debugmsg /
* get / jobid / clusterconf / ctrlstatus all alias the same 8 bytes). A
* failover-aborted reslist/joblist leaves ctx->reply.u.get pointing at the
* caller's (now-dead) stack `getreply`. When the next ctrlstatus reply lands on
* a worker thread, handle_reply must NOT treat that foreign, non-NULL slot as a
* CtrlStatus* and free it — doing so dereferenced a dead stack address inside
* free_CtrlStatus and crashed (ASan stack-use-after-return at
* controller.capnp.c:11638).
*
* This drives xce_handle_reply directly with a poisoned union slot and proves
* the foreign occupant is left untouched (no free, no deref). Pre-fix, the
* ctrlstatus branch frees the sentinel and the explicit free() below is a
* double-free that ASan aborts on. */
static void test_handle_reply_ctrlstatus_does_not_free_foreign_slot(void)
{
xce_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
/* A prior get/reslist leaked a non-CtrlStatus pointer into the shared slot.
* Sized as a zeroed xce_ctrlstatus_t so that, even on the buggy path,
* free_CtrlStatus(sentinel) only walks all-NULL fields (no wild deref) and
* the regression surfaces as a clean double-free rather than a hard crash. */
xce_ctrlstatus_t* sentinel = (xce_ctrlstatus_t*)calloc(1, sizeof(*sentinel));
TEST_ASSERT_NOT_NULL(sentinel);
ctx.reply.u.get = (dataset_t*)sentinel; /* same 8 bytes as u.ctrlstatus */
/* A genuine ctrlstatus reply arrives for this ctx. */
xce_ctrlstatus_t* cs = (xce_ctrlstatus_t*)calloc(1, sizeof(*cs));
TEST_ASSERT_NOT_NULL(cs);
cs->clusterName = strdup("eda1");
xce_reply_t reply;
memset(&reply, 0, sizeof(reply));
reply.opcode = OPCode_ctrlstatus;
reply.status = 0;
reply.data_which = RequestReply_data_ctrlstatus;
reply.data.ctrlstatus = cs;
int rc = xce_handle_reply(&ctx, &reply);
/* The genuine reply is now owned by the ctx (MOVE_PTR nulls the source) … */
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_EQUAL_PTR(cs, ctx.reply.u.ctrlstatus);
TEST_ASSERT_NULL(reply.data.ctrlstatus);
/* … and the foreign sentinel was NOT freed by the reply handler. Freeing it
* here is the sole owner's free; on the buggy code it is the second free. */
free(sentinel);
free_CtrlStatus(cs);
free(cs);
printf("test_handle_reply_ctrlstatus_does_not_free_foreign_slot: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ctrlstatus(void)
{
RUN_TEST(test_handle_reply_ctrlstatus_does_not_free_foreign_slot);
RUN_TEST(test_ctrlstatus_roundtrip_all_fields);
RUN_TEST(test_ctrlstatus_roundtrip_unknown_primary);
RUN_TEST(test_xcecli_ctrlstatus_null_ctx_typed_failure);
RUN_TEST(test_xcecli_ctrlstatus_primary_null_ctx_typed_failure);
RUN_TEST(test_xcecli_ctrlstatus_primary_null_out_typed_failure);
RUN_TEST(test_ctrlstatus_is_authoritative_decision);
RUN_TEST(test_ctrlstatus_primary_addr_from_peers);
RUN_TEST(test_ctrlstatus_primary_addr_unknown_or_missing);
}

95
src/lib/testsuite/test_delay.c
View file

@ -1,6 +1,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "delay.h"
@ -26,7 +27,7 @@ static void test_delay1()
int delay_period = 2;
time_t t_start = time(NULL);
delaymgr_init(&mgr, 1, NULL);
delaymgr_init(&mgr, 1000 /* precision_ms */, NULL);
delaymgr_start(&mgr);
delay_task(&mgr, delay_period, runcount, (delay_task_entry_fn_t)delay_work,
&count);
@ -48,9 +49,101 @@ static void test_delay1()
return;
}
static void test_delaymgr_precision_ms_field(void)
{
delaymgr_t mgr;
memset(&mgr, 0, sizeof(mgr));
/* precision_ms field must exist and be stored correctly */
int rc = delaymgr_init(&mgr, 200 /* precision_ms */, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(200, mgr.precision_ms);
delaymgr_fini(&mgr);
printf("test_delaymgr_precision_ms_field: PASS\n");
}
/* ---------- RED: test for delay_task_ms (millisecond-granularity) ---------- */
static int ms_work_count = 0;
static int ms_work(void* arg)
{
(void)arg;
ms_work_count++;
return 0;
}
static void test_delay_task_ms_fires_within_tolerance(void)
{
delaymgr_t mgr;
struct timespec t_start, t_end;
long elapsed_ms;
const int delay_ms = 500;
const int tolerance_ms = 200;
const int precision_ms = 50;
ms_work_count = 0;
delaymgr_init(&mgr, precision_ms, NULL);
delaymgr_start(&mgr);
clock_gettime(CLOCK_MONOTONIC, &t_start);
delay_task_ms(&mgr, delay_ms, 1, (delay_task_entry_fn_t)ms_work, NULL);
/* wait long enough: delay + tolerance + one extra precision tick */
{
struct timespec wait = {
.tv_sec = 0,
.tv_nsec = (long)(delay_ms + tolerance_ms + precision_ms) * 1000000L,
};
nanosleep(&wait, NULL);
}
clock_gettime(CLOCK_MONOTONIC, &t_end);
elapsed_ms = (t_end.tv_sec - t_start.tv_sec) * 1000L
+ (t_end.tv_nsec - t_start.tv_nsec) / 1000000L;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ms_work_count,
"ms task must have fired exactly once");
TEST_ASSERT_TRUE_MESSAGE(elapsed_ms >= (delay_ms - tolerance_ms),
"task fired too early");
TEST_ASSERT_TRUE_MESSAGE(elapsed_ms <= (delay_ms + tolerance_ms + precision_ms + 50),
"task fired too late");
delaymgr_fini(&mgr);
printf("test_delay_task_ms_fires_within_tolerance: PASS\n");
}
static void test_delay_task_ms_invalid_args(void)
{
delaymgr_t mgr;
int rc;
delaymgr_init(&mgr, 50, NULL);
/* NULL manager */
rc = delay_task_ms(NULL, 500, 1, (delay_task_entry_fn_t)ms_work, NULL);
TEST_ASSERT_TRUE(rc < 0);
/* zero delay */
rc = delay_task_ms(&mgr, 0, 1, (delay_task_entry_fn_t)ms_work, NULL);
TEST_ASSERT_TRUE(rc < 0);
/* NULL entry */
rc = delay_task_ms(&mgr, 500, 1, NULL, NULL);
TEST_ASSERT_TRUE(rc < 0);
delaymgr_fini(&mgr);
printf("test_delay_task_ms_invalid_args: PASS\n");
}
/* ---------- existing tests still exercise backward compat ---------- */
void test_delay(void)
{
printf("\n>> delay test\n\n");
RUN_TEST(test_delay1);
RUN_TEST(test_delaymgr_precision_ms_field);
RUN_TEST(test_delay_task_ms_fires_within_tolerance);
RUN_TEST(test_delay_task_ms_invalid_args);
}

43
src/lib/testsuite/test_error.c
View file

@ -35,11 +35,54 @@ static void test_strerror_r(void) {
}
}
#endif
/* Every XCE error code in [XCE_ERRNO_BASE, E_XCE_MAX) must map to a real string
* via xce_strerror_r — no NULL deref, no out-of-bounds read of xce_err_str[].
* Regression for the global-buffer-overflow at utils.c:335: the HA codes
* E_XCE_NOMASTER/REDIRECT/NOT_READY/STANDBY (10011-10014) were added to the
* enum but not to xce_err_str[], so formatting them (e.g. ctrlstatus on a
* failover-exhausted cluster returning E_XCE_NOT_READY) ran off the array.
* Under ASAN this crashes pre-fix. */
static void test_strerror_all_xce_codes(void)
{
char buf[128];
int code;
for (code = XCE_ERRNO_BASE; code < E_XCE_MAX; code++) {
buf[0] = '\0';
int rc = xce_strerror_r(code, buf, sizeof(buf));
TEST_ASSERT_EQUAL_INT(0, rc); /* every in-range code resolves */
TEST_ASSERT_TRUE(buf[0] != '\0'); /* to a non-empty string */
TEST_ASSERT_TRUE(strcmp(buf, "unknown error") != 0); /* a SPECIFIC one */
}
/* the specific HA codes that triggered the crash */
xce_strerror_r(E_XCE_NOT_READY, buf, sizeof(buf));
TEST_ASSERT_TRUE(buf[0] != '\0' && strcmp(buf, "unknown error") != 0);
xce_strerror_r(E_XCE_NOMASTER, buf, sizeof(buf));
TEST_ASSERT_TRUE(buf[0] != '\0' && strcmp(buf, "unknown error") != 0);
}
/* Out-of-range XCE codes (>= E_XCE_MAX) must fall back to "unknown error",
* never index past xce_err_str[] (defense in depth against future enum drift). */
static void test_strerror_out_of_range(void)
{
char buf[128];
TEST_ASSERT_EQUAL_INT(-1, xce_strerror_r(E_XCE_MAX, buf, sizeof(buf)));
TEST_ASSERT_TRUE(strcmp(buf, "unknown error") == 0);
TEST_ASSERT_EQUAL_INT(-1, xce_strerror_r(E_XCE_MAX + 50, buf, sizeof(buf)));
TEST_ASSERT_TRUE(strcmp(buf, "unknown error") == 0);
}
void test_error(void)
{
printf("\n>> error test\n\n");
RUN_TEST(test_error_setget);
RUN_TEST(test_strerror_all_xce_codes);
RUN_TEST(test_strerror_out_of_range);
#if 0
RUN_TEST(test_strerror_r);
#endif

167
src/lib/testsuite/test_hooks.c Normal file
View file

@ -0,0 +1,167 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include "hook.h"
#include <unity/unity.h>
/* Shared call counter for hook tests */
static int s_call_count = 0;
static int counting_fn(int type, int sit, int det, void* data, va_list ap)
{
(void)type;
(void)sit;
(void)det;
(void)ap;
(*(int*)data)++;
return 0;
}
/* --- test cases --- */
static void test_disable_hook_stops_invocation(void)
{
hooksys_t hs;
int rc;
s_call_count = 0;
rc = xce_hook_init(&hs, 2 /* maxtypes */);
TEST_ASSERT_EQUAL_INT(0, rc);
rc = xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &s_call_count);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Before disable: hook fires once */
xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
TEST_ASSERT_EQUAL_INT(1, s_call_count);
/* Disable the hook */
rc = xce_disable_hook(&hs, HOOK_AFTER_EVENT, counting_fn);
TEST_ASSERT_EQUAL_INT(0, rc);
/* After disable: hook must not fire */
xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
TEST_ASSERT_EQUAL_INT(1, s_call_count); /* unchanged */
xce_hook_fini(&hs);
printf("test_disable_hook_stops_invocation: PASS\n");
}
static void test_disable_hook_returns_enoent_if_not_found(void)
{
hooksys_t hs;
int rc;
rc = xce_hook_init(&hs, 2);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Disabling a hook never added must return non-zero (ENOENT) */
rc = xce_disable_hook(&hs, HOOK_AFTER_EVENT, counting_fn);
TEST_ASSERT_NOT_EQUAL(0, rc);
xce_hook_fini(&hs);
printf("test_disable_hook_returns_enoent_if_not_found: PASS\n");
}
static void test_add_hook_reuses_null_slot(void)
{
hooksys_t hs;
int rc;
int count_before_reuse;
s_call_count = 0;
rc = xce_hook_init(&hs, 2);
TEST_ASSERT_EQUAL_INT(0, rc);
rc = xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &s_call_count);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Record list length after first add */
count_before_reuse = hs.hooks[HOOK_AFTER_EVENT].count;
TEST_ASSERT_EQUAL_INT(1, count_before_reuse);
/* Disable the slot — fn becomes NULL atomically */
rc = xce_disable_hook(&hs, HOOK_AFTER_EVENT, counting_fn);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Re-add the same fn — must reuse the NULL slot, not grow the list */
rc = xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &s_call_count);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(count_before_reuse, hs.hooks[HOOK_AFTER_EVENT].count);
/* Fire: must get exactly 1 call (not 2 if slot was duplicated) */
xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
TEST_ASSERT_EQUAL_INT(1, s_call_count);
xce_hook_fini(&hs);
printf("test_add_hook_reuses_null_slot: PASS\n");
}
/* bug-845: re-registering the SAME (fn,data) on a type must be idempotent.
* xce_disable_hook removes only ONE matching slot and xce_run_hooks fires ALL
* slots, so a duplicate (fn,data) registration would (a) fire twice and (b)
* survive a single disable. This is exactly how the controller double-appended
* every WAL_JOB_SUBMIT: persist_start() and ha_register_persist_hooks() both
* register persist_job_hook_fun with the same persist_t closure. */
static void test_add_hook_dedups_same_fn_and_data(void)
{
hooksys_t hs;
int a = 0;
TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&hs, 2));
/* First add registers; the second add with the SAME fn AND data must be a
* no-op (still rc=0, still exactly one slot). */
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &a));
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &a));
TEST_ASSERT_EQUAL_INT(1, hs.hooks[HOOK_AFTER_EVENT].count);
/* Fires ONCE per event, never twice. */
xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
TEST_ASSERT_EQUAL_INT(1, a);
/* A single disable fully removes it — no duplicate slot lingers. */
TEST_ASSERT_EQUAL_INT(0, xce_disable_hook(&hs, HOOK_AFTER_EVENT, counting_fn));
xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
TEST_ASSERT_EQUAL_INT(1, a); /* unchanged */
xce_hook_fini(&hs);
printf("test_add_hook_dedups_same_fn_and_data: PASS\n");
}
/* The dedup is keyed on (fn,data): the SAME fn with DISTINCT closures is a
* legitimate pair of registrations and BOTH must fire. */
static void test_add_hook_allows_same_fn_distinct_data(void)
{
hooksys_t hs;
int a = 0, b = 0;
TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&hs, 2));
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &a));
TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &b));
TEST_ASSERT_EQUAL_INT(2, hs.hooks[HOOK_AFTER_EVENT].count);
xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
TEST_ASSERT_EQUAL_INT(1, a);
TEST_ASSERT_EQUAL_INT(1, b);
xce_hook_fini(&hs);
printf("test_add_hook_allows_same_fn_distinct_data: PASS\n");
}
/* --- suite entry point --- */
void test_hooks(void)
{
printf("\n>> hook system tests\n\n");
RUN_TEST(test_disable_hook_stops_invocation);
RUN_TEST(test_disable_hook_returns_enoent_if_not_found);
RUN_TEST(test_add_hook_reuses_null_slot);
RUN_TEST(test_add_hook_dedups_same_fn_and_data);
RUN_TEST(test_add_hook_allows_same_fn_distinct_data);
}

244
src/lib/testsuite/test_kvmgr_capture.c Normal file
View file

@ -0,0 +1,244 @@
/*
* Test suite for kvmgr_capture_keys() — P0-14a.
*
* Tests follow TDD: written BEFORE implementation.
* All tests exercise the public API declared in kvmgr.h.
*/
#include <rtti.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "kv.h"
#include "kvmgr.h"
#include "persist.h"
#include "utils.h"
/* ------------------------------------------------------------------ */
/* helpers */
/* ------------------------------------------------------------------ */
static int find_key_in_captured(slot_key_t* keys, size_t n,
const char* target_key, int* index_out)
{
size_t i;
for (i = 0; i < n; i++) {
if (strcmp(keys[i].key, target_key) == 0) {
if (index_out != NULL) {
*index_out = keys[i].index;
}
return 1;
}
}
return 0;
}
/* ------------------------------------------------------------------ */
/* test 1: capture from a string-keyed kvmgr with 3 entries */
/* ------------------------------------------------------------------ */
static void test_capture_keys_string_kvmgr(void)
{
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
.nslots = 8,
.delta = 4,
.datakiller = NULL };
var kvmgr = makeInstance(&xkvmgr_type, &params);
TEST_ASSERT_NOT_NULL(kvmgr);
/* Add 3 string-keyed entries whose values are arbitrary pointers;
the index returned by key_add is the slot index we capture. */
int idx_a = key_add(kvmgr, (void*)"alpha", (void*)"data_alpha");
int idx_b = key_add(kvmgr, (void*)"beta", (void*)"data_beta");
int idx_c = key_add(kvmgr, (void*)"gamma", (void*)"data_gamma");
TEST_ASSERT_TRUE(idx_a >= 0);
TEST_ASSERT_TRUE(idx_b >= 0);
TEST_ASSERT_TRUE(idx_c >= 0);
slot_key_t* keys = NULL;
size_t n_keys = 0;
int rc = kvmgr_capture_keys(kvmgr, &keys, &n_keys);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_size_t(3, n_keys);
TEST_ASSERT_NOT_NULL(keys);
/* Every inserted key must appear exactly once with the correct index */
int found_idx = -1;
TEST_ASSERT_TRUE(find_key_in_captured(keys, n_keys, "alpha", &found_idx));
TEST_ASSERT_EQUAL_INT(idx_a, found_idx);
TEST_ASSERT_TRUE(find_key_in_captured(keys, n_keys, "beta", &found_idx));
TEST_ASSERT_EQUAL_INT(idx_b, found_idx);
TEST_ASSERT_TRUE(find_key_in_captured(keys, n_keys, "gamma", &found_idx));
TEST_ASSERT_EQUAL_INT(idx_c, found_idx);
/* key_len must equal strlen(key)+1 */
size_t i;
for (i = 0; i < n_keys; i++) {
TEST_ASSERT_EQUAL_size_t(strlen(keys[i].key) + 1, keys[i].key_len);
}
free(keys);
destroy(kvmgr);
}
/* ------------------------------------------------------------------ */
/* test 2: empty kvmgr returns 0 entries and NULL keys_out */
/* ------------------------------------------------------------------ */
static void test_capture_keys_empty_kvmgr(void)
{
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
.nslots = 4,
.delta = 4,
.datakiller = NULL };
var kvmgr = makeInstance(&xkvmgr_type, &params);
TEST_ASSERT_NOT_NULL(kvmgr);
slot_key_t* keys = (slot_key_t*)(uintptr_t)0xdeadbeef; /* sentinel */
size_t n_keys = 99;
int rc = kvmgr_capture_keys(kvmgr, &keys, &n_keys);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_size_t(0, n_keys);
TEST_ASSERT_NULL(keys);
destroy(kvmgr);
}
/* ------------------------------------------------------------------ */
/* test 3: captured keys are deep copies — safe after kvmgr destroyed */
/* ------------------------------------------------------------------ */
static void test_capture_keys_are_deep_copies(void)
{
kvmgr_params_t params = { .key_type = KVMGR_KEY_STRING,
.nslots = 4,
.delta = 4,
.datakiller = NULL };
var kvmgr = makeInstance(&xkvmgr_type, &params);
TEST_ASSERT_NOT_NULL(kvmgr);
key_add(kvmgr, (void*)"keyA", (void*)"valA");
key_add(kvmgr, (void*)"keyB", (void*)"valB");
slot_key_t* keys = NULL;
size_t n_keys = 0;
int rc = kvmgr_capture_keys(kvmgr, &keys, &n_keys);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_size_t(2, n_keys);
/* Stash manual copies of the captured key strings */
char copy_a[256], copy_b[256];
int found_a = 0, found_b = 0;
size_t i;
for (i = 0; i < n_keys; i++) {
if (strcmp(keys[i].key, "keyA") == 0) {
strncpy(copy_a, keys[i].key, sizeof(copy_a) - 1);
copy_a[sizeof(copy_a) - 1] = '\0';
found_a = 1;
} else if (strcmp(keys[i].key, "keyB") == 0) {
strncpy(copy_b, keys[i].key, sizeof(copy_b) - 1);
copy_b[sizeof(copy_b) - 1] = '\0';
found_b = 1;
}
}
TEST_ASSERT_TRUE(found_a);
TEST_ASSERT_TRUE(found_b);
/* Destroy the kvmgr — this frees all kvtab entries including the
original key strings. Captured keys[] must remain valid. */
destroy(kvmgr);
TEST_ASSERT_EQUAL_STRING("keyA", copy_a);
TEST_ASSERT_EQUAL_STRING("keyB", copy_b);
/* Directly verify the captured keys array is still readable */
int still_a = 0, still_b = 0;
for (i = 0; i < n_keys; i++) {
if (strcmp(keys[i].key, "keyA") == 0) still_a = 1;
if (strcmp(keys[i].key, "keyB") == 0) still_b = 1;
}
TEST_ASSERT_TRUE(still_a);
TEST_ASSERT_TRUE(still_b);
free(keys);
}
/* ------------------------------------------------------------------ */
/* test 4: capture from an integer-keyed kvmgr with 3 entries */
/* ------------------------------------------------------------------ */
static void test_capture_keys_int_kvmgr(void)
{
kvmgr_params_t params = { .key_type = KVMGR_KEY_INT,
.nslots = 8,
.delta = 4,
.datakiller = NULL };
var kvmgr = makeInstance(&xkvmgr_type, &params);
TEST_ASSERT_NOT_NULL(kvmgr);
/* Add 3 integer-keyed entries; the integer keys are jobids.
For integer-keyed kvmgrs, key_add expects a (void*)(long)intval. */
int64_t jobid_1001 = 1001;
int64_t jobid_1002 = 1002;
int64_t jobid_1003 = 1003;
int idx_1001 =
key_add(kvmgr, (void*)(long)jobid_1001, (void*)"data_1001");
int idx_1002 =
key_add(kvmgr, (void*)(long)jobid_1002, (void*)"data_1002");
int idx_1003 =
key_add(kvmgr, (void*)(long)jobid_1003, (void*)"data_1003");
TEST_ASSERT_TRUE(idx_1001 >= 0);
TEST_ASSERT_TRUE(idx_1002 >= 0);
TEST_ASSERT_TRUE(idx_1003 >= 0);
slot_key_t* keys = NULL;
size_t n_keys = 0;
int rc = kvmgr_capture_keys(kvmgr, &keys, &n_keys);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_size_t(3, n_keys);
TEST_ASSERT_NOT_NULL(keys);
/* Every inserted integer key must appear as a decimal string with the
correct index */
int found_idx = -1;
TEST_ASSERT_TRUE(find_key_in_captured(keys, n_keys, "1001", &found_idx));
TEST_ASSERT_EQUAL_INT(idx_1001, found_idx);
TEST_ASSERT_TRUE(find_key_in_captured(keys, n_keys, "1002", &found_idx));
TEST_ASSERT_EQUAL_INT(idx_1002, found_idx);
TEST_ASSERT_TRUE(find_key_in_captured(keys, n_keys, "1003", &found_idx));
TEST_ASSERT_EQUAL_INT(idx_1003, found_idx);
/* key_len must equal strlen(key)+1 */
size_t i;
for (i = 0; i < n_keys; i++) {
TEST_ASSERT_EQUAL_size_t(strlen(keys[i].key) + 1, keys[i].key_len);
}
free(keys);
destroy(kvmgr);
}
/* ------------------------------------------------------------------ */
/* entry point called from test.c */
/* ------------------------------------------------------------------ */
void test_kvmgr_capture(void)
{
printf("\n>> kvmgr_capture_keys test\n");
RUN_TEST(test_capture_keys_string_kvmgr);
RUN_TEST(test_capture_keys_empty_kvmgr);
RUN_TEST(test_capture_keys_are_deep_copies);
RUN_TEST(test_capture_keys_int_kvmgr);
}

1720
src/lib/testsuite/test_libxce_retry.c Normal file
View file

File diff suppressed because it is too large Load diff

95
src/lib/testsuite/test_netio_arm_failure.c Normal file
View file

@ -0,0 +1,95 @@
/*
* test_netio_arm_failure.c — libxce-failover: a poll-arm failure must fail LOUD.
*
* Root cause (bug-646): endpoint_update_epoll() swallowed iomgr_update_poll()
* (EPOLL_CTL_MOD) failures — it logged "rc1=-1 < 0" but returned 0 and left the
* poll interest unarmed. On the synchronous client send path (endpoint_enq),
* a failed arm meant the request bytes were never flushed, no reply ever
* arrived, and the caller blocked forever in cbuf_get() while a perfectly
* healthy io_worker slept in epoll_wait(). (EPOLL_CTL_MOD fails with ENOENT
* when the fd is no longer registered — e.g. after a failed reconnect.)
*
* Contract under test: when the poll interest cannot be armed, endpoint_enq()
* must (1) return a negative, RETRIABLE error so the caller propagates it
* (request.c senders return it; the client failover loop rotates — bug-650),
* and (2) must NOT synchronously invoke the endpoint's err_handler. endpoint_enq
* runs on the caller's thread, which may already hold a lock the err_handler
* needs (the controller's primary_heartbeat_fn holds ha->lock and
* vote_ep_err_handler re-acquires it → self-deadlock, bug-652). The real
* teardown stays on the io_worker thread, off the next epoll error event.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <unity/unity.h>
#include "netio.h"
/* ── err_handler invocation tracker ──────────────────────────────── */
static int g_err_count = 0;
static int err_handler_stub(void *ep)
{
(void)ep;
g_err_count++;
return 0;
}
/*
* Arming EPOLL_CTL_MOD on an fd that was never EPOLL_CTL_ADD'd fails with
* ENOENT — the exact post-failed-reconnect condition. endpoint_enq() must
* surface it (negative rc) WITHOUT calling the err_handler (no re-entrancy).
*/
static void test_enq_arm_failure_is_loud_no_errhandler(void)
{
iomgr_t iomgr;
memset(&iomgr, 0, sizeof(iomgr));
TEST_ASSERT_EQUAL_INT(0, iomgr_init(&iomgr));
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
/* A real socket fd that is NOT registered in iomgr's epoll: EPOLL_CTL_MOD
* on it fails with ENOENT, mirroring a send after a failed reconnect. */
ep->fd = socket(AF_INET, SOCK_STREAM, 0);
TEST_ASSERT_TRUE(ep->fd >= 0);
ep->iomgr = &iomgr;
ep->ssl_handshake_done = 1; /* past handshake: calc_flags arms EPOLLIN */
ep->flags = 0; /* force flags != calc_flags → iomgr_update_poll */
endpoint_set_err_handler(ep, (err_handler_t)err_handler_stub);
g_err_count = 0;
/* One packet on the outq so there is real outbound work to strand. The
* endpoint owns it after enq; endpoint_fini drains it via outq_item_killer. */
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
int rc = endpoint_enq(ep, &pkt->link);
/* Loud failure (negative, retriable) so the caller propagates and rotates —
* but the err_handler must NOT be invoked from this synchronous send path
* (re-entrancy would self-deadlock the controller, bug-652). */
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_INT(0, g_err_count);
endpoint_fini(ep);
free(ep);
close(iomgr.epoll_fd);
printf("test_enq_arm_failure_is_loud_no_errhandler: PASS\n");
}
/* ── suite entry point ───────────────────────────────────────────── */
void test_netio_arm_failure(void)
{
printf("\n>> netio poll-arm failure tests\n\n");
RUN_TEST(test_enq_arm_failure_is_loud_no_errhandler);
}

77
src/lib/testsuite/test_netio_close_propagation.c Normal file
View file

@ -0,0 +1,77 @@
/*
* test_netio_close_propagation.c — bug-648: a peer close/EOF must propagate as a
* negative error, never as rc==0.
*
* io_fsm_do() moved the read FSM to FSM_STATE_ERROR on a peer close but returned
* rc = -errno, which on a clean EOF (errno==0) is 0. handle_input() then read
* rc>=0, never called handle_error, and a blocked cbuf_get() reply-waiter was
* never woken. io_classify_io_error() is the extracted, pure decision; it must
* return <0 for every close/error case and 0 only for retriable would-block.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <stdio.h>
#include <openssl/ssl.h>
#include <unity/unity.h>
/* Pure classifier defined (non-static) in netio.c, linked into this binary. */
extern int io_classify_io_error(int ssl_err, int ecode, int io_rc);
/* WANT_READ / WANT_WRITE are retriable would-block → rc==0, FSM stays put. */
static void test_wantread_writeready_is_retriable_zero(void)
{
TEST_ASSERT_EQUAL_INT(0, io_classify_io_error(SSL_ERROR_WANT_READ, 0, -1));
TEST_ASSERT_EQUAL_INT(0, io_classify_io_error(SSL_ERROR_WANT_WRITE, 0, -1));
printf("test_wantread_writeready_is_retriable_zero: PASS\n");
}
/* Non-blocking EAGAIN/EWOULDBLOCK syscall (io_rc<0) is retriable → rc==0. */
static void test_syscall_eagain_is_retriable_zero(void)
{
TEST_ASSERT_EQUAL_INT(0, io_classify_io_error(SSL_ERROR_SYSCALL, EAGAIN, -1));
TEST_ASSERT_EQUAL_INT(0,
io_classify_io_error(SSL_ERROR_SYSCALL, EWOULDBLOCK, -1));
printf("test_syscall_eagain_is_retriable_zero: PASS\n");
}
/*
* THE BUG: a clean EOF — SSL_read returns 0 with errno==0 — must yield a
* NEGATIVE error (not 0) so the close propagates and the waiter is woken.
*/
static void test_clean_eof_is_negative_not_zero(void)
{
int rc = io_classify_io_error(SSL_ERROR_SYSCALL, 0 /*errno*/, 0 /*io_rc EOF*/);
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc);
printf("test_clean_eof_is_negative_not_zero: PASS\n");
}
/* Clean TLS close_notify (ZERO_RETURN) with errno==0 must also be negative. */
static void test_zero_return_is_negative_not_zero(void)
{
int rc = io_classify_io_error(SSL_ERROR_ZERO_RETURN, 0, 0);
TEST_ASSERT_TRUE(rc < 0);
TEST_ASSERT_EQUAL_INT(-ECONNRESET, rc);
printf("test_zero_return_is_negative_not_zero: PASS\n");
}
/* A hard syscall error preserves its real errno (still negative). */
static void test_hard_syscall_error_preserves_errno(void)
{
TEST_ASSERT_EQUAL_INT(-ECONNRESET,
io_classify_io_error(SSL_ERROR_SYSCALL, ECONNRESET, -1));
printf("test_hard_syscall_error_preserves_errno: PASS\n");
}
void test_netio_close_propagation(void)
{
printf("\n>> netio peer-close propagation tests\n\n");
RUN_TEST(test_wantread_writeready_is_retriable_zero);
RUN_TEST(test_syscall_eagain_is_retriable_zero);
RUN_TEST(test_clean_eof_is_negative_not_zero);
RUN_TEST(test_zero_return_is_negative_not_zero);
RUN_TEST(test_hard_syscall_error_preserves_errno);
}

182
src/lib/testsuite/test_netio_on_connected.c Normal file
View file

@ -0,0 +1,182 @@
/*
* test_netio_on_connected.c — TDD tests for on_connected callback in endpoint_t.
*
* Task 1.2: election-iomgr-refactor spec
* Boundary: netio.c / netio.h
*
* Tests verify:
* 1. on_connected field exists in endpoint_t (compile-time check)
* 2. endpoint_set_on_connected() stores the callback
* 3. Callback fires exactly once when ssl_handshake_done is set to 1
* 4. Callback pointer is nulled after firing (fire-once semantics)
* 5. handle_ssl_handshake path: invokes on_connected via io_handler mock
*/
#define _GNU_SOURCE
#include <errno.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "netio.h"
/* ── Static state for callback tracking ──────────────────────────── */
static int g_cb_count = 0;
static endpoint_t *g_cb_ep = NULL;
static void on_connected_stub(endpoint_t *ep)
{
g_cb_count++;
g_cb_ep = ep;
}
/* Reset per-test globals */
static void reset_cb_state(void)
{
g_cb_count = 0;
g_cb_ep = NULL;
}
/* ── Test 1: Field exists and is zero-initialised after endpoint_get ── */
static void test_on_connected_field_exists_and_is_null(void)
{
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
/* Field must be accessible and NULL after fresh allocation */
TEST_ASSERT_NULL(ep->on_connected);
free(ep);
printf("test_on_connected_field_exists_and_is_null: PASS\n");
}
/* ── Test 2: endpoint_set_on_connected stores the callback ──────── */
static void test_endpoint_set_on_connected_stores_callback(void)
{
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
int rc = endpoint_set_on_connected(ep, on_connected_stub);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_PTR(on_connected_stub, ep->on_connected);
free(ep);
printf("test_endpoint_set_on_connected_stores_callback: PASS\n");
}
/* ── Test 3: endpoint_set_on_connected with NULL ep returns -EINVAL */
static void test_endpoint_set_on_connected_null_ep(void)
{
int rc = endpoint_set_on_connected(NULL, on_connected_stub);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
printf("test_endpoint_set_on_connected_null_ep: PASS\n");
}
/* ── Test 4: endpoint_set_on_connected accepts NULL callback ────── */
static void test_endpoint_set_on_connected_null_cb(void)
{
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
/* Setting to NULL is valid — it clears a prior callback */
int rc = endpoint_set_on_connected(ep, NULL);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_NULL(ep->on_connected);
free(ep);
printf("test_endpoint_set_on_connected_null_cb: PASS\n");
}
/*
* ── Test 5: Direct simulation of ssl_handshake_done path ─────────
*
* Because io_handler requires a live SSL+epoll context, we simulate
* the exact code path specified by the design:
*
* ep->ssl_handshake_done = 1;
* if (ep->on_connected) { ep->on_connected(ep); ep->on_connected = NULL; }
*
* This tests that the fire-once contract holds when the hook is invoked
* immediately after ssl_handshake_done is set.
*/
static void test_on_connected_fires_once_after_handshake_done(void)
{
reset_cb_state();
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
endpoint_set_on_connected(ep, on_connected_stub);
TEST_ASSERT_EQUAL_PTR(on_connected_stub, ep->on_connected);
/* Simulate the moment handle_ssl_handshake sets ssl_handshake_done = 1
* and the subsequent on_connected invocation in io_handler.
* This mirrors the exact logic added to netio.c.
*/
ep->ssl_handshake_done = 1;
if (ep->on_connected) {
ep->on_connected(ep);
ep->on_connected = NULL;
}
/* Callback must have fired exactly once */
TEST_ASSERT_EQUAL_INT(1, g_cb_count);
/* Callback must receive the correct endpoint pointer */
TEST_ASSERT_EQUAL_PTR(ep, g_cb_ep);
/* Pointer must be nulled (fire-once) */
TEST_ASSERT_NULL(ep->on_connected);
/* Simulate a second epoll event — must NOT fire again */
if (ep->on_connected) {
ep->on_connected(ep);
ep->on_connected = NULL;
}
TEST_ASSERT_EQUAL_INT(1, g_cb_count); /* still exactly 1 */
free(ep);
printf("test_on_connected_fires_once_after_handshake_done: PASS\n");
}
/* ── Test 6: No callback set — no crash when handshake completes ── */
static void test_no_callback_handshake_done_no_crash(void)
{
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
/* No on_connected set */
ep->ssl_handshake_done = 1;
if (ep->on_connected) {
ep->on_connected(ep);
ep->on_connected = NULL;
}
/* Must reach here without crash */
TEST_ASSERT_NULL(ep->on_connected);
free(ep);
printf("test_no_callback_handshake_done_no_crash: PASS\n");
}
/* ── Test suite entry point ───────────────────────────────────────── */
void test_netio_on_connected(void)
{
printf("\n>> netio on_connected callback tests\n\n");
RUN_TEST(test_on_connected_field_exists_and_is_null);
RUN_TEST(test_endpoint_set_on_connected_stores_callback);
RUN_TEST(test_endpoint_set_on_connected_null_ep);
RUN_TEST(test_endpoint_set_on_connected_null_cb);
RUN_TEST(test_on_connected_fires_once_after_handshake_done);
RUN_TEST(test_no_callback_handshake_done_no_crash);
}

9185
src/lib/testsuite/test_raft.c Normal file
View file

File diff suppressed because it is too large Load diff

117
src/lib/testsuite/test_request.c
View file

@ -16,6 +16,89 @@ static void test_hostctrl_req_null_ep(void)
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
}
/* ── HA heartbeat sender (xce_ha_heartbeat_req) ───────────────────────────
* The HA heartbeat is an OP_nop request carrying a HeartbeatMsg body. The
* sender builds that request and hands it to xce_request(); since unit tests
* have no live endpoint, these tests round-trip the same request shape the
* sender builds through xce_encode()/xce_decode() and assert the three
* committed-position fields survive the wire. A companion guard confirms the
* generic xce_nop_req still produces a body-less nop. */
/* Build the heartbeat request exactly as xce_ha_heartbeat_req does and encode
* it into a heap io_packet_t. xce_decode() takes ownership of the packet
* (free_packet + FREEUP), so the caller must NOT free the returned pkt — it is
* consumed by the subsequent xce_decode(). Mirrors the test_wal_wire pattern. */
static io_packet_t *encode_heartbeat(uint64_t term, uint64_t committed_lsn,
uint64_t committed_config_tag)
{
heartbeat_msg_t hb = {
.sender_priority = 0,
.term = term,
.committed_lsn = committed_lsn,
.committed_config_tag = committed_config_tag,
};
xce_request_t request = {
.opcode = OPCode_nop,
.body_which = Request_body_heartbeatmsg,
.body = { .heartbeatmsg = &hb },
};
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
/* xce_encode returns 1 on success (mirrors xce_request's encode path). */
TEST_ASSERT_EQUAL_INT(1, xce_encode(&request, pkt));
return pkt;
}
static void test_ha_heartbeat_req_null_ep(void)
{
/* Mirror the nop sender's NULL-endpoint guard. */
int rc = xce_ha_heartbeat_req(NULL, 1, 1, 1);
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
}
static void test_ha_heartbeat_req_roundtrip_fields(void)
{
/* Arrange/Act: encode a heartbeat carrying a committed position, decode. */
io_packet_t *pkt = encode_heartbeat(7, 9, 4);
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req)); /* consumes/frees pkt */
/* Assert: OP_nop + heartbeatmsg arm with the three fields intact. */
TEST_ASSERT_EQUAL_INT(OPCode_nop, req.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_heartbeatmsg, req.body_which);
TEST_ASSERT_NOT_NULL(req.body.heartbeatmsg);
TEST_ASSERT_EQUAL_UINT64(7, req.body.heartbeatmsg->term);
TEST_ASSERT_EQUAL_UINT64(9, req.body.heartbeatmsg->committed_lsn);
TEST_ASSERT_EQUAL_UINT64(4, req.body.heartbeatmsg->committed_config_tag);
xce_free_request(&req);
printf("test_ha_heartbeat_req_roundtrip_fields: PASS\n");
}
/* Guard: the generic nop sender still produces a body-less nop request. */
static void test_nop_req_remains_bodyless(void)
{
xce_request_t request
= { .opcode = OPCode_nop, .body_which = Request_body_nomsg };
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(1, xce_encode(&request, pkt));
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req)); /* consumes/frees pkt */
TEST_ASSERT_EQUAL_INT(OPCode_nop, req.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_nomsg, req.body_which);
xce_free_request(&req);
printf("test_nop_req_remains_bodyless: PASS\n");
}
static void test_hostctrl_req_null_ctrl(void)
{
/* non-NULL dummy ep is enough; the NULL ctrl check fires first */
@ -65,6 +148,34 @@ static void test_hostctrl_req_reject_negative_action(void)
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
}
/* init_reply_by_req must fully zero the reply so the generated
* encode_RequestReply never reads a stale `redirect_host` pointer (and the
* other common trailer fields) off an uninitialized stack reply. Regression
* guard for the heap-use-after-free in encode_RequestReply / strlen seen on a
* NOP/state-report reply whose redirect_host was stack garbage. */
static void test_init_reply_by_req_zeroes_reply(void)
{
/* Poison the whole reply with non-zero bytes — mirrors an uninitialized
* stack reply whose redirect_host happens to be a (dangling) non-NULL. */
xce_reply_t reply;
xce_request_t req = { .seq = 42, .opcode = OPCode_nop };
memset(&reply, 0xFF, sizeof(reply));
init_reply_by_req(&reply, &req);
/* seq/opcode carried from the request */
TEST_ASSERT_EQUAL_UINT64(42, reply.seq);
TEST_ASSERT_EQUAL_INT(OPCode_nop, reply.opcode);
/* Trailer/common fields the encoder always reads must be cleared so a
* handler that only sets data_which/data cannot leak garbage onto the wire
* (or dereference a dangling redirect_host). */
TEST_ASSERT_NULL(reply.redirect_host);
TEST_ASSERT_EQUAL_UINT(0, reply.redirect_port);
TEST_ASSERT_EQUAL_INT(0, reply.status);
TEST_ASSERT_EQUAL_INT(0, reply.data_which);
}
void test_request(void)
{
printf("\n>> host control request validation tests\n\n");
@ -76,4 +187,10 @@ void test_request(void)
RUN_TEST(test_hostctrl_req_reject_tag);
RUN_TEST(test_hostctrl_req_reject_untag);
RUN_TEST(test_hostctrl_req_reject_negative_action);
RUN_TEST(test_init_reply_by_req_zeroes_reply);
printf("\n>> HA heartbeat sender round-trip tests\n\n");
RUN_TEST(test_ha_heartbeat_req_null_ep);
RUN_TEST(test_ha_heartbeat_req_roundtrip_fields);
RUN_TEST(test_nop_req_remains_bodyless);
}

72
src/lib/testsuite/test_request_send_propagation.c Normal file
View file

@ -0,0 +1,72 @@
/*
* test_request_send_propagation.c — bug-650: xce_request() must PROPAGATE an
* endpoint_enq() send failure, not swallow it.
*
* xce_request() (and the other request.c senders) ignored endpoint_enq()'s
* return and always returned 0. So when the transport was dead — endpoint_enq()
* returns <0 and marks the endpoint dead (bug-646) — xce_request() still
* reported success. xce_jobsubmit() then fell through to a BLOCKING cbuf_get()
* and hung forever waiting for a reply to a request that was never transmitted
* (server idle, client fd unconnected). xce_request() must return the negative
* enq error so the failover loop rotates instead of blocking.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <unistd.h>
#include <unity/unity.h>
#include "netio.h"
#include "request.h"
/*
* A request that ENCODES successfully but whose send fails because the endpoint
* is dead (EPOLL_CTL_MOD on an fd that is not registered → ENOENT) must make
* xce_request() return a negative error, not 0.
*/
static void test_xce_request_propagates_send_failure(void)
{
iomgr_t iomgr;
memset(&iomgr, 0, sizeof(iomgr));
TEST_ASSERT_EQUAL_INT(0, iomgr_init(&iomgr));
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
ep->fd = socket(AF_INET, SOCK_STREAM, 0); /* valid fd, NOT in the epoll */
TEST_ASSERT_TRUE(ep->fd >= 0);
ep->iomgr = &iomgr;
ep->ssl_handshake_done = 1;
ep->flags = 0; /* force iomgr_update_poll MOD on the unregistered fd */
char nm[] = "tester";
char ky[] = "key";
xce_peerinfo_t peer = { .name = nm, .key = ky };
ep->peer = &peer;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_nop;
req.seq = 42;
int rc = xce_request(ep, &req);
/* Old code: always 0 (swallowed). Fixed: the negative enq error propagates. */
TEST_ASSERT_TRUE(rc < 0);
endpoint_fini(ep);
free(ep);
close(iomgr.epoll_fd);
printf("test_xce_request_propagates_send_failure: PASS\n");
}
void test_request_send_propagation(void)
{
printf("\n>> request send-failure propagation tests\n\n");
RUN_TEST(test_xce_request_propagates_send_failure);
}

25
src/lib/testsuite/test_select.c
View file

@ -726,12 +726,12 @@ static void
test_gc_sweep_unreachable_freed(void)
{
/* After a sweep with no live jobs, every node in the table is
* unreachable. Re-interning the same arguments must return a freshly
* allocated pointer — the old node was freed and removed from the table.
* unreachable. Re-interning the same arguments must return a fresh
* node — the old node was freed and removed from the table.
*
* The pointer-inequality assertion captures the contract; it becomes
* meaningful once Phase 4f wires the mark-and-sweep to the intern table.
*/
* We track node_id rather than pointer equality: calloc may lawfully
* reuse the freed address, so pointer-inequality is not a reliable
* indicator that GC swept correctly. */
sel_node_table_init();
sel_optimize_set(1);
@ -741,16 +741,18 @@ test_gc_sweep_unreachable_freed(void)
xce_select_node_t *node1 = makeSelectNode(SEL_NUM, bits, 0);
TEST_ASSERT_NOT_NULL(node1);
uint32_t node1_id = node1->node_id;
/* No live jobs → all nodes are unreachable */
gc_clear_marks();
gc_sweep_select_nodes(NULL, 0);
/* Re-intern: node1 was freed; a new node must be allocated */
/* Re-intern: node1 was freed; a new node must be allocated.
* node_id is monotonically assigned — a fresh intern produces a
* different id even if the heap address happens to be reused. */
xce_select_node_t *node2 = makeSelectNode(SEL_NUM, bits, 0);
TEST_ASSERT_NOT_NULL(node2);
/* Phase 4f: node2 must be a distinct allocation from the freed node1 */
TEST_ASSERT_TRUE(node2 != node1);
TEST_ASSERT_NOT_EQUAL(node1_id, node2->node_id);
sel_optimize_test_reset();
}
@ -2230,6 +2232,7 @@ test_gc_clear_marks_exported(void)
xce_select_node_t* n2 = makeSelectNode(SEL_NUM, b2, 0);
TEST_ASSERT_NOT_NULL(n1);
TEST_ASSERT_NOT_NULL(n2);
uint32_t n1_id = n1->node_id;
/* manually mark n1 */
n1->gc_mark = 1;
@ -2246,10 +2249,12 @@ test_gc_clear_marks_exported(void)
gc_sweep_select_nodes(NULL, 0);
TEST_ASSERT_EQUAL_INT(1, n2->gc_mark);
/* verify n1 was swept — re-interning gives a fresh allocation */
/* n1 was swept — re-interning must produce a fresh node_id.
* Pointer inequality would be unreliable because calloc may
* lawfully reuse the freed address. */
xce_select_node_t* n1_again = makeSelectNode(SEL_NUM, b1, 0);
TEST_ASSERT_NOT_NULL(n1_again);
TEST_ASSERT_NOT_EQUAL(n1, n1_again); /* different pointer = n1 was freed */
TEST_ASSERT_NOT_EQUAL(n1_id, n1_again->node_id);
sel_node_table_fini();
sel_optimize_test_reset();

290
src/lib/testsuite/test_wal_wire.c Normal file
View file

@ -0,0 +1,290 @@
/* test_wal_wire.c — Unity tests for the raw payload wire format
* (ha-repl-hardening task 7.1).
*
* Verifies true encode→decode round-trips for:
* - OPCode_walCatchupEntry batches (xce_wal_batch_encode_raw):
* per-sub-entry lsn/entry_type and raw_payload bytes survive the wire
* - OPCode_snapshotTransfer chunks (xce_snapshot_chunk_encode_raw):
* chunk envelope fields and raw_payload bytes survive the wire
*
* These messages bypass the generated encode path because their
* payload :Data fields cannot be carried through xce_data_t (same
* limitation as replConfigPush); xce_decode() pre-extracts the bytes
* into raw_payload/raw_payload_len before capn_free().
*/
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "request.h"
#include "xce/types.h"
/* ── Helpers ────────────────────────────────────────────────────────────── */
static xce_peerinfo_t test_peer = { 0 };
/* encode a batch into a heap io_packet_t (xce_decode frees it) */
static io_packet_t *encode_batch(wal_catchup_entry_t **entries, int n)
{
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
int rc = xce_wal_batch_encode_raw(entries, n, &test_peer, 42, pkt);
TEST_ASSERT_EQUAL_INT(0, rc);
return pkt;
}
/* ── WAL batch round-trip ───────────────────────────────────────────────── */
static void test_wal_batch_roundtrip_payload_bytes(void)
{
/* Arrange: 2 entries with distinct payloads */
uint8_t p0[] = { 0xDE, 0xAD, 0xBE, 0xEF };
uint8_t p1[] = "capnp job blob \x00 with embedded NUL";
wal_catchup_entry_t e0, e1;
memset(&e0, 0, sizeof(e0));
memset(&e1, 0, sizeof(e1));
e0.lsn = 101;
e0.entry_type = 1; /* WAL_JOB_SUBMIT */
e0.raw_payload = p0;
e0.raw_payload_len = sizeof(p0);
e1.lsn = 102;
e1.entry_type = 2;
e1.raw_payload = p1;
e1.raw_payload_len = sizeof(p1);
wal_catchup_entry_t *entries[2] = { &e0, &e1 };
/* Act: encode → decode */
io_packet_t *pkt = encode_batch(entries, 2);
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
/* Assert: opcode, body arm, batch shape */
TEST_ASSERT_EQUAL_INT(OPCode_walCatchupEntry, req.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_walcatchupentrymsg, req.body_which);
TEST_ASSERT_NOT_NULL(req.body.walcatchupentrymsg);
wal_catchup_entry_t *batch = req.body.walcatchupentrymsg;
TEST_ASSERT_EQUAL_INT(2, batch->n_entries);
TEST_ASSERT_NOT_NULL(batch->entries);
/* Entry 0: header + payload bytes intact */
TEST_ASSERT_NOT_NULL(batch->entries[0]);
TEST_ASSERT_EQUAL_UINT64(101, batch->entries[0]->lsn);
TEST_ASSERT_EQUAL_UINT8(1, batch->entries[0]->entry_type);
TEST_ASSERT_EQUAL_size_t(sizeof(p0), batch->entries[0]->raw_payload_len);
TEST_ASSERT_NOT_NULL(batch->entries[0]->raw_payload);
TEST_ASSERT_EQUAL_MEMORY(p0, batch->entries[0]->raw_payload, sizeof(p0));
/* Entry 1: embedded NUL survives (binary-safe) */
TEST_ASSERT_NOT_NULL(batch->entries[1]);
TEST_ASSERT_EQUAL_UINT64(102, batch->entries[1]->lsn);
TEST_ASSERT_EQUAL_UINT8(2, batch->entries[1]->entry_type);
TEST_ASSERT_EQUAL_size_t(sizeof(p1), batch->entries[1]->raw_payload_len);
TEST_ASSERT_EQUAL_MEMORY(p1, batch->entries[1]->raw_payload, sizeof(p1));
xce_free_request(&req);
printf("test_wal_batch_roundtrip_payload_bytes: PASS\n");
}
/* Production-scale batch: many entries each carrying a full-size (872-byte)
* capnp-Job-sized payload — mirrors the live catch-up stream where ctrl-1
* rejoined and received header-only entries (PLEN=0). The small 2-entry
* round-trip passes; this reproduces whatever the larger/multi-segment batch
* does on the wire (bug-847). */
static void test_wal_batch_roundtrip_large_payloads(void)
{
/* KNOWN-FAILING REPRODUCER for bug-847: a production-scale batch that spans
* more than one capnp segment loses its per-entry payload Data pointers on
* the wire (decode returns raw_payload_len=0; it also leaks the orphaned
* decode allocations under ASAN). IGNORE'd so the suite stays green; REMOVE
* this TEST_IGNORE line when bug-847 is fixed and assert the payloads
* survive. Decisive evidence: this fails "Expected 872 Was 0" while the
* 2-entry round-trip above passes. */
TEST_IGNORE_MESSAGE(
"bug-847: catch-up batch drops payloads across capnp segments — "
"un-ignore when fixed");
enum { N = 14, PSZ = 872 };
wal_catchup_entry_t e[N];
wal_catchup_entry_t *entries[N];
uint8_t* payloads[N];
for (int i = 0; i < N; i++) {
payloads[i] = malloc(PSZ);
TEST_ASSERT_NOT_NULL(payloads[i]);
/* distinct, non-zero content per entry so a dropped/zeroed payload is
* unambiguous */
for (int b = 0; b < PSZ; b++)
payloads[i][b] = (uint8_t)((i * 31 + b) & 0xff);
memset(&e[i], 0, sizeof(e[i]));
e[i].lsn = (uint64_t)(i + 1);
e[i].entry_type = 1; /* WAL_JOB_SUBMIT */
e[i].raw_payload = payloads[i];
e[i].raw_payload_len = PSZ;
entries[i] = &e[i];
}
io_packet_t* pkt = encode_batch(entries, N);
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
wal_catchup_entry_t* batch = req.body.walcatchupentrymsg;
TEST_ASSERT_NOT_NULL(batch);
TEST_ASSERT_EQUAL_INT(N, batch->n_entries);
for (int i = 0; i < N; i++) {
TEST_ASSERT_NOT_NULL(batch->entries[i]);
TEST_ASSERT_EQUAL_UINT64((uint64_t)(i + 1), batch->entries[i]->lsn);
/* the bug: these come back NULL / len 0 on the live path */
TEST_ASSERT_EQUAL_size_t(PSZ, batch->entries[i]->raw_payload_len);
TEST_ASSERT_NOT_NULL(batch->entries[i]->raw_payload);
TEST_ASSERT_EQUAL_MEMORY(payloads[i], batch->entries[i]->raw_payload,
PSZ);
}
xce_free_request(&req);
for (int i = 0; i < N; i++)
free(payloads[i]);
printf("test_wal_batch_roundtrip_large_payloads: PASS\n");
}
static void test_wal_batch_entry_without_payload(void)
{
/* An entry with no payload (e.g. header-only types) must decode with
* raw_payload == NULL and raw_payload_len == 0 — not garbage. */
wal_catchup_entry_t e;
memset(&e, 0, sizeof(e));
e.lsn = 7;
e.entry_type = 3;
wal_catchup_entry_t *entries[1] = { &e };
io_packet_t *pkt = encode_batch(entries, 1);
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
wal_catchup_entry_t *batch = req.body.walcatchupentrymsg;
TEST_ASSERT_NOT_NULL(batch);
TEST_ASSERT_EQUAL_INT(1, batch->n_entries);
TEST_ASSERT_EQUAL_UINT64(7, batch->entries[0]->lsn);
TEST_ASSERT_NULL(batch->entries[0]->raw_payload);
TEST_ASSERT_EQUAL_size_t(0, batch->entries[0]->raw_payload_len);
xce_free_request(&req);
printf("test_wal_batch_entry_without_payload: PASS\n");
}
static void test_wal_batch_encode_rejects_invalid_args(void)
{
io_packet_t pkt;
memset(&pkt, 0, sizeof(pkt));
wal_catchup_entry_t e;
memset(&e, 0, sizeof(e));
wal_catchup_entry_t *entries[1] = { &e };
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_wal_batch_encode_raw(NULL, 1, &test_peer, 0, &pkt));
/* n_entries == 0 is a valid heartbeat (ha-term-safety task 3.2): a 0-entry AE
* carries the AppendEntries metadata (term, prev tuple, leader_commit) on the
* top-level container to drive UpdateTerm/logOk. Only n_entries < 0 is invalid. */
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_wal_batch_encode_raw(entries, -1, &test_peer, 0, &pkt));
TEST_ASSERT_EQUAL_INT(-EINVAL,
xce_wal_batch_encode_raw(entries, 1, &test_peer, 0, NULL));
printf("test_wal_batch_encode_rejects_invalid_args: PASS\n");
}
/* ── Snapshot chunk round-trip ──────────────────────────────────────────── */
static void test_snapshot_chunk_roundtrip(void)
{
/* Arrange: a mid-transfer chunk with binary payload */
uint8_t payload[300];
for (size_t i = 0; i < sizeof(payload); i++)
payload[i] = (uint8_t)(i * 7);
snapshot_transfer_t chunk;
memset(&chunk, 0, sizeof(chunk));
chunk.snapshot_lsn = 5000;
chunk.chunk_index = 3;
chunk.chunk_offset = 3 * 65536;
chunk.total_size = 4 * 65536 + 300;
chunk.is_last = 1;
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0,
xce_snapshot_chunk_encode_raw(&chunk, payload, sizeof(payload),
&test_peer, 9, pkt));
/* Act */
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
/* Assert: envelope + payload bytes */
TEST_ASSERT_EQUAL_INT(OPCode_snapshotTransfer, req.opcode);
TEST_ASSERT_EQUAL_INT(Request_body_snapshottransfermsg, req.body_which);
snapshot_transfer_t *rx = req.body.snapshottransfermsg;
TEST_ASSERT_NOT_NULL(rx);
TEST_ASSERT_EQUAL_UINT64(5000, rx->snapshot_lsn);
TEST_ASSERT_EQUAL_UINT32(3, rx->chunk_index);
TEST_ASSERT_EQUAL_UINT64(3 * 65536, rx->chunk_offset);
TEST_ASSERT_EQUAL_UINT64(4 * 65536 + 300, rx->total_size);
TEST_ASSERT_TRUE(rx->is_last);
TEST_ASSERT_EQUAL_size_t(sizeof(payload), rx->raw_payload_len);
TEST_ASSERT_NOT_NULL(rx->raw_payload);
TEST_ASSERT_EQUAL_MEMORY(payload, rx->raw_payload, sizeof(payload));
xce_free_request(&req);
printf("test_snapshot_chunk_roundtrip: PASS\n");
}
static void test_snapshot_chunk_empty_payload(void)
{
/* Zero-length payload (defensive): envelope round-trips, raw NULL/0 */
snapshot_transfer_t chunk;
memset(&chunk, 0, sizeof(chunk));
chunk.snapshot_lsn = 1;
chunk.total_size = 0;
chunk.is_last = 1;
io_packet_t *pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0,
xce_snapshot_chunk_encode_raw(&chunk, NULL, 0, &test_peer, 0, pkt));
xce_request_t req;
memset(&req, 0, sizeof(req));
TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, &req));
snapshot_transfer_t *rx = req.body.snapshottransfermsg;
TEST_ASSERT_NOT_NULL(rx);
TEST_ASSERT_EQUAL_UINT64(1, rx->snapshot_lsn);
TEST_ASSERT_TRUE(rx->is_last);
TEST_ASSERT_NULL(rx->raw_payload);
TEST_ASSERT_EQUAL_size_t(0, rx->raw_payload_len);
xce_free_request(&req);
printf("test_snapshot_chunk_empty_payload: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_wal_wire(void)
{
RUN_TEST(test_wal_batch_roundtrip_payload_bytes);
RUN_TEST(test_wal_batch_roundtrip_large_payloads);
RUN_TEST(test_wal_batch_entry_without_payload);
RUN_TEST(test_wal_batch_encode_rejects_invalid_args);
RUN_TEST(test_snapshot_chunk_roundtrip);
RUN_TEST(test_snapshot_chunk_empty_payload);
}

136
src/lib/testsuite/test_zarray_cow.c Normal file
View file

@ -0,0 +1,136 @@
#include <assert.h>
#include <rtti.h>
#include <stdio.h>
#include <unity/unity.h>
#include "zarray.h"
/* xzarray_type is defined in zarray.c */
extern typeinfo_t xzarray_type;
/* Helper: create an xzarray instance */
static meta_obj_t* make_test_xzarray(void)
{
xzarray_params_t params = { .delta = 4, .datakiller = NULL };
return (meta_obj_t*)makeInstance(&xzarray_type, &params);
}
static void test_freeze_thaw_slots_exist(void)
{
/* After P0-2, vfreeze and vthaw must be wired (non-NULL). */
const intf_varray_t* iface
= (const intf_varray_t*)xzarray_type.interfaces.varray;
TEST_ASSERT_NOT_NULL(iface);
/* Access slots without crash — verifies struct layout has them */
(void)iface->vfreeze;
(void)iface->vthaw;
/* Slots must be non-NULL after P0-2 wires the real implementations */
TEST_ASSERT_NOT_NULL(iface->vfreeze);
TEST_ASSERT_NOT_NULL(iface->vthaw);
printf("test_freeze_thaw_slots_exist: PASS\n");
}
static void test_freeze_creates_shadow(void)
{
meta_obj_t* arr = make_test_xzarray();
TEST_ASSERT_NOT_NULL(arr);
/* Store 3 items */
char* items[3] = { "alpha", "beta", "gamma" };
for (int i = 0; i < 3; i++) {
int idx = vstore(arr, items[i]);
TEST_ASSERT_TRUE(idx >= 0);
}
/* Freeze should succeed */
TEST_ASSERT_EQUAL_INT(0, vfreeze(arr));
/* After freeze, vget must still return stored values (COW mode active) */
void* out = NULL;
TEST_ASSERT_EQUAL_INT(0, vget(arr, 0, &out));
TEST_ASSERT_NOT_NULL(out);
/* Thaw to clean up */
vthaw(arr, NULL, NULL);
destroy(arr);
}
static void test_freeze_thaw_preserves_all_entries(void)
{
meta_obj_t* arr = make_test_xzarray();
TEST_ASSERT_NOT_NULL(arr);
int N = 5;
char* items[5] = { "a", "b", "c", "d", "e" };
int idxs[5];
for (int i = 0; i < N; i++) {
idxs[i] = vstore(arr, items[i]);
TEST_ASSERT_TRUE(idxs[i] >= 0);
}
TEST_ASSERT_EQUAL_INT(0, vfreeze(arr));
TEST_ASSERT_EQUAL_INT(0, vthaw(arr, NULL, NULL));
/* All 5 entries must be reachable after thaw */
for (int i = 0; i < N; i++) {
void* out = NULL;
TEST_ASSERT_EQUAL_INT(0, vget(arr, idxs[i], &out));
TEST_ASSERT_EQUAL_PTR(items[i], out);
}
destroy(arr);
}
static void test_mutation_during_cow_visible_after_thaw(void)
{
meta_obj_t* arr = make_test_xzarray();
TEST_ASSERT_NOT_NULL(arr);
char* original = "original";
char* updated = "updated";
int idx = vstore(arr, original);
TEST_ASSERT_TRUE(idx >= 0);
TEST_ASSERT_EQUAL_INT(0, vfreeze(arr));
/* Overwrite index during COW — write to new active array */
int put_idxs[1] = { idx };
void* put_data[1] = { updated };
vput_vec(arr, 1, put_idxs, put_data);
TEST_ASSERT_EQUAL_INT(0, vthaw(arr, NULL, NULL));
/* After thaw, the updated value must win over the shadow value */
void* out = NULL;
TEST_ASSERT_EQUAL_INT(0, vget(arr, idx, &out));
TEST_ASSERT_EQUAL_PTR(updated, out);
destroy(arr);
}
static void test_double_freeze_fails(void)
{
meta_obj_t* arr = make_test_xzarray();
TEST_ASSERT_NOT_NULL(arr);
TEST_ASSERT_EQUAL_INT(0, vfreeze(arr));
/* Second freeze must fail (already in COW mode) */
TEST_ASSERT_TRUE(vfreeze(arr) != 0);
vthaw(arr, NULL, NULL);
destroy(arr);
}
void test_zarray_cow(void)
{
printf("\n>> zarray COW interface slot test\n\n");
RUN_TEST(test_freeze_thaw_slots_exist);
RUN_TEST(test_freeze_creates_shadow);
RUN_TEST(test_freeze_thaw_preserves_all_entries);
RUN_TEST(test_mutation_during_cow_visible_after_thaw);
RUN_TEST(test_double_freeze_fails);
}

83
src/lib/utils.c
View file

@ -309,30 +309,54 @@ void set_error(char* file, char* func, int line, int code)
ectx_t* get_ectx() { return &ectx; }
static char* xce_err_str[] = { "invalid job",
"invalid worker",
"duplicated jobs",
"duplicated workers",
"bad opcode",
"unknown role",
"unknown resource",
"unknown queue",
"unknown group",
"internal error",
NULL };
/* One string per E_XCE_* code, in enum order (errnox.h). Index = code -
* XCE_ERRNO_BASE. MUST stay in lockstep with the enum — the _Static_assert
* below fails the build if a code is added to errnox.h without a string here
* (that drift caused a global-buffer-overflow in xce_strerror_r: bug-NOT_READY,
* the HA codes 10011-10014 had no entries). */
static char* xce_err_str[] = { "invalid job", /* E_XCE_BAD_JOB */
"invalid worker", /* E_XCE_BAD_WORKER */
"duplicated jobs", /* E_XCE_DUP_JOB */
"duplicated workers", /* E_XCE_DUP_WORKER */
"bad opcode", /* E_XCE_BAD_OPCODE */
"unknown role", /* E_XCE_BAD_ROLE */
"unknown resource", /* E_XCE_BAD_RES */
"unknown queue", /* E_XCE_BAD_QUEUE */
"unknown group", /* E_XCE_BAD_GROUP */
"internal error", /* E_XCE_INTERNAL */
"no known primary", /* E_XCE_NOMASTER */
"redirect to primary", /* E_XCE_REDIRECT */
"controller not ready", /* E_XCE_NOT_READY */
"controller is standby" };/* E_XCE_STANDBY */
_Static_assert(sizeof(xce_err_str) / sizeof(xce_err_str[0])
== (size_t)(E_XCE_MAX - XCE_ERRNO_BASE),
"xce_err_str[] is out of sync with the E_XCE_* enum (errnox.h): "
"add a string for every new error code");
int xce_strerror_r(int errnum, char* buf, size_t bufsize)
{
if (bufsize == 0) {
return -1;
}
if (errnum < XCE_ERRNO_BASE) {
return strerror_r(errnum, buf, bufsize);
}
if (errnum >= E_XCE_MAX) {
/* Defense in depth: bound against the ACTUAL table size (not just E_XCE_MAX)
* and reject NULL slots, so a future enum/table drift degrades to "unknown
* error" instead of reading past xce_err_str[]. */
size_t idx = (size_t)(errnum - XCE_ERRNO_BASE);
size_t nstr = sizeof(xce_err_str) / sizeof(xce_err_str[0]);
if (errnum >= E_XCE_MAX || idx >= nstr || xce_err_str[idx] == NULL) {
strncpy(buf, "unknown error", bufsize);
buf[bufsize - 1] = '\0';
return -1;
}
strncpy(buf, xce_err_str[errnum - XCE_ERRNO_BASE], bufsize);
strncpy(buf, xce_err_str[idx], bufsize);
buf[bufsize - 1] = '\0';
return 0;
}
@ -385,36 +409,39 @@ int copy_strlist(xce_strlist_t* dst, xce_strlist_t src)
}
return 0;
}
void free_resval_with_idx_list(xce_val_with_idx_t*** vals, int count)
void free_name_val_list(xce_name_val_t*** vals, int *count_p)
{
xce_val_with_idx_t** resvals = *vals;
int i;
xce_name_val_t** resvals = *vals;
int count = *count_p;
int i;
if (resvals == NULL) {
return;
}
if (count <= 0) {
if (resvals == NULL || count <= 0) {
*count_p = 0;
return;
}
for (i = 0; i < count; i++) {
xce_val_with_idx_t* val = resvals[i];
xce_name_val_t* nv = resvals[i];
if (val == NULL) {
if (nv == NULL) {
continue;
}
if (val->val->which == Value_u_svalue) {
FREEUP(val->val->u.svalue);
FREEUP(nv->name);
if (nv->val != NULL) {
if (nv->val->which == Value_u_svalue) {
FREEUP(nv->val->u.svalue);
}
FREEUP(nv->val);
}
FREEUP(val->val);
FREEUP(val);
FREEUP(nv);
}
FREEUP(resvals);
*vals = NULL;
*vals = NULL;
*count_p = 0;
}
void free_strlist(xce_strlist_t* l)

122
src/lib/wobj.c
View file

@ -22,12 +22,34 @@ static size_t worker_datasize(va_list ap)
return sz;
}
static int worker_init(meta_obj_t* obj, va_list ap)
static void kill_worker(worker_t* w)
{
worker_t* dst = RTTI_OBJ_TO_DATA(worker_t, obj);
worker_private_t* priv = (worker_private_t*)RTTI_OBJ_TO_PRIV(obj);
xce_worker_t* src = va_arg(ap, xce_worker_t*);
dereference(w->host.name);
if (w->host.attrs != NULL) {
dereference(w->host.attrs->hosttype);
dereference(w->host.attrs->hostmodel);
FREEUP(w->host.attrs);
}
if (w->host.dispatch_window_config != NULL) {
free_timewindow_config(w->host.dispatch_window_config);
w->host.dispatch_window_config = NULL;
}
if (w->host.thresholds != NULL) {
free_resource_thresholds(w->host.thresholds, w->host.nthresholds);
w->host.thresholds = NULL;
w->host.nthresholds = 0;
}
zarray_fini(&(w->host.resvals));
return;
}
static int copy_worker(worker_t* dst, xce_worker_t* src)
{
if ((src == NULL) || (src->attrs == NULL)) {
return -EINVAL;
}
@ -55,6 +77,46 @@ static int worker_init(meta_obj_t* obj, va_list ap)
dst->host.nresvals_to_sched = 0;
dst->host.resvals_to_sched = NULL;
return 1;
}
static int override_worker(worker_t* dst, xce_worker_t* w)
{
#define XCE_WORKER_ADMIN_MASK \
(XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED | XCE_WORKER_HIDDEN \
| XCE_WORKER_CLOSED_BY_ADMIN)
uint32_t admin_status = 0;
int rc = 0;
int idx;
admin_status = dst->host.status & XCE_WORKER_ADMIN_MASK;
idx = dst->idx;
kill_worker(dst);
rc = copy_worker(dst, w);
if (rc >= 0) {
dst->host.status |= admin_status;
dst->ep = NULL;
dst->idx = idx;
}
return rc;
#undef XCE_WORKER_ADMIN_MASK
}
static int worker_init(meta_obj_t* obj, va_list ap)
{
worker_t* dst = RTTI_OBJ_TO_DATA(worker_t, obj);
worker_private_t* priv = (worker_private_t*)RTTI_OBJ_TO_PRIV(obj);
xce_worker_t* src = va_arg(ap, xce_worker_t*);
int rc = 0;
rc = copy_worker(dst, src);
if (rc < 0) {
return rc;
}
my_rwlock_init(&(priv->lock));
priv->refcount = 1;
@ -66,26 +128,7 @@ static void worker_destroy(meta_obj_t* obj)
worker_t* w = RTTI_OBJ_TO_DATA(worker_t, obj);
worker_private_t* priv = (worker_private_t*)RTTI_OBJ_TO_PRIV(obj);
dereference(w->host.name);
if (w->host.attrs != NULL) {
dereference(w->host.attrs->hosttype);
dereference(w->host.attrs->hostmodel);
FREEUP(w->host.attrs);
}
if (w->host.dispatch_window_config != NULL) {
free_timewindow_config(w->host.dispatch_window_config);
w->host.dispatch_window_config = NULL;
}
if (w->host.thresholds != NULL) {
free_resource_thresholds(w->host.thresholds, w->host.nthresholds);
w->host.thresholds = NULL;
w->host.nthresholds = 0;
}
zarray_fini(&(w->host.resvals));
kill_worker(w);
my_rwlock_fini(&(priv->lock));
memset(priv, 0x0, sizeof(*priv));
@ -163,9 +206,10 @@ static typeinfo_t worker_type = {
static worker_t* makeWorker_(var workermgr, xce_worker_t* w)
{
worker_t* worker = NULL;
worker_t* w1 = NULL;
int rc = 0;
rc = key_find(workermgr, (void*)w->name, (void**)&worker);
rc = key_find(workermgr, (void*)w->name, (void**)&w1);
if ((rc < 0) && (rc != -ENOENT)) {
errno = -rc;
@ -174,20 +218,26 @@ static worker_t* makeWorker_(var workermgr, xce_worker_t* w)
}
if (rc > 0) {
errno = EEXIST;
return NULL;
}
worker = makeInstance(&worker_type, w);
if (worker != NULL) {
rc = key_add(workermgr, (void*)worker->host.name, (void*)worker);
rc = override_worker(w1, w);
if (rc < 0) {
destroy(worker);
errno = -rc;
return NULL;
}
worker = w1;
} else {
worker = makeInstance(&worker_type, w);
if (worker != NULL) {
rc = key_add(workermgr, (void*)worker->host.name, (void*)worker);
if (rc < 0) {
destroy(worker);
errno = -rc;
return NULL;
}
worker->idx = rc;
}
}
return worker;

11
src/lib/xce_select.c
View file

@ -98,7 +98,16 @@ static uint32_t sel_key_hash_fn(void* k)
static int sel_key_cmp_fn(void* a, void* b)
{
return memcmp(a, b, sizeof(sel_node_key_t));
/* Field-by-field comparison: memcmp on the whole struct would
* compare padding bytes between type and op1, which can differ
* between the stack-local search key and the heap-allocated copy
* stored in the kvtab, causing false misses in key_del. */
sel_node_key_t* ka = (sel_node_key_t*)a;
sel_node_key_t* kb = (sel_node_key_t*)b;
if (ka->type != kb->type) return 1;
if (ka->op1 != kb->op1) return 1;
if (ka->op2 != kb->op2) return 1;
return 0;
}
static int sel_key_assign_fn(void** dst, void* src)

49
src/lib/xceapi.c
View file

@ -116,17 +116,17 @@ int xce_get_with_condition(xce_ctx_t* ctx, xce_get_t* get, void*** data)
}
memset(&getreply, 0x0, sizeof(getreply));
ctx->reply.u.get = &getreply;
rc = xce_get_req(ctx->ep, get);
if (rc < 0) {
return rc;
goto done;
}
struct timespec _dl; cbuf_deadline_10s(&_dl);
rc = cbuf_timedget(&(ctx->wbuf), &status, &_dl);
if (rc < 0) {
return rc;
goto done;
}
if (status >= 0) {
@ -136,6 +136,12 @@ int xce_get_with_condition(xce_ctx_t* ctx, xce_get_t* get, void*** data)
rc = status;
}
done:
/* Always clear the destination — including on the xce_get_req/cbuf_get
* failover error paths — so the single, untagged reply union never retains
* the address of this now-returning stack `getreply`. A later ctrlstatus
* reply would otherwise free that dead-stack pointer as a CtrlStatus*
* (bug-1077, ASan stack-use-after-return). */
ctx->reply.u.get = NULL;
return rc;
@ -200,13 +206,13 @@ int xce_resval(xce_ctx_t* ctx, char* host, char* res, xce_resval_t*** vals)
ctx->reply.u.get = &getreply;
rc = xce_get_req(ctx->ep, &get);
if (rc < 0) {
return rc;
goto done;
}
struct timespec _dl; cbuf_deadline_10s(&_dl);
rc = cbuf_timedget(&(ctx->wbuf), &status, &_dl);
if (rc < 0) {
return rc;
goto done;
}
if (status >= 0) {
@ -216,6 +222,9 @@ int xce_resval(xce_ctx_t* ctx, char* host, char* res, xce_resval_t*** vals)
rc = status;
}
done:
/* Clear on every path (incl. the failover error returns) so the shared
* reply union never keeps this stack `getreply` address — bug-1077. */
ctx->reply.u.get = NULL;
return rc;
@ -443,6 +452,36 @@ int xce_clusterconf(xce_ctx_t* ctx, xce_clusterconf_t** conf)
return 0;
}
int xce_ctrlstatus(xce_ctx_t* ctx, xce_ctrlstatus_t** status)
{
int rc, st;
if ((ctx == NULL) || (status == NULL)) {
return -EINVAL;
}
if (ctx->state != XCE_CTX_RUN) {
return -EINVAL;
}
rc = xce_ctrlstatus_req(ctx->ep);
if (rc < 0) {
return rc;
}
rc = cbuf_get(&(ctx->wbuf), &st);
if (rc < 0) {
return rc;
}
if (st < 0) {
return st;
}
MOVE_PTR((*status), ctx->reply.u.ctrlstatus);
return 0;
}
int xce_update_resvals(xce_ctx_t* ctx, int count, xce_resval_t** vals)
{
int rc;

141
src/lib/xcectx.c
View file

@ -14,6 +14,14 @@ static int handle_get_reply(xce_ctx_t* ctx, xce_reply_t* reply)
xce_data_t* getdata = reply->data.getdata;
dataset_t* dataset = ctx->reply.u.get;
/* No get is in flight (slot cleared by the consumer on every exit path):
* drop a late/duplicate reply rather than dereference a NULL or stale
* destination — bug-1077, the shared reply union must never be used as a
* stale typed pointer. */
if (dataset == NULL) {
return -1;
}
return extract_data(dataset, getdata);
}
@ -27,10 +35,39 @@ static int handle_sub_reply(xce_ctx_t* ctx, xce_reply_t* reply)
return 1;
}
static int handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
void xce_ctx_capture_redirect_hint(xce_ctx_t* ctx, const char* host,
uint16_t port)
{
if (ctx == NULL) {
return;
}
/* No usable redirect address → leave the hint untouched (empty). The
* capnp-owned host pointer may be NULL; an empty host or a zero port is
* likewise "no address". Advisory only — correctness never depends on it. */
if ((host == NULL) || (host[0] == '\0') || (port == 0)) {
return;
}
strncpy(ctx->redirect_hint_host, host, XCE_MAXHOSTNAME_LEN - 1);
ctx->redirect_hint_host[XCE_MAXHOSTNAME_LEN - 1] = '\0';
ctx->redirect_hint_port = port;
}
/* Exposed (non-static) so the libxce test suite can drive the reply path
* directly without a live endpoint (see test_handle_reply_* regression for the
* shared-reply-union type-confusion crash, bug-1077). */
int xce_handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
{
int rc = 0;
/* Capture the redirect address (if any) as a best-effort failover hint
* BEFORE the capnp-owned reply — including reply->redirect_host — is freed
* by the caller (Req 2.2, 8.3). The status still flows through the existing
* cbuf_tryput(&ctx->wbuf,&status) path unchanged, so xceapi.c is untouched. */
xce_ctx_capture_redirect_hint(ctx, reply->redirect_host,
reply->redirect_port);
if (reply->status < 0) {
return reply->status;
}
@ -45,10 +82,27 @@ static int handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
case OPCode_clusterconf:
rc = -1;
if (reply->data_which == RequestReply_data_clusterconf) {
/* MOVE_PTR overwrites the slot WITHOUT dereferencing the old value.
* Do NOT "free the prior occupant": ctx->reply.u is a single
* untagged union shared by every opcode, so a non-NULL slot does NOT
* imply a ClusterConf* — it may hold a dataset_t* (a dead stack
* `getreply` left by a failover-aborted get) or a jobid. Freeing it
* as a ClusterConf walked a foreign pointer and crashed (bug-1077).
* The consumer owns slot lifecycle and clears it on every path. */
MOVE_PTR(ctx->reply.u.clusterconf, reply->data.clusterconf);
rc = 1;
}
break;
case OPCode_ctrlstatus:
rc = -1;
if (reply->data_which == RequestReply_data_ctrlstatus) {
/* See the clusterconf note above: never free the shared union slot
* here as a CtrlStatus* — a foreign occupant would be type-confused
* and dereferenced (bug-1077, ASan stack-use-after-return). */
MOVE_PTR(ctx->reply.u.ctrlstatus, reply->data.ctrlstatus);
rc = 1;
}
break;
case OPCode_fetchfile:
rc = -1;
if (reply->data_which == RequestReply_data_fetchfilereply) {
@ -90,7 +144,7 @@ static int vtask_entry(vtask_t* task)
int need_wake = 1;
int status = request.body.replymsg->status;
need_wake = handle_reply(ctx, request.body.replymsg);
need_wake = xce_handle_reply(ctx, request.body.replymsg);
if (need_wake) {
rc = xce_wakeup(ctx, (int)status);
}
@ -140,6 +194,29 @@ static int handle_msg(endpoint_t* ep)
(void*)ep);
}
/* Called by netio when the TCP connection drops (EPOLLHUP / EPOLLERR / read EOF).
* Unblocks any thread stuck in cbuf_get (e.g. xce_hello waiting for a reply)
* so that checking_connection can detect the failure and attempt to rejoin. */
static int handle_disconnect(endpoint_t* ep)
{
if (ep == NULL)
return 0;
xce_ctx_t* ctx = (xce_ctx_t*)ep->owner;
if (ctx == NULL)
return 0;
ctx->state = XCE_CTX_ERR;
/* Inject an error into wbuf so any blocked cbuf_get returns immediately.
* cbuf_tryput returns -EAGAIN if the buffer already has an item — that is
* fine: the existing item will wake the waiter. */
int err = -ECONNRESET;
cbuf_tryput(&ctx->wbuf, &err);
return 0;
}
int xce_ctx_init(xce_ctx_t* ctx, char* name, char* key)
{
int rc;
@ -244,6 +321,7 @@ int xce_ctx_start(xce_ctx_t* ctx)
endpoint_set_owner(ctx->ep, ctx);
endpoint_set_msg_handler(ctx->ep, (msg_handler_t)handle_msg);
endpoint_set_err_handler(ctx->ep, (err_handler_t)handle_disconnect);
rc = iomgr_monitor(ctx->ep);
if (rc < 0) {
ctx->state = XCE_CTX_ERR;
@ -258,6 +336,65 @@ int xce_ctx_start(xce_ctx_t* ctx)
return 0;
}
/*
* xce_ctx_reconnect (libxce-failover) — re-establish ctx->ep on a fresh
* connection to host:port after the previous primary's connection dropped.
*
* This mirrors the connection-establishment part of xcecli_start/xce_ctx_start
* (the parts that DROP on endpoint_fini): tear down the dead fd/SSL, recreate
* the socket, re-bind the iomgr (endpoint_connect needs ep->iomgr->ssl_ctx),
* drive the SSL handshake, then re-arm the endpoint in the ALREADY-RUNNING
* iomgr with its owner + msg/err handlers and re-monitor the new fd. It does
* NOT restart iomgr/vtaskmgr (already running) and does NOT touch ctx->state
* (the failover loop's xcecli_ctx_recover owns the ERR→RUN reset).
*
* Without the iomgr re-bind, endpoint_connect dereferenced a NULL ep->iomgr
* (crash at netio.c endpoint_connect); without re-arming the handlers + monitor
* the reconnected endpoint would never route replies to ctx->wbuf.
*/
int xce_ctx_reconnect(xce_ctx_t* ctx, char* host, int port)
{
int rc;
if ((ctx == NULL) || (ctx->ep == NULL) || (host == NULL)) {
return -EINVAL;
}
/* Tear down the dead connection (closes fd, frees SSL, clears handlers and
* resets the endpoint struct — including ep->iomgr). */
endpoint_fini(ctx->ep);
rc = endpoint_init(ctx->ep, AF_INET, SOCK_STREAM, 0, 0);
if (rc < 0) {
return rc;
}
endpoint_set_peer(ctx->ep, &(ctx->myinfo));
/* Re-bind the iomgr so endpoint_connect can reach ctx->iomgr->ssl_ctx
* (xce_ctx_set_endpoint does this on the initial start). */
ctx->ep->iomgr = &(ctx->iomgr);
rc = endpoint_connect(ctx->ep, host, port);
if (rc < 0) {
return rc;
}
/* Re-arm the endpoint in the running iomgr: owner + reply/error handlers,
* then monitor the new fd. handle_disconnect re-registers RDHUP-driven
* failover for the new primary. */
endpoint_set_owner(ctx->ep, ctx);
endpoint_set_msg_handler(ctx->ep, (msg_handler_t)handle_msg);
endpoint_set_err_handler(ctx->ep, (err_handler_t)handle_disconnect);
rc = iomgr_monitor(ctx->ep);
if (rc < 0) {
return rc;
}
return 0;
}
int xce_ctx_stop(xce_ctx_t* ctx)
{
if (ctx == NULL) {

148
src/lib/zarray.c
View file

@ -1,5 +1,7 @@
#include <assert.h>
#include <errno.h>
#include <rtti.h>
#include <stdatomic.h>
#include <stdlib.h>
#include <string.h>
@ -351,13 +353,16 @@ void* zarray_get(zarray_t* array, int idx)
}
typedef struct {
mylock_t lock;
} xzarray_private_t;
_Atomic _Bool inuse;
void* data;
} xzarray_item_t;
typedef struct {
unsigned int inuse : 1;
void* data;
} xzarray_item_t;
mylock_t lock;
xzarray_item_t* shadow; /* COW snapshot (may be NULL if array was empty at freeze) */
size_t shadow_caps; /* capacity of shadow array */
bool in_cow; /* true while between freeze and thaw */
} xzarray_private_t;
typedef struct {
int count;
@ -392,6 +397,10 @@ static int xzarray_init(meta_obj_t* obj, va_list ap)
a->pos = 0;
a->killer = params->datakiller;
priv->shadow = NULL;
priv->shadow_caps = 0;
priv->in_cow = false;
mylock_init(&(priv->lock));
return 1;
@ -403,8 +412,16 @@ static void xzarray_destroy(meta_obj_t* obj)
xzarray_t* a = RTTI_OBJ_TO_DATA(xzarray_t, obj);
int i;
for (i = 0; i < a->caps; i++) {
if (a->data[i].inuse) {
/* Free shadow if still present (e.g. destroy called mid-COW) */
if (priv->in_cow) {
free(priv->shadow);
priv->shadow = NULL;
priv->shadow_caps = 0;
priv->in_cow = false;
}
for (i = 0; i < (int)a->caps; i++) {
if (atomic_load_explicit(&a->data[i].inuse, memory_order_acquire)) {
if (a->killer != NULL) {
a->killer(a->data[i].data);
}
@ -437,14 +454,14 @@ static int xzarray_ensure_items(xzarray_t* array, int mincap)
array->data = p;
for (i = array->caps; i < nc; i++) {
array->data[i].inuse = 0;
atomic_store_explicit(&array->data[i].inuse, false, memory_order_release);
array->data[i].data = NULL;
}
/* this will make we can get next free
* location quickly
*/
if (array->data[array->pos].inuse) {
if (atomic_load_explicit(&array->data[array->pos].inuse, memory_order_acquire)) {
array->pos = array->caps;
}
@ -467,9 +484,9 @@ static int xzarray_store(meta_obj_t* obj, void* data)
idx = a->pos;
for (i = 0; i < a->caps; i++) {
if (a->data[idx].inuse == 0) {
a->data[idx].inuse = 1;
a->data[idx].data = data;
if (!atomic_load_explicit(&a->data[idx].inuse, memory_order_acquire)) {
a->data[idx].data = data;
atomic_store_explicit(&a->data[idx].inuse, true, memory_order_release);
a->count++;
rc = idx;
@ -503,7 +520,15 @@ static int xzarray_get_vec(meta_obj_t* obj, int count, int* idxs, void** data)
for (i = 0; i < count; i++) {
int idx = idxs[i];
if ((idx >= a->caps) || (a->data[idx].inuse == 0)) {
/* During COW, check active first, then shadow */
bool in_active = (idx < (int)a->caps) &&
atomic_load_explicit(&a->data[idx].inuse, memory_order_acquire);
bool in_shadow = !in_active && priv->in_cow &&
priv->shadow != NULL &&
(idx < (int)priv->shadow_caps) &&
atomic_load_explicit(&priv->shadow[idx].inuse, memory_order_acquire);
if (!in_active && !in_shadow) {
rc = -ENOENT;
break;
}
@ -513,7 +538,13 @@ static int xzarray_get_vec(meta_obj_t* obj, int count, int* idxs, void** data)
for (i = 0; i < count; i++) {
int idx = idxs[i];
data[i] = a->data[idx].data;
/* Active takes precedence over shadow */
if ((idx < (int)a->caps) &&
atomic_load_explicit(&a->data[idx].inuse, memory_order_acquire)) {
data[i] = a->data[idx].data;
} else {
data[i] = priv->shadow[idx].data;
}
}
}
return rc;
@ -557,14 +588,16 @@ static int xzarray_put_vec(meta_obj_t* obj, int count, int* idxs, void** data)
for (i = 0; i < count; i++) {
int idx = idxs[i];
if (a->data[idx].inuse == 1) {
if (atomic_load_explicit(&a->data[idx].inuse, memory_order_acquire)) {
if (a->killer != NULL) {
a->killer(a->data[idx].data);
}
} else {
a->count++;
}
a->data[idx].data = data[i];
a->data[idx].inuse = 1;
a->data[idx].data = data[i];
atomic_store_explicit(&a->data[idx].inuse, true, memory_order_release);
}
}
@ -584,15 +617,18 @@ static int xzarray_clear(meta_obj_t* obj, int idx)
xzarray_private_t* priv = (xzarray_private_t*)RTTI_OBJ_TO_PRIV(obj);
int rc = -EINVAL;
if (idx < a->caps) {
assert(!priv->in_cow && "xzarray_clear called during COW — deferred cleanup required");
if (idx < (int)a->caps) {
rc = 0;
if (a->data[idx].inuse) {
if (atomic_load_explicit(&a->data[idx].inuse, memory_order_acquire)) {
if (a->killer != NULL) {
a->killer(a->data[idx].data);
}
a->count--;
}
a->data[idx].inuse = 0;
atomic_store_explicit(&a->data[idx].inuse, false, memory_order_release);
a->data[idx].data = NULL;
a->pos = idx; /* we can find the available idx next time */
}
@ -614,8 +650,8 @@ static int xzarray_iter_next(iterator_t* iter, void** data)
xzarray_t* a = RTTI_OBJ_TO_DATA(xzarray_t, iter->obj);
int i;
for (i = (int)(long)iter->data; i < a->caps; i++) {
if (a->data[i].inuse) {
for (i = (int)(long)iter->data; i < (int)a->caps; i++) {
if (atomic_load_explicit(&a->data[i].inuse, memory_order_acquire)) {
*data = a->data[i].data;
iter->data = (void*)(long)(i + 1);
@ -656,6 +692,70 @@ static void xzarray_iter_destroy(iterator_t* iter)
return;
}
/* P0-2: COW freeze — O(1) pointer swap, pre-grow active array */
static int xzarray_freeze(meta_obj_t* obj)
{
xzarray_private_t* priv = (xzarray_private_t*)RTTI_OBJ_TO_PRIV(obj);
xzarray_t* a = RTTI_OBJ_TO_DATA(xzarray_t, obj);
if (priv->in_cow)
return -EALREADY; /* already frozen */
/* Pre-grow active to 1.5x + 16 to avoid realloc during COW (A6-026) */
size_t new_caps = a->caps + (a->caps / 2) + 16;
xzarray_item_t* active = calloc(new_caps, sizeof(xzarray_item_t));
if (!active)
return -ENOMEM;
priv->shadow = a->data; /* may be NULL if array was empty */
priv->shadow_caps = a->caps;
priv->in_cow = true;
a->data = active;
a->caps = new_caps;
a->count = 0;
/* Start pos past shadow to prevent index collisions with shadow entries */
a->pos = (int)priv->shadow_caps;
return 0;
}
/* P0-2: COW thaw — merge shadow into active, free shadow */
static int xzarray_thaw(meta_obj_t* obj, retain_fn_t should_retain, void* arg)
{
xzarray_private_t* priv = (xzarray_private_t*)RTTI_OBJ_TO_PRIV(obj);
xzarray_t* a = RTTI_OBJ_TO_DATA(xzarray_t, obj);
if (!priv->in_cow)
return -EINVAL; /* not in COW mode */
/* Merge: for each shadow entry not already written to active, copy it */
for (size_t i = 0; i < priv->shadow_caps; i++) {
if (!atomic_load_explicit(&priv->shadow[i].inuse, memory_order_acquire))
continue;
void* item = priv->shadow[i].data;
/* GC: skip entries that should not be retained */
if (should_retain != NULL && !should_retain(item, arg))
continue;
/* Only copy to active if active slot not already updated during COW */
if (i < a->caps &&
!atomic_load_explicit(&a->data[i].inuse, memory_order_acquire)) {
a->data[i].data = item;
atomic_store_explicit(&a->data[i].inuse, true, memory_order_release);
a->count++;
}
/* If active[i].inuse == true: mutation during COW wins — keep it */
}
free(priv->shadow);
priv->shadow = NULL;
priv->shadow_caps = 0;
priv->in_cow = false;
a->pos = 0;
return 0;
}
typeinfo_t xzarray_type = {
.name = "xzarray",
.privateSize = sizeof(xzarray_private_t),
@ -668,7 +768,9 @@ typeinfo_t xzarray_type = {
.vget_vec = (vget_vec_fn_t)xzarray_get_vec,
.vput = (vput_fn_t)xzarray_put,
.vput_vec = (vput_vec_fn_t)xzarray_put_vec,
.vclear = (vclear_fn_t)xzarray_clear },
.vclear = (vclear_fn_t)xzarray_clear,
.vfreeze = (vfreeze_fn_t)xzarray_freeze,
.vthaw = (vthaw_fn_t)xzarray_thaw },
.iter
= &(intf_iterator_t) { .make = (iter_make_fn_t)xzarray_make_iter,
.next = (iter_next_fn_t)xzarray_iter_next,