lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'working' into job_peek_lchen

Browse source 
job_peek_lchen: Add fetchfile feature (job output/error peek proxy)
working: HA/Raft infrastructure (consensus, replication, config mgmt)

Conflict resolution summary:
- controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30
- cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK
- job.c: kept full fetchfile pending-request subsystem (~183 lines)
- request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS
- helper.c: added missing preVoteRequest/preVoteReply mappings
- All 15 new test registrations (test_cbuf.c + 14 HA tests)
This commit is contained in:
Lei Chen 2026-07-22 13:57:25 +08:00
commit 3a2094a461
529 changed files with 345174 additions and 5048 deletions
Download patch file Download diff file Expand all files Collapse all files

55
src/include/xce/client_failover.h Normal file
View file

@ -0,0 +1,55 @@
/*
* client_failover.h — public StatusClassifier surface for the libxce-failover
* spec.
*
* Task 2.1 (StatusClassifier): pure mapping from a reply status / transport rc
* to a failover action. No I/O, no globals — see design.md
* "Components and Interfaces → StatusClassifier".
*/
#ifndef XCE_CLIENT_FAILOVER_H
#define XCE_CLIENT_FAILOVER_H
#include <stdbool.h>
/* Failover action a round-trip outcome maps to. Order mirrors the harness'
* fo_expected_action_t (failover_harness.h) and design.md. */
typedef enum {
XCE_FO_SUCCESS, /* status == 0, or idempotent-replay success
(DUP on a re-sent write) */
XCE_FO_RETRY_NOW, /* E_XCE_REDIRECT / E_XCE_STANDBY / transport error
— rotate, no backoff */
XCE_FO_RETRY_BACKOFF, /* E_XCE_NOMASTER / E_XCE_NOT_READY — rotate after
jittered backoff */
XCE_FO_TERMINAL /* AUTH/PERM/INVALID/NOENT and other non-retriable
— surface to caller */
} xce_fo_action_t;
/*
* Map a reply status / transport rc to a failover action.
*
* status : the controller reply status or negative -errno transport rc, as
* the failover loop observes it. Controller codes (E_XCE_*) are
* accepted in either sign.
* is_write: true for mutating ops (e.g. job submit); false for reads.
* attempt : 0 for the first send; >0 for a re-send after failover.
*
* Invariant (2.5): E_XCE_REDIRECT/STANDBY/NOMASTER/NOT_READY never map to
* XCE_FO_SUCCESS.
*
* Idempotent-replay rule (5.2): E_XCE_DUP_JOB / EEXIST on a re-sent write
* (is_write && attempt > 0) → XCE_FO_SUCCESS; otherwise → XCE_FO_TERMINAL.
*
* Unknown / unrecognized statuses are treated conservatively as
* XCE_FO_TERMINAL (surfaced, not retried).
*
* Pure function: no I/O, no globals.
*/
xce_fo_action_t xce_failover_classify(int status, bool is_write, int attempt);
/*
* EndpointRotator (xce_failover_next_target / xce_failover_rotate) operates on
* the private api_ctx_t and is therefore declared in the internal header
* src/lib/client_internal.h, not in this public surface.
*/
#endif /* XCE_CLIENT_FAILOVER_H */

128
src/include/xce/cluster.h Normal file
View file

@ -0,0 +1,128 @@
/* src/include/xce/cluster.h — cluster discovery and reconnect API (P2.5-1)
*
* xce_connect_cluster() returns endpoint_t* (cast to void*).
* The SSL* is accessible as ((endpoint_t*)result)->ssl.
*/
#ifndef XCE_CLUSTER_H
#define XCE_CLUSTER_H
#include <stddef.h>
#include "config.h" /* MAX_HA_CANDIDATES, XCE_MAXHOSTNAME_LEN */
/**
* A single controller candidate.
* name: logical name from "name=host:port" syntax (empty if omitted).
* Matching against myconf->name allows multiple controllers on the same
* dev host to distinguish themselves by name rather than IP address.
*/
typedef struct {
char name[XCE_MAXNAME_LEN];
char hostname[XCE_MAXHOSTNAME_LEN];
int port;
} xce_candidate_t;
/**
* Cluster connection configuration.
* Populated by each caller (worker/scheduler/apiserver) from --peers and
* --ca-cert CLI flags. libxce never reads config files.
*
* candidates_count must be in [1, MAX_HA_CANDIDATES].
* A WARN is logged at startup if candidates_count > 3.
*/
typedef struct {
xce_candidate_t candidates[MAX_HA_CANDIDATES];
int candidates_count;
char ca_cert_path[4096]; /* path to CA cert for TLS verify */
char identity[256]; /* "from" name sent to controller */
char key[256]; /* authentication key */
} xce_cluster_config_t;
/**
* Reconnect state machine states.
*/
typedef enum {
XCE_RECONNECT_IDLE = 0,
XCE_RECONNECT_CONNECTING,
XCE_RECONNECT_BACKOFF,
XCE_RECONNECT_CONNECTED,
XCE_RECONNECT_FAILED
} xce_reconnect_state_t;
/**
* Reconnect context (use xce_reconnect_* functions to operate).
*/
typedef struct {
xce_cluster_config_t* config;
xce_reconnect_state_t state;
int attempt;
int backoff_ms; /* current backoff delay in ms */
void* endpoint; /* endpoint_t* when CONNECTED */
} xce_reconnect_ctx_t;
/* Redirect follow limit */
#define XCE_MAX_REDIRECTS 4
/* Backoff parameters — initial kept short so tests stay fast */
#define XCE_RECONNECT_INITIAL_BACKOFF_MS 100
#define XCE_RECONNECT_MAX_BACKOFF_MS 30000
#define XCE_RECONNECT_BACKOFF_MULTIPLIER 2
/* --- Public API --- */
/**
* xce_cluster_config_validate() — validate a populated xce_cluster_config_t.
*
* Returns 0 on success, -EINVAL if:
* - candidates_count < 1 or > MAX_HA_CANDIDATES
* - any candidate has empty hostname or port <= 0
* - ca_cert_path is empty
* - identity is empty
*/
int xce_cluster_config_validate(const xce_cluster_config_t* cfg);
/**
* xce_connect_cluster() — connect to the cluster via candidate list.
*
* Iterates candidates in order, attempting a TLS connection to each.
* Returns endpoint_t* (cast to void*) on success, NULL if all fail.
*/
void* xce_connect_cluster(xce_cluster_config_t* cfg);
/**
* xce_reconnect_init() — initialise a reconnect context.
* Returns 0 on success, -EINVAL on bad arguments.
*/
int xce_reconnect_init(xce_reconnect_ctx_t* rctx, xce_cluster_config_t* cfg);
/**
* xce_reconnect_step() — drive one step of the reconnect state machine.
*
* Call in a loop until state == XCE_RECONNECT_CONNECTED or FAILED.
* Sleeps backoff_ms in BACKOFF state (skipped when mock connect is set).
*
* Returns 0 to continue, -1 when in terminal state.
*/
int xce_reconnect_step(xce_reconnect_ctx_t* rctx);
/**
* xce_reconnect_fini() — release resources held by a reconnect context.
*/
void xce_reconnect_fini(xce_reconnect_ctx_t* rctx);
/**
* xce_parse_peers() — parse "--peers name=host:port,..." into cfg->candidates[].
*
* Each token may be "name=host:port" or bare "host:port".
* The name field is stored when present; it allows multiple daemons on the
* same host to identify themselves by logical name rather than IP address.
*
* Returns 0 on success, -1 on empty/NULL input or any parse error.
*/
int xce_parse_peers(const char* peers_str, xce_cluster_config_t* cfg);
/* Test seam: set to non-NULL to override the real connect (unit tests only) */
typedef void* (*xce_connect_fn_t)(const char* host, int port,
const char* ca_cert);
void xce_cluster_set_connect_fn(xce_connect_fn_t fn);
#endif /* XCE_CLUSTER_H */

37
src/include/xce/cluster_def.h
View file

@ -14,7 +14,12 @@
#include "myconf.h"
#include "mylock.h"
#include "netio.h"
#include "persist.h"
#include "vtaskmgr.h"
/* ha.h forward-declares cluster_t; including it here is safe (no circular dep) */
#include "commit.h"
#include "ha.h"
#include "xce/types.h"
#include "xlist.h"
#include "zarray.h"
@ -55,7 +60,9 @@ enum {
HOOK_DETAIL_QUEUE_UPDATE,
HOOK_DETAIL_USERGROUP_UPDATE,
HOOK_DETAIL_HOSTGROUP_UPDATE,
HOOK_DETAIL_JOB_RUSAGE_UPDATE
HOOK_DETAIL_JOB_RUSAGE_UPDATE,
HOOK_DETAIL_JOB_STOP,
HOOK_DETAIL_JOB_RESUME
};
#define XCE_NS_CONFIG (0)
@ -88,9 +95,10 @@ typedef struct {
endpoint_t* ep;
} cluster_apiserver_t;
typedef struct {
typedef struct cluster_s {
myconf_t* myconf;
xce_peerinfo_t* myhost;
xce_peerinfo_t* myhost; /* points at myhost_info once initialised */
xce_peerinfo_t myhost_info; /* storage; name set from myconf->name */
hooksys_t hooksys;
iomgr_t iomgr;
zarray_t endpoints;
@ -112,6 +120,29 @@ typedef struct {
vtaskmgr_t vtaskmgr;
cbuf_t wbuf;
int running;
persist_t persist; /* Persistence module — P0-F */
ha_state_t ha; /* HA role management — P1-5 */
commit_engine_t commit; /* Per-entry commit engine — ha-commit-engine 5.3 */
_Atomic int restart_requested; /* set by OP_upgradeConfig handler */
pthread_t heartbeat_listener_tid; /* pre-Phase-2 SSL listener thread */
_Atomic int heartbeat_listener_running;
_Atomic int lost_and_found_job_count; /* incremented during persist_recover() */
/* Performance-tuning parameters — initialized by init_controller_tunables()
* in cluster_init_phase1 so they are available before cluster_init_phase2.
* TODO: wire to cluster_config once config generation tool supports @13-@17. */
struct {
int jobmgr_nhash; /* hash slots for job table */
int jobmgr_delta; /* growth step for job table */
int workermgr_nhash; /* hash slots for worker table */
int workermgr_delta; /* growth step for worker table */
int qmgr_nhash; /* hash slots for queue table */
int qmgr_delta; /* growth step for queue table */
int resmgr_nhash; /* hash slots for resource table */
int resmgr_delta; /* growth step for resource table */
int resmgr_inst; /* growth step for shared-instance array */
int xstrings_nhash; /* hash slots for xstring table */
} tunables;
} cluster_t;
#endif /* XCE_CLUSTER_DEF_H */

9
src/include/xce/errnox.h
View file

@ -19,7 +19,16 @@ enum {
E_XCE_BAD_QUEUE,
E_XCE_BAD_GROUP,
E_XCE_INTERNAL,
E_XCE_NOMASTER, /* Standby has no known primary — client should retry */
E_XCE_REDIRECT, /* Standby is redirecting client to primary */
E_XCE_NOT_READY, /* Controller in CTRL_INIT or CTRL_ELECTING state */
E_XCE_STANDBY, /* Controller in CTRL_STANDBY; op is primary-only */
E_XCE_MAX
};
/* HA dispatch gate error variables — defined in ha.c, declared here so all
* dispatch call sites that include errnox.h get the right names. */
extern int XCE_ERR_NOT_READY; /* == E_XCE_NOT_READY */
extern int XCE_ERR_STANDBY; /* == E_XCE_STANDBY */
#endif

23
src/include/xce/types.h
View file

@ -172,10 +172,13 @@ typedef struct {
#define XCE_WORKER_LOCKEDW (0x00040000)
#define XCE_WORKER_BUSY (0x00080000)
#define XCE_WORKER_CLOSED (0x00100000)
#define XCE_WORKER_HIDDEN (0x00200000) /* Admin explicitly removed — invisible but not yet cleaned up */
#define XCE_WORKER_CLOSED_BY_ADMIN (0x00400000) /* Admin closed — host closed explicitly by admin */
#define IS_WORKER_UNAVAIL(status) (((status) & XCE_WORKER_UNAVAIL) != 0)
#define IS_WORKER_LOCKEDU(status) (((status) & XCE_WORKER_LOCKEDU) != 0)
#define IS_WORKER_LOCKEDW(status) (((status) & XCE_WORKER_LOCKEDW) != 0)
#define IS_WORKER_UNAVAIL(status) (((status) & XCE_WORKER_UNAVAIL) != 0)
#define IS_WORKER_LOCKEDU(status) (((status) & XCE_WORKER_LOCKEDU) != 0)
#define IS_WORKER_LOCKEDW(status) (((status) & XCE_WORKER_LOCKEDW) != 0)
#define IS_WORKER_CLOSED_BY_ADMIN(status) (((status) & XCE_WORKER_CLOSED_BY_ADMIN) != 0)
#define IS_WORKER_LOCKED(status) \
(IS_WORKER_LOCKEDU(status) || IS_WORKER_LOCKEDW(status))
#define IS_WORKER_BUSY(status) (((status) & XCE_WORKER_BUSY) != 0)
@ -212,8 +215,8 @@ typedef struct {
int nthresholds;
xce_res_threshold_t** thresholds;
/* following two vars used to push resource values to sched */
int nresvals_to_sched;
xce_val_with_idx_t** resvals_to_sched;
int nresvals_to_sched;
xce_name_val_t** resvals_to_sched;
} xce_worker_t;
typedef struct {
@ -353,7 +356,8 @@ typedef struct {
#define JOB_STAT_PDONE (0x80) /**< Post job process done successfully */
#define JOB_STAT_PERR (0x100) /**< Post job process has error */
#define JOB_STAT_WAIT (0x200) /**< Chunk job waiting its turn to exec */
#define JOB_STAT_WAIT (0x200) /**< Chunk job waiting its turn to exec */
#define JOB_STAT_TRANSIENT (0x400) /**< Job state uncertain after failover — awaiting worker re-join */
#define JOB_STAT_RUNKWN \
0x8000 /* Flag : Job status is UNKWN caused by losting contact with remote \
cluster */
@ -381,6 +385,7 @@ typedef struct {
#define QUEUE_STAT_DISC 0x10 /**< Remote queue status is disconnected. */
#define QUEUE_STAT_FAIRSHARE_DEFAULT \
0x20 /**< Fairshare queue with null user share defined*/
#define QUEUE_STAT_HIDDEN (0x0100) /**< Admin deleted — invisible but not yet cleaned up */
/* FIXME: improve following later */
#define JOB_IS_PEND(state) (((state) & JOB_STAT_PEND) != 0)
@ -472,10 +477,16 @@ typedef struct {
char* ckptinfo;
int64_t pending_ctrl;
xce_job_preason_t* pend_reasons;
uint64_t transient_since; /* epoch seconds; set when job enters JOB_STAT_TRANSIENT */
} xce_job_t;
typedef struct {
xce_job_t job;
/* ha-convergence-recovery (3.3): WAL entry LSN of this job's submit record.
* In-memory only — recomputed from the applied entry at replay; never
* serialized (wire type xce_job_t above is untouched). 0 = unknown /
* pre-feature = always exportable (see job_submit_committed, cluster.h). */
uint64_t submit_lsn;
} job_t;
typedef struct {