Merge branch 'working' into job_peek_lchen
job_peek_lchen: Add fetchfile feature (job output/error peek proxy) working: HA/Raft infrastructure (consensus, replication, config mgmt) Conflict resolution summary: - controller.capnp: fetchfile @41, fetchfilereply @18, fetchfilemsg @30 - cluster.c: bifurcated OPCode_reply for fetchfile + config-push ACK - job.c: kept full fetchfile pending-request subsystem (~183 lines) - request.c: kept both xce_fetchfile_req() and XCE_RETRY_DEFAULTS - helper.c: added missing preVoteRequest/preVoteReply mappings - All 15 new test registrations (test_cbuf.c + 14 HA tests)
This commit is contained in:
Lei Chen
commit
3a2094a461
529 changed files with 345174 additions and 5048 deletions
35
src/include/byteorder.h
Normal file
35
src/include/byteorder.h
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#pragma once
|
||||
/* Portable be64toh() — big-endian to host byte order conversion. */
|
||||
#if defined(__linux__)
|
||||
# ifndef _DEFAULT_SOURCE
|
||||
# define _DEFAULT_SOURCE 1
|
||||
# endif
|
||||
# ifndef _BSD_SOURCE
|
||||
# define _BSD_SOURCE 1
|
||||
# endif
|
||||
# include <endian.h>
|
||||
# ifndef be64toh
|
||||
/* musl and some older glibc: fall back to builtin */
|
||||
# include <stdint.h>
|
||||
static inline uint64_t be64toh(uint64_t x) {
|
||||
# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
return __builtin_bswap64(x);
|
||||
# else
|
||||
return x;
|
||||
# endif
|
||||
}
|
||||
# endif /* be64toh */
|
||||
#elif defined(__APPLE__)
|
||||
# include <machine/endian.h>
|
||||
# include <libkern/OSByteOrder.h>
|
||||
# define be64toh(x) OSSwapBigToHostInt64(x)
|
||||
#else
|
||||
# include <stdint.h>
|
||||
static inline uint64_t be64toh(uint64_t x) {
|
||||
# if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
||||
return __builtin_bswap64(x);
|
||||
# else
|
||||
return x;
|
||||
# endif
|
||||
}
|
||||
#endif
|
||||
|
|
@ -2,6 +2,8 @@
|
|||
|
||||
#define _CLIENT_H_ 1
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "netio.h"
|
||||
|
|
@ -11,6 +13,18 @@
|
|||
|
||||
typedef void* xcecli_ctx_t;
|
||||
|
||||
/*
|
||||
* xcecli_reconnect_cb: consumer rejoin hook invoked once per transparent
|
||||
* reconnect/failover, AFTER the reconnect/recovery and BEFORE the interrupted
|
||||
* op resumes. The callback CONTENT (join handshake, worker job re-sync) belongs
|
||||
* to the consumer; libxce owns only the mechanism + ordering. Runs on the
|
||||
* calling thread. Return 0 to proceed with the interrupted op; <0 to treat the
|
||||
* rejoin as failed (the new primary rejected the join) so the failover loop
|
||||
* rotates/backs off again instead of resuming. Pure clients register NO hook
|
||||
* (NULL) — the loop then resumes directly.
|
||||
*/
|
||||
typedef int (*xcecli_reconnect_cb)(xcecli_ctx_t ctx, void* arg);
|
||||
|
||||
#define N_MAX_MASTERS (3)
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -26,6 +40,8 @@ typedef struct {
|
|||
extern int xcecli_ctx_init(xcecli_ctx_t*, xce_client_config_t*);
|
||||
extern void xcecli_ctx_fini(xcecli_ctx_t);
|
||||
extern int xcecli_set_handler(xcecli_ctx_t, xce_request_handler_t, void*);
|
||||
extern int xcecli_set_reconnect_handler(xcecli_ctx_t, xcecli_reconnect_cb,
|
||||
void*);
|
||||
extern int xcecli_start(xcecli_ctx_t);
|
||||
extern int xcecli_stop(xcecli_ctx_t);
|
||||
extern int xcecli_wakeup(xcecli_ctx_t, int);
|
||||
|
|
@ -39,6 +55,30 @@ extern int xcecli_workerlist(xcecli_ctx_t, xce_worker_t***);
|
|||
extern int xcecli_jobdispatch(xcecli_ctx_t, xce_job_dispatch_t*);
|
||||
extern int xcecli_joblist(xcecli_ctx_t, xce_job_t***);
|
||||
extern int xcecli_clusterconf(xcecli_ctx_t, xce_clusterconf_t**);
|
||||
extern int xcecli_ctrlstatus(xcecli_ctx_t, xce_ctrlstatus_t**);
|
||||
/*
|
||||
* xcecli_ctrlstatus_primary (controller-status task 3.2): AUTHORITATIVE
|
||||
* controller status read. Returns ONLY a reply whose responder role is the
|
||||
* primary. When the first answer is from a non-primary, it follows the redirect
|
||||
* (targets the reported primary via the reply's peer list, else rotates the
|
||||
* configured controller list) until the primary answers or the failover budget
|
||||
* is exhausted. On budget exhaustion / no reachable primary it returns a typed
|
||||
* negative and NEVER hands back a non-primary hint reply (Req 9.1, 9.2). The
|
||||
* any-node hint view is still available via xcecli_ctrlstatus above.
|
||||
*/
|
||||
extern int xcecli_ctrlstatus_primary(xcecli_ctx_t, xce_ctrlstatus_t**);
|
||||
/*
|
||||
* Pure decision helpers backing the authoritative read (no I/O — unit-testable
|
||||
* and exposed for tests). xcecli_ctrlstatus_is_authoritative returns true iff
|
||||
* the reply's responderRole is the primary role (ctrl_role_t PRIMARY == 0).
|
||||
* xcecli_ctrlstatus_primary_addr maps the reply's primaryName to its host:port
|
||||
* via the reply's peers list: returns 0 and fills host/port on a hit, -ENOENT
|
||||
* when the primary is unknown/absent from peers, -EINVAL on a bad argument.
|
||||
*/
|
||||
extern bool xcecli_ctrlstatus_is_authoritative(const xce_ctrlstatus_t*);
|
||||
extern int xcecli_ctrlstatus_primary_addr(const xce_ctrlstatus_t*,
|
||||
char* host_out, size_t host_sz,
|
||||
uint16_t* port_out);
|
||||
extern int xcecli_notify_jobfinish(xcecli_ctx_t, xce_jobfinish_t*);
|
||||
extern int xcecli_notify_jobstart(xcecli_ctx_t, job_pidinfo_t*);
|
||||
extern int xcecli_notify_jobfail(xcecli_ctx_t c, job_failinfo_t*);
|
||||
|
|
@ -74,6 +114,10 @@ extern int xcecli_jobinfo(xcecli_ctx_t, uint64_t, xce_job_t**);
|
|||
#define xcecli_free_queues(nqueues, queues) xce_free_queues(nqueues, queues)
|
||||
#define xcecli_free_groups(ngroups, groups) xce_free_groups(ngroups, groups)
|
||||
#define xcecli_free_clusterconf(conf) xce_free_clusterconf(conf)
|
||||
/* ctrlstatus reply is freed by the generated codec free (free_CtrlStatus,
|
||||
* declared in the generated controller.capnp.h). */
|
||||
#define xce_free_ctrlstatus(s) free_CtrlStatus(s)
|
||||
#define xcecli_free_ctrlstatus(s) xce_free_ctrlstatus(s)
|
||||
|
||||
extern int xcecli_reply_fetchfile(xcecli_ctx_t, xce_request_t*, xce_fetchfile_reply_t*);
|
||||
extern int xcecli_fetchfile(xcecli_ctx_t, xce_fetchfile_t*, xce_fetchfile_reply_t**);
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ typedef struct {
|
|||
int inactive_heartbeat_count;
|
||||
int fetch_job_interval;
|
||||
int netio_timeout;
|
||||
int vcpus;
|
||||
int vcpus; /* deprecated: use --vcpus CLI flag; kept for config blob backward compatibility */
|
||||
int nres;
|
||||
xce_resource_t** resources;
|
||||
int nqueues;
|
||||
|
|
@ -57,6 +57,7 @@ typedef struct {
|
|||
scaler_config_t** hproviders;
|
||||
int nlimits;
|
||||
xce_limit_config_t** limits;
|
||||
uint64_t config_generation; /* monotonic audit counter from capnp schema */
|
||||
} cluster_config_t;
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -71,4 +71,27 @@ typedef uint64_t jobid_t;
|
|||
#define REGEX_JOBARRAY_ID \
|
||||
"[ \t]*([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)\\[([0-9]+)\\][ \t]*"
|
||||
|
||||
/* ======================================================
|
||||
* Persistence & HA system constants (Part 2.6, Appendix A.10)
|
||||
* ====================================================== */
|
||||
#define MAX_STANDBYS 3
|
||||
#define MAX_HA_CANDIDATES 4
|
||||
#define WAL_FORMAT_VERSION 1
|
||||
#define WAL_ENTRY_TYPE_MAX 100 /* WAL_CHECKPOINT = 100, highest known type */
|
||||
#define MAX_WAL_PAYLOAD_BYTES (1 << 20) /* 1 MB per replication entry */
|
||||
#define MAX_SNAPSHOT_BYTES (256 * 1024 * 1024) /* 256 MB max snapshot size */
|
||||
#define WAL_HDR_CRC_SIZE 32 /* sizeof(wal_entry_hdr_t) - 2*sizeof(uint32_t) */
|
||||
#define WAL_CRC_NONE 0xFFFFFFFFu /* sentinel: no predecessor in chain */
|
||||
|
||||
#define DEFAULT_WAL_SYNC_MS 10
|
||||
#define DEFAULT_SNAP_INTERVAL_S 60
|
||||
#define DEFAULT_SNAP_WAL_SIZE_MB 64
|
||||
#define DEFAULT_SNAP_KEEP_N 2
|
||||
#define DEFAULT_TRANSIENT_JOB_TIMEOUT_S 300
|
||||
#define DEFAULT_FINISHED_RETAIN_S 3600
|
||||
#define DEFAULT_WORKER_RETAIN_S 86400
|
||||
#define DEFAULT_SYNC_TIMEOUT_MS 500
|
||||
#define DEFAULT_HB_INTERVAL_MS 3000
|
||||
#define DEFAULT_MAX_WAL_RECORDS 10000
|
||||
|
||||
#endif
|
||||
|
|
|
|||
47
src/include/ctrl_metrics.h
Normal file
47
src/include/ctrl_metrics.h
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#pragma once
|
||||
/*
|
||||
* ctrl_metrics.h — P3-7: Operational metrics for the XCE controller.
|
||||
*
|
||||
* All metrics use C11 _Atomic uint64_t for lock-free, thread-safe access.
|
||||
* Metric names use the ctrl_ prefix.
|
||||
*
|
||||
* Design doc: Part P3-7, review A8-007.
|
||||
*/
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* ── Counter/gauge globals ──────────────────────────────────────────── */
|
||||
extern _Atomic uint64_t ctrl_repl_degraded_count; /* counter */
|
||||
extern _Atomic uint64_t ctrl_split_brain_detected; /* counter */
|
||||
extern _Atomic uint64_t ctrl_async_events_lost; /* counter */
|
||||
extern _Atomic uint64_t ctrl_transient_jobs_stuck; /* counter */
|
||||
extern _Atomic uint64_t ctrl_wal_bytes; /* gauge */
|
||||
extern _Atomic uint64_t ctrl_checkpoint_duration_ms; /* gauge */
|
||||
extern _Atomic uint64_t ctrl_repl_lag_lsn; /* gauge */
|
||||
|
||||
/* ── Macros ─────────────────────────────────────────────────────────── */
|
||||
|
||||
/* Increment a monotonic counter by 1 (relaxed ordering — counters are
|
||||
* sampled, not used for synchronization). */
|
||||
#define CTRL_METRIC_INC(name) \
|
||||
atomic_fetch_add_explicit(&(name), 1, memory_order_relaxed)
|
||||
|
||||
/* Overwrite a gauge with a new absolute value. */
|
||||
#define CTRL_METRIC_SET(name, val) \
|
||||
atomic_store_explicit(&(name), (uint64_t)(val), memory_order_relaxed)
|
||||
|
||||
/* ── Descriptor table for metric discovery ──────────────────────────── */
|
||||
typedef struct {
|
||||
const char* name;
|
||||
_Atomic uint64_t* value;
|
||||
const char* description;
|
||||
int is_counter; /* 1 = counter (monotonic), 0 = gauge */
|
||||
} ctrl_metric_desc_t;
|
||||
|
||||
/* Terminated by a { NULL, NULL, NULL, 0 } sentinel entry. */
|
||||
extern const ctrl_metric_desc_t ctrl_metric_descriptors[];
|
||||
|
||||
/* ── Lifecycle ──────────────────────────────────────────────────────── */
|
||||
|
||||
/* Reset all metrics to zero. Call from persist_start() and ha_init(). */
|
||||
void ctrl_metrics_init(void);
|
||||
|
|
@ -14,6 +14,7 @@ typedef int (*task_run_fn_t)(delay_task_entry_fn_t, void*);
|
|||
|
||||
typedef struct {
|
||||
time_t nextrun;
|
||||
struct timespec nextrun_ts; /* non-zero tv_nsec flags an ms-task */
|
||||
int run_period;
|
||||
int runcount;
|
||||
delay_task_entry_fn_t entry;
|
||||
|
|
@ -23,7 +24,7 @@ typedef struct {
|
|||
typedef struct {
|
||||
pthread_t worker;
|
||||
int running;
|
||||
int precision;
|
||||
int precision_ms;
|
||||
task_run_fn_t runner_fn;
|
||||
timer_t timerid;
|
||||
ordq_t tasks;
|
||||
|
|
@ -34,5 +35,6 @@ extern void delaymgr_stop(delaymgr_t*);
|
|||
extern void delaymgr_fini(delaymgr_t*);
|
||||
extern int delaymgr_start(delaymgr_t*);
|
||||
extern int delay_task(delaymgr_t*, int, int, delay_task_entry_fn_t, void*);
|
||||
extern int delay_task_ms(delaymgr_t*, int, int, delay_task_entry_fn_t, void*);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -19,5 +19,6 @@ extern void* xce_notify_data_ptr(xce_notify_data_t*);
|
|||
extern void* xce_jobctrl_data_ptr(xce_jobctrl_t*);
|
||||
extern void* xce_reply_data_ptr(xce_reply_t*);
|
||||
extern char* xce_dataset_table(int);
|
||||
extern char* xce_notify_event_name(int);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#define __HOOK_H__ 1
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#include "xlist.h"
|
||||
|
|
@ -12,8 +13,8 @@
|
|||
|
||||
typedef int (*hook_fn_t)(int, int, int, void*, va_list);
|
||||
typedef struct {
|
||||
hook_fn_t fn;
|
||||
void* data;
|
||||
_Atomic hook_fn_t fn;
|
||||
void* data;
|
||||
} hook_item_t;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -23,6 +24,7 @@ typedef struct {
|
|||
|
||||
extern int xce_hook_init(hooksys_t*, int);
|
||||
extern int xce_add_hook(hooksys_t*, int, hook_fn_t, void*);
|
||||
extern int xce_disable_hook(hooksys_t*, int, hook_fn_t);
|
||||
extern int xce_run_hooks(hooksys_t*, int, int, int, ...);
|
||||
extern void xce_hook_fini(hooksys_t*);
|
||||
|
||||
|
|
|
|||
|
|
@ -30,4 +30,22 @@ extern kv_key_ops_t int_key_ops;
|
|||
extern kv_key_ops_t string_key_ops;
|
||||
|
||||
extern int key_cmp_default(void*, void*, size_t);
|
||||
|
||||
/*
|
||||
* kvtab_foreach — iterate every live entry in the kvtab (P0-14).
|
||||
*
|
||||
* For each entry, cb(key, index, arg) is called.
|
||||
* - key : the raw key pointer stored in the entry (char* for
|
||||
* string-keyed tables, (void*)(long)intval for int-keyed).
|
||||
* - index : the slot index stored as the entry value.
|
||||
* - arg : caller-supplied context.
|
||||
*
|
||||
* Iteration order is unspecified.
|
||||
* Returns the number of entries visited, or -EINVAL if kvtab is NULL.
|
||||
*/
|
||||
typedef void (*kvtab_foreach_fn_t)(void* key, int index, void* arg);
|
||||
extern int kvtab_foreach(var kvtab,
|
||||
kvtab_foreach_fn_t cb,
|
||||
void* arg);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
#include "xce/types.h"
|
||||
#include "xlist.h"
|
||||
#include "zarray.h"
|
||||
#include "persist.h"
|
||||
|
||||
typedef enum {
|
||||
KVMGR_KEY_INT = 0,
|
||||
|
|
@ -22,4 +23,22 @@ typedef struct {
|
|||
|
||||
extern typeinfo_t xkvmgr_type;
|
||||
|
||||
/*
|
||||
* kvmgr_capture_keys — capture key→index mappings for checkpoint
|
||||
* serialization (Part 3.8).
|
||||
*
|
||||
* Caller MUST hold the kvmgr write lock for the full duration of
|
||||
* this call. The function allocates *keys_out and populates it with
|
||||
* *n_keys_out entries. Each entry's key[] field is an owned deep
|
||||
* copy that remains valid after the write lock is released.
|
||||
*
|
||||
* Caller must free(*keys_out) when done.
|
||||
*
|
||||
* Returns 0 on success.
|
||||
* Returns -ENOMEM if allocation fails.
|
||||
*/
|
||||
int kvmgr_capture_keys(var kvmgr,
|
||||
slot_key_t** keys_out,
|
||||
size_t* n_keys_out);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -4,9 +4,16 @@
|
|||
|
||||
#define MAX_MASTER_CANDIDATES (64)
|
||||
|
||||
#include <time.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <tlog.h>
|
||||
|
||||
#include "clusterconfig.h"
|
||||
#include "xce/cluster.h"
|
||||
|
||||
/* Forward declaration — avoids circular include with cluster_def.h */
|
||||
typedef struct cluster_s cluster_t;
|
||||
|
||||
struct logging {
|
||||
char* logdir;
|
||||
|
|
@ -14,13 +21,30 @@ struct logging {
|
|||
};
|
||||
|
||||
typedef struct {
|
||||
cluster_config_t* cluster_config;
|
||||
char* conffile;
|
||||
char* certfile;
|
||||
char* keyfile;
|
||||
cluster_config_t* cluster_config;
|
||||
char* conffile;
|
||||
char* certfile;
|
||||
char* keyfile;
|
||||
char* workdir; /* working directory for WAL/snapshot storage */
|
||||
char* name; /* controller name (hostname by default) */
|
||||
xce_cluster_config_t peers_cfg; /* HA peer list parsed from --peers; count==0 = single-node */
|
||||
uint64_t config_version_tag; /* first 8 bytes of SHA-256, host order */
|
||||
uint8_t config_sha256[32]; /* full SHA-256 of conffile bytes */
|
||||
int conf_keep_n; /* versioned file retention count (default 2) */
|
||||
uint64_t config_generation; /* monotonic audit counter */
|
||||
time_t conf_stale_timeout; /* seconds before saved config flagged stale (default 86400) */
|
||||
int conf_stale; /* 1 when saved config older than conf_stale_timeout */
|
||||
int vcpus; /* Req 4.3: --vcpus CLI value; 0 = use config default */
|
||||
uint32_t sync_timeout_ms; /* --sync-timeout-ms; 0 = use persist default */
|
||||
uint32_t wal_sync_ms; /* --wal-sync-ms; 0 = use persist default */
|
||||
} myconf_t;
|
||||
|
||||
extern cluster_config_t* get_default_conf(void*);
|
||||
extern int add_config(void*, char*, char*);
|
||||
|
||||
void myconf_fini(myconf_t *myconf);
|
||||
|
||||
int load_primary_config(cluster_t *c);
|
||||
int fetch_config_from_primary(cluster_t *c);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ typedef struct {
|
|||
size_t iocount;
|
||||
} io_fsm_t;
|
||||
|
||||
typedef struct {
|
||||
typedef struct endpoint_t {
|
||||
mylock_t lock;
|
||||
iomgr_t* iomgr;
|
||||
SSL* ssl;
|
||||
|
|
@ -85,6 +85,7 @@ typedef struct {
|
|||
err_handler_t err_handler;
|
||||
time_t ts_last_in;
|
||||
time_t ts_last_out;
|
||||
void (*on_connected)(struct endpoint_t *ep); /* fire-once; called in iomgr thread after ssl_handshake_done=1 */
|
||||
} endpoint_t;
|
||||
|
||||
#define INVOKE_MSG_HANDLER(ep) \
|
||||
|
|
@ -126,6 +127,7 @@ extern int endpoint_is_task_blocked(endpoint_t*);
|
|||
extern int endpoint_set_owner(endpoint_t*, void*);
|
||||
extern int endpoint_set_msg_handler(endpoint_t*, msg_handler_t);
|
||||
extern int endpoint_set_err_handler(endpoint_t*, err_handler_t);
|
||||
extern int endpoint_set_on_connected(endpoint_t*, void (*cb)(endpoint_t *));
|
||||
extern void endpoint_fini(endpoint_t*);
|
||||
extern int endpoint_set_peer(endpoint_t*, xce_peerinfo_t*);
|
||||
|
||||
|
|
|
|||
638
src/include/persist.h
Normal file
638
src/include/persist.h
Normal file
|
|
@ -0,0 +1,638 @@
|
|||
#pragma once
|
||||
#include <pthread.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "wal_types.h" /* wal_entry_hdr_t, wal_entry_type — P2-4: shared with repl.h */
|
||||
#include "repl.h" /* replicator_t — P2-4: embedded in persist_t */
|
||||
|
||||
/* Forward declaration */
|
||||
typedef struct cluster_s cluster_t;
|
||||
|
||||
/* ============================================================
|
||||
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1)
|
||||
*
|
||||
* The outcome of WAL replay at startup, distinguishing a structurally
|
||||
* intact log from one whose chain is broken. It is about the STRUCTURE
|
||||
* of the local log, NOT about being behind the cluster: a clean log that
|
||||
* simply ends below the cluster's committed high-water mark is still
|
||||
* RECOVER_CLEAN (the freshness/promotion fence handles "behind"). A
|
||||
* corruption-truncated recovery is NEVER reported as a clean recovery.
|
||||
* ============================================================ */
|
||||
typedef enum {
|
||||
RECOVER_CLEAN = 0, /* WAL forms a valid CRC chain to a clean entry
|
||||
* boundary: clean EOF, pre-allocated-zero tail,
|
||||
* snapshot-sentinel + clean replay, or an empty
|
||||
* first-boot WAL (no entries != corruption). */
|
||||
RECOVER_TRUNCATED = 1, /* replay stopped on structural corruption: a CRC
|
||||
* chain break, a CRC mismatch, or a short/partial
|
||||
* entry read — the log cannot be reconstructed
|
||||
* contiguously. Marks the member non-promotable
|
||||
* (consumed by the promotion gate, task 4.3). */
|
||||
} recover_verdict_t;
|
||||
|
||||
/* ============================================================
|
||||
* Boot-policy recovery decision (ha-commit-durability task 3.3, Req 2.4)
|
||||
*
|
||||
* The outcome of evaluating a member's boot/promotion-eligibility state
|
||||
* against the Boot Policy Decision Table (design.md "Boot Policy Decision
|
||||
* Table (task 3.2)"). This task (3.3) owns the FATAL terminal; the fence +
|
||||
* catch-up + release path is task 4.3.
|
||||
*
|
||||
* ELIGIBLE — may promote (no fence). Rows 1, 6.
|
||||
* FENCE — ineligible to promote; catch up from a healthy peer, regain
|
||||
* eligibility on success (Req 2.6). Rows 2, 3, 5. A reachable
|
||||
* healthy peer NEVER yields FENCE_UNRECOVERABLE.
|
||||
* FENCE_UNRECOVERABLE — loud, PERMANENT, NON-terminating fence: the member
|
||||
* cannot recover locally AND cannot catch up (no healthy peer).
|
||||
* It logs LOUDLY, stays PERMANENTLY non-promotable, and keeps
|
||||
* retrying catch-up indefinitely — but the process STAYS ALIVE.
|
||||
* Rows 4, 4b. This is NOT process exit: a controller MUST NEVER
|
||||
* self-terminate as a designed response to an operational
|
||||
* condition (user invariant, 2026-06-18). A self-exited controller
|
||||
* needs MANUAL admin recovery and leaves quorum degraded
|
||||
* meanwhile; a controller that stays alive and keeps retrying
|
||||
* SELF-HEALS when a peer returns. The fence RELEASES (→ ELIGIBLE,
|
||||
* Req 2.6) automatically when catch-up reaches a
|
||||
* committed-consistent state.
|
||||
*
|
||||
* AUTHORITATIVE INVARIANT (design.md): a truncated/behind member resolves to
|
||||
* either FENCE→ELIGIBLE (caught up) or a loud PERMANENT FENCE_UNRECOVERABLE
|
||||
* (stays alive, non-promotable, keeps retrying). It is NEVER silently divergent
|
||||
* (it refuses to lead) and the process NEVER self-exits.
|
||||
* ============================================================ */
|
||||
typedef enum {
|
||||
RECOVER_DECISION_ELIGIBLE = 0,
|
||||
RECOVER_DECISION_FENCE = 1,
|
||||
RECOVER_DECISION_FENCE_UNRECOVERABLE = 2,
|
||||
} recover_decision_t;
|
||||
|
||||
/* Policy inputs evaluated at boot / promotion-eligibility (design.md table).
|
||||
* Modelled as a plain value so the decision is a pure, unit-testable function
|
||||
* of its inputs — the live catch-up machinery (replication / task 4.3) supplies
|
||||
* these; this task does not build it. */
|
||||
typedef struct {
|
||||
recover_verdict_t verdict; /* RECOVER_CLEAN / RECOVER_TRUNCATED (task 3.1) */
|
||||
bool durable_below_hwm; /* local durable position < cluster committed HWM */
|
||||
bool healthy_peer_reachable; /* a healthy peer is reachable for catch-up */
|
||||
bool catch_up_succeeded; /* catch-up reached a committed-consistent state
|
||||
* within the bounded window (only meaningful
|
||||
* when a catch-up was attempted) */
|
||||
uint64_t durable_floor_lsn; /* highest intact replayed LSN (truncation boundary) */
|
||||
uint64_t cluster_hwm_lsn; /* cluster committed high-water mark, for logging */
|
||||
} recover_policy_inputs_t;
|
||||
|
||||
/* ============================================================
|
||||
* Persistence configuration (Part 2.6)
|
||||
* ============================================================ */
|
||||
typedef struct {
|
||||
uint32_t wal_sync_ms; /* fsync interval in ms (default 10) */
|
||||
uint32_t snap_interval_s; /* checkpoint interval in seconds (default 60) */
|
||||
uint32_t snap_wal_size_mb; /* checkpoint when WAL exceeds this MB (default 64) */
|
||||
uint32_t snap_keep_n; /* snapshots to retain (default 2) */
|
||||
uint32_t transient_job_timeout_s; /* seconds before TRANSIENT job auto-terminates (default 300) */
|
||||
uint32_t finished_retain_s; /* retain finished jobs N seconds (default 3600) */
|
||||
uint32_t worker_retain_s; /* retain unavailable workers N seconds (default 86400) */
|
||||
uint32_t sync_timeout_ms; /* sync replication ACK timeout ms (default 50) */
|
||||
uint32_t max_wal_records; /* checkpoint trigger: records since last snap (default 10000) */
|
||||
} persist_config_t;
|
||||
|
||||
static inline persist_config_t persist_config_defaults(void)
|
||||
{
|
||||
persist_config_t c;
|
||||
c.wal_sync_ms = DEFAULT_WAL_SYNC_MS;
|
||||
c.snap_interval_s = DEFAULT_SNAP_INTERVAL_S;
|
||||
c.snap_wal_size_mb = DEFAULT_SNAP_WAL_SIZE_MB;
|
||||
c.snap_keep_n = DEFAULT_SNAP_KEEP_N;
|
||||
c.transient_job_timeout_s = DEFAULT_TRANSIENT_JOB_TIMEOUT_S;
|
||||
c.finished_retain_s = DEFAULT_FINISHED_RETAIN_S;
|
||||
c.worker_retain_s = DEFAULT_WORKER_RETAIN_S;
|
||||
c.sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS;
|
||||
c.max_wal_records = DEFAULT_MAX_WAL_RECORDS;
|
||||
return c;
|
||||
}
|
||||
|
||||
/*
|
||||
* persist_config_set_wal_sync_ms — apply the --wal-sync-ms flush-cadence
|
||||
* override (ha-commit-durability Req 3.1).
|
||||
*
|
||||
* Single override point shared by both persist_config_defaults() callers
|
||||
* (cluster.c phase2 and myapp.c step 6), mirroring the --sync-timeout-ms
|
||||
* convention: override_ms == 0 means "flag not passed" and leaves the
|
||||
* default flush cadence untouched (Req 3.3 — the seam changes timing
|
||||
* controllability only, not the durability contract). A positive value
|
||||
* sets the cadence to exactly that, applied before persistence starts.
|
||||
*/
|
||||
static inline void persist_config_set_wal_sync_ms(persist_config_t *c,
|
||||
uint32_t override_ms)
|
||||
{
|
||||
if (override_ms > 0)
|
||||
c->wal_sync_ms = override_ms;
|
||||
}
|
||||
|
||||
/* ============================================================
|
||||
* Key→index mapping for checkpoint serialization (Part 3.8)
|
||||
* ============================================================ */
|
||||
typedef struct {
|
||||
int index; /* shadow slot index in the frozen zarray */
|
||||
char key[XCE_MAXNAME_LEN]; /* owned deep copy — not a borrowed kvtab pointer */
|
||||
size_t key_len; /* length including null terminator */
|
||||
} slot_key_t;
|
||||
|
||||
/* ============================================================
|
||||
* persist_t — concrete definition (exposed for stack/calloc allocation)
|
||||
* The opaque pattern can be restored in a later refactor once the
|
||||
* full API surface is stable.
|
||||
* ============================================================ */
|
||||
struct persist_s {
|
||||
/* Back-pointer to owning cluster (set by persist_init, used in persist_start) */
|
||||
cluster_t* cluster;
|
||||
|
||||
/* WAL state */
|
||||
int wal_fd;
|
||||
uint64_t current_lsn;
|
||||
uint32_t last_crc;
|
||||
uint32_t last_replicated_crc;
|
||||
uint64_t term;
|
||||
uint32_t seg_offset;
|
||||
uint64_t checkpoint_lsn;
|
||||
_Atomic uint64_t committed_lsn; /* highest LSN known committed; invariant: <= current_lsn - 1 */
|
||||
_Atomic uint64_t durable_lsn; /* highest LSN whose bytes are fdatasync'd locally (group-commit
|
||||
* prefix watermark). Advanced ONLY by the flusher after a
|
||||
* successful fdatasync. Invariants: monotonic non-decreasing
|
||||
* (except clamp-on-truncate); durable_lsn <= current_lsn - 1.
|
||||
* The commit majority counts self at this LSN, not the
|
||||
* appended tip (ha-commit-durability Req 1.1/1.2; consumed by
|
||||
* task 2.2). */
|
||||
uint64_t total_wal_bytes;
|
||||
char seg_path[4096];
|
||||
|
||||
/* Per-LSN recovered/live term index (ha-term-safety task 1.3, R1.5).
|
||||
* IN-MEMORY ONLY — no on-disk layout change (R1.7). Records each entry's
|
||||
* hdr.term keyed by LSN so the term can be read back after recovery and
|
||||
* fed to the reconcile/promotion logic (on_get_entry_term, task 2.2).
|
||||
*
|
||||
* term_index[lsn - 1] holds the RAW recorded term for a 1-based LSN (LSN 0
|
||||
* is reserved and never indexed). term_index_count is the highest LSN
|
||||
* recorded (== number of contiguous slots in use), used as the presence
|
||||
* watermark so persist_entry_term distinguishes an ABSENT LSN (returns 0)
|
||||
* from a PRESENT legacy-0 entry (mapped to the lowest-known term). Grown on
|
||||
* demand by record_entry_term during replay_wal and live appends; freed in
|
||||
* persist_fini. Guarded by p->lock (written under the append lock and the
|
||||
* single-threaded recovery path; read by accessors that take p->lock). */
|
||||
uint64_t* term_index; /* term_index[i] = term of LSN (i+1) */
|
||||
uint64_t term_index_count; /* highest recorded LSN (slots in use) */
|
||||
uint64_t term_index_cap; /* allocated slot capacity */
|
||||
|
||||
/* Durable committed-config reference carried in the checkpoint sentinel
|
||||
* (ha-term-safety task 4.2 remediation r1 — compaction-safe config derive).
|
||||
* A checkpoint trims WAL segments older than the open one (trim_wal_segments)
|
||||
* and the snapshot sentinel does NOT carry entity state, so a committed
|
||||
* WAL_CONFIG_MARKER in a trimmed segment would vanish and the derived
|
||||
* committed config (config_committed_marker) would regress. To prevent that,
|
||||
* checkpoint stamps the latest-committed marker's 40-byte REFERENCE
|
||||
* ({version_tag, sha256} — a reference, NOT the config bytes / entity
|
||||
* snapshot) into the sentinel; recovery reads it back here; and the derive
|
||||
* falls back to this when no marker survives the live WAL scan.
|
||||
* cfgref_present == 0 means "no committed config marker was known at the last
|
||||
* checkpoint" (a fresh store or a pre-config-marker checkpoint). Written
|
||||
* single-threaded at checkpoint/recovery; read by config_committed_marker. */
|
||||
int cfgref_present; /* 1 if cfgref_tag/sha are valid */
|
||||
uint64_t cfgref_tag; /* latest committed marker version_tag */
|
||||
uint8_t cfgref_sha[32]; /* latest committed marker content hash */
|
||||
|
||||
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1).
|
||||
* Set by persist_recover/replay_wal at startup: RECOVER_CLEAN when the
|
||||
* local WAL forms a valid CRC chain to a clean entry boundary (incl. an
|
||||
* empty first-boot WAL, a snapshot-sentinel + clean replay, and a
|
||||
* pre-allocated-zero tail), RECOVER_TRUNCATED when replay stopped on a
|
||||
* structural corruption — a CRC chain break, a CRC mismatch, or a
|
||||
* short/partial entry read — so the on-disk log cannot be reconstructed
|
||||
* contiguously. A node that cleanly ends BELOW the cluster high-water is
|
||||
* still CLEAN here (the freshness/promotion fence catches "behind", task
|
||||
* 4.2/4.3 — not this verdict). 3.1 only CLASSIFIES + sets this; it does
|
||||
* NOT FATAL (task 3.3) and does NOT fence promotion (task 4.3, which
|
||||
* CONSUMES this verdict). Defaults to RECOVER_CLEAN before recovery runs
|
||||
* (a fresh store is clean). Written single-threaded during recovery
|
||||
* before the node participates; read by the promotion gate thereafter. */
|
||||
recover_verdict_t recovery_verdict;
|
||||
|
||||
/* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4,
|
||||
* NEVER-SELF-EXIT rework 2026-06-18). Set by
|
||||
* persist_recover_loud_fence_if_unrecoverable when a member is
|
||||
* recovery-truncated AND/OR durable below the cluster committed high-water
|
||||
* mark with NO successful catch-up: the member stays ALIVE but is held
|
||||
* PERMANENTLY non-promotable and keeps retrying catch-up. A controller MUST
|
||||
* NEVER self-exit as a designed response to this operational condition; it
|
||||
* self-heals automatically when a healthy peer returns and catch-up reaches
|
||||
* a committed-consistent state — at which point this flag is CLEARED via
|
||||
* persist_clear_recovery_unrecoverable (wired into the fence-release /
|
||||
* catch-up path). Read by the promotion gate as belt-and-suspenders
|
||||
* (a member must never lead while this is set). Atomic: set on the recovery
|
||||
* path, cleared on the catch-up path, read by the promotion/heartbeat
|
||||
* threads. Defaults to false before recovery runs. */
|
||||
_Atomic bool recovery_unrecoverable;
|
||||
|
||||
/* Paths (derived from workdir + name via persist_set_workdir) */
|
||||
char wal_path[4096];
|
||||
char snap_path[4096];
|
||||
|
||||
/* Config */
|
||||
persist_config_t config;
|
||||
|
||||
/* Concurrency */
|
||||
pthread_mutex_t lock;
|
||||
atomic_uint wal_record_count; /* records appended since last checkpoint */
|
||||
atomic_int checkpoint_in_progress; /* CAS guard: 0=idle, 1=scheduled/running */
|
||||
atomic_int fsync_stop;
|
||||
atomic_int fsync_thread_running;
|
||||
#ifdef XCE_TESTING
|
||||
/* Fault-injection seam (ha-commit-durability task 2.1 error-path test):
|
||||
* when non-zero, every fdatasync site treats the sync as FAILED — it
|
||||
* still calls the real fdatasync but forces a non-zero rc, so durable_lsn
|
||||
* must NOT advance and the failure is logged loudly. Compiled out of
|
||||
* production (xce_controller is built without -DXCE_TESTING). */
|
||||
atomic_int test_force_fsync_fail;
|
||||
/* fdatasync observability counter (ha-commit-durability task 2.3):
|
||||
* incremented at the standby-path inline sync in
|
||||
* persist_wal_write_replicated, BEFORE that function returns (and thus
|
||||
* before the caller can ACK). A persist-before-ACK regression test reads
|
||||
* this to prove a durable sync ran inside the call — page-cache re-reads
|
||||
* cannot distinguish fsync'd from cache-only, so a direct sync signal is
|
||||
* required. Compiled out of production. */
|
||||
atomic_uint test_replicated_fdatasync_count;
|
||||
/* Liveness observability (ha-commit-durability task 5.1, Req 1.6):
|
||||
* the thread id of the last flusher fdatasync that advanced durable_lsn.
|
||||
* A test reads this to prove fdatasync runs on the dedicated flusher
|
||||
* thread, NOT on the caller/iomgr/heartbeat thread. Compiled out of
|
||||
* production. */
|
||||
pthread_t test_last_fdatasync_thread;
|
||||
atomic_int test_last_fdatasync_thread_set;
|
||||
#endif
|
||||
pthread_t fsync_thread;
|
||||
pthread_mutex_t fsync_mutex; /* guards the fsync timer wait */
|
||||
pthread_cond_t fsync_cond; /* signaled by persist_stop for prompt join */
|
||||
|
||||
/* Durable-advance wake hook (ha-commit-durability task 2.2).
|
||||
* Persistence owns this slot; the commit engine REGISTERS a callback
|
||||
* (ctx = the commit_engine_t) via persist_set_durable_cb. The flusher
|
||||
* invokes it AFTER advancing durable_lsn and AFTER releasing p->lock,
|
||||
* so a COMMIT_MODE_WAIT caller is woken promptly when its entry becomes
|
||||
* durable locally — without it, a commit whose standby ACKs all arrived
|
||||
* before the local fdatasync would stall until the WAIT timeout. The
|
||||
* dependency direction is preserved (commit depends on persist, never
|
||||
* the reverse): persist only stores+calls an opaque function pointer.
|
||||
* NULL until registered (every flush is then a no-op wake). */
|
||||
void (*on_durable_advance)(void* ctx);
|
||||
void* durable_cb_ctx;
|
||||
|
||||
/* Checkpoint (stubs for now — wired in Task 10) */
|
||||
atomic_int checkpoint_requested;
|
||||
pthread_cond_t checkpoint_done;
|
||||
pthread_mutex_t checkpoint_mutex;
|
||||
pthread_t snap_thread;
|
||||
|
||||
/* Fan-out replicator — P2-4 */
|
||||
replicator_t repl;
|
||||
};
|
||||
typedef struct persist_s persist_t;
|
||||
|
||||
/* Set workdir and controller name before persist_init().
|
||||
* Derives wal_path and snap_path as <workdir>/<name>/data/{wal,snap}. */
|
||||
void persist_set_workdir(persist_t* p, const char* workdir, const char* name);
|
||||
|
||||
/* ============================================================
|
||||
* WAL term adapter (ha-term-safety task 1.3, Req 1.1/1.5/1.6/1.7/1.8)
|
||||
*
|
||||
* Today every WAL append stamps hdr.term = p->term, but p->term is never
|
||||
* assigned, so all entries carry term 0 (the G1 bug). The on-disk
|
||||
* wal_entry_hdr_t already has a 40-byte `term` field; this is a VALUE fix, NOT
|
||||
* a layout change (R1.7).
|
||||
* ============================================================ */
|
||||
|
||||
/* Legacy-zero-term rule (R1.8) — REVIEW ISSUE 2, made precise:
|
||||
*
|
||||
* A WAL entry that predates this feature carries hdr.term == 0. For (term,lsn)
|
||||
* matching it must map to a SINGLE, well-defined "lowest-known term" so that
|
||||
* (a) recovery never fails on legacy-0 entries, and
|
||||
* (b) a legacy-0 entry is never spuriously treated as divergent from another
|
||||
* legacy-0 entry at the same LSN.
|
||||
* The rule: legacy-0 -> WAL_LEGACY_TERM (1, the lowest real Raft term — Raft
|
||||
* terms start at 1; term 0 is the pre-election "no term" placeholder). It is
|
||||
* applied UNIFORMLY at read-back (persist_entry_term), so every legacy-0 entry
|
||||
* answers the same term 1 and matches another legacy-0 entry at the same LSN.
|
||||
* The raw 0 is still stored in the index (so "present legacy-0" stays
|
||||
* distinguishable from "absent"); only the read-back value is mapped. */
|
||||
#define WAL_LEGACY_TERM 1u
|
||||
|
||||
/* Set the controller's live election term so subsequent WAL appends stamp it
|
||||
* into hdr.term (R1.1/R1.6). Called from the append callback (on_persist_entry,
|
||||
* task 2.2) and on term adoption, so no append records a zero placeholder while
|
||||
* a valid term is in effect. No on-disk layout change; value fix only (R1.7).
|
||||
* Safe on a NULL p (no-op). */
|
||||
void persist_set_term(persist_t* p, uint64_t term);
|
||||
|
||||
/* Recovered/live per-entry term for a 1-based LSN (lsn == Raft index), feeding
|
||||
* the task-2.2 on_get_entry_term callback and the library's init restore
|
||||
* (R1.5). Returns 0 when the LSN is ABSENT (never recorded). A PRESENT entry
|
||||
* that carries a legacy zero term is mapped to WAL_LEGACY_TERM (R1.8) so it is
|
||||
* never confused with an absent LSN and matches other legacy-0 entries.
|
||||
* Safe on a NULL p (returns 0). */
|
||||
uint64_t persist_entry_term(const persist_t* p, uint64_t lsn);
|
||||
|
||||
/* Lifecycle */
|
||||
int persist_init(persist_t* p, persist_config_t* config, cluster_t* c);
|
||||
int persist_start(persist_t* p);
|
||||
void persist_stop(persist_t* p);
|
||||
void persist_fini(persist_t* p);
|
||||
int persist_recover(persist_t* p);
|
||||
|
||||
/* WAL write paths */
|
||||
int persist_wal_async(persist_t* p, uint32_t type,
|
||||
const void* payload, uint32_t len);
|
||||
int persist_wal_sync_append(persist_t* p, uint32_t type,
|
||||
const void* payload, uint32_t len,
|
||||
int* sync_rc_out);
|
||||
|
||||
/* Write a REPLICATED entry (received from the primary) into the local WAL
|
||||
* at its GIVEN lsn — used by standbys to durably persist streamed entries
|
||||
* BEFORE acking them (persist-before-ACK; an ACK must imply durability).
|
||||
*
|
||||
* Unlike persist_wal_async it does NOT assign a new lsn (the lsn is supplied
|
||||
* by the primary) and does NOT repl_broadcast (standbys never re-replicate).
|
||||
* It builds a SELF-consistent CRC chain (the streamed entry carries no crc),
|
||||
* fsyncs the segment before returning, and is idempotent on re-stream: if the
|
||||
* lsn is already persisted (lsn+1 <= current_lsn) it is a no-op returning 0.
|
||||
*
|
||||
* WAL_CHECKPOINT is rejected with -EINVAL (never streamed to standbys).
|
||||
* Returns 0 on success (incl. dedup no-op), negative errno on failure. */
|
||||
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
|
||||
const void* payload, uint32_t len);
|
||||
|
||||
/* ============================================================
|
||||
* Write-path commit classification (ha-commit-barrier-rollout)
|
||||
* ============================================================ */
|
||||
typedef enum {
|
||||
WAL_MODE_WAIT, /* request-shaped: barrier before requester reply */
|
||||
WAL_MODE_RETAIN, /* fact-shaped: retained until committed, no timeout */
|
||||
WAL_MODE_LOCAL, /* not replicated (WAL_CHECKPOINT) */
|
||||
} wal_commit_mode_t;
|
||||
|
||||
/* Pure classification lookup. detail disambiguates WAL_WORKER_CTRL only
|
||||
* (HOOK_DETAIL_WORKER_JOIN/UPDATE -> WAIT, HOOK_DETAIL_WORKER_DEAD ->
|
||||
* RETAIN). Unknown type: XCE_TESTING assert (fail fast); production logs
|
||||
* ERROR and returns WAL_MODE_RETAIN (never dropped, never confirmed early). */
|
||||
wal_commit_mode_t wal_classify(uint32_t wal_type, int detail);
|
||||
|
||||
/* Commit barrier — handler-callable WAIT completion on everything appended
|
||||
* so far (ha-commit-barrier-rollout). Snapshots the appended watermark
|
||||
* (current_lsn - 1, under p->lock — matching persist_wal_sync_append's
|
||||
* read); returns 0 without blocking when committed_lsn already covers it
|
||||
* (or nothing was ever appended). Otherwise delegates to
|
||||
* commit_submit(COMMIT_MODE_WAIT) and returns its code unmodified:
|
||||
* 0 — committed on a majority
|
||||
* -ETIMEDOUT — not committed within the deadline (entries remain
|
||||
* in the log; outcome converges per umbrella 2.5)
|
||||
* -XCE_ERR_NOT_READY — engine admission cap exhausted (retriable)
|
||||
* timeout_ms = 0 falls back to p->config.sync_timeout_ms. n_peers <= 1
|
||||
* returns 0 immediately (engine self-quorum, standalone preserved).
|
||||
* Call only from vtask threads with no entity locks held. */
|
||||
int persist_commit_barrier(persist_t* p, uint32_t timeout_ms);
|
||||
|
||||
/* Force WAL fdatasync (for tests and clean shutdown) */
|
||||
int persist_flush(persist_t* p);
|
||||
|
||||
/* Highest LSN whose bytes are durably fdatasync'd locally — the
|
||||
* group-commit prefix watermark advanced by the demand-driven flusher
|
||||
* (ha-commit-durability task 2.1). The commit majority self-contribution
|
||||
* caps at this value (task 2.2 consumer). Lock-free atomic read; safe from
|
||||
* any thread. Returns 0 when nothing is yet durable. */
|
||||
uint64_t persist_durable_lsn(persist_t* p);
|
||||
|
||||
/* Durable freshness floor (ha-commit-durability task 3.4, Req 2.3).
|
||||
*
|
||||
* The member's durably-backed last-log index (WAL tip): the highest LSN that
|
||||
* is on stable storage locally. persist_recover re-derives it from the
|
||||
* recovered WAL tip (the highest INTACT replayed LSN) so the post-recovery
|
||||
* value is backed by durable on-disk state, NOT a volatile in-memory
|
||||
* applied_lsn / last_known_committed_lsn. The Raft freshness gate (task 4.2,
|
||||
* ha.c / cluster.c) consumes this so a truncated member cannot over-advertise
|
||||
* a tip it does not durably hold.
|
||||
*
|
||||
* Precision: this is the durable LAST-LOG INDEX, NOT the commit position
|
||||
* (committed_lsn). The post-election safe-commit point still comes from voter
|
||||
* replies, never from this floor (CLAUDE.md). After a truncating restart the
|
||||
* floor equals the lower intact tip, never a stale higher in-memory value and
|
||||
* never the commit position.
|
||||
*
|
||||
* Thin wrapper over persist_durable_lsn: at and after recovery the durable
|
||||
* watermark IS the recovered WAL tip (recovered on-disk entries are genuinely
|
||||
* durable), so a separate field would be redundant. Lock-free atomic read;
|
||||
* safe from any thread. Returns 0 when nothing is durable (empty first-boot
|
||||
* WAL). */
|
||||
uint64_t persist_durable_floor(persist_t* p);
|
||||
|
||||
/* Recovery-integrity verdict + flag (ha-commit-durability task 3.1, Req 2.1).
|
||||
* persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED
|
||||
* verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh
|
||||
* store before recovery runs). persist_recovery_truncated is the convenience
|
||||
* flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate
|
||||
* (task 4.3) consumes to mark the member non-promotable. Both are pure reads
|
||||
* of recovery state established single-threaded at startup. */
|
||||
recover_verdict_t persist_recovery_verdict(persist_t* p);
|
||||
bool persist_recovery_truncated(persist_t* p);
|
||||
|
||||
/* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4;
|
||||
* never-self-exit rework 2026-06-18).
|
||||
* The interval after which a still-unrecoverable, loudly-fenced member RE-EMITS
|
||||
* its operator alert (it does NOT terminate — see RECOVER_DECISION_FENCE_-
|
||||
* UNRECOVERABLE). The member stays alive and keeps retrying catch-up
|
||||
* indefinitely; this window only controls how often the loud operator-visible
|
||||
* alert is repeated. Default 30s. */
|
||||
#define DEFAULT_RECOVERY_CATCHUP_WINDOW_MS 30000u
|
||||
|
||||
/* persist_recover_decision (ha-commit-durability task 3.3, Req 2.4).
|
||||
*
|
||||
* Pure decision over the Boot Policy Decision Table: maps the policy inputs to
|
||||
* RECOVER_DECISION_{ELIGIBLE,FENCE,FENCE_UNRECOVERABLE}. No side effects, no
|
||||
* termination — it only classifies, so it is unit-testable in isolation. The
|
||||
* FENCE_UNRECOVERABLE terminal is acted on by
|
||||
* persist_recover_loud_fence_if_unrecoverable (which logs LOUDLY + sets the
|
||||
* sticky non-promotable flag — it NEVER terminates the process).
|
||||
*
|
||||
* Contract (design.md rows): a reachable healthy peer NEVER returns
|
||||
* FENCE_UNRECOVERABLE (rows 3, 5); it is returned ONLY when the member is
|
||||
* truncated or behind the HWM, has NO healthy peer, and catch-up did not reach
|
||||
* a committed-consistent state (rows 4, 4b). */
|
||||
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in);
|
||||
|
||||
/* persist_recover_loud_fence_if_unrecoverable (ha-commit-durability task 3.3,
|
||||
* Req 2.4; never-self-exit rework 2026-06-18).
|
||||
*
|
||||
* Evaluate persist_recover_decision over `in`; if the outcome is
|
||||
* FENCE_UNRECOVERABLE, log LOUDLY at ERROR — INCLUDING the truncation boundary
|
||||
* (durable_floor_lsn) and the cluster high-water mark — stating the member is
|
||||
* staying fenced + non-promotable and will keep retrying catch-up, then SET the
|
||||
* sticky `recovery_unrecoverable` flag on `p` (so the promotion gate refuses
|
||||
* leadership as belt-and-suspenders). Returns the decision.
|
||||
*
|
||||
* CRITICAL — the process STAYS ALIVE. There is NO abort()/exit()/tlog_exit()
|
||||
* here: a controller MUST NEVER self-terminate as a designed response to an
|
||||
* operational condition (user invariant, 2026-06-18). The member self-heals —
|
||||
* the fence releases (→ ELIGIBLE, Req 2.6) and the flag clears via
|
||||
* persist_clear_recovery_unrecoverable — automatically when a healthy peer
|
||||
* returns and catch-up reaches a committed-consistent state.
|
||||
*
|
||||
* `p` may be NULL (pure-decision callers that only want the classification +
|
||||
* loud log without touching a persist_t); the flag set is then skipped. */
|
||||
recover_decision_t
|
||||
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
|
||||
const recover_policy_inputs_t* in);
|
||||
|
||||
/* persist_recovery_unrecoverable / persist_clear_recovery_unrecoverable
|
||||
* (ha-commit-durability Req 2.4/2.6, never-self-exit rework 2026-06-18).
|
||||
*
|
||||
* Accessor + clear for the sticky "unrecoverable, loudly fenced" flag. The flag
|
||||
* is SET by persist_recover_loud_fence_if_unrecoverable and CLEARED on the
|
||||
* catch-up / fence-release path once the member reaches a committed-consistent
|
||||
* state (not truncated AND durable_floor >= HWM). While set, the member is held
|
||||
* permanently non-promotable but stays alive. Safe on a NULL p (returns false /
|
||||
* no-op). */
|
||||
bool persist_recovery_unrecoverable(persist_t* p);
|
||||
void persist_clear_recovery_unrecoverable(persist_t* p);
|
||||
|
||||
/* Register a durable-advance wake callback (ha-commit-durability task 2.2).
|
||||
* The group-commit flusher calls fn(ctx) after it advances durable_lsn and
|
||||
* after releasing p->lock, so a consumer (the commit engine) can re-evaluate
|
||||
* its commit majority and wake any COMMIT_MODE_WAIT caller whose entry just
|
||||
* became durable locally. fn MUST NOT take p->lock (it runs after release, on
|
||||
* the flusher thread) — the commit engine's callback takes only ce->lock,
|
||||
* keeping the lock order p->lock → ce->lock cycle-free. Pass fn = NULL to
|
||||
* unregister. Safe to call before persist_start (no flusher running yet). */
|
||||
void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx);
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* ============================================================
|
||||
* Durability-verification test seam (ha-commit-durability Req 3.2, 3.3)
|
||||
*
|
||||
* persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL
|
||||
* segments from STABLE STORAGE and return the highest fully-persisted
|
||||
* LSN, independent of the in-memory current_lsn / committed_lsn
|
||||
* watermarks. Lets a test assert that an acknowledged operation is
|
||||
* recoverable from disk after an immediate, ungraceful termination —
|
||||
* without relying on the background flush timer (Req 3.2):
|
||||
*
|
||||
* persist_wal_async(p, ...); // append to page cache
|
||||
* persist_flush(p); // fdatasync → durable on disk
|
||||
* lsn = persist_test_durable_lsn_on_disk(p); // highest durable LSN
|
||||
*
|
||||
* It scans the same on-disk format and CRC chain as recovery, stopping
|
||||
* at the first corruption / short read / pre-allocated zero region, so
|
||||
* it reports only entries that are genuinely on stable storage. It
|
||||
* never mutates persist_t state and fires no hooks (Req 3.3 — the seam
|
||||
* is purely inspective; the acknowledgment contract is unchanged).
|
||||
*
|
||||
* Returns the highest durable LSN, or 0 when nothing is durable (empty
|
||||
* first-boot WAL, no segments). Compiled out of production builds:
|
||||
* the xce_controller target is built without -DXCE_TESTING and links
|
||||
* without this symbol.
|
||||
* ============================================================ */
|
||||
uint64_t persist_test_durable_lsn_on_disk(persist_t* p);
|
||||
|
||||
/* Fault-injection toggle for the fdatasync error path (task 2.1).
|
||||
* When `fail` is non-zero, every fdatasync site in persist.c forces a
|
||||
* failed sync result: the real fdatasync still runs, but durable_lsn is
|
||||
* left UNCHANGED and the failure is logged loudly — proving an acknowledged
|
||||
* entry is never counted durable on a sync error (Req 1.1/1.2, fail-loud).
|
||||
* Lock-free atomic write; safe to flip from a test thread at any time.
|
||||
* Compiled out of production (xce_controller links without it). */
|
||||
void persist_test_set_force_fsync_fail(persist_t* p, int fail);
|
||||
|
||||
/* Read the standby-path fdatasync counter (task 2.3, persist-before-ACK
|
||||
* regression). Counts the inline fdatasync calls made by
|
||||
* persist_wal_write_replicated. A test snapshots this before the call and
|
||||
* asserts it incremented after the call returns — proving the durable sync
|
||||
* ran inside the persist primitive, BEFORE the caller can send its WalAck.
|
||||
* Lock-free atomic read; compiled out of production. */
|
||||
unsigned persist_test_replicated_fdatasync_count(persist_t* p);
|
||||
|
||||
/* Invoke the legacy sync-append commit store (task 2.4) with the durable
|
||||
* precondition applied — the same store persist_wal_sync_append performs on a
|
||||
* majority ACK in its engine-less / LSN-0 branch. Lets a test prove
|
||||
* committed_lsn never advances past durable_lsn on the ACK path without
|
||||
* standing up the replication wire. Compiled out of production. */
|
||||
void persist_test_sync_append_legacy_commit_store(persist_t* p,
|
||||
uint64_t written_lsn);
|
||||
|
||||
/* Return the thread id of the last flusher fdatasync that advanced
|
||||
* durable_lsn (task 5.1 / Req 1.6 liveness). A test asserts this is NOT the
|
||||
* caller/iomgr thread — proving durability (fdatasync) work runs on the
|
||||
* dedicated flusher thread and never stalls heartbeat/election handling.
|
||||
* Returns pthread_self() if no flusher sync has advanced durable_lsn yet
|
||||
* (so an unprimed read can never falsely look "off-thread"). Compiled out of
|
||||
* production. */
|
||||
pthread_t persist_test_last_fdatasync_thread(persist_t* p);
|
||||
|
||||
/* Close the active WAL segment and open a fresh one at the current LSN
|
||||
* (ha-term-safety task 4.2 r1 test seam). Forces a multi-segment layout so a
|
||||
* subsequent persist_checkpoint trims the older segment — exercising the
|
||||
* compaction hazard where a committed marker's segment is unlinked. Returns 0
|
||||
* on success, negative errno on I/O failure. Compiled out of production. */
|
||||
int persist_test_force_new_segment(persist_t* p);
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* Checkpoint */
|
||||
int persist_checkpoint(persist_t* p);
|
||||
|
||||
/* WAL truncation — remove all entries with LSN > cutoff_lsn.
|
||||
* Idempotent; returns 0 on success, negative errno on I/O error. */
|
||||
int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn);
|
||||
|
||||
/* Logging helper — never logs payload content (security invariant) */
|
||||
void persist_log_entry_metadata(const wal_entry_hdr_t* hdr);
|
||||
|
||||
/* ============================================================
|
||||
* AFTER_EVENT hook functions — registered by persist_start()
|
||||
*
|
||||
* Signature matches hook_fn_t: int fn(int type, int situation,
|
||||
* int detail, void* data, va_list ap)
|
||||
* cls/data points to the persist_t instance.
|
||||
*
|
||||
* P0: writes a minimal 4-byte payload (detail value) to the WAL.
|
||||
* Full Cap'n Proto serialization is wired in Task 15.
|
||||
* ============================================================ */
|
||||
int persist_job_hook_fun(int type, int situation, int detail,
|
||||
void* cls, va_list ap);
|
||||
int persist_queue_hook_fun(int type, int situation, int detail,
|
||||
void* cls, va_list ap);
|
||||
int persist_worker_hook_fun(int type, int situation, int detail,
|
||||
void* cls, va_list ap);
|
||||
|
||||
/* Apply a single WAL entry to in-memory state (LOGICAL entity restore only).
|
||||
* No hooks fire — hook system is empty on standbys until promotion.
|
||||
*
|
||||
* This restores entity state from (type, lsn, payload) ONLY. It does NOT touch
|
||||
* the on-disk WAL header, the CRC chain (p->last_crc), or current_lsn — those
|
||||
* are owned exclusively by the physical layer: persist_wal_write_replicated on
|
||||
* the live replicated path, and replay_wal/persist_recover on recovery. (bug-778)
|
||||
*
|
||||
* Returns 0 on success (including skip for checkpoint/unknown types),
|
||||
* negative errno on error (oversized payload, NULL persist). */
|
||||
int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
|
||||
const void* payload, uint32_t payload_len);
|
||||
|
||||
/* Load a snapshot file from path, set p->checkpoint_lsn and p->current_lsn.
|
||||
* lsn hint is informational — the actual LSN is read from the file header.
|
||||
* Returns 0 on success, -errno or -EINVAL/-EIO on error. */
|
||||
int persist_load_snapshot(persist_t* p, const char* path, uint64_t lsn);
|
||||
|
||||
/* GC retention predicates (P0-5a / P0-8 / P0-13) */
|
||||
#include <stdbool.h>
|
||||
#include "xce/types.h"
|
||||
bool job_is_terminal(const job_t* j);
|
||||
bool job_should_retain(const job_t* j, const persist_config_t* cfg);
|
||||
bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg);
|
||||
167
src/include/repl.h
Normal file
167
src/include/repl.h
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
#pragma once
|
||||
/*
|
||||
* repl.h — P2-1: Replicator structures and public API.
|
||||
* Fan-out TCP replication via existing iomgr/SSL endpoint pipeline.
|
||||
* ACKs processed in io_worker read callback (NOT vtaskmgr) to prevent deadlock.
|
||||
*/
|
||||
#include <pthread.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <time.h>
|
||||
#include "config.h"
|
||||
#include "mylock.h"
|
||||
#include "wal_types.h" /* wal_entry_hdr_t — P2-4: use wal_types.h to break circular include */
|
||||
|
||||
/* Max queued-but-unacknowledged sends per standby before eviction to catch-up */
|
||||
#define MAX_PENDING_ACKS 16
|
||||
|
||||
/* Endpoint is an opaque pointer — full type in netio.h (avoid heavy include) */
|
||||
typedef void* repl_ep_t;
|
||||
|
||||
/* Commit engine (controller/include/commit.h) — forward-declared so repl.h
|
||||
* keeps zero controller-header dependencies (ha-commit-engine task 5). */
|
||||
struct commit_engine;
|
||||
|
||||
/* Per-standby connection slot */
|
||||
typedef struct {
|
||||
repl_ep_t ep; /* NULL = slot empty */
|
||||
_Atomic uint64_t acked_lsn; /* Last WAL LSN ACKed (walAck) */
|
||||
_Atomic uint64_t acked_config_lsn; /* Last config LSN ACKed (upgradeConfigAck) */
|
||||
_Atomic int in_catch_up; /* Non-zero while catching up */
|
||||
uint64_t join_applied_lsn; /* LSN standby reported on reconnect */
|
||||
char host[XCE_MAXNAME_LEN];
|
||||
uint32_t port;
|
||||
uint64_t last_disconnected_lsn;
|
||||
time_t last_disconnect_time;
|
||||
uint32_t pending_ack_count; /* Sends not yet ACKed; evict if > MAX_PENDING_ACKS */
|
||||
} standby_conn_t;
|
||||
|
||||
/* Replicator — embedded in persist_t or cluster_t (P2-4).
|
||||
* Named tag replicator_s matches the forward declaration in persist.h. */
|
||||
typedef struct replicator_s {
|
||||
standby_conn_t standbys[MAX_STANDBYS];
|
||||
int nstandbys; /* Active standby count */
|
||||
mylock_t lock; /* Protects standbys[], nstandbys */
|
||||
pthread_cond_t ack_cond; /* Signaled when any ACK arrives */
|
||||
pthread_mutex_t ack_lock; /* Guards ack_cond */
|
||||
/* Commit engine feed (ha-commit-engine task 5.2). NULL = not wired
|
||||
* (legacy counting contexts). Set once via repl_set_commit_engine at
|
||||
* cluster init, before replication traffic — read without r->lock.
|
||||
* Lock ordering: engine paths take ce->lock then r->lock, so repl.c
|
||||
* must NEVER call into the engine while holding r->lock. */
|
||||
struct commit_engine *ce;
|
||||
} replicator_t;
|
||||
|
||||
/* Lifecycle */
|
||||
int repl_init(replicator_t *r);
|
||||
void repl_fini(replicator_t *r);
|
||||
|
||||
/* Standby management */
|
||||
int repl_add_standby(replicator_t *r, repl_ep_t ep, const char *host, uint32_t port);
|
||||
void repl_remove_standby(replicator_t *r, repl_ep_t ep);
|
||||
void repl_expire_disconnect_slots(replicator_t *r, time_t now_s, uint32_t snap_interval_s);
|
||||
|
||||
/* Write-admission quorum accounting (ha-commit-engine Req 3.1-3.3).
|
||||
* Counts only standbys able to receive new log entries: slots with
|
||||
* ep != NULL && !in_catch_up, evaluated under r->lock.
|
||||
* NULL registry counts zero. Consumed by ha_has_write_quorum (ha.c). */
|
||||
int repl_active_standby_count(replicator_t *r);
|
||||
|
||||
/* Replication send (stub in P2-1; wired in P2-4).
|
||||
*
|
||||
* ha-term-safety task 3.4 (Req 1.3/1.4): the LIVE primary append replicator.
|
||||
* Carries the leader's per-entry Raft term (`hdr->term`) and the AppendEntries
|
||||
* prev tuple (`prev_log_index = hdr->lsn - 1`, `prev_log_term` supplied by the
|
||||
* caller from its term index) on every broadcast, so the standby intake routes
|
||||
* the entry through `raft_on_append_entries_request` and persists the LEADER's
|
||||
* term (not a local substitute or 0). `prev_log_term` is the term recorded for
|
||||
* `hdr->lsn - 1` (0 when `hdr->lsn == 1`, i.e. no predecessor). The term-0
|
||||
* legacy direct-apply receive path is thereby retired for live entries. */
|
||||
int repl_broadcast(replicator_t *r, const wal_entry_hdr_t *hdr,
|
||||
const void *payload, uint32_t payload_len,
|
||||
uint64_t prev_log_term);
|
||||
|
||||
/* AppendEntries send (ha-term-safety task 2.3, Req 1.3/3.1).
|
||||
*
|
||||
* Build and enqueue ONE AppendEntries envelope to the standby at slot
|
||||
* `peer_idx` (== the library's raft peer index, which maps 1:1 to the repl
|
||||
* standby registry slot — the same index used by match_index/next_index).
|
||||
* The envelope carries the AppendEntries prev tuple (`prev_log_index`,
|
||||
* `prev_log_term`) + `leader_commit`; when `n_entries == 1` it also carries the
|
||||
* single entry at LSN `prev_log_index + 1` whose `entry_term` is the term the
|
||||
* leader recorded for that LSN (== persist_entry_term) and whose
|
||||
* `entry_type`/`payload` are read from the leader's own WAL. `n_entries == 0`
|
||||
* is a heartbeat (zero entries) carrying only the prev tuple + leader_commit.
|
||||
*
|
||||
* Non-blocking: enqueues via the existing async repl transport
|
||||
* (xce_wal_batch_req_raw), never blocks. Returns 0 on enqueue, or a negative
|
||||
* errno (e.g. -ENOENT when peer_idx has no live endpoint). */
|
||||
int repl_send_append_entries(replicator_t *r, int peer_idx,
|
||||
uint64_t prev_log_index, uint64_t prev_log_term,
|
||||
uint64_t leader_commit, int n_entries,
|
||||
uint64_t entry_lsn, uint8_t entry_type,
|
||||
uint64_t entry_term,
|
||||
const void *payload, uint32_t payload_len);
|
||||
|
||||
/* WAL ACK handling — call from io_worker read callback ONLY */
|
||||
void repl_on_ack(replicator_t *r, repl_ep_t ep, uint64_t acked_lsn);
|
||||
|
||||
/* Config ACK handling — call from io_worker read callback ONLY (task 8.1) */
|
||||
void repl_on_config_ack(replicator_t *r, repl_ep_t ep, uint64_t acked_config_lsn);
|
||||
|
||||
/* Sync replication (stub in P2-1; wired in P2-3) */
|
||||
int repl_wait_ack(replicator_t *r, uint64_t target_lsn, uint32_t timeout_ms);
|
||||
|
||||
/* Majority-quorum ACK wait (task 7.3; ha-commit-engine task 5 adapter;
|
||||
* configAck track hardened by ha-commit-barrier-rollout task 5.2).
|
||||
* entry_lsn: walAck — the WAL entry the caller waits on, delegated to the
|
||||
* commit engine for per-LSN evaluation; configAck — the in-flight config
|
||||
* push tag, counted as acked_config_lsn >= tag with majority
|
||||
* 1 (self) + count > n_peers / 2 over configured members (no second
|
||||
* parameter — the configAck branch reuses this one). */
|
||||
int repl_wait_majority_ack(replicator_t *r, uint64_t entry_lsn,
|
||||
uint32_t timeout_ms, uint8_t ack_op);
|
||||
|
||||
/* Wire the commit engine into the replicator so ACK and membership events
|
||||
* are fed to it (ha-commit-engine task 5.2; NULL = not wired). */
|
||||
void repl_set_commit_engine(replicator_t *r, struct commit_engine *ce);
|
||||
|
||||
/* WAL trim guard */
|
||||
uint64_t repl_trim_lsn(replicator_t *r, uint64_t checkpoint_lsn, uint32_t snap_interval_s);
|
||||
|
||||
/* Catch-up state management */
|
||||
|
||||
/* Mark standby as entering catch-up mode; stores join_applied_lsn */
|
||||
void repl_start_catchup(replicator_t *r, repl_ep_t ep, uint64_t join_applied_lsn);
|
||||
|
||||
/* Mark standby catch-up complete (called on OP_replCatchUpDone receipt) */
|
||||
void repl_complete_catchup(replicator_t *r, repl_ep_t ep);
|
||||
|
||||
/* Returns 1 if standby needs a full snapshot transfer (its LSN predates oldest WAL).
|
||||
* oldest_wal_lsn == 0 means no WAL exists — no snapshot needed for LSN 0 standbys. */
|
||||
int repl_needs_snapshot(replicator_t *r, uint64_t standby_lsn, uint64_t oldest_wal_lsn);
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/* AppendEntries send capture seam (ha-term-safety task 2.3).
|
||||
*
|
||||
* When `repl_send_ae_capture_hook` is non-NULL, repl_send_append_entries
|
||||
* records the FULL built envelope into this struct instead of enqueuing it on a
|
||||
* real endpoint — so a unit test asserts the prev tuple / leader_commit /
|
||||
* n_entries / entry term + payload without network I/O. The payload bytes are
|
||||
* copied into a fixed buffer (capped at sizeof(payload)). */
|
||||
typedef struct {
|
||||
int called;
|
||||
int peer_idx;
|
||||
uint64_t prev_log_index;
|
||||
uint64_t prev_log_term;
|
||||
uint64_t leader_commit;
|
||||
int n_entries;
|
||||
uint64_t entry_lsn;
|
||||
uint8_t entry_type;
|
||||
uint64_t entry_term;
|
||||
uint32_t payload_len;
|
||||
uint8_t payload[256];
|
||||
} repl_ae_capture_t;
|
||||
|
||||
extern repl_ae_capture_t *repl_send_ae_capture_hook;
|
||||
#endif
|
||||
|
|
@ -6,6 +6,8 @@
|
|||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include <openssl/ssl.h>
|
||||
|
||||
#include "netio.h"
|
||||
#include "typemap.h"
|
||||
#include "utils.h"
|
||||
|
|
@ -17,12 +19,20 @@ typedef struct {
|
|||
|
||||
typedef int (*xce_check_fn_t)(void*);
|
||||
|
||||
extern int xce_request(endpoint_t *ep, xce_request_t *req);
|
||||
extern int xce_encode(xce_request_t*, io_packet_t*);
|
||||
extern int xce_decode(io_packet_t*, xce_request_t*);
|
||||
|
||||
extern int xce_reply(endpoint_t*, xce_peerinfo_t*, xce_reply_t*);
|
||||
extern int xce_hello_req(endpoint_t*, char*);
|
||||
extern int xce_nop_req(endpoint_t*);
|
||||
/* HA heartbeat: an OP_nop request carrying a HeartbeatMsg body with the
|
||||
* primary's committed position. Non-blocking, no reply awaited (the standby's
|
||||
* StateReport rides the normal nop reply). senderPriority is unused (0).
|
||||
* Generic xce_nop_req is left unchanged (body-less). */
|
||||
extern int xce_ha_heartbeat_req(endpoint_t* ep, uint64_t term,
|
||||
uint64_t committed_lsn,
|
||||
uint64_t committed_config_tag);
|
||||
extern int xce_shutdown_req(endpoint_t*);
|
||||
extern void init_reply_by_req(xce_reply_t*, xce_request_t*);
|
||||
extern int xce_join_req(endpoint_t*, int, hostattrs_t*);
|
||||
|
|
@ -46,6 +56,7 @@ extern int xce_startjob_req(endpoint_t*, xce_job_exec_t*);
|
|||
extern int xce_clusterconf_req(endpoint_t*);
|
||||
extern int xce_reply_clusterconf(endpoint_t*, uint64_t, int, xce_peerinfo_t*,
|
||||
xce_clusterconf_t*);
|
||||
extern int xce_ctrlstatus_req(endpoint_t*);
|
||||
extern int xce_get_req(endpoint_t*, xce_get_t*);
|
||||
extern int xce_reply_get(endpoint_t*, xce_peerinfo_t*, xce_reply_t*,
|
||||
xce_check_fn_t, void*, int);
|
||||
|
|
@ -60,4 +71,134 @@ extern int xce_set_pend_reason_req(endpoint_t*, xce_preason_set_t*);
|
|||
extern int xce_clear_reason_req(endpoint_t*, xce_clear_reason_t*);
|
||||
extern int xce_fetchfile_req(endpoint_t*, xce_fetchfile_t*);
|
||||
|
||||
/*
|
||||
* xce_replconfigpush_req_raw: send OP_replConfigPush carrying raw byte arrays.
|
||||
* Bypasses the encode_ReplConfigPushRequest_ptr path (which maps sha256/bytes
|
||||
* through xce_data_t — wrong type for raw bytes) by constructing the capnp
|
||||
* message directly via capn_new_list8 + capn_setv8 + write_Request.
|
||||
*/
|
||||
extern int xce_replconfigpush_req_raw(endpoint_t *ep, xce_peerinfo_t *from,
|
||||
uint64_t version_tag,
|
||||
const uint8_t *sha256, size_t sha256_len,
|
||||
const uint8_t *config_bytes,
|
||||
size_t config_len);
|
||||
|
||||
/*
|
||||
* xce_replconfig_reply_raw: primary sends an OP_replConfig reply with
|
||||
* actual config bytes via endpoint (iomgr queued). Bypasses
|
||||
* encode_ReplConfigReply_ptr (which maps sha256/bytes through xce_data_t).
|
||||
*/
|
||||
extern int xce_replconfig_reply_raw(endpoint_t *ep, xce_peerinfo_t *from,
|
||||
uint64_t seq, uint64_t tag,
|
||||
const uint8_t *sha256, size_t sha256_len,
|
||||
const uint8_t *config, size_t config_len);
|
||||
|
||||
/*
|
||||
* xce_replconfig_req_ssl: standby sends OP_replConfig over a raw SSL
|
||||
* socket (no iomgr). Wire framing: uint64_t size + capnp payload
|
||||
* (matches netio.c io_fsm_perform).
|
||||
*/
|
||||
extern int xce_replconfig_req_ssl(SSL *ssl, uint64_t version_tag);
|
||||
|
||||
/*
|
||||
* xce_replconfig_reply_ssl_decode: standby reads + parses an
|
||||
* OP_replConfig reply from a raw SSL socket. Output buffers are
|
||||
* malloc'd and owned by the caller. Both *out_sha256 and *out_bytes
|
||||
* may be NULL on the up_to_date == 1 fast path.
|
||||
*/
|
||||
extern int xce_replconfig_reply_ssl_decode(SSL *ssl,
|
||||
int *out_up_to_date,
|
||||
uint64_t *out_tag,
|
||||
uint8_t **out_sha256,
|
||||
uint8_t **out_bytes,
|
||||
size_t *out_len);
|
||||
|
||||
/*
|
||||
* xce_free_request: free an xce_request_t and all owned sub-fields,
|
||||
* including the raw_sha256/raw_bytes fields on upgrade_config_req_t
|
||||
* that the generated free_Request() does not know about.
|
||||
* Use this everywhere instead of calling free_Request() directly.
|
||||
*/
|
||||
extern void xce_free_request(xce_request_t *req);
|
||||
|
||||
/*
|
||||
* xce_wal_batch_encode_raw / xce_wal_batch_req_raw: encode/send an
|
||||
* OPCode_walCatchupEntry batch carrying raw WAL payload bytes.
|
||||
*
|
||||
* Bypasses the generated encode path (payload :Data cannot be carried
|
||||
* through xce_data_t — same limitation as replConfigPush). Each
|
||||
* entries[i]->raw_payload/raw_payload_len is written as a capnp Data
|
||||
* (List(UInt8)); the legacy xce_data_t payload field is ignored.
|
||||
* The receiver's xce_decode() extracts the bytes back into
|
||||
* raw_payload/raw_payload_len on each decoded sub-entry.
|
||||
*
|
||||
* encode_raw fills pkt (caller frees via free_packet on error paths);
|
||||
* req_raw allocates the packet and enqueues it on ep (non-blocking).
|
||||
*/
|
||||
extern int xce_wal_batch_encode_raw(wal_catchup_entry_t **entries,
|
||||
int n_entries,
|
||||
xce_peerinfo_t *from,
|
||||
uint64_t seq,
|
||||
io_packet_t *pkt);
|
||||
extern int xce_wal_batch_req_raw(endpoint_t *ep,
|
||||
wal_catchup_entry_t **entries,
|
||||
int n_entries);
|
||||
|
||||
/*
|
||||
* xce_snapshot_chunk_encode_raw / xce_snapshot_chunk_req_raw: encode/send
|
||||
* an OPCode_snapshotTransfer chunk carrying raw snapshot file bytes.
|
||||
* Same raw-Data convention as xce_wal_batch_req_raw; chunk envelope
|
||||
* fields (snapshot_lsn, chunk_index, chunk_offset, total_size, is_last)
|
||||
* are taken from *chunk, payload bytes from payload/payload_len.
|
||||
*/
|
||||
extern int xce_snapshot_chunk_encode_raw(const snapshot_transfer_t *chunk,
|
||||
const uint8_t *payload,
|
||||
size_t payload_len,
|
||||
xce_peerinfo_t *from,
|
||||
uint64_t seq,
|
||||
io_packet_t *pkt);
|
||||
extern int xce_snapshot_chunk_req_raw(endpoint_t *ep,
|
||||
const snapshot_transfer_t *chunk,
|
||||
const uint8_t *payload,
|
||||
size_t payload_len);
|
||||
|
||||
/*
|
||||
* xce_upgrade_config_req_extract: extract sha256 + bytes from a decoded
|
||||
* OP_upgradeConfig request. xce_decode() stores malloc'd copies in
|
||||
* req->body.upgradeconfigrequestmsg->raw_sha256 / raw_bytes before
|
||||
* capn_free() runs. Returns shallow pointers (owned by req); caller
|
||||
* must NOT free them. Returns -EINVAL if fields are missing.
|
||||
*/
|
||||
extern int xce_upgrade_config_req_extract(const xce_request_t *req,
|
||||
uint8_t **out_bytes,
|
||||
size_t *out_len,
|
||||
uint8_t **out_sha256);
|
||||
|
||||
/*
|
||||
* xce_upgrade_config_req_extract_raw: parse OP_upgradeConfig request
|
||||
* bytes from a raw on-wire packet buffer. Output buffers (*out_bytes,
|
||||
* *out_sha256) are malloc'd and owned by the caller.
|
||||
*/
|
||||
extern int xce_upgrade_config_req_extract_raw(const uint8_t *pkt_data,
|
||||
size_t pkt_sz,
|
||||
uint8_t **out_bytes,
|
||||
size_t *out_len,
|
||||
uint8_t **out_sha256);
|
||||
|
||||
/* ── Client-Side Transparent Retry tunables (Req 6.1, 6.2) ──────────────────
|
||||
*
|
||||
* xce_retry_config_t: configurable retry parameters with safe production
|
||||
* defaults, consumed by the libxce client failover loop (client.c /
|
||||
* client_failover.c). XCE_RETRY_DEFAULTS supplies the defaults applied when no
|
||||
* override is configured.
|
||||
*/
|
||||
typedef struct {
|
||||
int max_retries; /* default 20 */
|
||||
int initial_retry_ms; /* default 50 */
|
||||
int max_retry_ms; /* default 5000 */
|
||||
double backoff_multiplier; /* default 2.0 (exponential) */
|
||||
} xce_retry_config_t;
|
||||
|
||||
extern const xce_retry_config_t XCE_RETRY_DEFAULTS;
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -6,6 +6,20 @@
|
|||
#include "clusterconfig.h"
|
||||
#include "xce/types.h"
|
||||
|
||||
/* Forward declarations for config-replication types used in
|
||||
* xce_reply_t and xce_request_t unions below. */
|
||||
typedef struct repl_config_reply_s repl_config_reply_t;
|
||||
typedef struct repl_config_push_reply_s repl_config_push_reply_t;
|
||||
typedef struct upgrade_config_reply_s upgrade_config_reply_t;
|
||||
typedef struct repl_config_req_s repl_config_req_t;
|
||||
typedef struct repl_config_push_req_s repl_config_push_req_t;
|
||||
typedef struct upgrade_config_req_s upgrade_config_req_t;
|
||||
typedef struct heartbeat_msg_s heartbeat_msg_t;
|
||||
typedef struct ctrl_hello_req_s ctrl_hello_req_t;
|
||||
typedef struct ctrl_hello_reply_s ctrl_hello_reply_t;
|
||||
typedef struct wal_catchup_entry wal_catchup_entry_t;
|
||||
typedef struct snapshot_transfer_s snapshot_transfer_t;
|
||||
|
||||
typedef struct {
|
||||
char* resname;
|
||||
char* hostname;
|
||||
|
|
@ -46,8 +60,8 @@ typedef struct {
|
|||
xce_group_config_t** ugroups;
|
||||
int n_hgroups;
|
||||
xce_group_config_t** hgroups;
|
||||
int nresvals;
|
||||
xce_val_with_idx_t** resvals;
|
||||
int nresvals;
|
||||
xce_name_val_t** resvals;
|
||||
int nlimits;
|
||||
xce_limit_config_t** limits;
|
||||
int nhproviders;
|
||||
|
|
@ -160,18 +174,93 @@ typedef struct {
|
|||
} data;
|
||||
} xce_update_data_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t term;
|
||||
char *candidate_id;
|
||||
uint64_t config_tag;
|
||||
uint64_t last_log_lsn;
|
||||
/* Raft (lastLogTerm, lastLogIndex) freshness tuple (ha-term-safety task 1.2):
|
||||
* feeds the library logOk gate, distinct from config_tag/last_log_lsn which
|
||||
* drive the application freshness gate. last_log_index == candidate log_len. */
|
||||
uint64_t last_log_term;
|
||||
uint64_t last_log_index;
|
||||
} request_vote_req_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t term;
|
||||
int vote_granted;
|
||||
uint64_t peer_config_tag;
|
||||
uint64_t peer_last_log_lsn;
|
||||
} request_vote_reply_t;
|
||||
|
||||
/* Heartbeat reply state report — carries standby's applied_lsn and config_tag */
|
||||
typedef struct {
|
||||
uint64_t applied_lsn;
|
||||
uint64_t config_tag;
|
||||
} state_report_t;
|
||||
|
||||
/* WAL apply ACK — sent by standby to primary after applying a WAL catch-up
|
||||
* entry (Task 3.1, ha-wal-replication). Must be declared before xce_reply_t
|
||||
* so the walackreplymsg union arm compiles correctly. */
|
||||
typedef struct {
|
||||
uint64_t term;
|
||||
uint64_t lsn;
|
||||
int success;
|
||||
} wal_ack_reply_t;
|
||||
|
||||
/* ctrlstatus reply payload (controller-status spec): one configured
|
||||
* controller (CtrlPeer) and the lean identity/health snapshot (CtrlStatus).
|
||||
* The peers list maps via $C.mapname("peers.list")/$C.maplistcount("peers.count")
|
||||
* to the nested { count, list } member below. */
|
||||
typedef struct {
|
||||
char* name;
|
||||
char* host;
|
||||
uint32_t port;
|
||||
} xce_ctrlpeer_t;
|
||||
|
||||
typedef struct {
|
||||
char* clusterName;
|
||||
char* primaryName;
|
||||
char* responderName;
|
||||
uint8_t responderRole;
|
||||
uint64_t term;
|
||||
uint64_t configVersion;
|
||||
char* configChecksum;
|
||||
uint64_t committedConfigVersion;
|
||||
uint64_t committedLsn;
|
||||
uint64_t appliedLsn;
|
||||
unsigned fenced;
|
||||
struct {
|
||||
int count;
|
||||
xce_ctrlpeer_t** list;
|
||||
} peers;
|
||||
uint32_t peerCount;
|
||||
uint64_t responder_time_ms; /* @13: responder wall-clock at sample time (epoch ms) */
|
||||
} xce_ctrlstatus_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t seq;
|
||||
uint32_t opcode;
|
||||
int32_t status;
|
||||
int data_which;
|
||||
union {
|
||||
char* debugmsg;
|
||||
uint64_t jobid;
|
||||
xce_data_t* getdata;
|
||||
xce_clusterconf_t* clusterconf;
|
||||
xce_fetchfile_reply_t* fetchfilereply;
|
||||
char* debugmsg;
|
||||
uint64_t jobid;
|
||||
xce_data_t* getdata;
|
||||
xce_clusterconf_t* clusterconf;
|
||||
repl_config_reply_t* replconfigreplymsg;
|
||||
repl_config_push_reply_t* replconfigpushreplymsg;
|
||||
upgrade_config_reply_t* upgradeconfigreplymsg;
|
||||
ctrl_hello_reply_t* ctrlhelloreplymsg;
|
||||
request_vote_reply_t* requestvotereplymsg;
|
||||
state_report_t* state_report;
|
||||
wal_ack_reply_t* walackreplymsg;
|
||||
xce_ctrlstatus_t* ctrlstatus;
|
||||
xce_fetchfile_reply_t* fetchfilereply;
|
||||
} data;
|
||||
/* Req 7: standby redirect fields — populated when status == XCE_ERR_STANDBY */
|
||||
char *redirect_host;
|
||||
uint16_t redirect_port;
|
||||
} xce_reply_t;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -195,11 +284,209 @@ typedef struct {
|
|||
xce_jobctrl_t* jobctrlmsg;
|
||||
xce_queuectrl_t* queuectrlmsg;
|
||||
xce_taskrun_t* runtaskmsg;
|
||||
xce_preason_set_t* preasonmsg;
|
||||
xce_clear_reason_t* clrpreasonmsg;
|
||||
xce_hostctrl_t* hostctrlmsg;
|
||||
xce_fetchfile_t* fetchfilemsg;
|
||||
xce_preason_set_t* preasonmsg;
|
||||
xce_clear_reason_t* clrpreasonmsg;
|
||||
xce_hostctrl_t* hostctrlmsg;
|
||||
repl_config_req_t* replconfigrequestmsg;
|
||||
repl_config_push_req_t* replconfigpushrequestmsg;
|
||||
upgrade_config_req_t* upgradeconfigrequestmsg;
|
||||
heartbeat_msg_t* heartbeatmsg;
|
||||
ctrl_hello_req_t* ctrlhellomsg;
|
||||
request_vote_req_t* requestvotemsg;
|
||||
wal_catchup_entry_t* walcatchupentrymsg;
|
||||
snapshot_transfer_t* snapshottransfermsg;
|
||||
xce_fetchfile_t* fetchfilemsg;
|
||||
} body;
|
||||
} xce_request_t;
|
||||
|
||||
/* ── HA election and replication Cap'n Proto mapped types ────────────────
|
||||
* C-side counterparts for the HA protocol capnp structs added in the
|
||||
* ha-config-bootstrap spec (controller.capnp structs with $C.mapname).
|
||||
* Must be declared here so controller.capnp.h can reference them.
|
||||
* request_vote_req_t and request_vote_reply_t are defined above
|
||||
* xce_reply_t so they can be used in the xce_request_t body union. */
|
||||
|
||||
typedef struct {
|
||||
char *name;
|
||||
} probe_config_tag_req_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t config_tag;
|
||||
} probe_config_tag_reply_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t term;
|
||||
uint64_t lsn;
|
||||
} wal_ack_req_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t term;
|
||||
uint64_t acked_tag;
|
||||
int success;
|
||||
} upgrade_config_ack_t;
|
||||
|
||||
struct snapshot_transfer_s {
|
||||
uint64_t snapshot_lsn; /* checkpoint LSN this snapshot corresponds to */
|
||||
xce_data_t *payload; /* legacy decoded field — broken for raw bytes
|
||||
* (decode_Data_ptr misinterprets List(UInt8));
|
||||
* kept NULL on send, do not use */
|
||||
uint32_t chunk_index; /* 0-based chunk number */
|
||||
uint64_t chunk_offset; /* byte offset in the full snapshot */
|
||||
uint64_t total_size; /* total snapshot size in bytes */
|
||||
int is_last; /* true (non-zero) for the final chunk */
|
||||
/* Raw chunk bytes — same pattern as repl_config_push_req_t.
|
||||
* Send side: filled by the sender, encoded by xce_snapshot_chunk_req_raw.
|
||||
* Receive side: extracted by xce_decode() before capn_free(); malloc'd,
|
||||
* owned by the struct, freed by xce_free_request(). */
|
||||
uint8_t *raw_payload;
|
||||
size_t raw_payload_len;
|
||||
};
|
||||
|
||||
struct wal_catchup_entry {
|
||||
uint64_t lsn;
|
||||
uint8_t entry_type;
|
||||
xce_data_t *payload; /* legacy decoded field — broken for
|
||||
* raw bytes; kept NULL, do not use */
|
||||
int n_entries; /* batch: number of sub-entries */
|
||||
struct wal_catchup_entry **entries; /* batch: array of sub-entry pointers */
|
||||
/* Raw WAL entry payload bytes — same pattern as repl_config_push_req_t.
|
||||
* Send side: filled from the WAL segment file, encoded per sub-entry by
|
||||
* xce_wal_batch_req_raw. Receive side: extracted by xce_decode() before
|
||||
* capn_free(); malloc'd, owned, freed by xce_free_request(). */
|
||||
uint8_t *raw_payload;
|
||||
size_t raw_payload_len;
|
||||
/* Raft AppendEntries term fields (ha-term-safety task 1.1, Req 1.3/1.4).
|
||||
* term == this entry's term stamp; prev_log_index == lsn-1; prev_log_term ==
|
||||
* term of the entry at prev_log_index (0 if absent). Consumed by the log
|
||||
* callbacks (tasks 2.x) for the AppendEntries logOk / truncation decision. */
|
||||
uint64_t term;
|
||||
uint64_t prev_log_index;
|
||||
uint64_t prev_log_term;
|
||||
/* ha-term-safety task 3.2 (Req 2.5/4.1): the leader's commit index carried on
|
||||
* the AppendEntries so the standby intake passes it into
|
||||
* raft_on_append_entries_request. Wire field leaderCommit @7. */
|
||||
uint64_t leader_commit;
|
||||
/* bug-1067: the streaming leader's CURRENT term — the Raft AppendEntries
|
||||
* HEADER term, DISTINCT from `term` above (the entry's STORED term). Equal for
|
||||
* a live append; for a HISTORICAL catch-up entry the stored term is lower than
|
||||
* the leader's current term. The standby intake uses this as the AE header
|
||||
* term (UpdateTerm/stale-check) so cross-term catch-up is not stale-rejected;
|
||||
* 0 = unset (legacy sender) → fall back to the stored term. Wire field
|
||||
* leaderCurrentTerm @8. */
|
||||
uint64_t leader_current_term;
|
||||
};
|
||||
|
||||
/* ── Persistence Cap'n Proto mapped types (P0-9) ─────────────────────
|
||||
* These are the C-side counterparts for the capnp encode/decode shims.
|
||||
* Named with capnp_ prefix to avoid collision with the plain-C structs
|
||||
* in persist.h (persist_config_t) and the on-disk binary formats. */
|
||||
typedef struct {
|
||||
uint32_t walSyncMs;
|
||||
uint32_t snapIntervalS;
|
||||
uint32_t snapWalSizeMb;
|
||||
uint32_t finishedRetainS;
|
||||
uint32_t workerRetainS;
|
||||
uint32_t syncTimeoutMs;
|
||||
uint32_t snapKeepN;
|
||||
uint32_t transientJobTimeoutS;
|
||||
} capnp_persist_cfg_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t lsn;
|
||||
uint64_t term;
|
||||
uint64_t timestamp;
|
||||
uint32_t version;
|
||||
uint32_t nJobs;
|
||||
uint32_t nQueues;
|
||||
uint32_t nWorkers;
|
||||
uint32_t checksum;
|
||||
uint32_t headerChecksum;
|
||||
} capnp_snap_hdr_t;
|
||||
|
||||
typedef struct {
|
||||
uint64_t term;
|
||||
uint64_t lsn;
|
||||
uint32_t entryType;
|
||||
xce_data_t* payload; /* owned; decoded via decode_Data_ptr */
|
||||
uint32_t crc;
|
||||
uint32_t prevCrc;
|
||||
} capnp_repl_envelope_t;
|
||||
|
||||
/* ── Config replication Cap'n Proto mapped types (Phase 1) ──────────────
|
||||
* C-side counterparts for the capnp encode/decode shims for config sync.
|
||||
* Struct tags match the forward declarations near the top of this file. */
|
||||
|
||||
struct repl_config_req_s {
|
||||
uint64_t version_tag; /* local config version tag */
|
||||
};
|
||||
|
||||
struct repl_config_reply_s {
|
||||
int up_to_date; /* 1 if standby is current */
|
||||
uint64_t version_tag; /* primary's current version tag */
|
||||
xce_data_t *sha256; /* 32-byte SHA-256 of config blob (may be NULL) */
|
||||
xce_data_t *bytes; /* serialised ClusterConfig blob (may be NULL) */
|
||||
};
|
||||
|
||||
struct repl_config_push_req_s {
|
||||
uint64_t version_tag; /* version tag of the pushed config */
|
||||
xce_data_t *sha256; /* 32-byte SHA-256 (decoded, not usable as raw bytes) */
|
||||
xce_data_t *bytes; /* serialised ClusterConfig blob (decoded, not usable) */
|
||||
/* Pre-extracted raw fields populated by xce_decode() before capn_free(),
|
||||
* mirroring the same pattern as upgrade_config_req_t. Owned by the struct. */
|
||||
uint8_t *raw_sha256; /* 32-byte SHA-256, malloc'd, or NULL */
|
||||
uint8_t *raw_bytes; /* serialised ClusterConfig blob, malloc'd, or NULL */
|
||||
size_t raw_bytes_len; /* length of raw_bytes */
|
||||
};
|
||||
|
||||
struct repl_config_push_reply_s {
|
||||
int accepted; /* 1 if standby accepted the push */
|
||||
uint64_t version_tag; /* standby's current version tag after decision */
|
||||
};
|
||||
|
||||
struct upgrade_config_req_s {
|
||||
/* Legacy decoded fields — populated by generated decode_UpgradeConfigRequest()
|
||||
* via decode_Data_ptr. These are structurally broken for List(UInt8) wire
|
||||
* fields (decode_Data_ptr reads a discriminant and misinterprets the bytes),
|
||||
* but must remain for the generated code to compile. Do NOT use these for
|
||||
* actual byte extraction. */
|
||||
xce_data_t *sha256; /* broken: always a bogus xce_jobs_data_t */
|
||||
xce_data_t *bytes; /* broken: always a bogus xce_jobs_data_t */
|
||||
|
||||
/* Correct raw bytes extracted by xce_decode() before capn_free() runs,
|
||||
* using UpgradeConfigRequest_get_sha256/bytes + capn_getv8.
|
||||
* Both are malloc'd and owned here; freed by free_upgrade_config_req_raw()
|
||||
* called from free_Request() after free_UpgradeConfigRequest().
|
||||
* raw_sha256 is exactly 32 bytes when non-NULL. */
|
||||
uint8_t *raw_sha256; /* 32-byte SHA-256, malloc'd, or NULL */
|
||||
uint8_t *raw_bytes; /* serialised ClusterConfig blob, malloc'd, or NULL */
|
||||
size_t raw_bytes_len; /* length of raw_bytes */
|
||||
};
|
||||
|
||||
struct upgrade_config_reply_s {
|
||||
uint64_t version_tag; /* new version tag after upgrade */
|
||||
char *error; /* NULL on success; error message on failure */
|
||||
uint32_t acked_count; /* number of standbys that acknowledged */
|
||||
};
|
||||
|
||||
struct heartbeat_msg_s {
|
||||
uint32_t sender_priority; /* sender's HA priority */
|
||||
uint64_t term; /* sender's current term */
|
||||
uint64_t committed_lsn; /* Req 5: primary's committed WAL LSN */
|
||||
uint64_t committed_config_tag; /* Req 8: primary's committed config tag */
|
||||
};
|
||||
|
||||
/* CtrlHello state sync (Req 6.5, 6.6, 6.7) */
|
||||
struct ctrl_hello_req_s {
|
||||
char *name; /* standby's logical name */
|
||||
xce_data_t *key; /* SHA-256(cluster_name + shared_secret) auth key */
|
||||
uint64_t applied_lsn; /* standby's last applied WAL LSN */
|
||||
uint64_t config_tag; /* standby's current config tag */
|
||||
};
|
||||
|
||||
struct ctrl_hello_reply_s {
|
||||
uint64_t committed_lsn; /* primary's committed WAL LSN */
|
||||
int needs_snapshot; /* 1 if standby LSN < oldest WAL segment */
|
||||
uint64_t config_tag; /* primary's current config tag */
|
||||
};
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ extern xce_timewindow_config_t* dup_timewindow_config(xce_timewindow_config_t*);
|
|||
extern void free_resource_thresholds(xce_res_threshold_t**, int);
|
||||
extern xce_res_threshold_t** dup_resource_thresholds(xce_res_threshold_t**,
|
||||
int);
|
||||
extern void free_resval_with_idx_list(xce_val_with_idx_t***, int);
|
||||
extern void free_name_val_list(xce_name_val_t***, int*);
|
||||
extern void free_strlist(xce_strlist_t*);
|
||||
extern int fmt_jobid_ex(char*, size_t, uint64_t, char*, char*);
|
||||
extern int fmt_jobid(char*, size_t, uint64_t);
|
||||
|
|
|
|||
105
src/include/wal_types.h
Normal file
105
src/include/wal_types.h
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
#pragma once
|
||||
/*
|
||||
* wal_types.h — WAL on-disk structures shared between persist.h and repl.h.
|
||||
* Extracted in P2-4 to break the persist.h ↔ repl.h circular include.
|
||||
*/
|
||||
#include <stdint.h>
|
||||
#include "config.h"
|
||||
|
||||
/* ── WAL payload helper structs ────────────────────────────────────────────
|
||||
* These are the binary formats embedded after wal_entry_hdr_t for each type.
|
||||
* All fields are native-endian (single-node WAL; replication uses capnp). */
|
||||
|
||||
/* WAL_JOB_START / FINISH / FAIL / STOP / RESUME / UPDATE / CKPT / PREASON / RUSAGE */
|
||||
typedef struct {
|
||||
uint64_t jobid;
|
||||
uint32_t state; /* xce_job state flags after this transition */
|
||||
uint32_t _pad;
|
||||
} wal_jobstate_t;
|
||||
|
||||
/* WAL_QUEUE_CTRL — queue name is a NUL-terminated string immediately after
|
||||
* this header. Total payload = sizeof(wal_queuehdr_t) + strlen(name)+1. */
|
||||
typedef struct {
|
||||
uint32_t status; /* xce_queue status flags after this transition */
|
||||
} wal_queuehdr_t;
|
||||
|
||||
/* ── Raft adapter entry-blob envelope (ha-term-safety task 2.1) ─────────────
|
||||
* The raft library (src/lib/raft/raft.c) holds NO entry bytes: it carries one
|
||||
* opaque (entry, len) blob through raft_log_append / on_persist_entry, leaving
|
||||
* its encoding entirely to the controller adapter. This struct IS that
|
||||
* encoding: a fixed 8-byte prefix recording the WAL entry `type` and the
|
||||
* `payload_len` of the bytes that immediately follow it, so on_persist_entry
|
||||
* can recover (type, payload, len) for the existing persist write path.
|
||||
*
|
||||
* This is purely an in-flight wire/blob format between the library callback
|
||||
* and persist.c — it is NEVER written to the WAL header (that stays the 40-byte
|
||||
* wal_entry_hdr_t, R1.7); only the decoded `type` + `payload` reach the WAL.
|
||||
*
|
||||
* The append/replication driver (tasks 3.1/3.2) packs this same prefix before
|
||||
* the payload when calling raft_log_append / routing an AppendEntries, so the
|
||||
* encoder and decoder share one definition (no drift). */
|
||||
typedef struct {
|
||||
uint32_t type; /* wal_entry_type enum value for this entry */
|
||||
uint32_t payload_len; /* bytes of payload immediately following this header */
|
||||
/* uint8_t payload[payload_len] follows */
|
||||
} raft_entry_blob_t;
|
||||
|
||||
/* ============================================================
|
||||
* WAL entry header — 40 bytes, 8-byte aligned (Part 2.2)
|
||||
* ============================================================ */
|
||||
typedef struct {
|
||||
uint64_t term; /* Election term (incremented on failover) */
|
||||
uint64_t lsn; /* Log sequence number within this term */
|
||||
uint64_t timestamp; /* Epoch microseconds */
|
||||
uint32_t type; /* wal_entry_type enum value */
|
||||
uint32_t payload_len; /* Bytes of payload following this header */
|
||||
uint32_t crc; /* CRC32(header[0..WAL_HDR_CRC_SIZE-1] + payload) */
|
||||
uint32_t prev_crc; /* CRC of previous record; WAL_CRC_NONE for first in segment */
|
||||
} wal_entry_hdr_t;
|
||||
_Static_assert(sizeof(wal_entry_hdr_t) == 40,
|
||||
"wal_entry_hdr_t must be 40 bytes for on-disk format stability");
|
||||
|
||||
/* ============================================================
|
||||
* WAL entry types (Part 2.3)
|
||||
* ============================================================ */
|
||||
typedef enum {
|
||||
WAL_JOB_SUBMIT = 1, /* SYNC replication */
|
||||
WAL_JOB_START = 2,
|
||||
WAL_JOB_FINISH = 3,
|
||||
WAL_JOB_FAIL = 4,
|
||||
WAL_JOB_STOP = 5,
|
||||
WAL_JOB_RESUME = 6,
|
||||
WAL_JOB_UPDATE = 7,
|
||||
WAL_JOB_RUSAGE = 8,
|
||||
WAL_JOB_CKPT = 9,
|
||||
WAL_JOB_PREASON = 10,
|
||||
WAL_QUEUE_CTRL = 20,
|
||||
WAL_WORKER_CTRL = 30,
|
||||
WAL_NOOP = 40, /* Internal: empty payload, replay no-op; appended only at promotion */
|
||||
WAL_CONFIG_MARKER = 50, /* Config-change marker: {version_tag, sha256} reference only —
|
||||
* NEVER the config bytes (R7.2/R7.4). Ordered + committed through
|
||||
* the same (term,lsn) log as job state (ha-term-safety). */
|
||||
WAL_CHECKPOINT = 100, /* Local WAL marker only — never replicated */
|
||||
} wal_entry_type;
|
||||
|
||||
/* ── WAL_CONFIG_MARKER payload (ha-term-safety task 4.1) ────────────────────
|
||||
* The marker payload is a *reference* to a configuration — its version tag and
|
||||
* full SHA-256 content hash — and carries NO configuration bytes (R7.2/R7.4).
|
||||
* The configuration bytes always travel out-of-band on the existing config-push
|
||||
* transport (OPCode_replConfigPush), keyed to {version_tag, sha256}; they are
|
||||
* NEVER placed in a WAL payload.
|
||||
*
|
||||
* This struct rides INSIDE the raft_entry_blob_t envelope with
|
||||
* type == WAL_CONFIG_MARKER; on persist it lands as the WAL entry payload. The
|
||||
* 40-byte wal_entry_hdr_t is unchanged (R1.7) — this is a payload, not a header.
|
||||
* Use config_marker_encode / config_marker_decode (config_repl.c) for a fixed,
|
||||
* padding-independent on-wire form. */
|
||||
typedef struct {
|
||||
uint64_t version_tag; /* first 8 bytes of the config SHA-256, host order */
|
||||
uint8_t sha256[32]; /* full SHA-256 of the config bytes (the reference) */
|
||||
} wal_config_marker_t;
|
||||
|
||||
/* Fixed serialized size of a WAL_CONFIG_MARKER payload: 8-byte version_tag +
|
||||
* 32-byte sha256, with NO config bytes ever appended. This is the exact byte
|
||||
* count config_marker_encode emits and config_marker_decode requires. */
|
||||
#define WAL_CONFIG_MARKER_PAYLOAD_LEN (8u + 32u)
|
||||
55
src/include/xce/client_failover.h
Normal file
55
src/include/xce/client_failover.h
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
/*
|
||||
* client_failover.h — public StatusClassifier surface for the libxce-failover
|
||||
* spec.
|
||||
*
|
||||
* Task 2.1 (StatusClassifier): pure mapping from a reply status / transport rc
|
||||
* to a failover action. No I/O, no globals — see design.md
|
||||
* "Components and Interfaces → StatusClassifier".
|
||||
*/
|
||||
#ifndef XCE_CLIENT_FAILOVER_H
|
||||
#define XCE_CLIENT_FAILOVER_H
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Failover action a round-trip outcome maps to. Order mirrors the harness'
|
||||
* fo_expected_action_t (failover_harness.h) and design.md. */
|
||||
typedef enum {
|
||||
XCE_FO_SUCCESS, /* status == 0, or idempotent-replay success
|
||||
(DUP on a re-sent write) */
|
||||
XCE_FO_RETRY_NOW, /* E_XCE_REDIRECT / E_XCE_STANDBY / transport error
|
||||
— rotate, no backoff */
|
||||
XCE_FO_RETRY_BACKOFF, /* E_XCE_NOMASTER / E_XCE_NOT_READY — rotate after
|
||||
jittered backoff */
|
||||
XCE_FO_TERMINAL /* AUTH/PERM/INVALID/NOENT and other non-retriable
|
||||
— surface to caller */
|
||||
} xce_fo_action_t;
|
||||
|
||||
/*
|
||||
* Map a reply status / transport rc to a failover action.
|
||||
*
|
||||
* status : the controller reply status or negative -errno transport rc, as
|
||||
* the failover loop observes it. Controller codes (E_XCE_*) are
|
||||
* accepted in either sign.
|
||||
* is_write: true for mutating ops (e.g. job submit); false for reads.
|
||||
* attempt : 0 for the first send; >0 for a re-send after failover.
|
||||
*
|
||||
* Invariant (2.5): E_XCE_REDIRECT/STANDBY/NOMASTER/NOT_READY never map to
|
||||
* XCE_FO_SUCCESS.
|
||||
*
|
||||
* Idempotent-replay rule (5.2): E_XCE_DUP_JOB / EEXIST on a re-sent write
|
||||
* (is_write && attempt > 0) → XCE_FO_SUCCESS; otherwise → XCE_FO_TERMINAL.
|
||||
*
|
||||
* Unknown / unrecognized statuses are treated conservatively as
|
||||
* XCE_FO_TERMINAL (surfaced, not retried).
|
||||
*
|
||||
* Pure function: no I/O, no globals.
|
||||
*/
|
||||
xce_fo_action_t xce_failover_classify(int status, bool is_write, int attempt);
|
||||
|
||||
/*
|
||||
* EndpointRotator (xce_failover_next_target / xce_failover_rotate) operates on
|
||||
* the private api_ctx_t and is therefore declared in the internal header
|
||||
* src/lib/client_internal.h, not in this public surface.
|
||||
*/
|
||||
|
||||
#endif /* XCE_CLIENT_FAILOVER_H */
|
||||
128
src/include/xce/cluster.h
Normal file
128
src/include/xce/cluster.h
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
/* src/include/xce/cluster.h — cluster discovery and reconnect API (P2.5-1)
|
||||
*
|
||||
* xce_connect_cluster() returns endpoint_t* (cast to void*).
|
||||
* The SSL* is accessible as ((endpoint_t*)result)->ssl.
|
||||
*/
|
||||
#ifndef XCE_CLUSTER_H
|
||||
#define XCE_CLUSTER_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include "config.h" /* MAX_HA_CANDIDATES, XCE_MAXHOSTNAME_LEN */
|
||||
|
||||
/**
|
||||
* A single controller candidate.
|
||||
* name: logical name from "name=host:port" syntax (empty if omitted).
|
||||
* Matching against myconf->name allows multiple controllers on the same
|
||||
* dev host to distinguish themselves by name rather than IP address.
|
||||
*/
|
||||
typedef struct {
|
||||
char name[XCE_MAXNAME_LEN];
|
||||
char hostname[XCE_MAXHOSTNAME_LEN];
|
||||
int port;
|
||||
} xce_candidate_t;
|
||||
|
||||
/**
|
||||
* Cluster connection configuration.
|
||||
* Populated by each caller (worker/scheduler/apiserver) from --peers and
|
||||
* --ca-cert CLI flags. libxce never reads config files.
|
||||
*
|
||||
* candidates_count must be in [1, MAX_HA_CANDIDATES].
|
||||
* A WARN is logged at startup if candidates_count > 3.
|
||||
*/
|
||||
typedef struct {
|
||||
xce_candidate_t candidates[MAX_HA_CANDIDATES];
|
||||
int candidates_count;
|
||||
char ca_cert_path[4096]; /* path to CA cert for TLS verify */
|
||||
char identity[256]; /* "from" name sent to controller */
|
||||
char key[256]; /* authentication key */
|
||||
} xce_cluster_config_t;
|
||||
|
||||
/**
|
||||
* Reconnect state machine states.
|
||||
*/
|
||||
typedef enum {
|
||||
XCE_RECONNECT_IDLE = 0,
|
||||
XCE_RECONNECT_CONNECTING,
|
||||
XCE_RECONNECT_BACKOFF,
|
||||
XCE_RECONNECT_CONNECTED,
|
||||
XCE_RECONNECT_FAILED
|
||||
} xce_reconnect_state_t;
|
||||
|
||||
/**
|
||||
* Reconnect context (use xce_reconnect_* functions to operate).
|
||||
*/
|
||||
typedef struct {
|
||||
xce_cluster_config_t* config;
|
||||
xce_reconnect_state_t state;
|
||||
int attempt;
|
||||
int backoff_ms; /* current backoff delay in ms */
|
||||
void* endpoint; /* endpoint_t* when CONNECTED */
|
||||
} xce_reconnect_ctx_t;
|
||||
|
||||
/* Redirect follow limit */
|
||||
#define XCE_MAX_REDIRECTS 4
|
||||
|
||||
/* Backoff parameters — initial kept short so tests stay fast */
|
||||
#define XCE_RECONNECT_INITIAL_BACKOFF_MS 100
|
||||
#define XCE_RECONNECT_MAX_BACKOFF_MS 30000
|
||||
#define XCE_RECONNECT_BACKOFF_MULTIPLIER 2
|
||||
|
||||
/* --- Public API --- */
|
||||
|
||||
/**
|
||||
* xce_cluster_config_validate() — validate a populated xce_cluster_config_t.
|
||||
*
|
||||
* Returns 0 on success, -EINVAL if:
|
||||
* - candidates_count < 1 or > MAX_HA_CANDIDATES
|
||||
* - any candidate has empty hostname or port <= 0
|
||||
* - ca_cert_path is empty
|
||||
* - identity is empty
|
||||
*/
|
||||
int xce_cluster_config_validate(const xce_cluster_config_t* cfg);
|
||||
|
||||
/**
|
||||
* xce_connect_cluster() — connect to the cluster via candidate list.
|
||||
*
|
||||
* Iterates candidates in order, attempting a TLS connection to each.
|
||||
* Returns endpoint_t* (cast to void*) on success, NULL if all fail.
|
||||
*/
|
||||
void* xce_connect_cluster(xce_cluster_config_t* cfg);
|
||||
|
||||
/**
|
||||
* xce_reconnect_init() — initialise a reconnect context.
|
||||
* Returns 0 on success, -EINVAL on bad arguments.
|
||||
*/
|
||||
int xce_reconnect_init(xce_reconnect_ctx_t* rctx, xce_cluster_config_t* cfg);
|
||||
|
||||
/**
|
||||
* xce_reconnect_step() — drive one step of the reconnect state machine.
|
||||
*
|
||||
* Call in a loop until state == XCE_RECONNECT_CONNECTED or FAILED.
|
||||
* Sleeps backoff_ms in BACKOFF state (skipped when mock connect is set).
|
||||
*
|
||||
* Returns 0 to continue, -1 when in terminal state.
|
||||
*/
|
||||
int xce_reconnect_step(xce_reconnect_ctx_t* rctx);
|
||||
|
||||
/**
|
||||
* xce_reconnect_fini() — release resources held by a reconnect context.
|
||||
*/
|
||||
void xce_reconnect_fini(xce_reconnect_ctx_t* rctx);
|
||||
|
||||
/**
|
||||
* xce_parse_peers() — parse "--peers name=host:port,..." into cfg->candidates[].
|
||||
*
|
||||
* Each token may be "name=host:port" or bare "host:port".
|
||||
* The name field is stored when present; it allows multiple daemons on the
|
||||
* same host to identify themselves by logical name rather than IP address.
|
||||
*
|
||||
* Returns 0 on success, -1 on empty/NULL input or any parse error.
|
||||
*/
|
||||
int xce_parse_peers(const char* peers_str, xce_cluster_config_t* cfg);
|
||||
|
||||
/* Test seam: set to non-NULL to override the real connect (unit tests only) */
|
||||
typedef void* (*xce_connect_fn_t)(const char* host, int port,
|
||||
const char* ca_cert);
|
||||
void xce_cluster_set_connect_fn(xce_connect_fn_t fn);
|
||||
|
||||
#endif /* XCE_CLUSTER_H */
|
||||
|
|
@ -14,7 +14,12 @@
|
|||
#include "myconf.h"
|
||||
#include "mylock.h"
|
||||
#include "netio.h"
|
||||
#include "persist.h"
|
||||
#include "vtaskmgr.h"
|
||||
|
||||
/* ha.h forward-declares cluster_t; including it here is safe (no circular dep) */
|
||||
#include "commit.h"
|
||||
#include "ha.h"
|
||||
#include "xce/types.h"
|
||||
#include "xlist.h"
|
||||
#include "zarray.h"
|
||||
|
|
@ -55,7 +60,9 @@ enum {
|
|||
HOOK_DETAIL_QUEUE_UPDATE,
|
||||
HOOK_DETAIL_USERGROUP_UPDATE,
|
||||
HOOK_DETAIL_HOSTGROUP_UPDATE,
|
||||
HOOK_DETAIL_JOB_RUSAGE_UPDATE
|
||||
HOOK_DETAIL_JOB_RUSAGE_UPDATE,
|
||||
HOOK_DETAIL_JOB_STOP,
|
||||
HOOK_DETAIL_JOB_RESUME
|
||||
};
|
||||
|
||||
#define XCE_NS_CONFIG (0)
|
||||
|
|
@ -88,9 +95,10 @@ typedef struct {
|
|||
endpoint_t* ep;
|
||||
} cluster_apiserver_t;
|
||||
|
||||
typedef struct {
|
||||
typedef struct cluster_s {
|
||||
myconf_t* myconf;
|
||||
xce_peerinfo_t* myhost;
|
||||
xce_peerinfo_t* myhost; /* points at myhost_info once initialised */
|
||||
xce_peerinfo_t myhost_info; /* storage; name set from myconf->name */
|
||||
hooksys_t hooksys;
|
||||
iomgr_t iomgr;
|
||||
zarray_t endpoints;
|
||||
|
|
@ -112,6 +120,29 @@ typedef struct {
|
|||
vtaskmgr_t vtaskmgr;
|
||||
cbuf_t wbuf;
|
||||
int running;
|
||||
persist_t persist; /* Persistence module — P0-F */
|
||||
ha_state_t ha; /* HA role management — P1-5 */
|
||||
commit_engine_t commit; /* Per-entry commit engine — ha-commit-engine 5.3 */
|
||||
_Atomic int restart_requested; /* set by OP_upgradeConfig handler */
|
||||
pthread_t heartbeat_listener_tid; /* pre-Phase-2 SSL listener thread */
|
||||
_Atomic int heartbeat_listener_running;
|
||||
_Atomic int lost_and_found_job_count; /* incremented during persist_recover() */
|
||||
|
||||
/* Performance-tuning parameters — initialized by init_controller_tunables()
|
||||
* in cluster_init_phase1 so they are available before cluster_init_phase2.
|
||||
* TODO: wire to cluster_config once config generation tool supports @13-@17. */
|
||||
struct {
|
||||
int jobmgr_nhash; /* hash slots for job table */
|
||||
int jobmgr_delta; /* growth step for job table */
|
||||
int workermgr_nhash; /* hash slots for worker table */
|
||||
int workermgr_delta; /* growth step for worker table */
|
||||
int qmgr_nhash; /* hash slots for queue table */
|
||||
int qmgr_delta; /* growth step for queue table */
|
||||
int resmgr_nhash; /* hash slots for resource table */
|
||||
int resmgr_delta; /* growth step for resource table */
|
||||
int resmgr_inst; /* growth step for shared-instance array */
|
||||
int xstrings_nhash; /* hash slots for xstring table */
|
||||
} tunables;
|
||||
} cluster_t;
|
||||
|
||||
#endif /* XCE_CLUSTER_DEF_H */
|
||||
|
|
|
|||
|
|
@ -19,7 +19,16 @@ enum {
|
|||
E_XCE_BAD_QUEUE,
|
||||
E_XCE_BAD_GROUP,
|
||||
E_XCE_INTERNAL,
|
||||
E_XCE_NOMASTER, /* Standby has no known primary — client should retry */
|
||||
E_XCE_REDIRECT, /* Standby is redirecting client to primary */
|
||||
E_XCE_NOT_READY, /* Controller in CTRL_INIT or CTRL_ELECTING state */
|
||||
E_XCE_STANDBY, /* Controller in CTRL_STANDBY; op is primary-only */
|
||||
E_XCE_MAX
|
||||
};
|
||||
|
||||
/* HA dispatch gate error variables — defined in ha.c, declared here so all
|
||||
* dispatch call sites that include errnox.h get the right names. */
|
||||
extern int XCE_ERR_NOT_READY; /* == E_XCE_NOT_READY */
|
||||
extern int XCE_ERR_STANDBY; /* == E_XCE_STANDBY */
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -172,10 +172,13 @@ typedef struct {
|
|||
#define XCE_WORKER_LOCKEDW (0x00040000)
|
||||
#define XCE_WORKER_BUSY (0x00080000)
|
||||
#define XCE_WORKER_CLOSED (0x00100000)
|
||||
#define XCE_WORKER_HIDDEN (0x00200000) /* Admin explicitly removed — invisible but not yet cleaned up */
|
||||
#define XCE_WORKER_CLOSED_BY_ADMIN (0x00400000) /* Admin closed — host closed explicitly by admin */
|
||||
|
||||
#define IS_WORKER_UNAVAIL(status) (((status) & XCE_WORKER_UNAVAIL) != 0)
|
||||
#define IS_WORKER_LOCKEDU(status) (((status) & XCE_WORKER_LOCKEDU) != 0)
|
||||
#define IS_WORKER_LOCKEDW(status) (((status) & XCE_WORKER_LOCKEDW) != 0)
|
||||
#define IS_WORKER_UNAVAIL(status) (((status) & XCE_WORKER_UNAVAIL) != 0)
|
||||
#define IS_WORKER_LOCKEDU(status) (((status) & XCE_WORKER_LOCKEDU) != 0)
|
||||
#define IS_WORKER_LOCKEDW(status) (((status) & XCE_WORKER_LOCKEDW) != 0)
|
||||
#define IS_WORKER_CLOSED_BY_ADMIN(status) (((status) & XCE_WORKER_CLOSED_BY_ADMIN) != 0)
|
||||
#define IS_WORKER_LOCKED(status) \
|
||||
(IS_WORKER_LOCKEDU(status) || IS_WORKER_LOCKEDW(status))
|
||||
#define IS_WORKER_BUSY(status) (((status) & XCE_WORKER_BUSY) != 0)
|
||||
|
|
@ -212,8 +215,8 @@ typedef struct {
|
|||
int nthresholds;
|
||||
xce_res_threshold_t** thresholds;
|
||||
/* following two vars used to push resource values to sched */
|
||||
int nresvals_to_sched;
|
||||
xce_val_with_idx_t** resvals_to_sched;
|
||||
int nresvals_to_sched;
|
||||
xce_name_val_t** resvals_to_sched;
|
||||
} xce_worker_t;
|
||||
|
||||
typedef struct {
|
||||
|
|
@ -353,7 +356,8 @@ typedef struct {
|
|||
|
||||
#define JOB_STAT_PDONE (0x80) /**< Post job process done successfully */
|
||||
#define JOB_STAT_PERR (0x100) /**< Post job process has error */
|
||||
#define JOB_STAT_WAIT (0x200) /**< Chunk job waiting its turn to exec */
|
||||
#define JOB_STAT_WAIT (0x200) /**< Chunk job waiting its turn to exec */
|
||||
#define JOB_STAT_TRANSIENT (0x400) /**< Job state uncertain after failover — awaiting worker re-join */
|
||||
#define JOB_STAT_RUNKWN \
|
||||
0x8000 /* Flag : Job status is UNKWN caused by losting contact with remote \
|
||||
cluster */
|
||||
|
|
@ -381,6 +385,7 @@ typedef struct {
|
|||
#define QUEUE_STAT_DISC 0x10 /**< Remote queue status is disconnected. */
|
||||
#define QUEUE_STAT_FAIRSHARE_DEFAULT \
|
||||
0x20 /**< Fairshare queue with null user share defined*/
|
||||
#define QUEUE_STAT_HIDDEN (0x0100) /**< Admin deleted — invisible but not yet cleaned up */
|
||||
|
||||
/* FIXME: improve following later */
|
||||
#define JOB_IS_PEND(state) (((state) & JOB_STAT_PEND) != 0)
|
||||
|
|
@ -472,10 +477,16 @@ typedef struct {
|
|||
char* ckptinfo;
|
||||
int64_t pending_ctrl;
|
||||
xce_job_preason_t* pend_reasons;
|
||||
uint64_t transient_since; /* epoch seconds; set when job enters JOB_STAT_TRANSIENT */
|
||||
} xce_job_t;
|
||||
|
||||
typedef struct {
|
||||
xce_job_t job;
|
||||
/* ha-convergence-recovery (3.3): WAL entry LSN of this job's submit record.
|
||||
* In-memory only — recomputed from the applied entry at replay; never
|
||||
* serialized (wire type xce_job_t above is untouched). 0 = unknown /
|
||||
* pre-feature = always exportable (see job_submit_committed, cluster.h). */
|
||||
uint64_t submit_lsn;
|
||||
} job_t;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ extern int xce_jobdispatch(xce_ctx_t*, xce_job_dispatch_t*);
|
|||
extern int xce_jobsubmit(xce_ctx_t*, xce_job_submit_t*, uint64_t*);
|
||||
extern int xce_joblist(xce_ctx_t*, xce_job_t***);
|
||||
extern int xce_clusterconf(xce_ctx_t*, xce_clusterconf_t**);
|
||||
extern int xce_ctrlstatus(xce_ctx_t*, xce_ctrlstatus_t**);
|
||||
extern int xce_push(xce_ctx_t*, xce_check_fn_t, void*, int, int, int, void**);
|
||||
extern int xce_update_rows(xce_ctx_t*, int, int, xlist_t*);
|
||||
extern int xce_resval(xce_ctx_t*, char*, char*, xce_resval_t***);
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
#define _XCECTX_H_ 1
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "config.h" /* XCE_MAXHOSTNAME_LEN */
|
||||
#include "netio.h"
|
||||
#include "request.h"
|
||||
#include "vtaskmgr.h"
|
||||
|
|
@ -28,12 +31,20 @@ typedef struct {
|
|||
cbuf_t wbuf;
|
||||
dataset_t* dataset;
|
||||
|
||||
/* Redirect-hint address captured from the last redirect reply by
|
||||
* handle_reply() (which only sees xce_ctx_t) and read by the failover
|
||||
* rotator on the next retry. Empty host ("") means no hint. Advisory only;
|
||||
* correctness never depends on it. */
|
||||
char redirect_hint_host[XCE_MAXHOSTNAME_LEN];
|
||||
uint16_t redirect_hint_port;
|
||||
|
||||
struct {
|
||||
union {
|
||||
char* debugmsg;
|
||||
dataset_t* get;
|
||||
uint64_t jobid;
|
||||
xce_clusterconf_t* clusterconf;
|
||||
xce_ctrlstatus_t* ctrlstatus;
|
||||
xce_fetchfile_reply_t* fetchfilereply;
|
||||
} u;
|
||||
} reply;
|
||||
|
|
@ -47,7 +58,29 @@ extern int xce_ctx_stop(xce_ctx_t*);
|
|||
extern int xce_wakeup(xce_ctx_t*, int);
|
||||
extern int xce_try_wakeup(xce_ctx_t*, int);
|
||||
extern int xce_ctx_set_endpoint(xce_ctx_t*, endpoint_t*);
|
||||
extern int xce_ctx_reconnect(xce_ctx_t*, char*, int);
|
||||
extern void xce_block_tasks(xce_ctx_t*);
|
||||
extern void xce_unblock_tasks(xce_ctx_t*);
|
||||
|
||||
/* Reply dispatch (normally driven by the vtask reply path). Exposed for the
|
||||
* libxce test suite so the reply union's type-safety can be verified without a
|
||||
* live endpoint (bug-1077: the shared, untagged reply union must never be
|
||||
* dereferenced/freed as the wrong type). */
|
||||
extern int xce_handle_reply(xce_ctx_t*, xce_reply_t*);
|
||||
|
||||
/*
|
||||
* xce_ctx_capture_redirect_hint: copy a redirect address carried on a reply
|
||||
* into the ctx's redirect_hint_host/port fields as a best-effort failover hint
|
||||
* (Req 2.2, 8.3). Called by handle_reply() BEFORE the capnp-owned reply (and
|
||||
* its redirect_host string) is freed.
|
||||
*
|
||||
* When `host` is NULL/empty or `port` is 0 there is no usable address: the hint
|
||||
* is left untouched (it stays empty). Otherwise the host is bounded-copied into
|
||||
* the fixed-size field (truncated + NUL-terminated, never overflowing) and the
|
||||
* port is stored. A NULL ctx is a safe no-op. Advisory only — correctness never
|
||||
* depends on the hint.
|
||||
*/
|
||||
extern void xce_ctx_capture_redirect_hint(xce_ctx_t*, const char* host,
|
||||
uint16_t port);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue