245 lines
10 KiB
C
245 lines
10 KiB
C
#ifndef HA_H
|
|
#define HA_H
|
|
|
|
/*
|
|
* XCE Controller HA — role management and election (P1).
|
|
*
|
|
* P1-1 audit: the existing create_ssl_context() + iomgr_set_ssl_ctx() path
|
|
* in cluster.c (line 559, set at line 583) covers all controller-to-controller SSL.
|
|
* HA listeners (heartbeat, election probe, replication accept) reuse the same
|
|
* ssl_ctx already installed on c->iomgr.
|
|
* No new TLS infrastructure is needed — XCE runs on a trusted management LAN.
|
|
* Peer identity is validated by host:port match against the --peers candidate list.
|
|
* See design doc Part 5.9 and memory: Persistence & HA Trusted LAN Decision.
|
|
*
|
|
* P1-2: full struct definitions and API.
|
|
* endpoint_t* (primary_ep) is intentionally omitted here — including netio.h
|
|
* would pull in heavy OpenSSL headers and create circular dependency risks.
|
|
* The watchdog (P1-3) will include netio.h directly where it needs endpoint_t.
|
|
*/
|
|
|
|
#include <pthread.h>
|
|
#include <stdatomic.h>
|
|
#include <stdint.h>
|
|
#include <time.h>
|
|
|
|
#include "config.h" /* XCE_MAXNAME_LEN, MAX_STANDBYS, etc. */
|
|
#include "xce/cluster.h" /* xce_candidate_t */
|
|
#include "zarray.h"
|
|
|
|
/* Forward declaration — avoids circular include with cluster_def.h */
|
|
typedef struct cluster_s cluster_t;
|
|
|
|
/* ── Inline PRNG for election jitter ───────────────────────── */
|
|
|
|
static inline uint64_t xorshift64_next(uint64_t *state)
|
|
{
|
|
uint64_t x = *state;
|
|
x ^= x << 13;
|
|
x ^= x >> 7;
|
|
x ^= x << 17;
|
|
*state = x;
|
|
return x;
|
|
}
|
|
|
|
/*
|
|
* ha_seed_jitter: seed the xorshift64 PRNG from /dev/urandom (8 bytes).
|
|
* Falls back to CLOCK_MONOTONIC ^ priority ^ pid if urandom is unavailable.
|
|
* Not inline — contains syscalls (open/read/close/clock_gettime/getpid).
|
|
*/
|
|
void ha_seed_jitter(uint64_t *seed, int my_priority);
|
|
|
|
/* ── Role enum ─────────────────────────────────────────────── */
|
|
typedef enum {
|
|
CTRL_PRIMARY = 0,
|
|
CTRL_STANDBY = 1
|
|
} ctrl_role_t;
|
|
|
|
/* ── HA configuration ──────────────────────────────────────── */
|
|
|
|
/* Default values used by cluster_init(). Tuned for a LAN with ≤5ms RTT. */
|
|
#define HA_DEFAULT_HEARTBEAT_INTERVAL_MS 3000
|
|
#define HA_DEFAULT_HEARTBEAT_TIMEOUT_MS 10000
|
|
#define HA_DEFAULT_ELECTION_TIMEOUT_MS 2000
|
|
#define HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS 500
|
|
#define HA_DEFAULT_MAX_PROMOTION_RETRIES 5
|
|
#define HA_DEFAULT_ELECTION_JITTER_MAX_MS 200
|
|
#define HA_DEFAULT_ELECTION_PRIORITY_STEP_MS 500
|
|
|
|
typedef struct {
|
|
uint32_t heartbeat_interval_ms; /* Primary heartbeat send interval (min 2000 ms) */
|
|
uint32_t heartbeat_timeout_ms; /* Standby timeout before promoting (>= interval) */
|
|
uint32_t election_timeout_ms; /* Election probe shared deadline (min 500 ms) */
|
|
uint32_t probe_connect_timeout_ms; /* Per-probe TCP connect timeout */
|
|
uint32_t max_promotion_retries; /* Max consecutive failed promotion attempts */
|
|
uint32_t election_jitter_max_ms; /* random jitter added to election timeout (default 200) */
|
|
uint32_t election_priority_step_ms; /* extra ms per priority level (default 500) */
|
|
} ha_config_t;
|
|
|
|
typedef struct {
|
|
xce_candidate_t *candidate; /* peer to probe */
|
|
void *ssl_ctx; /* SSL_CTX* (void* to avoid openssl header) */
|
|
uint32_t timeout_ms;
|
|
atomic_int *election_done;
|
|
int got_ack;
|
|
pthread_t thread;
|
|
} probe_ctx_t;
|
|
|
|
/* ── HA runtime state (embedded in cluster_t as `ha_state_t ha` in P1-5) ── */
|
|
typedef struct {
|
|
_Atomic uint64_t term; /* Election term — fencing token */
|
|
_Atomic int32_t role; /* Cast to ctrl_role_t at access sites */
|
|
int my_priority; /* Index in --peers list (lower = higher priority) */
|
|
_Atomic uint64_t applied_lsn; /* Standby: last applied WAL LSN */
|
|
char primary_host[XCE_MAXNAME_LEN]; /* Hostname of current primary */
|
|
uint32_t primary_port; /* Port of current primary */
|
|
struct timespec last_heartbeat; /* CLOCK_MONOTONIC timestamp of last heartbeat */
|
|
_Atomic int promotion_failed;
|
|
_Atomic int promotion_retry_count;
|
|
_Atomic int in_catch_up; /* Non-zero while processing historical WAL entries */
|
|
_Atomic int failover_requested; /* Set by SIGUSR1 handler; watchdog executes */
|
|
int hooks_registered; /* Idempotency guard for xce_add_hook calls */
|
|
uint32_t last_received_crc; /* Chained CRC of last received replicated entry */
|
|
ha_config_t config;
|
|
pthread_mutex_t lock; /* Protects primary_host/port + last_heartbeat */
|
|
/* Watchdog thread — started in P1-3 */
|
|
_Atomic int watchdog_stop;
|
|
pthread_t watchdog_thread;
|
|
void *ssl_ctx; /* SSL_CTX* cast; avoids pulling openssl headers here */
|
|
uint64_t jitter_seed; /* xorshift64 PRNG state for election jitter */
|
|
zarray_t *standby_endpoints; /* connected standby controller endpoints for config push */
|
|
} ha_state_t;
|
|
|
|
/* ── Public API ────────────────────────────────────────────── */
|
|
|
|
/*
|
|
* ha_init: zero-initialise *ha, validate cfg, store config and my_priority.
|
|
* Returns 0 on success, -EINVAL on bad config.
|
|
* Initial role is always CTRL_STANDBY.
|
|
*/
|
|
int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority);
|
|
|
|
/*
|
|
* ha_fini: destroy mutex. Watchdog thread join added in P1-3.
|
|
*/
|
|
void ha_fini(ha_state_t *ha);
|
|
|
|
/*
|
|
* am_i_primary: stub — priority-0 node is primary.
|
|
* Full election probing added in P1-3.
|
|
*/
|
|
int am_i_primary(cluster_t *c);
|
|
|
|
/*
|
|
* ha_promote_to_primary: stub — atomically sets role to PRIMARY.
|
|
* Full promotion sequence (hooks, WAL flush, peer notify) added in P1-3a.
|
|
*/
|
|
int ha_promote_to_primary(cluster_t *c);
|
|
|
|
/*
|
|
* ha_demote_to_standby: stub — atomically sets role to STANDBY.
|
|
* Full demotion sequence added in P1-3a.
|
|
*/
|
|
int ha_demote_to_standby(cluster_t *c);
|
|
|
|
/* ── Internal promotion/demotion (P1-3a) ───────────────────── */
|
|
|
|
/*
|
|
* ha_do_promote: increment term, set role=PRIMARY, reset last_received_crc.
|
|
* No cluster_t dependency — fully testable standalone.
|
|
* Callers: ha_promote_to_primary() (P1-5), watchdog failover path.
|
|
*/
|
|
void ha_do_promote(ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_do_demote: set role=STANDBY, reset hooks_registered idempotency guard.
|
|
* Hook disable requires cluster_t and is done in ha_demote_to_standby() (P1-5).
|
|
*/
|
|
void ha_do_demote(ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_install_sigusr1: register a SIGUSR1 handler that atomically sets
|
|
* ha->failover_requested. The handler uses only async-signal-safe operations
|
|
* (atomic store on an int-width type — POSIX.1-2017 §2.4.3).
|
|
* Must be called once, before any signal can arrive.
|
|
*/
|
|
void ha_install_sigusr1(ha_state_t *ha);
|
|
|
|
/* ── Watchdog thread API (P1-3) ────────────────────────────── */
|
|
|
|
/*
|
|
* ha_start_watchdog: spawn the heartbeat watchdog pthread.
|
|
* Must be called after ha_init(). Returns 0 on success, -errno on failure.
|
|
*/
|
|
int ha_start_watchdog(ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_stop_watchdog: set watchdog_stop flag and pthread_join the thread.
|
|
* Safe to call if watchdog was never started (thread == 0).
|
|
*/
|
|
void ha_stop_watchdog(ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_check_heartbeat_nocluster: check whether the primary heartbeat has
|
|
* timed out. Sets failover_requested if timeout exceeded and primary_host
|
|
* is non-empty. Test-only entry point — does not need cluster_t.
|
|
*/
|
|
void ha_check_heartbeat_nocluster(ha_state_t *ha);
|
|
|
|
/* ── Redirect helpers used by join.c ──────────────────────── */
|
|
|
|
/*
|
|
* ha_should_redirect: returns 1 if this controller should redirect join
|
|
* requests to the primary.
|
|
* Returns 0 if: (a) role is PRIMARY, or (b) primary_host is empty /
|
|
* primary_port is 0 (no primary known yet — client should retry).
|
|
*/
|
|
int ha_should_redirect(const ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_no_primary_known: returns 1 if role is STANDBY and primary_host is
|
|
* empty or primary_port is 0. Caller should reply E_XCE_NOMASTER so the
|
|
* client knows to retry later rather than treating the error as permanent.
|
|
*/
|
|
int ha_no_primary_known(const ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_is_primary: returns 1 if ha->role is CTRL_PRIMARY, 0 otherwise.
|
|
* Lock-free. Safe to call from any thread.
|
|
*/
|
|
int ha_is_primary(const ha_state_t *ha);
|
|
|
|
/*
|
|
* ha_check_priority_stepdown: called from the heartbeat handler when
|
|
* term == 0 (simultaneous boot). If sender_priority < ha->my_priority
|
|
* (higher priority), steps down the local node from PRIMARY to STANDBY.
|
|
* No-op if term > 0 or sender is not higher priority.
|
|
*/
|
|
void ha_check_priority_stepdown(ha_state_t *ha, int sender_priority,
|
|
uint64_t sender_term);
|
|
|
|
/*
|
|
* start_heartbeat_listener: start a minimal pre-Phase-2 SSL listener
|
|
* that responds to OP_heartbeat probes. Called after am_i_primary()
|
|
* returns 1. Records tid in c->heartbeat_listener_tid.
|
|
*/
|
|
void start_heartbeat_listener(cluster_t *c);
|
|
|
|
/*
|
|
* stop_heartbeat_listener: signal and join the heartbeat listener thread.
|
|
* Must be called before iomgr_init() to free the listening port (EADDRINUSE).
|
|
*/
|
|
void stop_heartbeat_listener(cluster_t *c);
|
|
|
|
/* ── P3-6: Test-only failover shim ────────────────────────────── */
|
|
|
|
#ifdef XCE_TESTING
|
|
/* Test-only shim: demote then re-promote; returns new term.
|
|
* Exercises the manual failover state machine without a live cluster_t.
|
|
* Calls ha_do_demote() then ha_do_promote(); validates role transitions
|
|
* and term monotonicity. See design doc P3-6 and src/docs/persistence-and-ha-design.md.
|
|
*/
|
|
uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha);
|
|
#endif
|
|
|
|
#endif /* HA_H */
|