refactor(ha): passive standby model — primary connects outbound to all peers
Remove fetch_config_from_primary, ha_find_and_connect_to_primary, and the tier-1/2/3 discovery path. Standbys never initiate outbound connections to the primary; the primary owns all inter-controller channels. Key changes: - ha.c: add primary heartbeat loop (ha_start_primary_heartbeat) that sends OPCode_nop to connected standbys every heartbeat_interval_ms; loop self-cancels on demotion via role check before re-arm. - ha.c: add outbound reconnect in primary_heartbeat_fn — for peers absent from standby_endpoints, open fresh outbound TLS connection (on_standby_connected sends ctrlhello; standby_ep_ctx_t mirrors vote_ep_ctx_t layout). - ha.c: remove ha_find_and_connect_to_primary + helpers (ha_raw_tcp_connect, ha_tcp_connect_hook); remove election_timeout_fn (superseded by raft_election_timeout_fn); remove fetch_config_from_primary call from ha_elect_evaluate_and_signal STANDBY branch. - cluster.c: OPCode_requestVote denial no longer reuses inbound connection; late joiners are bootstrapped via primary heartbeat outbound connect. - config_repl.c: remove connect_ssl_raw, do_one_config_fetch, fetch_config_from_primary (~250 LOC). - myapp.c: remove my_tag > 0 fetch branch; all standbys wait passively. - ha.h: remove ha_find_and_connect_to_primary declaration + macros + ha_tcp_connect_hook seam; add ha_start_primary_heartbeat declaration. - meson.build: remove test_ha_connect.c (tests deleted function). - controller_test.c: remove test_ha_connect registration. - CLAUDE.md, src/controller/CLAUDE.md: document passive standby model, inbound-vs-outbound connection ownership, async I/O constraint, heartbeat model. - request.h: add xce_request declaration (removes scattered local externs).
This commit is contained in:
parent
842b9a838e
commit
7b39650229
11 changed files with 716 additions and 919 deletions
|
|
@ -180,6 +180,9 @@ static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term)
|
|||
return 1;
|
||||
}
|
||||
|
||||
/* Track endpoint for promotion/close after election result */
|
||||
ectx->votes[votes_idx].ep = ep;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -381,6 +384,24 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
|||
/* Reset ack counter for this committing round */
|
||||
atomic_store(&ha->repl_ack_count, 0);
|
||||
|
||||
/* Promote surviving vote endpoints to standby replication channels.
|
||||
* Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to
|
||||
* ha->standby_endpoints so push_config and ctrlhello reach them. */
|
||||
{
|
||||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||||
&ha->elect_ctx, memory_order_acquire);
|
||||
if (ectx && ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
for (int i = 0; i < ectx->n_peers; i++) {
|
||||
if (ectx->votes[i].ep != NULL) {
|
||||
zarray_store(ha->standby_endpoints, ectx->votes[i].ep);
|
||||
INFO("HA: promoted vote ep[%d] → standby channel", i);
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
}
|
||||
}
|
||||
|
||||
if (c) {
|
||||
/* Push config to standbys with lower config_tag */
|
||||
#ifdef XCE_TESTING
|
||||
|
|
@ -447,19 +468,31 @@ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
|
|||
ha->phase2_done = 1;
|
||||
}
|
||||
|
||||
/* Close all outgoing vote endpoints (loser path).
|
||||
* The primary will push config and ctrlhello via promoted connections;
|
||||
* the standby waits passively on its listener for the inbound connection. */
|
||||
{
|
||||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||||
&ha->elect_ctx, memory_order_acquire);
|
||||
if (ectx) {
|
||||
for (int i = 0; i < ectx->n_peers; i++) {
|
||||
if (ectx->votes[i].ep != NULL) {
|
||||
INFO("HA: closing vote ep[%d] (loser path)", i);
|
||||
endpoint_fini(ectx->votes[i].ep);
|
||||
ectx->votes[i].ep = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
|
||||
memory_order_release);
|
||||
INFO("HA FSM (raft): → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
|
||||
term, is_first_boot);
|
||||
|
||||
/* Trigger async config fetch from primary so standby can load config */
|
||||
#ifndef XCE_TESTING
|
||||
if (c != NULL && is_first_boot) {
|
||||
extern int fetch_config_from_primary(cluster_t *c);
|
||||
delay_task_ms(&c->delaymgr, 100, 1,
|
||||
(delay_task_entry_fn_t)fetch_config_from_primary, c);
|
||||
}
|
||||
#endif
|
||||
/* Arm heartbeat watchdog — fires ha_on_primary_ctrl_fd_rdhup if primary
|
||||
* sends no ctrlhello within election_timeout_ms. */
|
||||
ha_arm_heartbeat_timeout_vtask(ha, c);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1142,6 +1175,11 @@ int ha_promote_to_primary(cluster_t *c)
|
|||
(hook_fn_t)persist_worker_hook_fun, &c->persist);
|
||||
c->ha.hooks_registered = 1;
|
||||
}
|
||||
|
||||
/* Arm periodic heartbeat to standbys — keeps their watchdog alive when
|
||||
* the cluster is idle (no WAL entries or config pushes). */
|
||||
ha_start_primary_heartbeat(c);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1495,6 +1533,7 @@ int (*ha_send_ctrlhello_hook)(void *c) = NULL;
|
|||
* (config_repl.c's push_config_to_standbys needs a real cluster). Req 7.1.
|
||||
*/
|
||||
int (*ha_push_config_hook)(void *c) = NULL;
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Election Evaluation (Req 2.1, 2.7, 2.8) ───────────────────────────────
|
||||
|
|
@ -1697,17 +1736,8 @@ apply_role:
|
|||
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
|
||||
atomic_load_explicit(&ha->term, memory_order_acquire),
|
||||
is_first_boot);
|
||||
|
||||
/* Trigger async config fetch from primary so standby can load
|
||||
* cluster_config, establish persistent control channel, and
|
||||
* participate in RDHUP failover detection. */
|
||||
#ifndef XCE_TESTING
|
||||
if (c != NULL) {
|
||||
extern int fetch_config_from_primary(cluster_t *c);
|
||||
delay_task_ms(&c->delaymgr, 100, 1,
|
||||
(delay_task_entry_fn_t)fetch_config_from_primary, c);
|
||||
}
|
||||
#endif
|
||||
/* Primary connects outbound and pushes config via ctrlhello.
|
||||
* Standby is fully passive here — no fetch_config_from_primary. */
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1796,16 +1826,15 @@ void handle_request_vote_reply(ha_state_t *ha,
|
|||
|
||||
/* vote_ep_ctx_t: typedef moved to Task 6.2 forward-declaration block above */
|
||||
|
||||
/* xce_request: defined in request.c, not yet in any public header */
|
||||
extern int xce_request(endpoint_t *ep, xce_request_t *req);
|
||||
|
||||
/*
|
||||
* vote_reply_msg_handler: forward declaration.
|
||||
* Implemented in task 2.3. Registered on the endpoint by on_vote_connected
|
||||
* so that the next iomgr EPOLLIN delivers the vote reply to this handler.
|
||||
* Signature matches msg_handler_t = int (*)(void*) from netio.h.
|
||||
* peer_ctrl_msg_handler / vote_reply_decode_and_dispatch /
|
||||
* repl_ack_decode_and_dispatch: forward declarations.
|
||||
* peer_ctrl_msg_handler is the single iomgr handler set in on_vote_connected
|
||||
* and never reassigned — it routes by ha->role at dispatch time.
|
||||
*/
|
||||
static int vote_reply_msg_handler(void *ep);
|
||||
static int peer_ctrl_msg_handler(void *ep_raw);
|
||||
static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx);
|
||||
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx);
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
/*
|
||||
|
|
@ -1865,8 +1894,9 @@ static void on_vote_connected(endpoint_t *ep)
|
|||
xce_request(ep, &req);
|
||||
#endif
|
||||
|
||||
/* Switch to reply-reading mode: next inbound packet → vote_reply_msg_handler */
|
||||
ep->msg_handler = vote_reply_msg_handler;
|
||||
/* Set state-aware dispatcher — routes by ha->role at message-arrival time.
|
||||
* Never reassigned: ELECTING → vote reply; COMMITTING/PRIMARY → repl ack. */
|
||||
ep->msg_handler = peer_ctrl_msg_handler;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1910,51 +1940,32 @@ static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx,
|
|||
}
|
||||
|
||||
/*
|
||||
* vote_reply_msg_handler: iomgr message handler for incoming RequestVoteReply.
|
||||
*
|
||||
* Called by iomgr (via INVOKE_MSG_HANDLER) when a full packet arrives on an
|
||||
* outgoing vote endpoint after on_vote_connected fired.
|
||||
*
|
||||
* Steps:
|
||||
* 1. Cast ep to endpoint_t*; get packet from ep->fsm_in.pkt (owned by us).
|
||||
* 2. Raw capnp decode: init capn_ctx, read outer Request wrapper.
|
||||
* 3. read_RequestReply() into rep_s (capnp pointer — still valid).
|
||||
* 4. read_RequestVoteReply() into vr_raw (scalars) BEFORE capn_free.
|
||||
* 5. capn_free() — capnp segment released; vr_raw still valid (scalars only).
|
||||
* 6. free_packet() + free(pkt) — packet memory released.
|
||||
* 7. Term fencing: if vr_raw.term > ha->term, update ha->term atomically.
|
||||
* 8. Call vote_reply_dispatch() with vr_raw scalar fields.
|
||||
*
|
||||
* On any decode error: treat peer as denied (grant=0, tag=0, lsn=0).
|
||||
*
|
||||
* ASAN safety: all capnp fields read before capn_free() — no heap-use-after-free.
|
||||
* Requirements: 3.1 (iomgr packet), 3.2 (no capnp UAF), 3.3 (extract fields),
|
||||
* 3.4 (term fencing).
|
||||
* vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch
|
||||
* to vote_reply_dispatch. Factored from vote_reply_msg_handler so that
|
||||
* peer_ctrl_msg_handler can invoke the same logic without code duplication.
|
||||
* Packet ownership: caller passes ep->fsm_in.pkt; this function frees it.
|
||||
*/
|
||||
static int vote_reply_msg_handler(void *ep_raw)
|
||||
static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx)
|
||||
{
|
||||
endpoint_t *ep = (endpoint_t *)ep_raw;
|
||||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||||
ha_state_t *ha = vctx->ha;
|
||||
ha_elect_ctx_t *ctx = vctx->ctx;
|
||||
int idx = vctx->peer_idx;
|
||||
io_packet_t *pkt = ep->fsm_in.pkt; /* ownership transferred by INVOKE_MSG_HANDLER */
|
||||
int ok = 0;
|
||||
ha_state_t *ha = vctx->ha;
|
||||
ha_elect_ctx_t *ctx = vctx->ctx;
|
||||
int idx = vctx->peer_idx;
|
||||
io_packet_t *pkt = ep->fsm_in.pkt;
|
||||
int ok = 0;
|
||||
|
||||
/* Decoded reply fields — populated on success path */
|
||||
uint64_t vr_term = 0;
|
||||
int vr_granted = 0;
|
||||
uint64_t vr_tag = 0;
|
||||
uint64_t vr_lsn = 0;
|
||||
uint64_t vr_term = 0;
|
||||
int vr_granted = 0;
|
||||
uint64_t vr_tag = 0;
|
||||
uint64_t vr_lsn = 0;
|
||||
|
||||
if (!pkt || !pkt->data || pkt->sz == 0)
|
||||
goto done;
|
||||
|
||||
{
|
||||
struct capn capn_ctx;
|
||||
struct Request outer_rq;
|
||||
Request_ptr outer_rp;
|
||||
struct RequestReply rep_s;
|
||||
struct capn capn_ctx;
|
||||
struct Request outer_rq;
|
||||
Request_ptr outer_rp;
|
||||
struct RequestReply rep_s;
|
||||
struct RequestVoteReply vr_raw;
|
||||
int capn_ok;
|
||||
|
||||
|
|
@ -1979,13 +1990,10 @@ static int vote_reply_msg_handler(void *ep_raw)
|
|||
goto done;
|
||||
}
|
||||
|
||||
/* CRITICAL: copy all scalars BEFORE capn_free to avoid UAF.
|
||||
* struct RequestVoteReply fields: term, voteGranted, peerConfigTag,
|
||||
* peerLastLogLsn — all scalars; safe to copy before freeing segment. */
|
||||
memset(&vr_raw, 0, sizeof(vr_raw));
|
||||
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
|
||||
|
||||
capn_free(&capn_ctx); /* capnp segment released; vr_raw.* still valid */
|
||||
capn_free(&capn_ctx);
|
||||
|
||||
vr_term = (uint64_t)vr_raw.term;
|
||||
vr_granted = vr_raw.voteGranted ? 1 : 0;
|
||||
|
|
@ -1995,14 +2003,9 @@ static int vote_reply_msg_handler(void *ep_raw)
|
|||
}
|
||||
|
||||
done:
|
||||
/* Release packet — INVOKE_MSG_HANDLER will null ep->fsm_in.pkt after return */
|
||||
if (pkt) { free_packet(pkt); free(pkt); }
|
||||
|
||||
if (ok) {
|
||||
/* Term fencing: if peer's term > ours, atomically advance ha->term.
|
||||
* Must be done here (before vote_reply_dispatch) so it applies on both
|
||||
* the production path (raft_on_vote_reply) and the XCE_TESTING hook path
|
||||
* where raft_cb_become_follower is bypassed. */
|
||||
uint64_t cur = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||||
while (vr_term > cur) {
|
||||
if (atomic_compare_exchange_weak_explicit(&ha->term, &cur, vr_term,
|
||||
|
|
@ -2012,7 +2015,6 @@ done:
|
|||
}
|
||||
vote_reply_dispatch(ha, ctx, idx, vr_term, vr_granted, vr_tag, vr_lsn);
|
||||
} else {
|
||||
/* Decode error: treat peer as denied */
|
||||
vote_reply_dispatch(ha, ctx, idx,
|
||||
atomic_load_explicit(&ha->term, memory_order_acquire),
|
||||
0, 0, 0);
|
||||
|
|
@ -2020,11 +2022,87 @@ done:
|
|||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* repl_ack_decode_and_dispatch: decode an inbound packet on a promoted vote
|
||||
* endpoint while role == CTRL_PRIMARY_COMMITTING or CTRL_PRIMARY.
|
||||
* Expects OPCode_replCatchUpDone; calls ha_repl_ack_received on success.
|
||||
* Drops unexpected opcodes with a WARN log.
|
||||
*/
|
||||
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx)
|
||||
{
|
||||
ha_state_t *ha = vctx->ha;
|
||||
cluster_t *c = vctx->c;
|
||||
io_packet_t *pkt = ep->fsm_in.pkt;
|
||||
|
||||
if (!pkt || !pkt->data || pkt->sz == 0) {
|
||||
if (pkt) { free_packet(pkt); free(pkt); }
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct capn capn_ctx;
|
||||
struct Request outer_rq;
|
||||
Request_ptr outer_rp;
|
||||
int capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
|
||||
(size_t)pkt->sz, 0);
|
||||
free_packet(pkt);
|
||||
free(pkt);
|
||||
ep->fsm_in.pkt = NULL;
|
||||
|
||||
if (capn_ok != 0)
|
||||
return 0;
|
||||
|
||||
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
|
||||
read_Request(&outer_rq, outer_rp);
|
||||
int opcode = (int)outer_rq.opcode;
|
||||
capn_free(&capn_ctx);
|
||||
|
||||
if (opcode == OPCode_replCatchUpDone) {
|
||||
ha_repl_ack_received(ha, c);
|
||||
} else {
|
||||
WARN("repl_ack_decode_and_dispatch: unexpected opcode %d on promoted ep",
|
||||
opcode);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* peer_ctrl_msg_handler: state-aware iomgr handler set once in on_vote_connected.
|
||||
* Routes by ha->role at dispatch time — no explicit handler reassignment needed.
|
||||
* CTRL_ELECTING → vote_reply_decode_and_dispatch
|
||||
* CTRL_PRIMARY_COMMITTING /
|
||||
* CTRL_PRIMARY → repl_ack_decode_and_dispatch
|
||||
* any other role → drop packet (WARN)
|
||||
*/
|
||||
static int peer_ctrl_msg_handler(void *ep_raw)
|
||||
{
|
||||
endpoint_t *ep = (endpoint_t *)ep_raw;
|
||||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||||
ha_state_t *ha = vctx->ha;
|
||||
|
||||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||||
memory_order_acquire);
|
||||
switch (role) {
|
||||
case CTRL_ELECTING:
|
||||
return vote_reply_decode_and_dispatch(ep, vctx);
|
||||
|
||||
case CTRL_PRIMARY_COMMITTING:
|
||||
case CTRL_PRIMARY:
|
||||
return repl_ack_decode_and_dispatch(ep, vctx);
|
||||
|
||||
default: {
|
||||
io_packet_t *pkt = ep->fsm_in.pkt;
|
||||
if (pkt) { free_packet(pkt); free(pkt); ep->fsm_in.pkt = NULL; }
|
||||
WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
||||
/*
|
||||
* vote_reply_build_pkt_for_test: test helper that encodes a RequestVoteReply
|
||||
* into a newly allocated io_packet_t that vote_reply_msg_handler can decode.
|
||||
* into a newly allocated io_packet_t that vote_reply_msg_handler_for_test can decode.
|
||||
*
|
||||
* Wire format mirrors the real peer reply:
|
||||
* Request{opcode=reply, body.replymsg=RequestReply{
|
||||
|
|
@ -2096,15 +2174,23 @@ io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
|
|||
}
|
||||
|
||||
/*
|
||||
* vote_reply_msg_handler_for_test: expose vote_reply_msg_handler under
|
||||
* XCE_TESTING so tests can call it directly (it is otherwise static).
|
||||
* Sets ep->fsm_in.pkt = pkt then calls vote_reply_msg_handler(ep).
|
||||
* vote_reply_msg_handler_for_test: test entry point for vote-reply decode path.
|
||||
* Temporarily forces role to CTRL_ELECTING so peer_ctrl_msg_handler routes to
|
||||
* vote_reply_decode_and_dispatch, then restores the original role.
|
||||
* Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
|
||||
*/
|
||||
int vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
|
||||
{
|
||||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||||
ha_state_t *ha = vctx ? vctx->ha : NULL;
|
||||
int32_t saved_role = ha ? atomic_load_explicit(&ha->role, memory_order_acquire) : 0;
|
||||
if (ha)
|
||||
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||||
ep->fsm_in.pkt = pkt;
|
||||
return vote_reply_msg_handler(ep);
|
||||
int rc = peer_ctrl_msg_handler(ep);
|
||||
if (ha)
|
||||
atomic_store_explicit(&ha->role, saved_role, memory_order_release);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -2181,28 +2267,85 @@ int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c,
|
|||
ha_promote_to_primary(c);
|
||||
return 0;
|
||||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Task 2.4: election_timeout_fn — delaymgr callback ─────────────────────
|
||||
*
|
||||
* Fires ha_elect_evaluate_and_signal after election_timeout_ms if not cancelled.
|
||||
* The cancelled flag is set atomically when quorum is already reached, making
|
||||
* this callback a no-op. Owns the election_timeout_ctx_t — frees it on exit.
|
||||
/*
|
||||
* repl_catchup_done_pkt_for_test: build a minimal io_packet_t encoding
|
||||
* Request{opcode=replCatchUpDone, body_which=nodata}.
|
||||
* Used by peer_ctrl_msg_handler tests to verify the COMMITTING dispatch path.
|
||||
*/
|
||||
static int election_timeout_fn(void *arg)
|
||||
io_packet_t *repl_catchup_done_pkt_for_test(void)
|
||||
{
|
||||
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
|
||||
if (atomic_load(&t->cancelled)) {
|
||||
free(t);
|
||||
return 0;
|
||||
}
|
||||
pthread_mutex_lock(&t->ctx->lock);
|
||||
ha_elect_evaluate_and_signal(t->ha, t->ctx);
|
||||
pthread_mutex_unlock(&t->ctx->lock);
|
||||
free(t);
|
||||
return 0;
|
||||
struct capn cs;
|
||||
struct capn_segment *seg;
|
||||
struct Request rq_capn;
|
||||
Request_ptr rq_ptr;
|
||||
io_packet_t *pkt;
|
||||
int rc;
|
||||
|
||||
capn_init_malloc(&cs);
|
||||
seg = capn_root(&cs).seg;
|
||||
|
||||
memset(&rq_capn, 0, sizeof(rq_capn));
|
||||
rq_capn.opcode = OPCode_replCatchUpDone;
|
||||
rq_capn.seq = 1;
|
||||
rq_capn.body_which = Request_body_nomsg;
|
||||
rq_ptr = new_Request(seg);
|
||||
write_Request(&rq_capn, rq_ptr);
|
||||
capn_setp(capn_root(&cs), 0, rq_ptr.p);
|
||||
|
||||
pkt = calloc(1, sizeof(*pkt));
|
||||
if (!pkt) { capn_free(&cs); return NULL; }
|
||||
pkt->sz = capn_size(&cs);
|
||||
rc = alloc_packet(pkt);
|
||||
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
|
||||
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
|
||||
capn_free(&cs);
|
||||
return pkt;
|
||||
}
|
||||
|
||||
/*
|
||||
* vote_ep_err_handler_for_test: call the static vote_ep_err_handler on ep.
|
||||
* Allows tests to invoke the err_handler without going through iomgr/endpoint machinery.
|
||||
*/
|
||||
int vote_ep_err_handler_for_test(endpoint_t *ep)
|
||||
{
|
||||
return vote_ep_err_handler((void *)ep);
|
||||
}
|
||||
|
||||
/*
|
||||
* peer_ctrl_msg_handler_for_test: expose peer_ctrl_msg_handler for tests.
|
||||
* Sets ep->fsm_in.pkt then invokes the handler. Packet ownership transferred.
|
||||
*/
|
||||
int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
|
||||
{
|
||||
ep->fsm_in.pkt = pkt;
|
||||
return peer_ctrl_msg_handler(ep);
|
||||
}
|
||||
|
||||
/*
|
||||
* raft_cb_become_leader_for_test: drive the leader promotion path without
|
||||
* a live cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress
|
||||
* production-only calls (phase2, push_config, ctrlhello, committing timer).
|
||||
*/
|
||||
void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term)
|
||||
{
|
||||
ha->raft_c = NULL;
|
||||
raft_cb_become_leader((void *)ha, term);
|
||||
}
|
||||
|
||||
/*
|
||||
* raft_cb_become_follower_for_test: drive the follower path without a live
|
||||
* cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production
|
||||
* calls (phase2, fetch_config); ha_arm_heartbeat_timeout_vtask is a no-op
|
||||
* when c=NULL so no watchdog thread is spawned.
|
||||
*/
|
||||
void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term)
|
||||
{
|
||||
ha->raft_c = NULL;
|
||||
raft_cb_become_follower((void *)ha, term, -1);
|
||||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Task 4.1: committing_timeout_fn — delaymgr callback ───────────────────
|
||||
*
|
||||
* Fires ha_promote_to_primary (degraded mode) if the majority ack threshold
|
||||
|
|
@ -2260,15 +2403,32 @@ static int vote_ep_err_handler(void *ep_raw)
|
|||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||||
if (!vctx) return 0;
|
||||
|
||||
handle_request_vote_reply(vctx->ha, vctx->ctx, vctx->peer_idx,
|
||||
vctx->my_term,
|
||||
0, /* vote_granted = false */
|
||||
0, 0);
|
||||
/* Fix 5 (HIGH): do NOT free vctx here.
|
||||
* ep->data (vctx) is also referenced by vote_reply_msg_handler via ep->data.
|
||||
* The endpoint lifetime is managed by iomgr; vctx is freed in
|
||||
* ha_election_cleanup() (task 5.1) when all per-peer contexts are released.
|
||||
* Freeing here would cause a use-after-free in vote_reply_msg_handler. */
|
||||
ha_state_t *ha = vctx->ha;
|
||||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||||
memory_order_acquire);
|
||||
|
||||
if (role == CTRL_PRIMARY_COMMITTING || role == CTRL_PRIMARY) {
|
||||
/* Promoted connection dropped — remove from standby_endpoints */
|
||||
if (ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||||
if (zarray_get(ha->standby_endpoints, i) == ep) {
|
||||
zarray_del(ha->standby_endpoints, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
}
|
||||
} else {
|
||||
/* Election phase: count as denied vote */
|
||||
handle_request_vote_reply(ha, vctx->ctx, vctx->peer_idx,
|
||||
vctx->my_term,
|
||||
0, /* vote_granted = false */
|
||||
0, 0);
|
||||
}
|
||||
|
||||
/* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data.
|
||||
* vctx is freed in ha_election_cleanup() when all per-peer contexts release. */
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -2443,17 +2603,7 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
|
|||
n_peers);
|
||||
/* Returns immediately — election result delivered via raft on_become_leader/follower */
|
||||
}
|
||||
/* ── Task 9.2: Persistent Control Channel ──────────────────────────────────
|
||||
*
|
||||
* ha_find_and_connect_to_primary: three-tier primary discovery + TCP connect.
|
||||
*
|
||||
* Requirements: 6.1
|
||||
* Boundary: Persistent Control Channel (ha.c)
|
||||
*/
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
int (*ha_tcp_connect_hook)(const char *host, int port) = NULL;
|
||||
|
||||
/*
|
||||
* ha_rdhup_election_hook: test-only seam for ha_start_election_async in the
|
||||
* RDHUP failover path. When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls
|
||||
|
|
@ -2465,158 +2615,6 @@ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
|
|||
const char *my_name) = NULL;
|
||||
#endif
|
||||
|
||||
/* ha_tcp_connect_raw: non-blocking connect + poll, returns connected fd or -errno */
|
||||
static int ha_tcp_connect_raw(const char *host, int port, int timeout_ms)
|
||||
{
|
||||
#ifdef XCE_TESTING
|
||||
if (ha_tcp_connect_hook)
|
||||
return ha_tcp_connect_hook(host, port);
|
||||
#endif
|
||||
|
||||
struct addrinfo hints, *res = NULL;
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
char port_str[16];
|
||||
snprintf(port_str, sizeof(port_str), "%d", port);
|
||||
if (getaddrinfo(host, port_str, &hints, &res) != 0)
|
||||
return -EHOSTUNREACH;
|
||||
|
||||
int sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
|
||||
if (sock < 0) { int e = errno; freeaddrinfo(res); return -e; }
|
||||
|
||||
int one = 1;
|
||||
setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
|
||||
|
||||
int flags = fcntl(sock, F_GETFL, 0);
|
||||
fcntl(sock, F_SETFL, flags | O_NONBLOCK);
|
||||
connect(sock, res->ai_addr, res->ai_addrlen);
|
||||
freeaddrinfo(res);
|
||||
|
||||
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
|
||||
int pr;
|
||||
do { pr = poll(&pfd, 1, timeout_ms); } while (pr < 0 && errno == EINTR);
|
||||
|
||||
if (pr == 0) { close(sock); return -ETIMEDOUT; }
|
||||
if (pr < 0 || (pfd.revents & (POLLERR | POLLHUP))) {
|
||||
close(sock); return -ECONNREFUSED;
|
||||
}
|
||||
int err = 0; socklen_t el = sizeof(err);
|
||||
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
|
||||
if (err) { close(sock); return -err; }
|
||||
|
||||
fcntl(sock, F_SETFL, flags & ~O_NONBLOCK);
|
||||
return sock;
|
||||
}
|
||||
|
||||
static int find_candidate_by_name(const xce_cluster_config_t *peers,
|
||||
const char *name)
|
||||
{
|
||||
for (int i = 0; i < peers->candidates_count; i++) {
|
||||
if (strcmp(peers->candidates[i].name, name) == 0)
|
||||
return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
static void register_fd_with_iomgr(cluster_t *c, int fd)
|
||||
{
|
||||
if (!c) return;
|
||||
struct epoll_event ev;
|
||||
memset(&ev, 0, sizeof(ev));
|
||||
ev.events = EPOLLIN | EPOLLRDHUP;
|
||||
ev.data.fd = fd;
|
||||
if (epoll_ctl(c->iomgr.epoll_fd, EPOLL_CTL_ADD, fd, &ev) != 0)
|
||||
WARN("ha_find_and_connect: epoll_ctl: %s", strerror(errno));
|
||||
}
|
||||
|
||||
static void ms_sleep_ha(long delay_ms)
|
||||
{
|
||||
if (delay_ms <= 0) return;
|
||||
struct timespec ts = { .tv_sec = delay_ms / 1000,
|
||||
.tv_nsec = (delay_ms % 1000) * 1000000L };
|
||||
nanosleep(&ts, NULL);
|
||||
}
|
||||
|
||||
int ha_find_and_connect_to_primary(ha_state_t *ha,
|
||||
const ha_config_t *cfg,
|
||||
const xce_cluster_config_t *peers,
|
||||
cluster_t *c)
|
||||
{
|
||||
if (!ha || !cfg || !peers || peers->candidates_count <= 0)
|
||||
return -EINVAL;
|
||||
|
||||
int timeout_ms = (int)cfg->probe_connect_timeout_ms;
|
||||
if (timeout_ms <= 0)
|
||||
timeout_ms = HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS;
|
||||
|
||||
/* Tier 1: voted_for_name → direct connect */
|
||||
if (ha->voted_for_name[0] != '\0') {
|
||||
int idx = find_candidate_by_name(peers, ha->voted_for_name);
|
||||
if (idx >= 0) {
|
||||
const xce_candidate_t *p1 = &peers->candidates[idx];
|
||||
for (int r = 0; r < HA_FIND_PRIMARY_T1_MAX_RETRIES; r++) {
|
||||
int fd = ha_tcp_connect_raw(p1->hostname, p1->port, timeout_ms);
|
||||
if (fd >= 0) {
|
||||
INFO("HA: Tier 1 connected to voted_for '%s' (retry=%d)",
|
||||
ha->voted_for_name, r);
|
||||
ha->primary_ctrl_fd = fd;
|
||||
register_fd_with_iomgr(c, fd);
|
||||
if (c) ha_arm_heartbeat_timeout_vtask(ha, c);
|
||||
return fd;
|
||||
}
|
||||
if (fd == -ETIMEDOUT || fd == -EAGAIN) {
|
||||
WARN("HA: Tier 1 '%s' not ready (retry %d/%d), backoff %dms",
|
||||
ha->voted_for_name, r + 1,
|
||||
HA_FIND_PRIMARY_T1_MAX_RETRIES, HA_FIND_PRIMARY_T1_BACKOFF_MS);
|
||||
ms_sleep_ha(HA_FIND_PRIMARY_T1_BACKOFF_MS);
|
||||
continue;
|
||||
}
|
||||
break; /* other errors → Tier 2 */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Tier 2: probe remaining candidates */
|
||||
{
|
||||
int t1_idx = find_candidate_by_name(peers, ha->voted_for_name);
|
||||
for (int i = 0; i < peers->candidates_count; i++) {
|
||||
if (i == t1_idx) continue;
|
||||
const xce_candidate_t *p2 = &peers->candidates[i];
|
||||
int fd = ha_tcp_connect_raw(p2->hostname, p2->port, timeout_ms);
|
||||
if (fd >= 0) {
|
||||
INFO("HA: Tier 2 connected to '%s'", p2->name);
|
||||
ha->primary_ctrl_fd = fd;
|
||||
register_fd_with_iomgr(c, fd);
|
||||
if (c) ha_arm_heartbeat_timeout_vtask(ha, c);
|
||||
return fd;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Tier 3: exponential backoff scan (no exit) */
|
||||
INFO("HA: entering Tier 3 primary discovery");
|
||||
long backoff_ms = HA_FIND_PRIMARY_T3_INIT_MS;
|
||||
for (;;) {
|
||||
for (int i = 0; i < peers->candidates_count; i++) {
|
||||
const xce_candidate_t *p3 = &peers->candidates[i];
|
||||
int fd = ha_tcp_connect_raw(p3->hostname, p3->port, timeout_ms);
|
||||
if (fd >= 0) {
|
||||
INFO("HA: Tier 3 connected to '%s' after backoff", p3->name);
|
||||
ha->primary_ctrl_fd = fd;
|
||||
register_fd_with_iomgr(c, fd);
|
||||
if (c) ha_arm_heartbeat_timeout_vtask(ha, c);
|
||||
return fd;
|
||||
}
|
||||
}
|
||||
WARN("HA: Tier 3 — no primary, sleep %ldms", backoff_ms);
|
||||
ms_sleep_ha(backoff_ms);
|
||||
backoff_ms = backoff_ms * 2 > HA_FIND_PRIMARY_T3_MAX_MS
|
||||
? HA_FIND_PRIMARY_T3_MAX_MS : backoff_ms * 2;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ───────────────────
|
||||
*
|
||||
* Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback),
|
||||
|
|
@ -2800,3 +2798,259 @@ void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)
|
|||
free(s_hb_vtask);
|
||||
s_hb_vtask = NULL;
|
||||
}
|
||||
|
||||
/* ── Primary heartbeat sender + late-joiner outbound connect ─────────────── */
|
||||
|
||||
/*
|
||||
* standby_ep_ctx_t: per-peer context for outbound standby connections opened
|
||||
* by the primary during the heartbeat task.
|
||||
*
|
||||
* Layout deliberately matches vote_ep_ctx_t for the first 4 fields so that
|
||||
* peer_ctrl_msg_handler (which casts ep->data to vote_ep_ctx_t*) can safely
|
||||
* read ha, c, and peer_idx. ctx is always NULL here (no active election).
|
||||
*/
|
||||
typedef struct {
|
||||
ha_state_t *ha;
|
||||
cluster_t *c;
|
||||
ha_elect_ctx_t *ctx; /* always NULL — not a vote endpoint */
|
||||
int peer_idx; /* index into c->myconf->peers_cfg.candidates[] */
|
||||
} standby_ep_ctx_t;
|
||||
|
||||
/*
|
||||
* on_standby_connected: iomgr on_connected callback for outbound standby
|
||||
* endpoints created by the heartbeat task.
|
||||
*
|
||||
* Sends OPCode_hello (CtrlHelloRequest) so the receiving standby runs
|
||||
* handle_ctrlhello and triggers WAL catch-up + config push. Registers the
|
||||
* endpoint in ha->standby_endpoints so future heartbeats reach it.
|
||||
*/
|
||||
static void on_standby_connected(endpoint_t *ep)
|
||||
{
|
||||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
|
||||
ha_state_t *ha = sctx->ha;
|
||||
cluster_t *c = sctx->c;
|
||||
|
||||
/* Register in standby_endpoints under lock (idempotency guard) */
|
||||
if (ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
int already = 0;
|
||||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||||
if (zarray_get(ha->standby_endpoints, i) == ep) { already = 1; break; }
|
||||
}
|
||||
if (!already)
|
||||
zarray_store(ha->standby_endpoints, ep);
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
}
|
||||
|
||||
/* Send ctrlhello — triggers handle_ctrlhello on the standby side */
|
||||
uint64_t lsn = atomic_load_explicit(&c->persist.committed_lsn,
|
||||
memory_order_acquire);
|
||||
uint64_t tag = c->myconf ? c->myconf->config_version_tag : 0;
|
||||
char *name = c->myconf ? c->myconf->name : "";
|
||||
|
||||
ctrl_hello_req_t hello = { .name = name, .key = NULL,
|
||||
.applied_lsn = lsn, .config_tag = tag };
|
||||
xce_request_t req;
|
||||
memset(&req, 0, sizeof(req));
|
||||
req.opcode = OPCode_hello;
|
||||
req.body_which = Request_body_ctrlhellomsg;
|
||||
req.body.ctrlhellomsg = &hello;
|
||||
|
||||
int rc = xce_request(ep, &req);
|
||||
if (rc < 0)
|
||||
WARN("HA: ctrlhello to peer[%d] failed: %d", sctx->peer_idx, rc);
|
||||
else
|
||||
INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent",
|
||||
sctx->peer_idx);
|
||||
|
||||
/* Set state-aware dispatcher for subsequent messages on this endpoint */
|
||||
ep->msg_handler = peer_ctrl_msg_handler;
|
||||
}
|
||||
|
||||
/*
|
||||
* standby_ep_err_handler_primary: error handler for outbound standby
|
||||
* endpoints created by the heartbeat task. Removes from standby_endpoints
|
||||
* and frees the context — the next heartbeat tick will attempt reconnect.
|
||||
*/
|
||||
static int standby_ep_err_handler_primary(void *ep_raw)
|
||||
{
|
||||
endpoint_t *ep = (endpoint_t *)ep_raw;
|
||||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
|
||||
if (!sctx) return 0;
|
||||
|
||||
ha_state_t *ha = sctx->ha;
|
||||
if (ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||||
if (zarray_get(ha->standby_endpoints, i) == ep) {
|
||||
zarray_del(ha->standby_endpoints, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
}
|
||||
free(sctx);
|
||||
ep->data = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* ha_connect_to_peer: open a new outbound TLS connection to a peer candidate
|
||||
* and register it as a standby channel. Called from primary_heartbeat_fn
|
||||
* for peers that are absent from standby_endpoints.
|
||||
*/
|
||||
static void ha_connect_to_peer(cluster_t *c, int peer_idx)
|
||||
{
|
||||
ha_state_t *ha = &c->ha;
|
||||
const xce_candidate_t *peer =
|
||||
&c->myconf->peers_cfg.candidates[peer_idx];
|
||||
|
||||
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
|
||||
if (!sctx) return;
|
||||
sctx->ha = ha;
|
||||
sctx->c = c;
|
||||
sctx->peer_idx = peer_idx;
|
||||
|
||||
endpoint_t *ep = endpoint_get();
|
||||
if (!ep) { free(sctx); return; }
|
||||
|
||||
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
|
||||
endpoint_set_peer(ep, c->myhost);
|
||||
ep->data = sctx;
|
||||
ep->iomgr = &c->iomgr;
|
||||
|
||||
if (ha->client_ssl_ctx) {
|
||||
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
|
||||
ep->ssl = SSL_new(client_ctx);
|
||||
if (ep->ssl)
|
||||
SSL_set_connect_state(ep->ssl);
|
||||
}
|
||||
|
||||
endpoint_set_on_connected(ep, on_standby_connected);
|
||||
endpoint_set_err_handler(ep, standby_ep_err_handler_primary);
|
||||
|
||||
int rc = endpoint_connect(ep, peer->hostname, peer->port);
|
||||
if (rc < 0) {
|
||||
WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
|
||||
peer_idx, peer->hostname, peer->port, rc);
|
||||
endpoint_fini(ep);
|
||||
free(sctx);
|
||||
return;
|
||||
}
|
||||
rc = iomgr_monitor(ep);
|
||||
if (rc < 0) {
|
||||
endpoint_fini(ep);
|
||||
free(sctx);
|
||||
return;
|
||||
}
|
||||
INFO("HA: connecting outbound to peer[%d] (%s:%u)",
|
||||
peer_idx, peer->hostname, peer->port);
|
||||
}
|
||||
|
||||
/*
|
||||
* primary_heartbeat_fn: self-perpetuating one-shot delaymgr task.
|
||||
*
|
||||
* On each tick:
|
||||
* 1. Sends OPCode_nop to all connected standbys (keeps watchdog alive).
|
||||
* 2. Scans the peers list; for any peer NOT in standby_endpoints, opens a
|
||||
* fresh outbound TLS connection so late joiners are bootstrapped without
|
||||
* the standby ever initiating an outbound connect.
|
||||
*
|
||||
* Stops naturally on demotion: the role check prevents re-arming.
|
||||
*/
|
||||
static int primary_heartbeat_fn(void *arg)
|
||||
{
|
||||
cluster_t *c = (cluster_t *)arg;
|
||||
ha_state_t *ha = &c->ha;
|
||||
|
||||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||||
memory_order_acquire);
|
||||
if (role != CTRL_PRIMARY && role != CTRL_PRIMARY_COMMITTING)
|
||||
return 0; /* demoted — stop the heartbeat loop */
|
||||
|
||||
/* 1. Heartbeat all connected standbys */
|
||||
if (ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
int n = ha->standby_endpoints->count;
|
||||
for (int i = 0; i < n; i++) {
|
||||
endpoint_t *ep = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
|
||||
if (ep) {
|
||||
int rc = xce_nop_req(ep);
|
||||
if (rc < 0)
|
||||
WARN("HA: heartbeat nop to standby[%d] failed: %d", i, rc);
|
||||
}
|
||||
}
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
}
|
||||
|
||||
/* 2. Reconnect to any peer missing from standby_endpoints */
|
||||
if (c->myconf) {
|
||||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||||
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||||
if (i == ha->my_priority) continue; /* skip self */
|
||||
|
||||
/* Check if this peer already has a standby endpoint.
|
||||
*
|
||||
* Two endpoint types can be in standby_endpoints:
|
||||
* - Promoted vote eps: ep->data = vote_ep_ctx_t, peer_idx is
|
||||
* votes_idx (relative, self excluded). Convert absolute i to
|
||||
* votes_idx using the same formula as raft_cb_send_vote_request.
|
||||
* - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL),
|
||||
* peer_idx is absolute candidate index. Compare directly with i.
|
||||
*/
|
||||
int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1;
|
||||
int found = 0;
|
||||
if (ha->standby_endpoints) {
|
||||
pthread_mutex_lock(&ha->lock);
|
||||
for (int j = 0; j < ha->standby_endpoints->count; j++) {
|
||||
endpoint_t *sep = (endpoint_t *)zarray_get(
|
||||
ha->standby_endpoints, j);
|
||||
if (!sep || !sep->data) continue;
|
||||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)sep->data;
|
||||
int match_idx = (sctx->ctx != NULL)
|
||||
? votes_idx_for_i /* promoted vote ep: relative */
|
||||
: i; /* standby ep: absolute */
|
||||
if (sctx->peer_idx == match_idx) { found = 1; break; }
|
||||
}
|
||||
pthread_mutex_unlock(&ha->lock);
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
INFO("HA: peer[%d] (%s) not connected — initiating outbound connect",
|
||||
i, pcfg->candidates[i].name ? pcfg->candidates[i].name : "?");
|
||||
ha_connect_to_peer(c, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Re-arm for next interval — only reached when still PRIMARY */
|
||||
int interval_ms = (int)ha->config.heartbeat_interval_ms;
|
||||
if (interval_ms <= 0)
|
||||
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
|
||||
delay_task_ms(&c->delaymgr, interval_ms, 1,
|
||||
(delay_task_entry_fn_t)primary_heartbeat_fn, c);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ha_start_primary_heartbeat(cluster_t *c)
|
||||
{
|
||||
if (!c) return;
|
||||
ha_state_t *ha = &c->ha;
|
||||
int interval_ms = (int)ha->config.heartbeat_interval_ms;
|
||||
if (interval_ms <= 0)
|
||||
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
|
||||
|
||||
/* Guard: skip if delaymgr is not running (e.g. unit tests with zeroed cluster_t).
|
||||
* In production the delaymgr is always started before any promotion. */
|
||||
if (!c->delaymgr.running)
|
||||
return;
|
||||
|
||||
/* One-shot: the task re-arms itself each time while role == PRIMARY */
|
||||
int rc = delay_task_ms(&c->delaymgr, interval_ms, 1,
|
||||
(delay_task_entry_fn_t)primary_heartbeat_fn, c);
|
||||
if (rc < 0)
|
||||
WARN("ha_start_primary_heartbeat: delay_task_ms failed: %d", rc);
|
||||
else
|
||||
INFO("HA: primary heartbeat armed (interval=%dms)", interval_ms);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue