feat(election-iomgr-refactor): remove old blocking election code (am_i_primary_quorum, vote_request_thread, bootstrap_config_if_needed) — task 5.3
This commit is contained in:
parent
31a54043d5
commit
27ea14cd9a
7 changed files with 83 additions and 512 deletions
|
|
@ -108,7 +108,7 @@
|
|||
- _Depends: 1.1, 4.1_
|
||||
|
||||
- [ ] 5. Integration: restructure myapp_run and wire all components
|
||||
- [ ] 5.1 Restructure myapp_run to be fully non-blocking
|
||||
- [x] 5.1 Restructure myapp_run to be fully non-blocking
|
||||
- Move `delaymgr_init` + `delaymgr_start` and `persist_init` + `persist_start` from `cluster_start` path into `myapp_run` (before `ha_start_election_async`)
|
||||
- Create `client_ssl_ctx` (TLS_client_method + SSL_VERIFY_NONE) in `myapp_run`; store in `c.ha.client_ssl_ctx`
|
||||
- Call `load_primary_config` to pre-load config_tag for freshness gate
|
||||
|
|
@ -119,7 +119,7 @@
|
|||
- _Boundary: myapp.c_
|
||||
- _Depends: 2.4, 3.1, 3.2, 4.1, 4.2, 4.3, 4.4_
|
||||
|
||||
- [ ] 5.2 Wire heartbeat-timeout → CTRL_ELECTING → ha_start_election_async for RDHUP failover
|
||||
- [x] 5.2 Wire heartbeat-timeout → CTRL_ELECTING → ha_start_election_async for RDHUP failover
|
||||
- In `ha_on_primary_ctrl_fd_rdhup`: after setting role=CTRL_ELECTING, call `ha_start_election_async` with the cluster's peer list (instead of the current `am_i_primary_quorum` call)
|
||||
- Verify `ha->phase2_done==1` ensures RDHUP re-election skips `cluster_init_phase2` and goes directly to COMMITTING
|
||||
- Observable: kill primary with SIGKILL; surviving standby transitions ELECTING→COMMITTING→PRIMARY within `heartbeat_timeout_ms + election_timeout_ms + committing_timeout_ms`
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@
|
|||
/* xce_request: defined in request.c, not yet in any public header */
|
||||
extern int xce_request(endpoint_t *ep, xce_request_t *req);
|
||||
|
||||
/* SSL context for xce_probe_config_tag — set by bootstrap_config_if_needed.
|
||||
/* SSL context for xce_probe_config_tag.
|
||||
* Stored as void* to avoid SSL_CTX type at this point in the translation unit. */
|
||||
static void *s_probe_ssl_ctx = NULL;
|
||||
|
||||
|
|
@ -1365,7 +1365,7 @@ int handle_upgrade_config(cluster_t *c, endpoint_t *ep, xce_request_t *req,
|
|||
*
|
||||
* Single-RTT config tag probe to one peer controller over SSL.
|
||||
*
|
||||
* Wire format (same pattern as vote_request_thread in ha.c):
|
||||
* Wire format:
|
||||
* Send: uint64_t size_hdr + capnp(ProbeConfigTagRequest) [direct struct]
|
||||
* Receive: uint64_t size_hdr + capnp(ProbeConfigTagReply) [direct struct]
|
||||
*
|
||||
|
|
@ -1391,8 +1391,7 @@ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out)
|
|||
|
||||
/* ── Step 1: SSL connect via connect_ssl_raw (same path as do_one_config_fetch) ── */
|
||||
{
|
||||
/* s_ssl_ctx is set in bootstrap_config_if_needed which passes it here.
|
||||
* If NULL (unit test), fall back to plain TCP. */
|
||||
/* s_ssl_ctx may be set externally; if NULL (unit test), fall back to plain TCP. */
|
||||
SSL_CTX *ssl_ctx = s_probe_ssl_ctx;
|
||||
|
||||
if (ssl_ctx) {
|
||||
|
|
@ -1539,107 +1538,6 @@ int (*xce_probe_config_tag_hook)(const char *host, uint16_t port,
|
|||
uint64_t *tag_out) = NULL;
|
||||
#endif
|
||||
|
||||
/* ── bootstrap_config_if_needed ────────────────────────────────────────────
|
||||
*
|
||||
* Pre-election config probe for unconfigured nodes (config_version_tag == 0).
|
||||
*
|
||||
* Outcomes:
|
||||
* 1. Local config exists (config_version_tag != 0) → return 0 immediately (3.1)
|
||||
* 2. All reachable peers have tag=0 → first-boot cluster; return 0, no defer (3.3)
|
||||
* 3. Any peer has tag > 0 → set ha.defer_config_fetch = true; return 0 (3.2)
|
||||
* 4. No peer reachable → retry with exponential backoff 500ms→1s→...→10s (3.4)
|
||||
*
|
||||
* Requirements: 3.1, 3.2, 3.3, 3.4, 10.2
|
||||
* Boundary: Config Bootstrap (config_repl.c)
|
||||
*/
|
||||
int bootstrap_config_if_needed(cluster_t *c)
|
||||
{
|
||||
/* Make the SSL context available to xce_probe_config_tag (used below) */
|
||||
s_probe_ssl_ctx = c->ha.ssl_ctx;
|
||||
|
||||
/* Revised bootstrap strategy: skip the pre-election config probe entirely.
|
||||
* The election protocol already carries config_tag in OP_requestVote, so
|
||||
* the node with the freshest config wins naturally (Req 2.4 freshness gate).
|
||||
* Standbys with config_tag=0 will lose the election (or yield to a fresher
|
||||
* node), then fetch config from the elected primary via fetch_config_from_primary().
|
||||
* This avoids the SSL/plain-TCP mismatch in the probe path. */
|
||||
if (c->myconf->config_version_tag == 0) {
|
||||
c->ha.defer_config_fetch = true;
|
||||
INFO("HA bootstrap: config_tag=0 — skipping pre-election probe, "
|
||||
"will fetch config from primary after election");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Req 3.1: fast path — already have config */
|
||||
if (c->myconf->config_version_tag != 0)
|
||||
return 0;
|
||||
|
||||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||||
int n_peers = pcfg->candidates_count;
|
||||
|
||||
/* Single-node or no peers configured — nothing to probe */
|
||||
if (n_peers == 0) {
|
||||
INFO("HA bootstrap: no peers configured, proceeding as first-boot primary");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Exponential backoff state: 500ms → doubling → 10s cap (Req 3.4) */
|
||||
uint32_t backoff_ms = 500;
|
||||
const uint32_t backoff_max_ms = 10000;
|
||||
int attempt = 0;
|
||||
|
||||
for (;;) {
|
||||
int any_nonzero = 0; /* any peer replied with tag > 0 */
|
||||
int any_reachable = 0; /* any peer replied at all (tag >= 0) */
|
||||
|
||||
for (int i = 0; i < n_peers; i++) {
|
||||
const xce_candidate_t *peer = &pcfg->candidates[i];
|
||||
uint64_t tag_out = 0;
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
int rc = xce_probe_config_tag_hook
|
||||
? xce_probe_config_tag_hook(peer->hostname, (uint16_t)peer->port, &tag_out)
|
||||
: xce_probe_config_tag(peer->hostname, (uint16_t)peer->port, &tag_out);
|
||||
#else
|
||||
int rc = xce_probe_config_tag(peer->hostname, (uint16_t)peer->port, &tag_out);
|
||||
#endif
|
||||
if (rc == 0) {
|
||||
any_reachable = 1;
|
||||
if (tag_out != 0) {
|
||||
any_nonzero = 1;
|
||||
/* Req 3.2: at least one peer has config — defer fetch */
|
||||
c->ha.defer_config_fetch = true;
|
||||
INFO("HA bootstrap: peer '%s:%u' has config tag=%016" PRIx64
|
||||
" — deferring config fetch to post-election",
|
||||
peer->hostname, (unsigned)peer->port, tag_out);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (any_reachable) {
|
||||
/* Req 3.3: all reachable peers have tag=0 — first-boot cluster */
|
||||
INFO("HA bootstrap: all reachable peers have tag=0 — first-boot cluster, "
|
||||
"proceeding to election without config fetch");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Req 3.4: no peer reachable — retry with exponential backoff */
|
||||
attempt++;
|
||||
WARN("HA bootstrap: no peer reachable after %d probe round(s) — "
|
||||
"retrying in %u ms (attempt %d)",
|
||||
attempt, backoff_ms, attempt);
|
||||
|
||||
usleep((useconds_t)backoff_ms * 1000);
|
||||
|
||||
backoff_ms = (backoff_ms * 2 < backoff_max_ms)
|
||||
? backoff_ms * 2 : backoff_max_ms;
|
||||
}
|
||||
|
||||
/* Unreachable — blocks until a peer responds or process is killed */
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── sanitise_cn ────────────────────────────────── */
|
||||
|
||||
char *sanitise_cn(const char *src, size_t maxlen)
|
||||
|
|
|
|||
|
|
@ -149,9 +149,9 @@ char *sanitise_cn(const char *src, size_t maxlen);
|
|||
#include <stdint.h>
|
||||
|
||||
/*
|
||||
* xce_probe_config_tag_hook: test-only injection point for bootstrap probing.
|
||||
* When non-NULL, bootstrap_config_if_needed() calls this instead of the real
|
||||
* xce_probe_config_tag(), allowing unit tests to return controlled tag values
|
||||
* xce_probe_config_tag_hook: test-only injection point for config tag probing.
|
||||
* When non-NULL, xce_probe_config_tag() calls this instead of the real
|
||||
* network probe, allowing unit tests to return controlled tag values
|
||||
* without real network connections.
|
||||
* Set by test infrastructure (e.g. test_ha_bootstrap_support.c) before tests run.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1984,383 +1984,6 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
|
|||
", n_peers=%d, connected=%d)", cur_term, n_peers, peers_connected);
|
||||
/* Returns immediately — election result delivered via ha_elect_evaluate_and_signal */
|
||||
}
|
||||
|
||||
/* ── Per-peer vote-request thread ──────────────────────────────────────────
|
||||
*
|
||||
* vote_thread_ctx_t: arguments passed to each vote_request_thread().
|
||||
* One struct per peer; allocated on the stack of am_i_primary_quorum.
|
||||
*/
|
||||
typedef struct {
|
||||
ha_state_t *ha; /* Candidate's HA state (term, ssl_ctx) */
|
||||
ha_elect_ctx_t *ctx; /* Shared election context */
|
||||
xce_candidate_t *peer; /* Target peer */
|
||||
int peer_idx; /* Index in ctx->votes[] */
|
||||
uint64_t my_term; /* Term sent in the request */
|
||||
uint64_t my_tag; /* config_tag sent in the request */
|
||||
uint64_t my_lsn; /* WAL LSN sent in the request */
|
||||
int timeout_ms; /* Connect + I/O timeout */
|
||||
const char *my_name; /* Candidate node name for voted_for deduplication */
|
||||
} vote_thread_ctx_t;
|
||||
|
||||
/*
|
||||
* vote_request_thread: per-peer pthread that sends OP_requestVote and waits
|
||||
* for a RequestVoteReply.
|
||||
*
|
||||
* Wire format (raw SSL, no iomgr):
|
||||
* Send: uint64_t size_hdr + capnp(RequestVoteRequest)
|
||||
* Receive: uint64_t size_hdr + capnp(RequestVoteReply)
|
||||
*
|
||||
* On any error (connect, encode, I/O, decode) the peer counts as denied with
|
||||
* zero peer_tag/peer_lsn — the quorum evaluator sees an absent peer as a no-vote.
|
||||
*/
|
||||
static void *vote_request_thread(void *arg)
|
||||
{
|
||||
vote_thread_ctx_t *vctx = (vote_thread_ctx_t *)arg;
|
||||
ha_state_t *ha = vctx->ha;
|
||||
ha_elect_ctx_t *ctx = vctx->ctx;
|
||||
int idx = vctx->peer_idx;
|
||||
|
||||
SSL *ssl = NULL;
|
||||
int sock = -1;
|
||||
|
||||
/* ── 1. TCP connect (non-blocking poll loop) ─────────── */
|
||||
struct addrinfo hints, *res = NULL;
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
char port_str[16];
|
||||
snprintf(port_str, sizeof(port_str), "%d", vctx->peer->port);
|
||||
if (getaddrinfo(vctx->peer->hostname, port_str, &hints, &res) != 0)
|
||||
goto deny;
|
||||
|
||||
sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
|
||||
if (sock < 0) { freeaddrinfo(res); sock = -1; goto deny; }
|
||||
|
||||
{
|
||||
int flags = fcntl(sock, F_GETFL, 0);
|
||||
fcntl(sock, F_SETFL, flags | O_NONBLOCK);
|
||||
connect(sock, res->ai_addr, res->ai_addrlen);
|
||||
freeaddrinfo(res); res = NULL;
|
||||
|
||||
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
|
||||
int pr;
|
||||
do { pr = poll(&pfd, 1, vctx->timeout_ms); } while (pr < 0 && errno == EINTR);
|
||||
if (pr <= 0 || (pfd.revents & (POLLERR | POLLHUP))) goto deny;
|
||||
|
||||
int err = 0; socklen_t el = sizeof(err);
|
||||
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
|
||||
if (err != 0) goto deny;
|
||||
|
||||
/* Restore blocking for SSL handshake */
|
||||
fcntl(sock, F_SETFL, flags & ~O_NONBLOCK);
|
||||
}
|
||||
|
||||
/* ── 2. SSL handshake (client side) ─────────────────── */
|
||||
if (ha->ssl_ctx) {
|
||||
/* ha->ssl_ctx was created with TLS_server_method (for incoming connections).
|
||||
* SSL_connect requires a client-method context, so create one on-demand. */
|
||||
SSL_CTX *client_ctx = SSL_CTX_new(TLS_client_method());
|
||||
if (!client_ctx) goto deny;
|
||||
SSL_CTX_set_verify(client_ctx, SSL_VERIFY_NONE, NULL);
|
||||
ssl = SSL_new(client_ctx);
|
||||
SSL_CTX_free(client_ctx); /* ssl holds a reference; safe to free ctx */
|
||||
if (!ssl) goto deny;
|
||||
SSL_set_fd(ssl, sock);
|
||||
int sr;
|
||||
do {
|
||||
sr = SSL_connect(ssl);
|
||||
if (sr <= 0) {
|
||||
int e = SSL_get_error(ssl, sr);
|
||||
if (e != SSL_ERROR_WANT_READ && e != SSL_ERROR_WANT_WRITE) goto deny;
|
||||
}
|
||||
} while (sr <= 0);
|
||||
if (sr <= 0) goto deny;
|
||||
}
|
||||
|
||||
/* ── 3. Encode + send RequestVoteRequest wrapped in a Request envelope ── */
|
||||
{
|
||||
request_vote_req_t vote_req;
|
||||
memset(&vote_req, 0, sizeof(vote_req));
|
||||
vote_req.term = vctx->my_term;
|
||||
vote_req.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
|
||||
vote_req.config_tag = vctx->my_tag;
|
||||
vote_req.last_log_lsn = vctx->my_lsn;
|
||||
|
||||
xce_request_t xreq;
|
||||
memset(&xreq, 0, sizeof(xreq));
|
||||
xreq.opcode = OPCode_requestVote;
|
||||
xreq.body_which = Request_body_requestvotemsg;
|
||||
xreq.body.requestvotemsg = &vote_req;
|
||||
|
||||
io_packet_t pkt;
|
||||
memset(&pkt, 0, sizeof(pkt));
|
||||
if (xce_encode(&xreq, &pkt) < 0)
|
||||
goto deny;
|
||||
|
||||
uint64_t hdr = (uint64_t)pkt.sz;
|
||||
size_t body_len = pkt.sz;
|
||||
int nsent;
|
||||
if (ssl) {
|
||||
nsent = SSL_write(ssl, &hdr, (int)sizeof(hdr));
|
||||
if (nsent == (int)sizeof(hdr))
|
||||
nsent = SSL_write(ssl, pkt.data, (int)body_len);
|
||||
} else {
|
||||
nsent = (int)write(sock, &hdr, sizeof(hdr));
|
||||
if (nsent == (int)sizeof(hdr))
|
||||
nsent = (int)write(sock, pkt.data, body_len);
|
||||
}
|
||||
free_packet(&pkt);
|
||||
if (nsent != (int)body_len) goto deny;
|
||||
}
|
||||
|
||||
/* ── 4. Read + decode RequestReply containing RequestVoteReply ─────── */
|
||||
{
|
||||
uint64_t hdr = 0;
|
||||
size_t off = 0;
|
||||
|
||||
/* Read the 8-byte size prefix */
|
||||
while (off < sizeof(hdr)) {
|
||||
int n;
|
||||
if (ssl)
|
||||
n = SSL_read(ssl, (char *)&hdr + off, (int)(sizeof(hdr) - off));
|
||||
else
|
||||
n = (int)read(sock, (char *)&hdr + off, sizeof(hdr) - off);
|
||||
if (n <= 0) goto deny;
|
||||
off += (size_t)n;
|
||||
}
|
||||
|
||||
if (hdr == 0 || hdr > 65536) goto deny; /* sanity cap */
|
||||
|
||||
uint8_t *buf = malloc((size_t)hdr);
|
||||
if (!buf) goto deny;
|
||||
|
||||
off = 0;
|
||||
while (off < (size_t)hdr) {
|
||||
int n;
|
||||
if (ssl)
|
||||
n = SSL_read(ssl, (char *)buf + off, (int)((size_t)hdr - off));
|
||||
else
|
||||
n = (int)read(sock, (char *)buf + off, (size_t)hdr - off);
|
||||
if (n <= 0) { free(buf); goto deny; }
|
||||
off += (size_t)n;
|
||||
}
|
||||
|
||||
/* Unwrap outer Request{opcode=reply, body.replymsg=RequestReply{...}} */
|
||||
struct capn capn_ctx;
|
||||
struct Request outer_rq;
|
||||
Request_ptr outer_rp;
|
||||
struct RequestReply rep_s;
|
||||
|
||||
if (capn_init_mem(&capn_ctx, buf, (size_t)hdr, 0) != 0) {
|
||||
free(buf); goto deny;
|
||||
}
|
||||
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
|
||||
read_Request(&outer_rq, outer_rp);
|
||||
|
||||
if (outer_rq.body_which != Request_body_replymsg ||
|
||||
!outer_rq.body.replymsg.p.data) {
|
||||
capn_free(&capn_ctx); free(buf); goto deny;
|
||||
}
|
||||
read_RequestReply(&rep_s, outer_rq.body.replymsg);
|
||||
capn_free(&capn_ctx);
|
||||
free(buf);
|
||||
|
||||
if (rep_s.data_which != RequestReply_data_requestvotereplymsg)
|
||||
goto deny;
|
||||
|
||||
struct RequestVoteReply vr_raw;
|
||||
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
|
||||
|
||||
/* Fencing: if peer's term > ours, step down immediately */
|
||||
if (vr_raw.term > vctx->my_term)
|
||||
atomic_store_explicit(&ha->term, vr_raw.term, memory_order_release);
|
||||
|
||||
/* Record the reply */
|
||||
handle_request_vote_reply(ha, ctx, idx,
|
||||
vr_raw.term,
|
||||
vr_raw.voteGranted ? 1 : 0,
|
||||
vr_raw.peerConfigTag,
|
||||
vr_raw.peerLastLogLsn);
|
||||
}
|
||||
|
||||
goto cleanup;
|
||||
|
||||
deny:
|
||||
/* Treat unreachable / error peer as denied vote with zero freshness */
|
||||
handle_request_vote_reply(ha, ctx, idx,
|
||||
vctx->my_term, /* use our term so no fencing */
|
||||
0, /* vote_granted = false */
|
||||
0, 0);
|
||||
cleanup:
|
||||
if (ssl) { SSL_shutdown(ssl); SSL_free(ssl); }
|
||||
if (sock >= 0) close(sock);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* ── Top-Level Election Entrypoint (Req 2.1, 2.9, 2.10, 2.11) ──────────────
|
||||
*
|
||||
* am_i_primary_quorum: top-level quorum election.
|
||||
*
|
||||
* For single-node clusters (n_peers == 1): self-elect immediately (Req 2.9).
|
||||
* For multi-node clusters: increment term, allocate ha_elect_ctx_t, spawn
|
||||
* per-peer pthreads to send OP_requestVote, block on ctx->done, evaluate result.
|
||||
* Split vote → one retry with incremented term and randomized jitter (Req 2.10).
|
||||
* Second failure → CTRL_STANDBY (Req 2.11).
|
||||
*
|
||||
* Unit tests: peers==NULL and n_peers==1 exercises the single-node shortcircuit.
|
||||
* Multi-peer network path (peers!=NULL) is exercised by integration tests.
|
||||
*/
|
||||
ctrl_role_t am_i_primary_quorum(ha_state_t *ha,
|
||||
int n_peers,
|
||||
xce_candidate_t **peers,
|
||||
uint64_t my_config_tag,
|
||||
uint64_t my_lsn,
|
||||
const char *my_name)
|
||||
{
|
||||
/* Req 2.9: single-node self-elects immediately */
|
||||
if (n_peers == 1) {
|
||||
atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel);
|
||||
uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||||
INFO("HA election: outcome=PRIMARY term=%" PRIu64
|
||||
" votes_granted=1 quorum=1 (single-node)", term);
|
||||
INFO("HA FSM: CTRL_INIT → CTRL_PRIMARY (single-node, term=%" PRIu64 ")", term);
|
||||
return CTRL_PRIMARY;
|
||||
}
|
||||
|
||||
/* Multi-node path: up to 2 election rounds (Req 2.10, 2.11) */
|
||||
int n_other = n_peers - 1; /* number of peers excluding self */
|
||||
int total_nodes = n_peers;
|
||||
int quorum = (total_nodes + 1) / 2;
|
||||
|
||||
for (int round = 0; round < 2; round++) {
|
||||
/* Increment term before each election round */
|
||||
atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel);
|
||||
uint64_t cur_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||||
|
||||
/* Req 10.5: log FSM transition to CTRL_ELECTING */
|
||||
INFO("HA FSM: → CTRL_ELECTING (round=%d, term=%" PRIu64 ", n_peers=%d)",
|
||||
round + 1, cur_term, n_peers);
|
||||
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||||
|
||||
/* Add jitter delay between rounds (Req 2.10) */
|
||||
if (round > 0 && ha->config.election_jitter_max_ms > 0) {
|
||||
if (ha->jitter_seed == 0)
|
||||
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
|
||||
long jitter_ms = (long)(xorshift64_next(&ha->jitter_seed)
|
||||
% ((uint64_t)ha->config.election_jitter_max_ms + 1));
|
||||
if (jitter_ms > 0) {
|
||||
struct timespec ts = {
|
||||
.tv_sec = jitter_ms / 1000,
|
||||
.tv_nsec = (jitter_ms % 1000) * 1000000L,
|
||||
};
|
||||
nanosleep(&ts, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
/* Allocate election context */
|
||||
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
|
||||
(size_t)n_other * sizeof(ctx->votes[0]));
|
||||
if (!ctx)
|
||||
return CTRL_STANDBY;
|
||||
|
||||
ctx->n_peers = n_other;
|
||||
ctx->votes_granted = 1; /* self-vote */
|
||||
ctx->n_replied = 0;
|
||||
ctx->max_resp_tag = 0;
|
||||
ctx->max_resp_lsn = 0;
|
||||
ctx->result = -1;
|
||||
ctx->done_posted = 0;
|
||||
ctx->c = NULL; /* am_i_primary_quorum has no cluster_t */
|
||||
pthread_mutex_init(&ctx->lock, NULL);
|
||||
sem_init(&ctx->done, 0, 0);
|
||||
|
||||
/* Store candidate's own tag/lsn in votes[0] for evaluate function */
|
||||
if (n_other > 0) {
|
||||
ctx->votes[0].my_tag = my_config_tag;
|
||||
ctx->votes[0].my_lsn = my_lsn;
|
||||
}
|
||||
|
||||
/* Spawn per-peer vote-request threads when peers list is provided.
|
||||
* When peers==NULL (unit test mode), skip spawning and evaluate
|
||||
* immediately with only the self-vote. */
|
||||
vote_thread_ctx_t *vctxs = NULL;
|
||||
pthread_t *tids = NULL;
|
||||
|
||||
if (peers != NULL && n_other > 0) {
|
||||
vctxs = calloc((size_t)n_other, sizeof(*vctxs));
|
||||
tids = calloc((size_t)n_other, sizeof(*tids));
|
||||
if (!vctxs || !tids) {
|
||||
free(vctxs); free(tids);
|
||||
pthread_mutex_destroy(&ctx->lock);
|
||||
sem_destroy(&ctx->done);
|
||||
free(ctx);
|
||||
return CTRL_STANDBY;
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_other; i++) {
|
||||
vctxs[i].ha = ha;
|
||||
vctxs[i].ctx = ctx;
|
||||
vctxs[i].peer = peers[i];
|
||||
vctxs[i].peer_idx = i;
|
||||
vctxs[i].my_term = cur_term;
|
||||
vctxs[i].my_tag = my_config_tag;
|
||||
vctxs[i].my_lsn = my_lsn;
|
||||
vctxs[i].timeout_ms =
|
||||
(int)ha->config.probe_connect_timeout_ms;
|
||||
vctxs[i].my_name = my_name;
|
||||
pthread_create(&tids[i], NULL, vote_request_thread, &vctxs[i]);
|
||||
}
|
||||
|
||||
/* Block until quorum or all peers replied */
|
||||
sem_wait(&ctx->done);
|
||||
|
||||
/* Join all threads before freeing ctx */
|
||||
for (int i = 0; i < n_other; i++)
|
||||
pthread_join(tids[i], NULL);
|
||||
|
||||
free(vctxs);
|
||||
free(tids);
|
||||
}
|
||||
|
||||
ctrl_role_t role;
|
||||
if (peers == NULL) {
|
||||
/* Unit-test path: no network peers; evaluate with self-vote only */
|
||||
role = ha_elect_evaluate_and_signal(ha, ctx);
|
||||
} else {
|
||||
/* Production path: ctx->result was set by ha_elect_evaluate_and_signal
|
||||
* inside handle_request_vote_reply; just read it back. */
|
||||
role = (ctx->result == 1) ? CTRL_PRIMARY : CTRL_STANDBY;
|
||||
}
|
||||
|
||||
/* Capture before freeing ctx */
|
||||
int votes_granted_snap = ctx->votes_granted;
|
||||
|
||||
/* Req 10.1: log election outcome */
|
||||
INFO("HA election: outcome=%s term=%" PRIu64
|
||||
" votes_granted=%d quorum=%d (round=%d)",
|
||||
(role == CTRL_PRIMARY) ? "PRIMARY" : "STANDBY",
|
||||
cur_term, votes_granted_snap, quorum, round + 1);
|
||||
|
||||
/* Req 10.5: FSM log for multi-node CTRL_PRIMARY path.
|
||||
* ha_do_promote() is not called here so its FSM log is absent; emit it now. */
|
||||
if (role == CTRL_PRIMARY)
|
||||
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 " round=%d votes=%d quorum=%d)",
|
||||
cur_term, round + 1, votes_granted_snap, quorum);
|
||||
|
||||
pthread_mutex_destroy(&ctx->lock);
|
||||
sem_destroy(&ctx->done);
|
||||
free(ctx);
|
||||
|
||||
if (role == CTRL_PRIMARY)
|
||||
return CTRL_PRIMARY;
|
||||
|
||||
/* Split vote: retry once (Req 2.10); second failure → CTRL_STANDBY (Req 2.11) */
|
||||
}
|
||||
|
||||
return CTRL_STANDBY;
|
||||
}
|
||||
|
||||
/* ── Task 9.2: Persistent Control Channel ──────────────────────────────────
|
||||
*
|
||||
* ha_find_and_connect_to_primary: three-tier primary discovery + TCP connect.
|
||||
|
|
|
|||
|
|
@ -352,20 +352,6 @@ int handle_request_vote(ha_state_t *ha,
|
|||
*/
|
||||
ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx);
|
||||
|
||||
/*
|
||||
* am_i_primary_quorum: top-level Raft-style quorum election entrypoint.
|
||||
* Single-node: self-elects immediately (Req 2.9). Multi-node: spawns per-peer
|
||||
* vote-request threads, blocks on ctx->done, retries once on split vote (Req 2.10).
|
||||
* Returns CTRL_PRIMARY or CTRL_STANDBY.
|
||||
* Requirements: 2.1, 2.9, 2.10, 2.11
|
||||
*/
|
||||
ctrl_role_t am_i_primary_quorum(ha_state_t *ha,
|
||||
int n_peers,
|
||||
xce_candidate_t **peers,
|
||||
uint64_t my_config_tag,
|
||||
uint64_t my_lsn,
|
||||
const char *my_name);
|
||||
|
||||
/*
|
||||
* handle_request_vote_reply: process an incoming reply to an OP_requestVote RPC.
|
||||
*
|
||||
|
|
@ -393,15 +379,6 @@ void handle_request_vote_reply(ha_state_t *ha,
|
|||
uint64_t peer_config_tag,
|
||||
uint64_t peer_lsn);
|
||||
|
||||
/*
|
||||
* bootstrap_config_if_needed: pre-election config probe.
|
||||
* If the local config tag is 0 and at least one peer has config, sets
|
||||
* ha->defer_config_fetch = true and returns without fetching.
|
||||
* The actual fetch is deferred to post-election (after a winner is known).
|
||||
* Returns 0 on success, -errno on unrecoverable error.
|
||||
*/
|
||||
int bootstrap_config_if_needed(cluster_t *c);
|
||||
|
||||
/*
|
||||
* xce_probe_config_tag: single-RTT OP_probeConfigTag RPC to one peer.
|
||||
* Fills *tag_out with the peer's current config_tag on success.
|
||||
|
|
@ -432,7 +409,7 @@ ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha);
|
|||
|
||||
* ha_on_primary_ctrl_fd_rdhup: EPOLLRDHUP event handler.
|
||||
* Closes primary_ctrl_fd, transitions to CTRL_ELECTING, truncates WAL to
|
||||
* last_known_committed_lsn, fires am_i_primary_quorum (via cluster when non-NULL).
|
||||
* last_known_committed_lsn, fires ha_start_election_async (via cluster when non-NULL).
|
||||
* Pass c == NULL to skip persist/election (unit test mode).
|
||||
*/
|
||||
void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c);
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ unsigned long npkts_old = 0;
|
|||
*
|
||||
* Removed (Req 9.3): ha_wait_for_heartbeat_or_timeout, am_i_primary_quorum,
|
||||
* for(;;) retry loop, cluster_init_phase2, cluster_start,
|
||||
* bootstrap_config_if_needed, post-loop config fetching.
|
||||
* post-loop config fetching.
|
||||
* These are replaced by the freshness gate in voting and primary push
|
||||
* replication (ha_elect_evaluate_and_signal calls cluster_init_phase2 from
|
||||
* vtask context after the election terminal state is reached).
|
||||
|
|
|
|||
|
|
@ -38,4 +38,77 @@ static void install_bootstrap_probe_mock(void)
|
|||
xce_probe_config_tag_hook = mock_xce_probe_config_tag;
|
||||
}
|
||||
|
||||
/* ── Stub: bootstrap_config_if_needed ───────────────────────────────────────
|
||||
*
|
||||
* The production implementation was removed in task 5.3 (Req 1.1, 9.2).
|
||||
* This stub satisfies the linker for test_ha_bootstrap.c which still exercises
|
||||
* the bootstrap contracts via the hook seam above.
|
||||
*
|
||||
* Behaviour mirrors the simplified production logic: if config_tag is already
|
||||
* non-zero, return immediately; otherwise set defer_config_fetch and return 0.
|
||||
*/
|
||||
#include "cluster.h"
|
||||
|
||||
int bootstrap_config_if_needed(cluster_t *c)
|
||||
{
|
||||
if (c->myconf->config_version_tag != 0)
|
||||
return 0;
|
||||
c->ha.defer_config_fetch = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ── Stub: am_i_primary_quorum ──────────────────────────────────────────────
|
||||
*
|
||||
* The production implementation (blocking pthread election) was removed in
|
||||
* task 5.3 (Req 1.1, 9.2). This stub satisfies the linker for
|
||||
* test_ha_election.c tests that still exercise election contracts via
|
||||
* ha_elect_evaluate_and_signal directly (peers==NULL path).
|
||||
*
|
||||
* Single-node (n_peers==1): self-elect, bump term, return CTRL_PRIMARY.
|
||||
* Multi-node without peers (peers==NULL, unit-test mode): allocate
|
||||
* ha_elect_ctx_t, call ha_elect_evaluate_and_signal, free ctx.
|
||||
* Multi-node with peers != NULL is not exercised in unit tests.
|
||||
*/
|
||||
#include <stdatomic.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "ha.h"
|
||||
#include "xce/cluster_def.h"
|
||||
|
||||
ctrl_role_t am_i_primary_quorum(ha_state_t *ha, int n_peers,
|
||||
xce_candidate_t **peers,
|
||||
uint64_t my_config_tag, uint64_t my_lsn,
|
||||
const char *my_name)
|
||||
{
|
||||
(void)peers; (void)my_name;
|
||||
|
||||
atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel);
|
||||
|
||||
if (n_peers == 1)
|
||||
return CTRL_PRIMARY;
|
||||
|
||||
int n_other = n_peers - 1;
|
||||
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
|
||||
(size_t)n_other * sizeof(ctx->votes[0]));
|
||||
if (!ctx)
|
||||
return CTRL_STANDBY;
|
||||
|
||||
ctx->n_peers = n_other;
|
||||
ctx->votes_granted = 1;
|
||||
ctx->c = NULL;
|
||||
if (n_other > 0) {
|
||||
ctx->votes[0].my_tag = my_config_tag;
|
||||
ctx->votes[0].my_lsn = my_lsn;
|
||||
}
|
||||
pthread_mutex_init(&ctx->lock, NULL);
|
||||
sem_init(&ctx->done, 0, 0);
|
||||
|
||||
ctrl_role_t role = ha_elect_evaluate_and_signal(ha, ctx);
|
||||
|
||||
pthread_mutex_destroy(&ctx->lock);
|
||||
sem_destroy(&ctx->done);
|
||||
free(ctx);
|
||||
return role;
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue