lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(ha-bootstrap): skip pre-election config probe; unconfigured nodes defer to post-election fetch

Browse source
This commit is contained in:
Rongsong Shen 2026-05-29 13:15:46 +00:00
parent 4e9924e95c
commit c7badd3f96
2 changed files with 67 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

119
src/controller/config_repl.c
View file

@ -40,6 +40,10 @@
/* xce_request: defined in request.c, not yet in any public header */
extern int xce_request(endpoint_t *ep, xce_request_t *req);
/* SSL context for xce_probe_config_tag — set by bootstrap_config_if_needed.
* Stored as void* to avoid SSL_CTX type at this point in the translation unit. */
static void *s_probe_ssl_ctx = NULL;
/* ── config load helpers (shared with myconf.c logic) ─────── */
#define SET_DEFAULT_IF_UNSET(var, unset, val) \
@ -1359,16 +1363,12 @@ int handle_upgrade_config(cluster_t *c, endpoint_t *ep, xce_request_t *req,
/* ── xce_probe_config_tag ───────────────────────────────────────────────────
*
* Single-RTT config tag probe to one peer controller over plain TCP.
* Single-RTT config tag probe to one peer controller over SSL.
*
* Wire format (same pattern as vote_request_thread in ha.c):
* Send: uint64_t size_hdr + capnp(ProbeConfigTagRequest) [direct struct]
* Receive: uint64_t size_hdr + capnp(ProbeConfigTagReply) [direct struct]
*
* The ProbeConfigTagRequest and ProbeConfigTagReply are sent as top-level
* capnp structs (not wrapped in Request/RequestReply), consistent with the
* OP_requestVote wire format used by vote_request_thread.
*
* Returns:
* 0 — success; *tag_out holds peer's config_tag (0 = no config)
* -ECONNREFUSED — peer refused the TCP connection (port not open)
@ -1385,64 +1385,51 @@ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out)
*tag_out = 0;
/* ── Step 1: Non-blocking TCP connect ── */
struct addrinfo hints, *res = NULL;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
SSL *ssl = NULL;
int sock = -1;
int flags_saved = 0;
char port_str[16];
snprintf(port_str, sizeof(port_str), "%u", (unsigned)port);
if (getaddrinfo(host, port_str, &hints, &res) != 0)
return -ENONET;
int sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sock < 0) {
int e = errno;
freeaddrinfo(res);
return -e;
}
int flags = fcntl(sock, F_GETFL, 0);
fcntl(sock, F_SETFL, flags | O_NONBLOCK);
connect(sock, res->ai_addr, res->ai_addrlen);
freeaddrinfo(res);
res = NULL;
/* Poll for connect completion — up to PROBE_CONNECT_TIMEOUT_MS */
/* ── Step 1: SSL connect via connect_ssl_raw (same path as do_one_config_fetch) ── */
{
const int PROBE_CONNECT_TIMEOUT_MS = 500;
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
int pr;
do { pr = poll(&pfd, 1, PROBE_CONNECT_TIMEOUT_MS); } while (pr < 0 && errno == EINTR);
if (pr == 0) {
close(sock);
return -ETIMEDOUT;
}
if (pr < 0 || (pfd.revents & (POLLERR | POLLHUP))) {
close(sock);
return -ECONNREFUSED;
}
if (!(pfd.revents & POLLOUT)) {
close(sock);
return -ECONNREFUSED;
/* s_ssl_ctx is set in bootstrap_config_if_needed which passes it here.
* If NULL (unit test), fall back to plain TCP. */
SSL_CTX *ssl_ctx = s_probe_ssl_ctx;
if (ssl_ctx) {
int rc = connect_ssl_raw(ssl_ctx, host, port, 500, NULL, &ssl, &sock);
if (rc != 0)
return rc;
} else {
struct addrinfo hints, *res = NULL;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
char port_str[16];
snprintf(port_str, sizeof(port_str), "%u", (unsigned)port);
if (getaddrinfo(host, port_str, &hints, &res) != 0)
return -EHOSTUNREACH;
sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
if (sock < 0) { int e = errno; freeaddrinfo(res); return -e; }
flags_saved = fcntl(sock, F_GETFL, 0);
fcntl(sock, F_SETFL, flags_saved | O_NONBLOCK);
connect(sock, res->ai_addr, res->ai_addrlen);
freeaddrinfo(res);
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
int pr;
do { pr = poll(&pfd, 1, 500); } while (pr < 0 && errno == EINTR);
if (pr == 0) { close(sock); return -ETIMEDOUT; }
if (pr < 0 || (pfd.revents & (POLLERR | POLLHUP))) { close(sock); return -ECONNREFUSED; }
int err = 0; socklen_t el = sizeof(err);
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
if (err) { close(sock); return -err; }
fcntl(sock, F_SETFL, flags_saved & ~O_NONBLOCK);
}
}
/* Verify TCP connect succeeded via SO_ERROR */
{
int err = 0;
socklen_t el = sizeof(err);
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
if (err != 0) {
close(sock);
return -err;
}
}
/* Restore blocking for send/recv */
fcntl(sock, F_SETFL, flags & ~O_NONBLOCK);
/* ── Step 2: Encode + send ProbeConfigTagRequest (direct struct) ── */
{
struct capn c;
@ -1567,6 +1554,22 @@ int (*xce_probe_config_tag_hook)(const char *host, uint16_t port,
*/
int bootstrap_config_if_needed(cluster_t *c)
{
/* Make the SSL context available to xce_probe_config_tag (used below) */
s_probe_ssl_ctx = c->ha.ssl_ctx;
/* Revised bootstrap strategy: skip the pre-election config probe entirely.
* The election protocol already carries config_tag in OP_requestVote, so
* the node with the freshest config wins naturally (Req 2.4 freshness gate).
* Standbys with config_tag=0 will lose the election (or yield to a fresher
* node), then fetch config from the elected primary via fetch_config_from_primary().
* This avoids the SSL/plain-TCP mismatch in the probe path. */
if (c->myconf->config_version_tag == 0) {
c->ha.defer_config_fetch = true;
INFO("HA bootstrap: config_tag=0 — skipping pre-election probe, "
"will fetch config from primary after election");
return 0;
}
/* Req 3.1: fast path — already have config */
if (c->myconf->config_version_tag != 0)
return 0;