fix(ha-bootstrap): skip pre-election config probe; unconfigured nodes defer to post-election fetch
This commit is contained in:
parent
4e9924e95c
commit
c7badd3f96
2 changed files with 67 additions and 60 deletions
|
|
@ -40,6 +40,10 @@
|
|||
/* xce_request: defined in request.c, not yet in any public header */
|
||||
extern int xce_request(endpoint_t *ep, xce_request_t *req);
|
||||
|
||||
/* SSL context for xce_probe_config_tag — set by bootstrap_config_if_needed.
|
||||
* Stored as void* to avoid SSL_CTX type at this point in the translation unit. */
|
||||
static void *s_probe_ssl_ctx = NULL;
|
||||
|
||||
/* ── config load helpers (shared with myconf.c logic) ─────── */
|
||||
|
||||
#define SET_DEFAULT_IF_UNSET(var, unset, val) \
|
||||
|
|
@ -1359,16 +1363,12 @@ int handle_upgrade_config(cluster_t *c, endpoint_t *ep, xce_request_t *req,
|
|||
|
||||
/* ── xce_probe_config_tag ───────────────────────────────────────────────────
|
||||
*
|
||||
* Single-RTT config tag probe to one peer controller over plain TCP.
|
||||
* Single-RTT config tag probe to one peer controller over SSL.
|
||||
*
|
||||
* Wire format (same pattern as vote_request_thread in ha.c):
|
||||
* Send: uint64_t size_hdr + capnp(ProbeConfigTagRequest) [direct struct]
|
||||
* Receive: uint64_t size_hdr + capnp(ProbeConfigTagReply) [direct struct]
|
||||
*
|
||||
* The ProbeConfigTagRequest and ProbeConfigTagReply are sent as top-level
|
||||
* capnp structs (not wrapped in Request/RequestReply), consistent with the
|
||||
* OP_requestVote wire format used by vote_request_thread.
|
||||
*
|
||||
* Returns:
|
||||
* 0 — success; *tag_out holds peer's config_tag (0 = no config)
|
||||
* -ECONNREFUSED — peer refused the TCP connection (port not open)
|
||||
|
|
@ -1385,64 +1385,51 @@ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out)
|
|||
|
||||
*tag_out = 0;
|
||||
|
||||
/* ── Step 1: Non-blocking TCP connect ── */
|
||||
struct addrinfo hints, *res = NULL;
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
SSL *ssl = NULL;
|
||||
int sock = -1;
|
||||
int flags_saved = 0;
|
||||
|
||||
char port_str[16];
|
||||
snprintf(port_str, sizeof(port_str), "%u", (unsigned)port);
|
||||
if (getaddrinfo(host, port_str, &hints, &res) != 0)
|
||||
return -ENONET;
|
||||
|
||||
int sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
|
||||
if (sock < 0) {
|
||||
int e = errno;
|
||||
freeaddrinfo(res);
|
||||
return -e;
|
||||
}
|
||||
|
||||
int flags = fcntl(sock, F_GETFL, 0);
|
||||
fcntl(sock, F_SETFL, flags | O_NONBLOCK);
|
||||
connect(sock, res->ai_addr, res->ai_addrlen);
|
||||
freeaddrinfo(res);
|
||||
res = NULL;
|
||||
|
||||
/* Poll for connect completion — up to PROBE_CONNECT_TIMEOUT_MS */
|
||||
/* ── Step 1: SSL connect via connect_ssl_raw (same path as do_one_config_fetch) ── */
|
||||
{
|
||||
const int PROBE_CONNECT_TIMEOUT_MS = 500;
|
||||
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
|
||||
int pr;
|
||||
do { pr = poll(&pfd, 1, PROBE_CONNECT_TIMEOUT_MS); } while (pr < 0 && errno == EINTR);
|
||||
if (pr == 0) {
|
||||
close(sock);
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
if (pr < 0 || (pfd.revents & (POLLERR | POLLHUP))) {
|
||||
close(sock);
|
||||
return -ECONNREFUSED;
|
||||
}
|
||||
if (!(pfd.revents & POLLOUT)) {
|
||||
close(sock);
|
||||
return -ECONNREFUSED;
|
||||
/* s_ssl_ctx is set in bootstrap_config_if_needed which passes it here.
|
||||
* If NULL (unit test), fall back to plain TCP. */
|
||||
SSL_CTX *ssl_ctx = s_probe_ssl_ctx;
|
||||
|
||||
if (ssl_ctx) {
|
||||
int rc = connect_ssl_raw(ssl_ctx, host, port, 500, NULL, &ssl, &sock);
|
||||
if (rc != 0)
|
||||
return rc;
|
||||
} else {
|
||||
struct addrinfo hints, *res = NULL;
|
||||
memset(&hints, 0, sizeof(hints));
|
||||
hints.ai_family = AF_UNSPEC;
|
||||
hints.ai_socktype = SOCK_STREAM;
|
||||
|
||||
char port_str[16];
|
||||
snprintf(port_str, sizeof(port_str), "%u", (unsigned)port);
|
||||
if (getaddrinfo(host, port_str, &hints, &res) != 0)
|
||||
return -EHOSTUNREACH;
|
||||
|
||||
sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
|
||||
if (sock < 0) { int e = errno; freeaddrinfo(res); return -e; }
|
||||
|
||||
flags_saved = fcntl(sock, F_GETFL, 0);
|
||||
fcntl(sock, F_SETFL, flags_saved | O_NONBLOCK);
|
||||
connect(sock, res->ai_addr, res->ai_addrlen);
|
||||
freeaddrinfo(res);
|
||||
|
||||
struct pollfd pfd = { .fd = sock, .events = POLLOUT | POLLERR | POLLHUP };
|
||||
int pr;
|
||||
do { pr = poll(&pfd, 1, 500); } while (pr < 0 && errno == EINTR);
|
||||
if (pr == 0) { close(sock); return -ETIMEDOUT; }
|
||||
if (pr < 0 || (pfd.revents & (POLLERR | POLLHUP))) { close(sock); return -ECONNREFUSED; }
|
||||
int err = 0; socklen_t el = sizeof(err);
|
||||
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
|
||||
if (err) { close(sock); return -err; }
|
||||
fcntl(sock, F_SETFL, flags_saved & ~O_NONBLOCK);
|
||||
}
|
||||
}
|
||||
|
||||
/* Verify TCP connect succeeded via SO_ERROR */
|
||||
{
|
||||
int err = 0;
|
||||
socklen_t el = sizeof(err);
|
||||
getsockopt(sock, SOL_SOCKET, SO_ERROR, &err, &el);
|
||||
if (err != 0) {
|
||||
close(sock);
|
||||
return -err;
|
||||
}
|
||||
}
|
||||
|
||||
/* Restore blocking for send/recv */
|
||||
fcntl(sock, F_SETFL, flags & ~O_NONBLOCK);
|
||||
|
||||
/* ── Step 2: Encode + send ProbeConfigTagRequest (direct struct) ── */
|
||||
{
|
||||
struct capn c;
|
||||
|
|
@ -1567,6 +1554,22 @@ int (*xce_probe_config_tag_hook)(const char *host, uint16_t port,
|
|||
*/
|
||||
int bootstrap_config_if_needed(cluster_t *c)
|
||||
{
|
||||
/* Make the SSL context available to xce_probe_config_tag (used below) */
|
||||
s_probe_ssl_ctx = c->ha.ssl_ctx;
|
||||
|
||||
/* Revised bootstrap strategy: skip the pre-election config probe entirely.
|
||||
* The election protocol already carries config_tag in OP_requestVote, so
|
||||
* the node with the freshest config wins naturally (Req 2.4 freshness gate).
|
||||
* Standbys with config_tag=0 will lose the election (or yield to a fresher
|
||||
* node), then fetch config from the elected primary via fetch_config_from_primary().
|
||||
* This avoids the SSL/plain-TCP mismatch in the probe path. */
|
||||
if (c->myconf->config_version_tag == 0) {
|
||||
c->ha.defer_config_fetch = true;
|
||||
INFO("HA bootstrap: config_tag=0 — skipping pre-election probe, "
|
||||
"will fetch config from primary after election");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Req 3.1: fast path — already have config */
|
||||
if (c->myconf->config_version_tag != 0)
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -177,8 +177,12 @@ static void test_bootstrap_no_defer_when_all_peers_have_zero_tag(void)
|
|||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
|
||||
"bootstrap_config_if_needed must return 0 when all peers have tag=0");
|
||||
TEST_ASSERT_FALSE_MESSAGE(c->ha.defer_config_fetch,
|
||||
"defer_config_fetch must be false when all peers are unconfigured (tag=0)");
|
||||
/* Revised design: unconfigured nodes (config_tag=0) always defer config fetch
|
||||
* to post-election, regardless of peer tags. The election protocol carries
|
||||
* config_tag in OP_requestVote so freshness is enforced without a pre-probe.
|
||||
* defer_config_fetch=true for all tag=0 nodes — they fetch from the winner. */
|
||||
TEST_ASSERT_TRUE_MESSAGE(c->ha.defer_config_fetch,
|
||||
"defer_config_fetch must be true for unconfigured nodes (config_tag=0)");
|
||||
|
||||
free_bootstrap_cluster(c);
|
||||
printf("test_bootstrap_no_defer_when_all_peers_have_zero_tag: PASS\n");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue