#include "config.h" #include #include #include #include "cluster.h" #include "ha.h" #include "persist.h" #include "xce/cluster_def.h" #include "xce/errnox.h" static int validate_host(cluster_t* c, const char* host, const char* key) { return 1; } /* * fill_redirect_reply: populate an E_XCE_REDIRECT reply with the primary's * host and port from ha state so the libxce client can use it as a one-hop * hint (libxce-failover Req 8.1). The redirect_host pointer aliases the * ha_state_t.primary_host buffer (capnp serialises it when non-NULL — see * request.c); the caller must keep *ha live until xce_reply() runs, which it * does (ha is embedded in cluster_t). Pure aside from the field writes — no * locking, no allocation. */ void fill_redirect_reply(xce_reply_t* reply, const ha_state_t* ha) { reply->redirect_host = (char*)ha->primary_host; reply->redirect_port = (uint16_t)ha->primary_port; } static int handle_client_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) { xce_reply_t reply; init_reply_by_req(&reply, request); reply.status = 0; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); return 0; } static int handle_worker_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) { int idx = 0; ep_private_t* priv = (ep_private_t*)ep->data; xce_reply_t reply; xce_peerinfo_t* from = request->from; int rc = 0; xce_joindata_t* joinmsg = NULL; worker_t* w = NULL; xce_hostconfig_t* hconfig = NULL; TRACE("%s worker %s joining...", __func__, from->name); init_reply_by_req(&reply, request); if (!validate_host(c, from->name, from->key)) { ERROR("%s reject join of host %s", __func__, from->name); rc = -EPERM; reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); return rc; } joinmsg = request->body.joinmsg; /* Fail fast before any lock: reject if no hostconfig matches (req 4.4) */ hconfig = find_matched_hostconfig(c, from->name); if (hconfig == NULL) { rc = (errno != 0) ? -errno : -EINVAL; reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); return rc; } /* BEFORE_EVENT fires before the existence check (req 5.1) */ xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_BEFORE_EVENT, HOOK_DETAIL_WORKER_JOIN, c, from->name, (worker_t*)NULL); /* Explicit existence check under brief READ lock (req 1.1) */ lock(c->workermgr, RTTI_LOCK_READ); rc = key_find(c->workermgr, (void*)from->name, (void**)&w); if (w != NULL) reference(w); unlock(c->workermgr); if (rc < 0 && rc != -ENOENT) { /* Non-ENOENT error: reject join (req 1.3) */ ERROR("%s workermgr lookup error for %s, rc %d", __func__, from->name, rc); idx = rc; } else if (w == NULL) { /* Host not yet registered: first-time join (req 1.2, req 2) */ idx = init_worker(c, ep, from->name, joinmsg->data.attrs, hconfig); if (idx < 0) ERROR("%s host %s first join failed, rc %d", __func__, from->name, -idx); } else { /* Host already registered: rejoin (req 1.2, req 3) */ idx = rejoin_worker(c, w, ep, joinmsg->data.attrs, hconfig); dereference(w); w = NULL; if (idx < 0) ERROR("%s host %s rejoin failed, rc %d", __func__, from->name, -idx); } /* Look up worker for AFTER_EVENT persist hook (w is NULL on error) */ if (idx >= 0) vget(c->workermgr, idx, (void**)&w); /* AFTER_EVENT fires after create/update, outside any lock (req 5.1) */ xce_run_hooks(&(c->hooksys), XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c, from->name, w); /* Commit barrier (ha-commit-barrier-rollout Req 1.1-1.4): the success * reply is sent only after the join's WAL entry is majority-committed; * barrier failures (-ETIMEDOUT, ...) become the error reply and skip * on_worker_join below. Join errors (idx < 0) reply immediately — no * barrier wait. No lock is held here (AFTER_EVENT runs outside any * lock). timeout 0 = configured sync_timeout_ms. */ if (idx >= 0) { rc = persist_commit_barrier(&c->persist, 0); if (rc != 0) { WARN("%s commit barrier failed for %s, rc %d", __func__, from->name, rc); idx = rc; } } reply.status = idx; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); if (idx < 0) return idx; /* Set priv->u.idx before on_worker_join (req 2.3, req 5.2) */ priv->u.idx = idx; on_worker_join(c, ep); return 0; } static int handle_scheduler_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) { xce_reply_t reply; ep_private_t* priv = (ep_private_t*)ep->data; int rc = 0; TRACE("%s scheduler join, ep %p", __func__, ep); init_reply_by_req(&reply, request); c->sched.ep = ep; priv->u.idx = 0; rc = on_scheduler_join(c); reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); return rc; } /* * handle_controller_join: reject a peer-controller join request. * * Controller-to-controller connection establishment is NOT this path — per * ha-async-io.md's Connection Ownership, it is always primary-initiated * outbound (primary_heartbeat_fn -> on_standby_connected -> ctrlhello -> * repl_add_standby), with the resulting endpoint owned by * vote_ep_err_handler / standby_ep_err_handler_primary. This arm used to * accept an INBOUND role=HostRole_controller join and store it directly * into standby_endpoints, bypassing both documented owners and their * err-handler-driven eviction — a live ownership-model violation, even * though no current sender in the tree ever builds such a request * (xce_join_req structurally rejects role==HostRole_controller). Reject * outright rather than leave the bypass reachable from the wire. */ static int handle_controller_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) { xce_reply_t reply; init_reply_by_req(&reply, request); reply.status = -E_XCE_BAD_ROLE; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); WARN("%s rejecting controller-role join from %s — controller peering is " "primary-initiated only, never via join", __func__, request->from ? request->from->name : "?"); return -E_XCE_BAD_ROLE; } static int handle_apiserver_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) { xce_reply_t reply; xce_peerinfo_t* from = request->from; int rc = 0; TRACE("%s api server join, ep %p", __func__, ep); init_reply_by_req(&reply, request); rc = add_apiserver(c, from->name, ep); if (rc >= 0) { ep_private_t* priv = (ep_private_t*)ep->data; priv->u.idx = rc; on_apiserver_join(c, ep); } reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); if (rc < 0) { ERROR("api server %s join failed, rc %d", from->name, rc); return rc; } return 0; } static int handle_join_(cluster_t* c, endpoint_t* ep, xce_request_t* request) { ep_private_t* priv; int rc = -EINVAL; int role; xce_reply_t reply; xce_peerinfo_t* from = request->from; init_reply_by_req(&reply, request); priv = (ep_private_t*)ep->data; if (priv == NULL) { rc = -E_XCE_INTERNAL; ERROR("priv data of endpoint should not be NULL"); reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); return rc; } role = request->body.joinmsg->role; priv->role = role; TRACE("%s role %d join from %s", __func__, role, from->name); /* ── HA guard (P1-6) ──────────────────────────────────────── * A standby must never accept joins as if it were the primary. * If the primary address is known, tell the client where to go * (E_XCE_REDIRECT). If no primary is known yet, tell the client * to retry later (E_XCE_NOMASTER). */ if (ha_should_redirect(&c->ha)) { WARN("%s standby redirecting role-%d join from %s to %s:%u", __func__, role, from->name, c->ha.primary_host, c->ha.primary_port); rc = -E_XCE_REDIRECT; reply.status = rc; reply.data_which = RequestReply_data_nodata; fill_redirect_reply(&reply, &c->ha); xce_reply(ep, c->myhost, &reply); return rc; } if (ha_no_primary_known(&c->ha)) { WARN("%s standby has no primary yet, rejecting role-%d join from %s", __func__, role, from->name); rc = -E_XCE_NOMASTER; reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); return rc; } switch (role) { case HostRole_client: rc = handle_client_join(c, ep, request); break; case HostRole_worker: rc = handle_worker_join(c, ep, request); break; case HostRole_scheduler: rc = handle_scheduler_join(c, ep, request); break; case HostRole_controller: rc = handle_controller_join(c, ep, request); break; case HostRole_apiserver: rc = handle_apiserver_join(c, ep, request); break; default: rc = -E_XCE_BAD_ROLE; ERROR("unknown role %d from host %s", role, from->name); reply.status = rc; reply.data_which = RequestReply_data_nodata; xce_reply(ep, c->myhost, &reply); break; } return rc; } int handle_join(cluster_t* c, endpoint_t* ep, xce_request_t* request) { int rc = 0; rc = handle_join_(c, ep, request); return rc; }