forked from lei.chen/myxce
feat(ha-commit-barrier-rollout): config barrier with production ACK ingestion (task 5.2 + r1)
configAck branch: per-push majority (acked_config_lsn >= push_tag, 1+count > n_peers/2 over r->ce->n_peers, n_peers<=1 immediate, ce==NULL -EINVAL); legacy >0 counting and n_active==0 shortcut dead. push_config_to_standbys returns the barrier result; upgrade failure -> send_upgrade_error (rc forwarded), no success ACK, no restart, committed_config_tag success-only. Remediation r1 (two CRITICAL review findings): - F1: wire production config-ACK ingestion — replconfigpushreplymsg decode branch in repl_ack_decode_and_dispatch -> repl_on_config_ack on the promoted-vote ep; + defensive cluster.c OPCode_reply case. Closes the unlanded half of ha-config-bootstrap 8.1; without this the barrier would brick every multi-member config upgrade. No schema change. - F2: keep the barrier off the iomgr thread — export push_config_transport_only (non-blocking) for the 5 ha.c + join.c streaming sites; barrier path now vtask-only (do_upgrade_config). Wire-path test drives real capnp ACK bytes through the production decode branch. 30/30 rollout, 6/6 repl_majority, 46/46 config_repl; census 0 FAIL; pre-existing fixture leak only (less than baseline).
This commit is contained in:
parent
9cdeba5816
commit
5a2fae1d37
12 changed files with 1034 additions and 399 deletions
|
|
@ -754,15 +754,18 @@ int handle_repl_config_push(cluster_t *c, endpoint_t *ep, xce_request_t *req)
|
|||
|
||||
/* ── send_upgrade_error ─────────────────────────── */
|
||||
|
||||
/* status: the negative errno carried to the requester (task 5.2 r1,
|
||||
* reviewer finding 3: validation failures keep -EINVAL; the barrier
|
||||
* failure path forwards its own rc, e.g. -ETIMEDOUT). */
|
||||
static void send_upgrade_error(endpoint_t *ep, xce_request_t *req,
|
||||
cluster_t *c, const char *msg)
|
||||
cluster_t *c, int status, const char *msg)
|
||||
{
|
||||
xce_reply_t reply;
|
||||
init_reply_by_req(&reply, req);
|
||||
reply.status = -EINVAL;
|
||||
reply.status = (status < 0) ? status : -EINVAL;
|
||||
reply.data_which = RequestReply_data_nodata;
|
||||
xce_reply(ep, c->myhost, &reply);
|
||||
ERROR("OP_upgradeConfig rejected: %s", msg);
|
||||
ERROR("OP_upgradeConfig rejected (status=%d): %s", reply.status, msg);
|
||||
}
|
||||
|
||||
/* ── push_config_to_standbys ─────────────────────── */
|
||||
|
|
@ -863,21 +866,32 @@ int wait_all_with_timeout_for_test(pthread_t *threads, push_ctx_t *ctxs,
|
|||
}
|
||||
#endif
|
||||
|
||||
int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
|
||||
const uint8_t sha256[32], uint64_t new_tag,
|
||||
int *acked_out)
|
||||
/* push_config_transport_only: transport fan-out half of
|
||||
* push_config_to_standbys (exported — task 5.2 remediation r1): spawn one
|
||||
* push thread per registered standby endpoint and wait for the sends.
|
||||
* Best-effort streaming, NO barrier wait — the sends are non-blocking
|
||||
* endpoint_enq()s, so this is safe on iomgr-reachable paths (election
|
||||
* promotion, heartbeat-driven convergence, controller join). Alloc or
|
||||
* thread failures degrade to fewer pushes, never to a fabricated quorum
|
||||
* (the barrier wait in push_config_to_standbys owns the quorum verdict).
|
||||
* Stores the registered standby count in *n_out (may be NULL); returns
|
||||
* the number of standbys that ACKed the transport push. */
|
||||
int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
|
||||
size_t blen, const uint8_t sha256[32],
|
||||
uint64_t new_tag, int *n_out)
|
||||
{
|
||||
/* acked_out: per-standby transport push count (task 5.1 signature;
|
||||
* behavior-preserving — the return value keeps the legacy push-count
|
||||
* semantics until task 5.2 re-plumbs it to the barrier result). */
|
||||
if (acked_out)
|
||||
*acked_out = 0;
|
||||
int n;
|
||||
int acked;
|
||||
|
||||
if (n_out)
|
||||
*n_out = 0;
|
||||
if (!c->ha.standby_endpoints)
|
||||
return 0;
|
||||
|
||||
pthread_mutex_lock(&c->ha.lock);
|
||||
int n = c->ha.standby_endpoints->count;
|
||||
n = c->ha.standby_endpoints->count;
|
||||
if (n_out)
|
||||
*n_out = n;
|
||||
if (n == 0) {
|
||||
pthread_mutex_unlock(&c->ha.lock);
|
||||
return 0;
|
||||
|
|
@ -885,6 +899,8 @@ int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
|
|||
push_ctx_t *ctxs = calloc((size_t)n, sizeof(push_ctx_t));
|
||||
if (!ctxs) {
|
||||
pthread_mutex_unlock(&c->ha.lock);
|
||||
WARN("push_config_transport_only: ctx alloc failed (n=%d)"
|
||||
" — skipping transport push", n);
|
||||
return 0;
|
||||
}
|
||||
for (int i = 0; i < n; i++) {
|
||||
|
|
@ -900,32 +916,54 @@ int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
|
|||
pthread_t *threads = malloc((size_t)n * sizeof(pthread_t));
|
||||
if (!threads) {
|
||||
free(ctxs);
|
||||
WARN("push_config_transport_only: thread alloc failed (n=%d)"
|
||||
" — skipping transport push", n);
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
ctxs[i].started = (pthread_create(&threads[i], NULL, push_thread, &ctxs[i]) == 0);
|
||||
if (!ctxs[i].started)
|
||||
WARN("push_config_to_standbys: pthread_create failed for standby %d", i);
|
||||
WARN("push_config_transport_only: pthread_create failed for standby %d", i);
|
||||
}
|
||||
|
||||
atomic_int push_done;
|
||||
atomic_init(&push_done, 0);
|
||||
|
||||
int acked = wait_all_with_timeout(threads, ctxs, n,
|
||||
CONF_PUSH_TIMEOUT_MS, &push_done);
|
||||
acked = wait_all_with_timeout(threads, ctxs, n,
|
||||
CONF_PUSH_TIMEOUT_MS, &push_done);
|
||||
|
||||
free(ctxs);
|
||||
free(threads);
|
||||
return acked;
|
||||
}
|
||||
|
||||
int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
|
||||
const uint8_t sha256[32], uint64_t new_tag,
|
||||
int *acked_out)
|
||||
{
|
||||
/* Returns the majority-ACK barrier result (0 / -ETIMEDOUT / -EINVAL);
|
||||
* the per-standby transport push count travels via acked_out (task 5.2
|
||||
* re-plumb). The transport fan-out is best-effort — push failures and
|
||||
* an empty standby set fall through to the barrier wait, which owns
|
||||
* the quorum policy (n_peers <= 1 → immediate success; multi-member
|
||||
* with no covering acks → -ETIMEDOUT; no n_active shortcut).
|
||||
* VTASK-CONTEXT ONLY (task 5.2 remediation r1): the barrier blocks up
|
||||
* to CONF_PUSH_TIMEOUT_MS — iomgr-reachable callers must use
|
||||
* push_config_transport_only instead (see config_repl.h). */
|
||||
int n = 0;
|
||||
int acked = push_config_transport_only(c, bytes, blen, sha256, new_tag, &n);
|
||||
|
||||
if (acked_out)
|
||||
*acked_out = acked;
|
||||
|
||||
/* Req 8.1–8.3: wait for majority-quorum OP_upgradeConfigAck.
|
||||
* On quorum: advance committed_config_tag atomically (Req 8.5). */
|
||||
/* entry_lsn = 0: the configAck branch ignores it (walAck-only adapter
|
||||
* parameter, ha-commit-engine task 5.1). */
|
||||
int rc = repl_wait_majority_ack(&c->persist.repl, 0, CONF_PUSH_TIMEOUT_MS,
|
||||
/* Req 8.1–8.3 / barrier-rollout Req 4.1-4.3: wait for majority-quorum
|
||||
* OP_upgradeConfigAck, passing the in-flight push tag for per-push
|
||||
* evaluation (entry_lsn carries the tag — task 5.2). On quorum:
|
||||
* advance committed_config_tag atomically (Req 8.5 / 4.3 — the
|
||||
* advance stays inside this success branch only). */
|
||||
int rc = repl_wait_majority_ack(&c->persist.repl, new_tag,
|
||||
CONF_PUSH_TIMEOUT_MS,
|
||||
(uint8_t)OPCode_upgradeConfigAck);
|
||||
if (rc == 0) {
|
||||
atomic_store_explicit(&c->ha.committed_config_tag, new_tag,
|
||||
|
|
@ -933,12 +971,11 @@ int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
|
|||
INFO("Config committed on majority (tag=%016" PRIx64
|
||||
", pushed=%d/%d standbys)", new_tag, acked, n);
|
||||
} else {
|
||||
WARN("Config push: timeout waiting for majority ACK"
|
||||
" (tag=%016" PRIx64 ", pushed=%d/%d standbys)",
|
||||
new_tag, acked, n);
|
||||
WARN("Config push: no majority ACK (rc=%d, tag=%016" PRIx64
|
||||
", pushed=%d/%d standbys)", rc, new_tag, acked, n);
|
||||
}
|
||||
|
||||
return acked;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ── handle_upgrade_config ──────────────────────── */
|
||||
|
|
@ -956,8 +993,9 @@ int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen,
|
|||
* 4. Increment generation, re-encode with updated counter
|
||||
* 5. Save to disk atomically
|
||||
* 6. Update in-memory generation
|
||||
* 7. Push to standbys (best-effort)
|
||||
* 8. ACK admin tool
|
||||
* 7. Push to standbys and wait for the majority-ACK barrier; on failure
|
||||
* reply with send_upgrade_error and return (no success ACK, no restart)
|
||||
* 8. ACK admin tool (success ⇔ majority-acked config)
|
||||
* 9. Wait for in-flight checkpoint, then signal restart
|
||||
*/
|
||||
static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
||||
|
|
@ -968,11 +1006,11 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
{
|
||||
/* 0. Validate size */
|
||||
if (in_len == 0 || in_bytes == NULL || in_sha == NULL) {
|
||||
send_upgrade_error(ep, req, c, "Missing bytes or sha256 field");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "Missing bytes or sha256 field");
|
||||
return -EINVAL;
|
||||
}
|
||||
if (in_len > CONF_MAX_SIZE) {
|
||||
send_upgrade_error(ep, req, c, "Config blob exceeds CONF_MAX_SIZE");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "Config blob exceeds CONF_MAX_SIZE");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
|
|
@ -980,7 +1018,7 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
uint8_t computed[32];
|
||||
SHA256(in_bytes, in_len, computed);
|
||||
if (memcmp(computed, in_sha, 32) != 0) {
|
||||
send_upgrade_error(ep, req, c,
|
||||
send_upgrade_error(ep, req, c, -EINVAL,
|
||||
"SHA-256 mismatch -- blob corrupted in transit");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
|
@ -994,13 +1032,13 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
|
||||
int fd = open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||||
if (fd < 0) {
|
||||
send_upgrade_error(ep, req, c, "Failed to create temp decode file");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "Failed to create temp decode file");
|
||||
return -errno;
|
||||
}
|
||||
if (write(fd, in_bytes, in_len) != (ssize_t)in_len) {
|
||||
close(fd);
|
||||
unlink(tmp_path);
|
||||
send_upgrade_error(ep, req, c, "Failed to write temp decode file");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "Failed to write temp decode file");
|
||||
return -EIO;
|
||||
}
|
||||
close(fd);
|
||||
|
|
@ -1014,7 +1052,7 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
free_ClusterConfig(new_cfg);
|
||||
free(new_cfg);
|
||||
}
|
||||
send_upgrade_error(ep, req, c, "Failed to parse Cap'n Proto blob");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "Failed to parse Cap'n Proto blob");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
|
|
@ -1022,7 +1060,7 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
if (config_contains_reserved_name(new_cfg)) {
|
||||
free_ClusterConfig(new_cfg);
|
||||
free(new_cfg);
|
||||
send_upgrade_error(ep, req, c,
|
||||
send_upgrade_error(ep, req, c, -EINVAL,
|
||||
"Config contains reserved name 'lost_and_found'");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
|
@ -1046,7 +1084,7 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
uint8_t *out_bytes = (uint8_t *)malloc(out_len);
|
||||
if (!out_bytes) {
|
||||
capn_free(&out_cap);
|
||||
send_upgrade_error(ep, req, c, "OOM re-encoding config");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "OOM re-encoding config");
|
||||
return -ENOMEM;
|
||||
}
|
||||
capn_write_mem(&out_cap, out_bytes, out_len, 0);
|
||||
|
|
@ -1058,19 +1096,37 @@ static int do_upgrade_config_with_bytes(cluster_t *c, endpoint_t *ep,
|
|||
rc = conf_save_config(c->myconf, out_bytes, out_len, computed);
|
||||
if (rc != 0) {
|
||||
free(out_bytes);
|
||||
send_upgrade_error(ep, req, c, "Failed to save new config to disk");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "Failed to save new config to disk");
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* 6. Update in-memory generation */
|
||||
c->myconf->config_generation = new_gen;
|
||||
|
||||
/* 7. Push to standbys (best-effort; see push_config_to_standbys comment) */
|
||||
int acked = push_config_to_standbys(c, out_bytes, out_len, computed,
|
||||
c->myconf->config_version_tag, NULL);
|
||||
/* 7. Push to standbys — the majority-ACK barrier result is
|
||||
* authoritative for the upgrade outcome (barrier-rollout Req 4.1/4.2):
|
||||
* non-zero means the new tag never reached quorum, so the requester
|
||||
* gets an explicit error and the cluster stays on the old committed
|
||||
* tag (no success ACK, no restart — Req 4.3/4.4). Disk save and
|
||||
* generation bump have already happened (pre-existing sequence);
|
||||
* convergence of a half-pushed config remains the OP_replConfig
|
||||
* anti-entropy behavior. */
|
||||
int acked = 0;
|
||||
rc = push_config_to_standbys(c, out_bytes, out_len, computed,
|
||||
c->myconf->config_version_tag, &acked);
|
||||
free(out_bytes);
|
||||
|
||||
/* 8. ACK the admin tool */
|
||||
if (rc != 0) {
|
||||
/* Forward the barrier rc (-ETIMEDOUT / -EINVAL) as the reply
|
||||
* status so the admin tool sees WHY the commit failed (task 5.2
|
||||
* r1, reviewer finding 3). */
|
||||
send_upgrade_error(ep, req, c, rc,
|
||||
"Config push failed to reach majority quorum"
|
||||
" -- upgrade not committed, no restart");
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* 8. ACK the admin tool (success ⇔ majority-acked config) */
|
||||
upgrade_config_reply_t urep;
|
||||
memset(&urep, 0, sizeof(urep));
|
||||
urep.version_tag = c->myconf->config_version_tag;
|
||||
|
|
@ -1118,12 +1174,12 @@ int handle_upgrade_config(cluster_t *c, endpoint_t *ep, xce_request_t *req)
|
|||
|
||||
int rc = xce_upgrade_config_req_extract(req, &in_bytes, &in_len, &in_sha);
|
||||
if (rc != 0 || !in_bytes || !in_sha) {
|
||||
send_upgrade_error(ep, req, c,
|
||||
send_upgrade_error(ep, req, c, -EINVAL,
|
||||
"Failed to extract bytes from UpgradeConfigRequest");
|
||||
return rc != 0 ? rc : -EINVAL;
|
||||
}
|
||||
if (in_len == 0) {
|
||||
send_upgrade_error(ep, req, c, "bytes or sha256 field missing");
|
||||
send_upgrade_error(ep, req, c, -EINVAL, "bytes or sha256 field missing");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue