feat(config-upgrade-snapshot): live config-upgrade soak infra + fix bug-1468/bug-1469 (task 8.6)
Build the live 3-node chaotic-multi-kill config-upgrade soak scenario (run-ha-soak.sh --config-upgrade) required by Requirements 4.1, 4.3, 6.1, 6.4. This needed genuinely new client-side infrastructure: a hand-rolled capnp send path for OP_upgradeConfig (the auto-generated encoder is a no-op for its Data fields, worked around the same way xce_replconfigpush_req_raw already does), plus cmd_hostctrl/cmd_upgradeconfig CLI commands. Building and running this live surfaced two real production bugs, both fixed here: - bug-1468: an unrelated prior commit inverted OPCode_upgradeConfig's primary/standby dispatch guard in cluster.c, making config upgrades unreachable from any network client. - bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never called config_activate_committed(), so the node that originates a config upgrade never refreshed its own in-memory state as part of committing it. Killing that node before some later, unrelated event happened to refresh it left its checkpoint permanently stale -- the first live, empirical reproduction of bug-857's originally hypothesized failure mode. Fixed by activating before the marker-triggered checkpoint write, since checkpoint content is built from live runtime tables that only reflect the reload after activation runs. A separate, narrower checkpoint divergence (bug-1471, not config-related) remains open and still blocks a clean soak PASS -- task 8.6 stays in-progress until it's resolved.
This commit is contained in:
parent
2bd44e8e4d
commit
959b0cdacb
13 changed files with 1022 additions and 10 deletions
|
|
@ -778,6 +778,25 @@ int xcecli_hostctrl(xcecli_ctx_t c, xce_hostctrl_t* ctrlinfo)
|
|||
return rc;
|
||||
}
|
||||
|
||||
int xcecli_upgradeconfig(xcecli_ctx_t c, const uint8_t *sha256,
|
||||
size_t sha256_len, const uint8_t *bytes,
|
||||
size_t bytes_len)
|
||||
{
|
||||
api_ctx_t* ctx = (api_ctx_t*)c;
|
||||
int rc;
|
||||
|
||||
if ((ctx == NULL) || (sha256 == NULL) || (sha256_len == 0)
|
||||
|| (bytes == NULL) || (bytes_len == 0)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
|
||||
xce_upgradeconfig(&(ctx->ctx), sha256,
|
||||
sha256_len, bytes,
|
||||
bytes_len));
|
||||
return rc;
|
||||
}
|
||||
|
||||
int xcecli_runtask(xcecli_ctx_t c, xce_taskrun_t* info)
|
||||
{
|
||||
api_ctx_t* ctx = (api_ctx_t*)c;
|
||||
|
|
|
|||
|
|
@ -499,6 +499,94 @@ int xce_replconfigpush_req_raw(endpoint_t *ep, xce_peerinfo_t *from,
|
|||
return (enq_rc < 0) ? enq_rc : 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* xce_upgradeconfig_req_raw: send OP_upgradeConfig with raw byte arrays.
|
||||
*
|
||||
* Bypasses encode_UpgradeConfigRequest (a literal no-op for the sha256/bytes
|
||||
* Data fields — see the doc comment above upgrade_config_req_s in typemap.h;
|
||||
* confirmed by inspection of the generated xce.build/lib/controller.capnp.c).
|
||||
* Instead, constructs the struct Request / struct UpgradeConfigRequest
|
||||
* directly using the capnp wire-format API (capn_new_list8 + capn_setv8),
|
||||
* mirroring xce_replconfigpush_req_raw's exact bypass pattern.
|
||||
*
|
||||
* Unlike that controller-to-controller helper, this is an ordinary CLIENT
|
||||
* request: seq and from are populated the same way the generic xce_request()
|
||||
* populates them for every other client op (endpoint_getseq(ep), ep->peer),
|
||||
* not passed in by the caller.
|
||||
*/
|
||||
int xce_upgradeconfig_req_raw(endpoint_t *ep,
|
||||
const uint8_t *sha256, size_t sha256_len,
|
||||
const uint8_t *bytes, size_t bytes_len)
|
||||
{
|
||||
struct capn c;
|
||||
struct capn_segment *cs;
|
||||
struct Request rq;
|
||||
struct UpgradeConfigRequest up_req;
|
||||
UpgradeConfigRequest_ptr up_ptr;
|
||||
Request_ptr rp;
|
||||
io_packet_t *pkt;
|
||||
int rc;
|
||||
|
||||
if (ep == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
if (sha256 == NULL || sha256_len == 0 || bytes == NULL || bytes_len == 0)
|
||||
return -EINVAL;
|
||||
|
||||
capn_init_malloc(&c);
|
||||
cs = capn_root(&c).seg;
|
||||
|
||||
memset(&rq, 0, sizeof(rq));
|
||||
memset(&up_req, 0, sizeof(up_req));
|
||||
|
||||
/* sha256: write as a List(UInt8) — capn_list8 and capn_data share {capn_ptr p} */
|
||||
{
|
||||
capn_list8 sha_list = capn_new_list8(cs, (int)sha256_len);
|
||||
capn_setv8(sha_list, 0, sha256, (int)sha256_len);
|
||||
up_req.sha256.p = sha_list.p;
|
||||
}
|
||||
|
||||
/* config bytes */
|
||||
{
|
||||
capn_list8 bytes_list = capn_new_list8(cs, (int)bytes_len);
|
||||
capn_setv8(bytes_list, 0, bytes, (int)bytes_len);
|
||||
up_req.bytes.p = bytes_list.p;
|
||||
}
|
||||
|
||||
up_ptr = new_UpgradeConfigRequest(cs);
|
||||
write_UpgradeConfigRequest(&up_req, up_ptr);
|
||||
|
||||
rq.opcode = OPCode_upgradeConfig;
|
||||
rq.seq = endpoint_getseq(ep);
|
||||
rq.body_which = Request_body_upgradeconfigrequestmsg;
|
||||
rq.body.upgradeconfigrequestmsg = up_ptr;
|
||||
encode_PeerInfo_ptr(cs, &rq.from, ep->peer);
|
||||
|
||||
rp = new_Request(cs);
|
||||
write_Request(&rq, rp);
|
||||
capn_setp(capn_root(&c), 0, rp.p);
|
||||
|
||||
pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
|
||||
if (!pkt) {
|
||||
capn_free(&c);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
pkt->sz = capn_size(&c);
|
||||
rc = alloc_packet(pkt);
|
||||
if (rc < 0) {
|
||||
free(pkt);
|
||||
capn_free(&c);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
capn_write_mem(&c, (uint8_t *)pkt->data, pkt->sz, 0);
|
||||
capn_free(&c);
|
||||
|
||||
int enq_rc = endpoint_enq(ep, &(pkt->link));
|
||||
return (enq_rc < 0) ? enq_rc : 0;
|
||||
}
|
||||
|
||||
/* wal_entry_to_wire: fill a wire WalCatchupEntry from a C entry's header
|
||||
* fields and raw payload bytes. Sub-entry nesting is one level deep —
|
||||
* elements written here never carry their own entries list. */
|
||||
|
|
|
|||
|
|
@ -647,6 +647,36 @@ int xce_hostctrl(xce_ctx_t* ctx, xce_hostctrl_t* ctrl)
|
|||
return status;
|
||||
}
|
||||
|
||||
int xce_upgradeconfig(xce_ctx_t* ctx, const uint8_t *sha256,
|
||||
size_t sha256_len, const uint8_t *bytes,
|
||||
size_t bytes_len)
|
||||
{
|
||||
int rc;
|
||||
int status;
|
||||
|
||||
if (ctx == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (ctx->state != XCE_CTX_RUN) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rc = xce_upgradeconfig_req_raw(ctx->ep, sha256, sha256_len, bytes,
|
||||
bytes_len);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
struct timespec _dl; cbuf_deadline_10s(&_dl);
|
||||
rc = cbuf_timedget(&(ctx->wbuf), &status, &_dl);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
int xce_runtask(xce_ctx_t* ctx, xce_taskrun_t* runinfo)
|
||||
{
|
||||
int rc;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue