feat(config-upgrade-snapshot): live config-upgrade soak infra + fix bug-1468/bug-1469 (task 8.6)

Build the live 3-node chaotic-multi-kill config-upgrade soak scenario
(run-ha-soak.sh --config-upgrade) required by Requirements 4.1, 4.3, 6.1,
6.4. This needed genuinely new client-side infrastructure: a hand-rolled
capnp send path for OP_upgradeConfig (the auto-generated encoder is a
no-op for its Data fields, worked around the same way
xce_replconfigpush_req_raw already does), plus cmd_hostctrl/cmd_upgradeconfig
CLI commands.

Building and running this live surfaced two real production bugs, both
fixed here:

- bug-1468: an unrelated prior commit inverted OPCode_upgradeConfig's
  primary/standby dispatch guard in cluster.c, making config upgrades
  unreachable from any network client.
- bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never called
  config_activate_committed(), so the node that originates a config
  upgrade never refreshed its own in-memory state as part of committing
  it. Killing that node before some later, unrelated event happened to
  refresh it left its checkpoint permanently stale -- the first live,
  empirical reproduction of bug-857's originally hypothesized failure
  mode. Fixed by activating before the marker-triggered checkpoint write,
  since checkpoint content is built from live runtime tables that only
  reflect the reload after activation runs.

A separate, narrower checkpoint divergence (bug-1471, not config-related)
remains open and still blocks a clean soak PASS -- task 8.6 stays
in-progress until it's resolved.
This commit is contained in:
Rongsong Shen 2026-07-25 15:48:20 +00:00
parent 2bd44e8e4d
commit 959b0cdacb
13 changed files with 1022 additions and 10 deletions

View file

@ -778,6 +778,25 @@ int xcecli_hostctrl(xcecli_ctx_t c, xce_hostctrl_t* ctrlinfo)
return rc;
}
int xcecli_upgradeconfig(xcecli_ctx_t c, const uint8_t *sha256,
size_t sha256_len, const uint8_t *bytes,
size_t bytes_len)
{
api_ctx_t* ctx = (api_ctx_t*)c;
int rc;
if ((ctx == NULL) || (sha256 == NULL) || (sha256_len == 0)
|| (bytes == NULL) || (bytes_len == 0)) {
return -EINVAL;
}
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
xce_upgradeconfig(&(ctx->ctx), sha256,
sha256_len, bytes,
bytes_len));
return rc;
}
int xcecli_runtask(xcecli_ctx_t c, xce_taskrun_t* info)
{
api_ctx_t* ctx = (api_ctx_t*)c;

View file

@ -499,6 +499,94 @@ int xce_replconfigpush_req_raw(endpoint_t *ep, xce_peerinfo_t *from,
return (enq_rc < 0) ? enq_rc : 0;
}
/*
* xce_upgradeconfig_req_raw: send OP_upgradeConfig with raw byte arrays.
*
* Bypasses encode_UpgradeConfigRequest (a literal no-op for the sha256/bytes
* Data fields — see the doc comment above upgrade_config_req_s in typemap.h;
* confirmed by inspection of the generated xce.build/lib/controller.capnp.c).
* Instead, constructs the struct Request / struct UpgradeConfigRequest
* directly using the capnp wire-format API (capn_new_list8 + capn_setv8),
* mirroring xce_replconfigpush_req_raw's exact bypass pattern.
*
* Unlike that controller-to-controller helper, this is an ordinary CLIENT
* request: seq and from are populated the same way the generic xce_request()
* populates them for every other client op (endpoint_getseq(ep), ep->peer),
* not passed in by the caller.
*/
int xce_upgradeconfig_req_raw(endpoint_t *ep,
const uint8_t *sha256, size_t sha256_len,
const uint8_t *bytes, size_t bytes_len)
{
struct capn c;
struct capn_segment *cs;
struct Request rq;
struct UpgradeConfigRequest up_req;
UpgradeConfigRequest_ptr up_ptr;
Request_ptr rp;
io_packet_t *pkt;
int rc;
if (ep == NULL)
return -EINVAL;
if (sha256 == NULL || sha256_len == 0 || bytes == NULL || bytes_len == 0)
return -EINVAL;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
memset(&rq, 0, sizeof(rq));
memset(&up_req, 0, sizeof(up_req));
/* sha256: write as a List(UInt8) — capn_list8 and capn_data share {capn_ptr p} */
{
capn_list8 sha_list = capn_new_list8(cs, (int)sha256_len);
capn_setv8(sha_list, 0, sha256, (int)sha256_len);
up_req.sha256.p = sha_list.p;
}
/* config bytes */
{
capn_list8 bytes_list = capn_new_list8(cs, (int)bytes_len);
capn_setv8(bytes_list, 0, bytes, (int)bytes_len);
up_req.bytes.p = bytes_list.p;
}
up_ptr = new_UpgradeConfigRequest(cs);
write_UpgradeConfigRequest(&up_req, up_ptr);
rq.opcode = OPCode_upgradeConfig;
rq.seq = endpoint_getseq(ep);
rq.body_which = Request_body_upgradeconfigrequestmsg;
rq.body.upgradeconfigrequestmsg = up_ptr;
encode_PeerInfo_ptr(cs, &rq.from, ep->peer);
rp = new_Request(cs);
write_Request(&rq, rp);
capn_setp(capn_root(&c), 0, rp.p);
pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
if (!pkt) {
capn_free(&c);
return -ENOMEM;
}
pkt->sz = capn_size(&c);
rc = alloc_packet(pkt);
if (rc < 0) {
free(pkt);
capn_free(&c);
return -errno;
}
capn_write_mem(&c, (uint8_t *)pkt->data, pkt->sz, 0);
capn_free(&c);
int enq_rc = endpoint_enq(ep, &(pkt->link));
return (enq_rc < 0) ? enq_rc : 0;
}
/* wal_entry_to_wire: fill a wire WalCatchupEntry from a C entry's header
* fields and raw payload bytes. Sub-entry nesting is one level deep —
* elements written here never carry their own entries list. */

View file

@ -647,6 +647,36 @@ int xce_hostctrl(xce_ctx_t* ctx, xce_hostctrl_t* ctrl)
return status;
}
int xce_upgradeconfig(xce_ctx_t* ctx, const uint8_t *sha256,
size_t sha256_len, const uint8_t *bytes,
size_t bytes_len)
{
int rc;
int status;
if (ctx == NULL) {
return -EINVAL;
}
if (ctx->state != XCE_CTX_RUN) {
return -EINVAL;
}
rc = xce_upgradeconfig_req_raw(ctx->ep, sha256, sha256_len, bytes,
bytes_len);
if (rc < 0) {
return rc;
}
struct timespec _dl; cbuf_deadline_10s(&_dl);
rc = cbuf_timedget(&(ctx->wbuf), &status, &_dl);
if (rc < 0) {
return rc;
}
return status;
}
int xce_runtask(xce_ctx_t* ctx, xce_taskrun_t* runinfo)
{
int rc;