feat(config-upgrade-snapshot): live config-upgrade soak infra + fix bug-1468/bug-1469 (task 8.6)
Build the live 3-node chaotic-multi-kill config-upgrade soak scenario (run-ha-soak.sh --config-upgrade) required by Requirements 4.1, 4.3, 6.1, 6.4. This needed genuinely new client-side infrastructure: a hand-rolled capnp send path for OP_upgradeConfig (the auto-generated encoder is a no-op for its Data fields, worked around the same way xce_replconfigpush_req_raw already does), plus cmd_hostctrl/cmd_upgradeconfig CLI commands. Building and running this live surfaced two real production bugs, both fixed here: - bug-1468: an unrelated prior commit inverted OPCode_upgradeConfig's primary/standby dispatch guard in cluster.c, making config upgrades unreachable from any network client. - bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never called config_activate_committed(), so the node that originates a config upgrade never refreshed its own in-memory state as part of committing it. Killing that node before some later, unrelated event happened to refresh it left its checkpoint permanently stale -- the first live, empirical reproduction of bug-857's originally hypothesized failure mode. Fixed by activating before the marker-triggered checkpoint write, since checkpoint content is built from live runtime tables that only reflect the reload after activation runs. A separate, narrower checkpoint divergence (bug-1471, not config-related) remains open and still blocks a clean soak PASS -- task 8.6 stays in-progress until it's resolved.
This commit is contained in:
parent
2bd44e8e4d
commit
959b0cdacb
13 changed files with 1022 additions and 10 deletions
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include <cjson/cJSON.h>
|
||||
#include <cjson/s2j.h>
|
||||
#include <openssl/sha.h> /* SHA256() — cmd_upgradeconfig hashes the config blob */
|
||||
|
||||
#include "client.h"
|
||||
#include "config.h" /* XCE_MAXNAME_LEN */
|
||||
|
|
@ -122,6 +123,8 @@ extern int cmd_resval(int, char*[]);
|
|||
extern int cmd_jobkill(int, char*[]);
|
||||
extern int cmd_jobckpt(int, char*[]);
|
||||
extern int cmd_qctrl(int, char*[]);
|
||||
extern int cmd_hostctrl(int, char*[]);
|
||||
extern int cmd_upgradeconfig(int, char*[]);
|
||||
extern int cmd_runtasks(int, char*[]);
|
||||
extern int cmd_setpend(int, char*[]);
|
||||
extern int cmd_jobinfo(int, char*[]);
|
||||
|
|
@ -230,6 +233,9 @@ static command_t client_commands[] = {
|
|||
{ "reslist", cmd_reslist, "Display cluster resources" },
|
||||
{ "workerlist", cmd_workerlist, "list workers" },
|
||||
{ "qctrl", cmd_qctrl, "queue control" },
|
||||
{ "hostctrl", cmd_hostctrl, "host control" },
|
||||
{ "upgradeconfig", cmd_upgradeconfig,
|
||||
"submit a config upgrade (cluster.bin file)" },
|
||||
{ "joblist", cmd_joblist, "list jobs" },
|
||||
{ "expect-jobs", cmd_expect_jobs, "assert current job count" },
|
||||
{ "expect-job", cmd_expect_job, "assert a JobId is present" },
|
||||
|
|
@ -1020,7 +1026,15 @@ int cmd_reslist(int argc, char* argv[])
|
|||
|
||||
static char* worker_status_str(uint32_t status)
|
||||
{
|
||||
if ((status & XCE_WORKER_UNAVAIL) != 0) {
|
||||
/* Admin-set CLOSED takes priority over the dynamic UNAVAIL bit: the
|
||||
* config-upgrade-snapshot soak (task 8.6) needs to assert a host's
|
||||
* administratively-set status specifically, and a host closed via
|
||||
* hostctrl that later disconnects (e.g. its xce_test `worker`/`join`
|
||||
* connection exits after the script ends) would otherwise print only
|
||||
* "UNAVAIL", masking the admin bit this check exists to verify. */
|
||||
if ((status & XCE_WORKER_CLOSED) != 0) {
|
||||
return "CLOSED";
|
||||
} else if ((status & XCE_WORKER_UNAVAIL) != 0) {
|
||||
return "UNAVAIL";
|
||||
} else if ((status & XCE_WORKER_LOCKEDU) != 0) {
|
||||
return "LOCKU";
|
||||
|
|
@ -2224,6 +2238,139 @@ int cmd_qctrl(int argc, char* argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* hostctrl -o|-c -h <hostname> — administratively open/close a host
|
||||
* (xce_hostctrl_t{action, hostname} via xcecli_hostctrl). Mirrors cmd_qctrl's
|
||||
* exact getopt shape; only -o/-c are wired (matching xce_hostctrl_req's own
|
||||
* accepted action range, request.c — HOST_TAG/HOST_UNTAG exist in the enum
|
||||
* but are rejected -EINVAL server-side today, so no flag is offered for them
|
||||
* here). Used by the config-upgrade chaotic soak (task 8.6) to set a host's
|
||||
* admin status before/around a config upgrade and assert it survives. */
|
||||
int cmd_hostctrl(int argc, char* argv[])
|
||||
{
|
||||
int opt;
|
||||
int rc;
|
||||
xce_hostctrl_t hostctrl;
|
||||
int action = HOST_DO_NONE;
|
||||
char* hostname = NULL;
|
||||
|
||||
optind = 1;
|
||||
|
||||
while ((opt = getopt(argc, argv, "och:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'o':
|
||||
action = HOST_OPEN;
|
||||
break;
|
||||
case 'c':
|
||||
action = HOST_CLOSE;
|
||||
break;
|
||||
case 'h':
|
||||
hostname = strdup(optarg);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (hostname == NULL) {
|
||||
fprintf(stderr, "you need to specify host name with -h\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (action == HOST_DO_NONE) {
|
||||
fprintf(stderr, "you need to specify -o (open) or -c (close)\n");
|
||||
free(hostname);
|
||||
return -1;
|
||||
}
|
||||
|
||||
hostctrl.action = action;
|
||||
hostctrl.hostname = hostname;
|
||||
|
||||
rc = xcecli_hostctrl(active_ctx->ctx, &hostctrl);
|
||||
if (rc < 0) {
|
||||
perror("xcecli_hostctrl");
|
||||
free(hostname);
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("Operation on host %s done\n", hostname);
|
||||
|
||||
free(hostname);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* upgradeconfig <cluster.bin> — submit a configuration upgrade
|
||||
* (config-upgrade-snapshot task 8.6). Reads the given capnp-encoded
|
||||
* ClusterConfig blob (the same format `xceadmin config build` produces and
|
||||
* deploy-ha.sh installs as cluster.bin), computes its SHA-256 (OpenSSL
|
||||
* SHA256(), matching config_repl.c's do_upgrade_config_with_bytes transit
|
||||
* verification), and drives it through xcecli_upgradeconfig ->
|
||||
* xce_upgradeconfig_req_raw (OP_upgradeConfig). Mirrors cmd_snapdiff's
|
||||
* bounds-checked file-reading style. */
|
||||
int cmd_upgradeconfig(int argc, char* argv[])
|
||||
{
|
||||
if (argc != 2) {
|
||||
fprintf(stderr, "usage: upgradeconfig <cluster.bin>\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char* path = argv[1];
|
||||
FILE* fp = fopen(path, "rb");
|
||||
if (fp == NULL) {
|
||||
fprintf(stderr, "upgradeconfig: cannot open %s: %s\n", path,
|
||||
strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (fseek(fp, 0, SEEK_END) != 0) {
|
||||
fprintf(stderr, "upgradeconfig: fseek failed on %s: %s\n", path,
|
||||
strerror(errno));
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
long fsz = ftell(fp);
|
||||
if (fsz <= 0) {
|
||||
fprintf(stderr, "upgradeconfig: %s is empty or ftell failed\n", path);
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
rewind(fp);
|
||||
|
||||
uint8_t* bytes = (uint8_t*)malloc((size_t)fsz);
|
||||
if (bytes == NULL) {
|
||||
fprintf(stderr, "upgradeconfig: out of memory reading %s\n", path);
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
size_t nread = fread(bytes, 1, (size_t)fsz, fp);
|
||||
fclose(fp);
|
||||
if (nread != (size_t)fsz) {
|
||||
fprintf(stderr, "upgradeconfig: short read on %s (%zu of %ld bytes)\n",
|
||||
path, nread, fsz);
|
||||
free(bytes);
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint8_t sha256[SHA256_DIGEST_LENGTH];
|
||||
SHA256(bytes, (size_t)fsz, sha256);
|
||||
|
||||
int rc = xcecli_upgradeconfig(active_ctx->ctx, sha256,
|
||||
sizeof(sha256), bytes, (size_t)fsz);
|
||||
free(bytes);
|
||||
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "upgradeconfig: failed (rc=%d): %s\n", rc,
|
||||
strerror(-rc));
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("Config upgrade submitted from %s (%ld bytes)\n", path, fsz);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmd_runtasks(int argc, char* argv[])
|
||||
{
|
||||
xce_taskhost_t hosts[MAX_TASK_HOSTS];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue