lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(config-upgrade-snapshot): live config-upgrade soak infra + fix bug-1468/bug-1469 (task 8.6)

Browse source 
Build the live 3-node chaotic-multi-kill config-upgrade soak scenario
(run-ha-soak.sh --config-upgrade) required by Requirements 4.1, 4.3, 6.1,
6.4. This needed genuinely new client-side infrastructure: a hand-rolled
capnp send path for OP_upgradeConfig (the auto-generated encoder is a
no-op for its Data fields, worked around the same way
xce_replconfigpush_req_raw already does), plus cmd_hostctrl/cmd_upgradeconfig
CLI commands.

Building and running this live surfaced two real production bugs, both
fixed here:

- bug-1468: an unrelated prior commit inverted OPCode_upgradeConfig's
  primary/standby dispatch guard in cluster.c, making config upgrades
  unreachable from any network client.
- bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never called
  config_activate_committed(), so the node that originates a config
  upgrade never refreshed its own in-memory state as part of committing
  it. Killing that node before some later, unrelated event happened to
  refresh it left its checkpoint permanently stale -- the first live,
  empirical reproduction of bug-857's originally hypothesized failure
  mode. Fixed by activating before the marker-triggered checkpoint write,
  since checkpoint content is built from live runtime tables that only
  reflect the reload after activation runs.

A separate, narrower checkpoint divergence (bug-1471, not config-related)
remains open and still blocks a clean soak PASS -- task 8.6 stays
in-progress until it's resolved.
This commit is contained in:
Rongsong Shen 2026-07-25 15:48:20 +00:00
parent 2bd44e8e4d
commit 959b0cdacb
13 changed files with 1022 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

149
src/controller/testsuite/test.c
View file

@ -15,6 +15,7 @@
#include <cjson/cJSON.h>
#include <cjson/s2j.h>
#include <openssl/sha.h> /* SHA256() — cmd_upgradeconfig hashes the config blob */
#include "client.h"
#include "config.h" /* XCE_MAXNAME_LEN */
@ -122,6 +123,8 @@ extern int cmd_resval(int, char*[]);
extern int cmd_jobkill(int, char*[]);
extern int cmd_jobckpt(int, char*[]);
extern int cmd_qctrl(int, char*[]);
extern int cmd_hostctrl(int, char*[]);
extern int cmd_upgradeconfig(int, char*[]);
extern int cmd_runtasks(int, char*[]);
extern int cmd_setpend(int, char*[]);
extern int cmd_jobinfo(int, char*[]);
@ -230,6 +233,9 @@ static command_t client_commands[] = {
{ "reslist", cmd_reslist, "Display cluster resources" },
{ "workerlist", cmd_workerlist, "list workers" },
{ "qctrl", cmd_qctrl, "queue control" },
{ "hostctrl", cmd_hostctrl, "host control" },
{ "upgradeconfig", cmd_upgradeconfig,
"submit a config upgrade (cluster.bin file)" },
{ "joblist", cmd_joblist, "list jobs" },
{ "expect-jobs", cmd_expect_jobs, "assert current job count" },
{ "expect-job", cmd_expect_job, "assert a JobId is present" },
@ -1020,7 +1026,15 @@ int cmd_reslist(int argc, char* argv[])
static char* worker_status_str(uint32_t status)
{
if ((status & XCE_WORKER_UNAVAIL) != 0) {
/* Admin-set CLOSED takes priority over the dynamic UNAVAIL bit: the
* config-upgrade-snapshot soak (task 8.6) needs to assert a host's
* administratively-set status specifically, and a host closed via
* hostctrl that later disconnects (e.g. its xce_test `worker`/`join`
* connection exits after the script ends) would otherwise print only
* "UNAVAIL", masking the admin bit this check exists to verify. */
if ((status & XCE_WORKER_CLOSED) != 0) {
return "CLOSED";
} else if ((status & XCE_WORKER_UNAVAIL) != 0) {
return "UNAVAIL";
} else if ((status & XCE_WORKER_LOCKEDU) != 0) {
return "LOCKU";
@ -2224,6 +2238,139 @@ int cmd_qctrl(int argc, char* argv[])
return 0;
}
/* hostctrl -o|-c -h <hostname> — administratively open/close a host
* (xce_hostctrl_t{action, hostname} via xcecli_hostctrl). Mirrors cmd_qctrl's
* exact getopt shape; only -o/-c are wired (matching xce_hostctrl_req's own
* accepted action range, request.c — HOST_TAG/HOST_UNTAG exist in the enum
* but are rejected -EINVAL server-side today, so no flag is offered for them
* here). Used by the config-upgrade chaotic soak (task 8.6) to set a host's
* admin status before/around a config upgrade and assert it survives. */
int cmd_hostctrl(int argc, char* argv[])
{
int opt;
int rc;
xce_hostctrl_t hostctrl;
int action = HOST_DO_NONE;
char* hostname = NULL;
optind = 1;
while ((opt = getopt(argc, argv, "och:")) != -1) {
switch (opt) {
case 'o':
action = HOST_OPEN;
break;
case 'c':
action = HOST_CLOSE;
break;
case 'h':
hostname = strdup(optarg);
break;
default:
break;
}
}
if (hostname == NULL) {
fprintf(stderr, "you need to specify host name with -h\n");
return -1;
}
if (action == HOST_DO_NONE) {
fprintf(stderr, "you need to specify -o (open) or -c (close)\n");
free(hostname);
return -1;
}
hostctrl.action = action;
hostctrl.hostname = hostname;
rc = xcecli_hostctrl(active_ctx->ctx, &hostctrl);
if (rc < 0) {
perror("xcecli_hostctrl");
free(hostname);
return rc;
}
printf("Operation on host %s done\n", hostname);
free(hostname);
return 0;
}
/* upgradeconfig <cluster.bin> — submit a configuration upgrade
* (config-upgrade-snapshot task 8.6). Reads the given capnp-encoded
* ClusterConfig blob (the same format `xceadmin config build` produces and
* deploy-ha.sh installs as cluster.bin), computes its SHA-256 (OpenSSL
* SHA256(), matching config_repl.c's do_upgrade_config_with_bytes transit
* verification), and drives it through xcecli_upgradeconfig ->
* xce_upgradeconfig_req_raw (OP_upgradeConfig). Mirrors cmd_snapdiff's
* bounds-checked file-reading style. */
int cmd_upgradeconfig(int argc, char* argv[])
{
if (argc != 2) {
fprintf(stderr, "usage: upgradeconfig <cluster.bin>\n");
return -1;
}
const char* path = argv[1];
FILE* fp = fopen(path, "rb");
if (fp == NULL) {
fprintf(stderr, "upgradeconfig: cannot open %s: %s\n", path,
strerror(errno));
return -1;
}
if (fseek(fp, 0, SEEK_END) != 0) {
fprintf(stderr, "upgradeconfig: fseek failed on %s: %s\n", path,
strerror(errno));
fclose(fp);
return -1;
}
long fsz = ftell(fp);
if (fsz <= 0) {
fprintf(stderr, "upgradeconfig: %s is empty or ftell failed\n", path);
fclose(fp);
return -1;
}
rewind(fp);
uint8_t* bytes = (uint8_t*)malloc((size_t)fsz);
if (bytes == NULL) {
fprintf(stderr, "upgradeconfig: out of memory reading %s\n", path);
fclose(fp);
return -1;
}
size_t nread = fread(bytes, 1, (size_t)fsz, fp);
fclose(fp);
if (nread != (size_t)fsz) {
fprintf(stderr, "upgradeconfig: short read on %s (%zu of %ld bytes)\n",
path, nread, fsz);
free(bytes);
return -1;
}
uint8_t sha256[SHA256_DIGEST_LENGTH];
SHA256(bytes, (size_t)fsz, sha256);
int rc = xcecli_upgradeconfig(active_ctx->ctx, sha256,
sizeof(sha256), bytes, (size_t)fsz);
free(bytes);
if (rc < 0) {
fprintf(stderr, "upgradeconfig: failed (rc=%d): %s\n", rc,
strerror(-rc));
return rc;
}
printf("Config upgrade submitted from %s (%ld bytes)\n", path, fsz);
return 0;
}
int cmd_runtasks(int argc, char* argv[])
{
xce_taskhost_t hosts[MAX_TASK_HOSTS];

169
src/controller/testsuite/test_config_repl.c
View file

@ -1819,9 +1819,25 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
uint64_t tagN; { uint64_t t; memcpy(&t, shaN, 8); tagN = be64toh(t); }
TEST_ASSERT_FALSE(memcmp(shaO, shaN, 32) == 0);
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances). */
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances).
* bug-1469 fix interaction: flushing makes durable_lsn advance, and this
* single-member fixture's commit engine (majority=1) reacts to ANY
* durable advance by racing committed_lsn forward on its OWN background
* fsync-worker thread (commit_on_durable_advance) -- which, since
* apply_committed_entry's WAL_CONFIG_MARKER case now ALSO activates
* config (bug-1469), could apply-and-activate this NEW marker for real
* before this test's own thread gets a chance to pin committed_lsn back
* down below it. PIN IMMEDIATELY, right here, before any other work
* (previously this pin lived after conf_save_config below, leaving a
* multi-syscall window — several real fsyncs — for that background race
* to win against a single-member fixture's over-eager auto-commit,
* something a real multi-node standby never does since its
* committed_lsn is solely driven by the primary's commit-advance, never
* local durability). */
uint64_t new_mlsn = t43_append_marker_uncommitted(c, tagN, shaN);
TEST_ASSERT_TRUE(new_mlsn > committed_at_old);
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
memory_order_release);
/* The OLD config is currently LIVE (activated in upgrade 1). Capture the live
* decoded-config pointer as the activation witness: load_primary_config frees
@ -1837,9 +1853,10 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
* commit barrier, which would advance committed_lsn to the durable WAL tip, and
* it bumps the advertised config_version_tag to tagN (this is the very reason
* the primary's config-behind re-push net does not re-fire — Finding D); we
* save FIRST, then PIN committed_lsn back below the NEW marker to model a
* standby whose committed prefix (driven by the primary's commit-advance, not
* local durability) has NOT yet advanced past the NEW marker. */
* already pinned committed_lsn back below the NEW marker immediately above (to
* model a standby whose committed prefix, driven by the primary's commit-
* advance, not local durability, has NOT yet advanced past the NEW marker) —
* re-pin again here for defense-in-depth against the same race. */
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesN, blenN, shaN));
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
memory_order_release);
@ -2329,6 +2346,147 @@ static void test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss(voi
printf("test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss: PASS\n");
}
/* ── bug-1469 regression: WAL_CONFIG_MARKER apply must ACTIVATE config on
* the ORIGINATING member, not just receiving standbys ──────────────────────
*
* Live 3-node chaotic-kill soak (config-upgrade-snapshot task 8.6,
* .wolf/buglog.json bug-1469): a config upgrade's WAL_CONFIG_MARKER commits
* and even checkpoints successfully on the ORIGINATING member (the primary
* that submitted the upgrade via do_upgrade_config_with_bytes), but
* apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) never called
* config_activate_committed()/load_primary_config() -- so that member's
* c->myconf stayed parsed from the OLD config bytes until some LATER,
* independent event (a restart, or an incoming out-of-band
* handle_repl_config_push) happened to refresh it. The live soak killed the
* originating primary shortly after it committed the upgrade, before any
* such later event, freezing a stale c->myconf into its last checkpoint --
* a genuine cross-member materialized-state divergence (ctrl-1's checkpoint
* ended up a different byte size than ctrl-2/ctrl-3's, both LSN 11).
* config_activate_committed WAS already being called correctly on
* RECEIVING standbys, via handle_repl_config_push -- this gap is specific
* to the member that originates/commits the marker on itself.
*/
/* Build a valid ClusterConfig blob distinguishable by config_generation +
* clustername (mirrors t43_make_config_blob, parameterized so this test can
* commit two genuinely different, independently-decodable configs -- an
* "old" pre-upgrade config and a "new" post-upgrade config -- without
* relying on a byte-mutation of an opaque capnp blob, which risks
* corrupting the encoding; config-upgrade-snapshot's own standing gotcha
* notes t43_make_config_blob() returns byte-IDENTICAL content on repeat
* calls, so a genuinely different SECOND blob needs its own encode call). */
static uint8_t *t1469_make_config_blob(uint64_t gen, const char *name,
size_t *len_out)
{
cluster_config_t cfg;
struct capn cap;
struct capn_segment *cs;
ClusterConfig_ptr p;
uint8_t *buf;
size_t sz;
memset(&cfg, 0, sizeof(cfg));
cfg.clustername = (char *)name;
cfg.config_generation = gen;
capn_init_malloc(&cap);
cs = capn_root(&cap).seg;
encode_ClusterConfig_ptr(cs, &p, &cfg);
capn_setp(capn_root(&cap), 0, p.p);
sz = (size_t)capn_size(&cap);
buf = (uint8_t *)malloc(sz);
TEST_ASSERT_NOT_NULL(buf);
capn_write_mem(&cap, buf, sz, 0);
capn_free(&cap);
*len_out = sz;
return buf;
}
/* This test drives the upgrade marker through the REAL production apply
* path (config_append_marker + commit_submit + wait-for-applied, exactly
* like test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss
* above) on a single-member fixture acting as its OWN primary -- i.e. the
* ORIGINATING member applying its own committed marker, never receiving an
* out-of-band handle_repl_config_push. It NEVER calls
* config_activate_committed() for the upgrade marker itself, so the only
* way mc->cluster_config can reflect the new config is via
* apply_committed_entry's own WAL_CONFIG_MARKER dispatch.
*
* RED (before the fix): mc->cluster_config stays pinned to the OLD config
* (config_generation==1, "t1469-old") after the new marker's real,
* successful commit+apply -- proving the marker-triggered dispatch never
* activates config on the member that originated it.
* GREEN (after the fix): mc->cluster_config reflects the NEW config
* (config_generation==2, "t1469-new") immediately after the marker's apply
* returns, with no restart and no out-of-band push. */
static void test_wal_config_marker_apply_activates_originating_member_config(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cr1469_activate_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/69, /*n_peers*/1);
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, 1, 8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
myconf_t *mc = t43_attach_myconf(c, tmp);
/* Establish the pre-upgrade steady state: an OLD config already
* committed and ACTIVATED via the legitimate, already-tested
* config_activate_committed path (mirrors 4.3-a above) -- this member
* is already running under config_generation=1 before any upgrade is
* ever submitted, exactly like a real cluster member. */
size_t oldlen = 0;
uint8_t *oldbytes = t1469_make_config_blob(1, "t1469-old", &oldlen);
uint8_t oldsha[32];
SHA256(oldbytes, oldlen, oldsha);
uint64_t oldtag; { uint64_t t; memcpy(&t, oldsha, 8); oldtag = be64toh(t); }
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, oldbytes, oldlen, oldsha));
free(oldbytes);
t43_commit_marker(c, oldtag, oldsha);
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
TEST_ASSERT_NOT_NULL(mc->cluster_config);
TEST_ASSERT_EQUAL_UINT64(1, mc->cluster_config->config_generation);
/* Now the upgrade: NEW config bytes saved locally exactly as
* do_upgrade_config_with_bytes's own step 5 (conf_save_config) does on
* the ORIGINATING primary, then a NEW marker committed through the
* exact same real path bug-1469's live soak exercised. Deliberately
* NEVER calls config_activate_committed() for this marker anywhere
* below -- activation must come from apply_committed_entry's own
* dispatch, or this assertion fails. */
size_t newlen = 0;
uint8_t *newbytes = t1469_make_config_blob(2, "t1469-new", &newlen);
uint8_t newsha[32];
SHA256(newbytes, newlen, newsha);
uint64_t newtag; { uint64_t t; memcpy(&t, newsha, 8); newtag = be64toh(t); }
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, newbytes, newlen, newsha));
free(newbytes);
t43_commit_marker(c, newtag, newsha);
/* THE ASSERTION THAT PROVES THE FIX: mc->cluster_config must now
* reflect the NEW config -- purely from apply_committed_entry's own
* WAL_CONFIG_MARKER dispatch, exactly the production path the
* ORIGINATING primary runs on itself. Before the fix this stays pinned
* at generation 1 ("t1469-old") forever -- exactly bug-1469's observed
* cross-member divergence, reproduced at the unit level. */
TEST_ASSERT_NOT_NULL(mc->cluster_config);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, mc->cluster_config->config_generation,
"bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never "
"activated the new config on the ORIGINATING member -- c->myconf "
"stayed pinned to the OLD config");
TEST_ASSERT_EQUAL_STRING("t1469-new", mc->cluster_config->clustername);
TEST_ASSERT_EQUAL_HEX64(newtag, mc->config_version_tag);
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
t43_free_myconf(c);
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_wal_config_marker_apply_activates_originating_member_config: PASS\n");
}
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
#ifdef XCE_TESTING
@ -5144,6 +5302,9 @@ void test_config_repl(void)
* production trigger (apply_committed_entry's WAL_CONFIG_MARKER case),
* not a direct test call -- proves the fence is reachable at all. */
RUN_TEST(test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss);
/* bug-1469: WAL_CONFIG_MARKER apply must activate config on the
* ORIGINATING member too, not just receiving standbys. */
RUN_TEST(test_wal_config_marker_apply_activates_originating_member_config);
#ifdef XCE_TESTING
RUN_TEST(test_load_primary_config_missing_returns_error);
RUN_TEST(test_wait_all_with_timeout_slow_threads);