feat(config-upgrade-snapshot): live config-upgrade soak infra + fix bug-1468/bug-1469 (task 8.6)
Build the live 3-node chaotic-multi-kill config-upgrade soak scenario (run-ha-soak.sh --config-upgrade) required by Requirements 4.1, 4.3, 6.1, 6.4. This needed genuinely new client-side infrastructure: a hand-rolled capnp send path for OP_upgradeConfig (the auto-generated encoder is a no-op for its Data fields, worked around the same way xce_replconfigpush_req_raw already does), plus cmd_hostctrl/cmd_upgradeconfig CLI commands. Building and running this live surfaced two real production bugs, both fixed here: - bug-1468: an unrelated prior commit inverted OPCode_upgradeConfig's primary/standby dispatch guard in cluster.c, making config upgrades unreachable from any network client. - bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never called config_activate_committed(), so the node that originates a config upgrade never refreshed its own in-memory state as part of committing it. Killing that node before some later, unrelated event happened to refresh it left its checkpoint permanently stale -- the first live, empirical reproduction of bug-857's originally hypothesized failure mode. Fixed by activating before the marker-triggered checkpoint write, since checkpoint content is built from live runtime tables that only reflect the reload after activation runs. A separate, narrower checkpoint divergence (bug-1471, not config-related) remains open and still blocks a clean soak PASS -- task 8.6 stays in-progress until it's resolved.
This commit is contained in:
parent
2bd44e8e4d
commit
959b0cdacb
13 changed files with 1022 additions and 10 deletions
|
|
@ -1819,9 +1819,25 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
|
|||
uint64_t tagN; { uint64_t t; memcpy(&t, shaN, 8); tagN = be64toh(t); }
|
||||
TEST_ASSERT_FALSE(memcmp(shaO, shaN, 32) == 0);
|
||||
|
||||
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances). */
|
||||
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances).
|
||||
* bug-1469 fix interaction: flushing makes durable_lsn advance, and this
|
||||
* single-member fixture's commit engine (majority=1) reacts to ANY
|
||||
* durable advance by racing committed_lsn forward on its OWN background
|
||||
* fsync-worker thread (commit_on_durable_advance) -- which, since
|
||||
* apply_committed_entry's WAL_CONFIG_MARKER case now ALSO activates
|
||||
* config (bug-1469), could apply-and-activate this NEW marker for real
|
||||
* before this test's own thread gets a chance to pin committed_lsn back
|
||||
* down below it. PIN IMMEDIATELY, right here, before any other work
|
||||
* (previously this pin lived after conf_save_config below, leaving a
|
||||
* multi-syscall window — several real fsyncs — for that background race
|
||||
* to win against a single-member fixture's over-eager auto-commit,
|
||||
* something a real multi-node standby never does since its
|
||||
* committed_lsn is solely driven by the primary's commit-advance, never
|
||||
* local durability). */
|
||||
uint64_t new_mlsn = t43_append_marker_uncommitted(c, tagN, shaN);
|
||||
TEST_ASSERT_TRUE(new_mlsn > committed_at_old);
|
||||
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
|
||||
memory_order_release);
|
||||
|
||||
/* The OLD config is currently LIVE (activated in upgrade 1). Capture the live
|
||||
* decoded-config pointer as the activation witness: load_primary_config frees
|
||||
|
|
@ -1837,9 +1853,10 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
|
|||
* commit barrier, which would advance committed_lsn to the durable WAL tip, and
|
||||
* it bumps the advertised config_version_tag to tagN (this is the very reason
|
||||
* the primary's config-behind re-push net does not re-fire — Finding D); we
|
||||
* save FIRST, then PIN committed_lsn back below the NEW marker to model a
|
||||
* standby whose committed prefix (driven by the primary's commit-advance, not
|
||||
* local durability) has NOT yet advanced past the NEW marker. */
|
||||
* already pinned committed_lsn back below the NEW marker immediately above (to
|
||||
* model a standby whose committed prefix, driven by the primary's commit-
|
||||
* advance, not local durability, has NOT yet advanced past the NEW marker) —
|
||||
* re-pin again here for defense-in-depth against the same race. */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesN, blenN, shaN));
|
||||
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
|
||||
memory_order_release);
|
||||
|
|
@ -2329,6 +2346,147 @@ static void test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss(voi
|
|||
printf("test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1469 regression: WAL_CONFIG_MARKER apply must ACTIVATE config on
|
||||
* the ORIGINATING member, not just receiving standbys ──────────────────────
|
||||
*
|
||||
* Live 3-node chaotic-kill soak (config-upgrade-snapshot task 8.6,
|
||||
* .wolf/buglog.json bug-1469): a config upgrade's WAL_CONFIG_MARKER commits
|
||||
* and even checkpoints successfully on the ORIGINATING member (the primary
|
||||
* that submitted the upgrade via do_upgrade_config_with_bytes), but
|
||||
* apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) never called
|
||||
* config_activate_committed()/load_primary_config() -- so that member's
|
||||
* c->myconf stayed parsed from the OLD config bytes until some LATER,
|
||||
* independent event (a restart, or an incoming out-of-band
|
||||
* handle_repl_config_push) happened to refresh it. The live soak killed the
|
||||
* originating primary shortly after it committed the upgrade, before any
|
||||
* such later event, freezing a stale c->myconf into its last checkpoint --
|
||||
* a genuine cross-member materialized-state divergence (ctrl-1's checkpoint
|
||||
* ended up a different byte size than ctrl-2/ctrl-3's, both LSN 11).
|
||||
* config_activate_committed WAS already being called correctly on
|
||||
* RECEIVING standbys, via handle_repl_config_push -- this gap is specific
|
||||
* to the member that originates/commits the marker on itself.
|
||||
*/
|
||||
|
||||
/* Build a valid ClusterConfig blob distinguishable by config_generation +
|
||||
* clustername (mirrors t43_make_config_blob, parameterized so this test can
|
||||
* commit two genuinely different, independently-decodable configs -- an
|
||||
* "old" pre-upgrade config and a "new" post-upgrade config -- without
|
||||
* relying on a byte-mutation of an opaque capnp blob, which risks
|
||||
* corrupting the encoding; config-upgrade-snapshot's own standing gotcha
|
||||
* notes t43_make_config_blob() returns byte-IDENTICAL content on repeat
|
||||
* calls, so a genuinely different SECOND blob needs its own encode call). */
|
||||
static uint8_t *t1469_make_config_blob(uint64_t gen, const char *name,
|
||||
size_t *len_out)
|
||||
{
|
||||
cluster_config_t cfg;
|
||||
struct capn cap;
|
||||
struct capn_segment *cs;
|
||||
ClusterConfig_ptr p;
|
||||
uint8_t *buf;
|
||||
size_t sz;
|
||||
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.clustername = (char *)name;
|
||||
cfg.config_generation = gen;
|
||||
|
||||
capn_init_malloc(&cap);
|
||||
cs = capn_root(&cap).seg;
|
||||
encode_ClusterConfig_ptr(cs, &p, &cfg);
|
||||
capn_setp(capn_root(&cap), 0, p.p);
|
||||
sz = (size_t)capn_size(&cap);
|
||||
buf = (uint8_t *)malloc(sz);
|
||||
TEST_ASSERT_NOT_NULL(buf);
|
||||
capn_write_mem(&cap, buf, sz, 0);
|
||||
capn_free(&cap);
|
||||
*len_out = sz;
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* This test drives the upgrade marker through the REAL production apply
|
||||
* path (config_append_marker + commit_submit + wait-for-applied, exactly
|
||||
* like test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss
|
||||
* above) on a single-member fixture acting as its OWN primary -- i.e. the
|
||||
* ORIGINATING member applying its own committed marker, never receiving an
|
||||
* out-of-band handle_repl_config_push. It NEVER calls
|
||||
* config_activate_committed() for the upgrade marker itself, so the only
|
||||
* way mc->cluster_config can reflect the new config is via
|
||||
* apply_committed_entry's own WAL_CONFIG_MARKER dispatch.
|
||||
*
|
||||
* RED (before the fix): mc->cluster_config stays pinned to the OLD config
|
||||
* (config_generation==1, "t1469-old") after the new marker's real,
|
||||
* successful commit+apply -- proving the marker-triggered dispatch never
|
||||
* activates config on the member that originated it.
|
||||
* GREEN (after the fix): mc->cluster_config reflects the NEW config
|
||||
* (config_generation==2, "t1469-new") immediately after the marker's apply
|
||||
* returns, with no restart and no out-of-band push. */
|
||||
static void test_wal_config_marker_apply_activates_originating_member_config(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr1469_activate_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/69, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Establish the pre-upgrade steady state: an OLD config already
|
||||
* committed and ACTIVATED via the legitimate, already-tested
|
||||
* config_activate_committed path (mirrors 4.3-a above) -- this member
|
||||
* is already running under config_generation=1 before any upgrade is
|
||||
* ever submitted, exactly like a real cluster member. */
|
||||
size_t oldlen = 0;
|
||||
uint8_t *oldbytes = t1469_make_config_blob(1, "t1469-old", &oldlen);
|
||||
uint8_t oldsha[32];
|
||||
SHA256(oldbytes, oldlen, oldsha);
|
||||
uint64_t oldtag; { uint64_t t; memcpy(&t, oldsha, 8); oldtag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, oldbytes, oldlen, oldsha));
|
||||
free(oldbytes);
|
||||
t43_commit_marker(c, oldtag, oldsha);
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, mc->cluster_config->config_generation);
|
||||
|
||||
/* Now the upgrade: NEW config bytes saved locally exactly as
|
||||
* do_upgrade_config_with_bytes's own step 5 (conf_save_config) does on
|
||||
* the ORIGINATING primary, then a NEW marker committed through the
|
||||
* exact same real path bug-1469's live soak exercised. Deliberately
|
||||
* NEVER calls config_activate_committed() for this marker anywhere
|
||||
* below -- activation must come from apply_committed_entry's own
|
||||
* dispatch, or this assertion fails. */
|
||||
size_t newlen = 0;
|
||||
uint8_t *newbytes = t1469_make_config_blob(2, "t1469-new", &newlen);
|
||||
uint8_t newsha[32];
|
||||
SHA256(newbytes, newlen, newsha);
|
||||
uint64_t newtag; { uint64_t t; memcpy(&t, newsha, 8); newtag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, newbytes, newlen, newsha));
|
||||
free(newbytes);
|
||||
t43_commit_marker(c, newtag, newsha);
|
||||
|
||||
/* THE ASSERTION THAT PROVES THE FIX: mc->cluster_config must now
|
||||
* reflect the NEW config -- purely from apply_committed_entry's own
|
||||
* WAL_CONFIG_MARKER dispatch, exactly the production path the
|
||||
* ORIGINATING primary runs on itself. Before the fix this stays pinned
|
||||
* at generation 1 ("t1469-old") forever -- exactly bug-1469's observed
|
||||
* cross-member divergence, reproduced at the unit level. */
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, mc->cluster_config->config_generation,
|
||||
"bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never "
|
||||
"activated the new config on the ORIGINATING member -- c->myconf "
|
||||
"stayed pinned to the OLD config");
|
||||
TEST_ASSERT_EQUAL_STRING("t1469-new", mc->cluster_config->clustername);
|
||||
TEST_ASSERT_EQUAL_HEX64(newtag, mc->config_version_tag);
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_wal_config_marker_apply_activates_originating_member_config: PASS\n");
|
||||
}
|
||||
|
||||
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
@ -5144,6 +5302,9 @@ void test_config_repl(void)
|
|||
* production trigger (apply_committed_entry's WAL_CONFIG_MARKER case),
|
||||
* not a direct test call -- proves the fence is reachable at all. */
|
||||
RUN_TEST(test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss);
|
||||
/* bug-1469: WAL_CONFIG_MARKER apply must activate config on the
|
||||
* ORIGINATING member too, not just receiving standbys. */
|
||||
RUN_TEST(test_wal_config_marker_apply_activates_originating_member_config);
|
||||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_load_primary_config_missing_returns_error);
|
||||
RUN_TEST(test_wait_all_with_timeout_slow_threads);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue