feat(config-upgrade-snapshot): live config-upgrade soak infra + fix bug-1468/bug-1469 (task 8.6)
Build the live 3-node chaotic-multi-kill config-upgrade soak scenario (run-ha-soak.sh --config-upgrade) required by Requirements 4.1, 4.3, 6.1, 6.4. This needed genuinely new client-side infrastructure: a hand-rolled capnp send path for OP_upgradeConfig (the auto-generated encoder is a no-op for its Data fields, worked around the same way xce_replconfigpush_req_raw already does), plus cmd_hostctrl/cmd_upgradeconfig CLI commands. Building and running this live surfaced two real production bugs, both fixed here: - bug-1468: an unrelated prior commit inverted OPCode_upgradeConfig's primary/standby dispatch guard in cluster.c, making config upgrades unreachable from any network client. - bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never called config_activate_committed(), so the node that originates a config upgrade never refreshed its own in-memory state as part of committing it. Killing that node before some later, unrelated event happened to refresh it left its checkpoint permanently stale -- the first live, empirical reproduction of bug-857's originally hypothesized failure mode. Fixed by activating before the marker-triggered checkpoint write, since checkpoint content is built from live runtime tables that only reflect the reload after activation runs. A separate, narrower checkpoint divergence (bug-1471, not config-related) remains open and still blocks a clean soak PASS -- task 8.6 stays in-progress until it's resolved.
This commit is contained in:
parent
2bd44e8e4d
commit
959b0cdacb
13 changed files with 1022 additions and 10 deletions
|
|
@ -669,10 +669,30 @@ static int myreq_handler(vtask_t* task)
|
|||
break;
|
||||
case OPCode_upgradeConfig:
|
||||
#if CONF_ENABLE_UPGRADE_CONFIG
|
||||
/* OPCode_upgradeConfig is sent BY the primary TO standbys to push config.
|
||||
* Standbys must apply it; only a primary sending this to itself is invalid. */
|
||||
if (ha_is_primary(&c->ha)) {
|
||||
/* Primary should not receive its own config push — ignore */
|
||||
/* OPCode_upgradeConfig is sent BY an admin client (or xceadmin's
|
||||
* config-push tool) TO the primary controller, which decodes the
|
||||
* blob, verifies it, and drives it through config_commit_change()
|
||||
* (the (term,lsn) marker consensus path) — see
|
||||
* do_upgrade_config_with_bytes's doc comment, config_repl.c. Only the
|
||||
* primary can commit a WAL entry, so a standby cannot service this
|
||||
* request; ha_dispatch_gate already denies OPCode_upgradeConfig for
|
||||
* every non-primary role (it is absent from the CTRL_STANDBY /
|
||||
* CTRL_INIT / CTRL_ELECTING allow-lists, ha.c), so this check is a
|
||||
* defensive belt-and-suspenders guard, not the primary gate.
|
||||
*
|
||||
* bug-fix (2026-07-25): this check was inverted from
|
||||
* commit fd93997f ("election-iomgr-refactor", 2026-05-31) onward —
|
||||
* it rejected the request WHEN primary, unconditionally reachable
|
||||
* only on primary (per the dispatch-gate note above), making
|
||||
* handle_upgrade_config() permanently unreachable from any network
|
||||
* caller (only unit tests calling it directly ever exercised it).
|
||||
* Independently diagnosed and documented as a blocking risk by the
|
||||
* admin-config-tool spec (design.md "Boundary Commitments" /
|
||||
* "Risks") before config-upgrade-snapshot task 8.6 needed a live
|
||||
* OP_upgradeConfig sender and hit the same wall. Restored to the
|
||||
* pre-regression (correct) polarity. */
|
||||
if (!ha_is_primary(&c->ha)) {
|
||||
/* Standby cannot commit a config change — reject loud. */
|
||||
xce_reply_t ureply;
|
||||
init_reply_by_req(&ureply, &request);
|
||||
ureply.status = -EAGAIN;
|
||||
|
|
|
|||
|
|
@ -56,6 +56,20 @@ extern int restore_host_state(cluster_t* c, const char* name,
|
|||
* initialized default. */
|
||||
extern int upgrade_materialization_verify(cluster_t* c);
|
||||
|
||||
/* config_activate_committed defined in config_repl.c (config-upgrade-snapshot
|
||||
* task 5/6.2 remediation, bug-1469, Req 1.8/3.2/3.4) -- re-parses the
|
||||
* locally-held config bytes referenced by the latest committed
|
||||
* WAL_CONFIG_MARKER into c->myconf (load_primary_config), hash-verifying
|
||||
* them first. Called unconditionally from apply_committed_entry's
|
||||
* WAL_CONFIG_MARKER case below, BEFORE upgrade_materialization_verify: every
|
||||
* member's own live apply of a marker it just committed must refresh its
|
||||
* own c->myconf, not only members that separately receive an out-of-band
|
||||
* handle_repl_config_push (see the call site's own doc comment for the full
|
||||
* rationale). Declared here via extern, mirroring upgrade_materialization_
|
||||
* verify just above, rather than pulling in config_repl.h's full include
|
||||
* surface for one symbol. */
|
||||
extern int config_activate_committed(cluster_t* c);
|
||||
|
||||
/* raft-apply-decoupling task 3.2: pending command-apply hand-off list node
|
||||
* (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is
|
||||
* the consumer). Declared here (rather than persist.h) so the list stays
|
||||
|
|
@ -3807,6 +3821,102 @@ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn,
|
|||
* checkpoint. `lsn` (this function's own parameter) is
|
||||
* already the marker's exact position, so it is passed
|
||||
* through explicitly instead of re-derived. */
|
||||
/* bug-1469 fix (live HA soak, config-upgrade-snapshot task 8.6):
|
||||
* unconditionally re-run config activation right here, BEFORE the
|
||||
* forced checkpoint below, so EVERY member -- primary, standby, and
|
||||
* any future newly-elected leader -- is guaranteed to have
|
||||
* refreshed c->myconf (and the runtime queue/host tables
|
||||
* load_primary_config provisions from it) from disk BEFORE the
|
||||
* checkpoint writer walks that same runtime state to build the
|
||||
* queue-entity list. Before this call existed at all,
|
||||
* config_activate_committed() was reachable from exactly two
|
||||
* places: handle_repl_config_push (a STANDBY receiving a push) and
|
||||
* the boot/recovery path -- never from a member's own live apply of
|
||||
* the marker it just committed. A live 3-node chaotic-kill soak
|
||||
* caught the gap empirically: the primary that ORIGINATED a config
|
||||
* upgrade (do_upgrade_config_with_bytes, which saves the new bytes
|
||||
* to disk and bumps config_generation but never re-activates them
|
||||
* into c->myconf on itself) forced a checkpoint at the marker's own
|
||||
* LSN whose queue-entity section still reflected the OLD
|
||||
* (pre-upgrade) queue set, then was killed before any LATER,
|
||||
* independent event (restart, or a racing inbound push) happened to
|
||||
* call config_activate_committed -- freezing a stale
|
||||
* queue-entity list into that checkpoint forever (a real
|
||||
* cross-member materialized-state divergence,
|
||||
* .wolf/buglog.json bug-1469's byte-level trace: the checkpoint's
|
||||
* embedded raw config-bytes section correctly carried the new
|
||||
* bytes, but the SEPARATE queue-entity list section -- built by
|
||||
* walking runtime state, not the raw bytes -- did not, because that
|
||||
* runtime state was never refreshed before the write).
|
||||
*
|
||||
* MUST run BEFORE persist_checkpoint_marker_triggered(), not merely
|
||||
* before upgrade_materialization_verify: the checkpoint writer
|
||||
* (write_snapshot -> collect_queue_frames/collect_host_frames)
|
||||
* builds its queue/host-entity sections by walking c->qmgr/
|
||||
* c->workermgr -- LIVE RUNTIME TABLES, populated at boot from
|
||||
* c->myconf->cluster_config and never otherwise re-synced from a
|
||||
* later config change on this call path -- not by re-parsing
|
||||
* mc->cluster_config directly. So activation must settle BEFORE
|
||||
* persist_checkpoint_marker_triggered() runs, or the checkpoint
|
||||
* this marker forces still captures whatever runtime state existed
|
||||
* prior to activation, regardless of how soon afterward activation
|
||||
* runs. (This ordering constraint is logically necessary given
|
||||
* write_snapshot's own read path, confirmed by reading
|
||||
* collect_queue_frames/write_snapshot directly -- not yet
|
||||
* separately confirmed against a live soak re-run at the time this
|
||||
* comment was written; see the task's own final live-soak
|
||||
* verification step for that.)
|
||||
*
|
||||
* Symmetric with task 6.2's own fix shape below: one unconditional
|
||||
* call at the single apply seam every member's marker-apply is
|
||||
* guaranteed to pass through, rather than patching only the
|
||||
* originating code path.
|
||||
*
|
||||
* Safe to call here for the same reason task 6.2's own call
|
||||
* (upgrade_materialization_verify, below) is safe at this exact
|
||||
* seam: neither config_activate_committed nor anything it
|
||||
* transitively calls (config_committed_marker, local_config_sha256,
|
||||
* conf_read_current_filename, conf_read_raw, load_primary_config)
|
||||
* acquires apply_pump_lock, p->apply_pump_lock, or any other lock
|
||||
* (verified -- config_repl.c's only two mutexes, c->ha.lock and
|
||||
* c->persist.lock, are both taken in unrelated functions never on
|
||||
* this call path) -- so no reentrant-lock hazard even though this
|
||||
* runs while persist_apply_pump_run holds apply_pump_lock across
|
||||
* its entire dispatch. The disk I/O this performs (WAL segment
|
||||
* scan, config file read, Cap'n Proto decode) is the same class
|
||||
* already unconditionally performed at this exact call site by
|
||||
* upgrade_materialization_verify below (config_committed_marker's
|
||||
* own scan + a config-bytes read/hash), so this adds no new
|
||||
* blocking-I/O hazard class. Return value intentionally ignored
|
||||
* (mirrors upgrade_materialization_verify below): a
|
||||
* mismatch/absence return (-EAGAIN/-ENOENT) means this member's own
|
||||
* serve-fence (config_serve_fenced, a SEPARATE mechanism from the
|
||||
* Activation Fence below) is set instead, and that fence already
|
||||
* has its own established retry path (config_schedule_activate_retry)
|
||||
* driven off commit-advance -- this call's only job here is to give
|
||||
* every marker-apply one unconditional attempt so a member never
|
||||
* depends solely on a later, independent event to activate.
|
||||
*
|
||||
* Guarded on c->myconf != NULL: in production myconf is always
|
||||
* wired during cluster init, well before any commit/apply can run,
|
||||
* so this is a no-op check on every real code path -- but
|
||||
* config_activate_committed's own -ENOENT fallback calls
|
||||
* load_primary_config(c) unconditionally on c->myconf without its
|
||||
* own null check (pre-existing in config_repl.c, out of this fix's
|
||||
* scope to touch), and this call site is reachable from a
|
||||
* bare/otherwise-empty test cluster_t that exercises
|
||||
* WAL_CONFIG_MARKER apply mechanics without wiring myconf
|
||||
* (test_upgrade_snapshot_apply.c's tusa_t fixture,
|
||||
* test_marker_apply_via_live_pump_checkpoints_at_marker_own_lsn --
|
||||
* confirmed via a live SEGV in load_primary_config, address 0x28,
|
||||
* before this guard was added) -- mirrors this same file's existing
|
||||
* tolerance of a bare fixture elsewhere (e.g. collect_queue_frames'
|
||||
* `c->qmgr == NULL` check, read_active_config_bytes' "no live
|
||||
* cluster back-pointer" handling) rather than crashing a test that
|
||||
* predates and is unrelated to config activation. */
|
||||
if (c->myconf != NULL)
|
||||
(void)config_activate_committed(c);
|
||||
|
||||
int rc;
|
||||
#ifdef XCE_TESTING
|
||||
if (persist_marker_checkpoint_fault_hook != NULL &&
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include <cjson/cJSON.h>
|
||||
#include <cjson/s2j.h>
|
||||
#include <openssl/sha.h> /* SHA256() — cmd_upgradeconfig hashes the config blob */
|
||||
|
||||
#include "client.h"
|
||||
#include "config.h" /* XCE_MAXNAME_LEN */
|
||||
|
|
@ -122,6 +123,8 @@ extern int cmd_resval(int, char*[]);
|
|||
extern int cmd_jobkill(int, char*[]);
|
||||
extern int cmd_jobckpt(int, char*[]);
|
||||
extern int cmd_qctrl(int, char*[]);
|
||||
extern int cmd_hostctrl(int, char*[]);
|
||||
extern int cmd_upgradeconfig(int, char*[]);
|
||||
extern int cmd_runtasks(int, char*[]);
|
||||
extern int cmd_setpend(int, char*[]);
|
||||
extern int cmd_jobinfo(int, char*[]);
|
||||
|
|
@ -230,6 +233,9 @@ static command_t client_commands[] = {
|
|||
{ "reslist", cmd_reslist, "Display cluster resources" },
|
||||
{ "workerlist", cmd_workerlist, "list workers" },
|
||||
{ "qctrl", cmd_qctrl, "queue control" },
|
||||
{ "hostctrl", cmd_hostctrl, "host control" },
|
||||
{ "upgradeconfig", cmd_upgradeconfig,
|
||||
"submit a config upgrade (cluster.bin file)" },
|
||||
{ "joblist", cmd_joblist, "list jobs" },
|
||||
{ "expect-jobs", cmd_expect_jobs, "assert current job count" },
|
||||
{ "expect-job", cmd_expect_job, "assert a JobId is present" },
|
||||
|
|
@ -1020,7 +1026,15 @@ int cmd_reslist(int argc, char* argv[])
|
|||
|
||||
static char* worker_status_str(uint32_t status)
|
||||
{
|
||||
if ((status & XCE_WORKER_UNAVAIL) != 0) {
|
||||
/* Admin-set CLOSED takes priority over the dynamic UNAVAIL bit: the
|
||||
* config-upgrade-snapshot soak (task 8.6) needs to assert a host's
|
||||
* administratively-set status specifically, and a host closed via
|
||||
* hostctrl that later disconnects (e.g. its xce_test `worker`/`join`
|
||||
* connection exits after the script ends) would otherwise print only
|
||||
* "UNAVAIL", masking the admin bit this check exists to verify. */
|
||||
if ((status & XCE_WORKER_CLOSED) != 0) {
|
||||
return "CLOSED";
|
||||
} else if ((status & XCE_WORKER_UNAVAIL) != 0) {
|
||||
return "UNAVAIL";
|
||||
} else if ((status & XCE_WORKER_LOCKEDU) != 0) {
|
||||
return "LOCKU";
|
||||
|
|
@ -2224,6 +2238,139 @@ int cmd_qctrl(int argc, char* argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* hostctrl -o|-c -h <hostname> — administratively open/close a host
|
||||
* (xce_hostctrl_t{action, hostname} via xcecli_hostctrl). Mirrors cmd_qctrl's
|
||||
* exact getopt shape; only -o/-c are wired (matching xce_hostctrl_req's own
|
||||
* accepted action range, request.c — HOST_TAG/HOST_UNTAG exist in the enum
|
||||
* but are rejected -EINVAL server-side today, so no flag is offered for them
|
||||
* here). Used by the config-upgrade chaotic soak (task 8.6) to set a host's
|
||||
* admin status before/around a config upgrade and assert it survives. */
|
||||
int cmd_hostctrl(int argc, char* argv[])
|
||||
{
|
||||
int opt;
|
||||
int rc;
|
||||
xce_hostctrl_t hostctrl;
|
||||
int action = HOST_DO_NONE;
|
||||
char* hostname = NULL;
|
||||
|
||||
optind = 1;
|
||||
|
||||
while ((opt = getopt(argc, argv, "och:")) != -1) {
|
||||
switch (opt) {
|
||||
case 'o':
|
||||
action = HOST_OPEN;
|
||||
break;
|
||||
case 'c':
|
||||
action = HOST_CLOSE;
|
||||
break;
|
||||
case 'h':
|
||||
hostname = strdup(optarg);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (hostname == NULL) {
|
||||
fprintf(stderr, "you need to specify host name with -h\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (action == HOST_DO_NONE) {
|
||||
fprintf(stderr, "you need to specify -o (open) or -c (close)\n");
|
||||
free(hostname);
|
||||
return -1;
|
||||
}
|
||||
|
||||
hostctrl.action = action;
|
||||
hostctrl.hostname = hostname;
|
||||
|
||||
rc = xcecli_hostctrl(active_ctx->ctx, &hostctrl);
|
||||
if (rc < 0) {
|
||||
perror("xcecli_hostctrl");
|
||||
free(hostname);
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("Operation on host %s done\n", hostname);
|
||||
|
||||
free(hostname);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* upgradeconfig <cluster.bin> — submit a configuration upgrade
|
||||
* (config-upgrade-snapshot task 8.6). Reads the given capnp-encoded
|
||||
* ClusterConfig blob (the same format `xceadmin config build` produces and
|
||||
* deploy-ha.sh installs as cluster.bin), computes its SHA-256 (OpenSSL
|
||||
* SHA256(), matching config_repl.c's do_upgrade_config_with_bytes transit
|
||||
* verification), and drives it through xcecli_upgradeconfig ->
|
||||
* xce_upgradeconfig_req_raw (OP_upgradeConfig). Mirrors cmd_snapdiff's
|
||||
* bounds-checked file-reading style. */
|
||||
int cmd_upgradeconfig(int argc, char* argv[])
|
||||
{
|
||||
if (argc != 2) {
|
||||
fprintf(stderr, "usage: upgradeconfig <cluster.bin>\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char* path = argv[1];
|
||||
FILE* fp = fopen(path, "rb");
|
||||
if (fp == NULL) {
|
||||
fprintf(stderr, "upgradeconfig: cannot open %s: %s\n", path,
|
||||
strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (fseek(fp, 0, SEEK_END) != 0) {
|
||||
fprintf(stderr, "upgradeconfig: fseek failed on %s: %s\n", path,
|
||||
strerror(errno));
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
long fsz = ftell(fp);
|
||||
if (fsz <= 0) {
|
||||
fprintf(stderr, "upgradeconfig: %s is empty or ftell failed\n", path);
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
rewind(fp);
|
||||
|
||||
uint8_t* bytes = (uint8_t*)malloc((size_t)fsz);
|
||||
if (bytes == NULL) {
|
||||
fprintf(stderr, "upgradeconfig: out of memory reading %s\n", path);
|
||||
fclose(fp);
|
||||
return -1;
|
||||
}
|
||||
|
||||
size_t nread = fread(bytes, 1, (size_t)fsz, fp);
|
||||
fclose(fp);
|
||||
if (nread != (size_t)fsz) {
|
||||
fprintf(stderr, "upgradeconfig: short read on %s (%zu of %ld bytes)\n",
|
||||
path, nread, fsz);
|
||||
free(bytes);
|
||||
return -1;
|
||||
}
|
||||
|
||||
uint8_t sha256[SHA256_DIGEST_LENGTH];
|
||||
SHA256(bytes, (size_t)fsz, sha256);
|
||||
|
||||
int rc = xcecli_upgradeconfig(active_ctx->ctx, sha256,
|
||||
sizeof(sha256), bytes, (size_t)fsz);
|
||||
free(bytes);
|
||||
|
||||
if (rc < 0) {
|
||||
fprintf(stderr, "upgradeconfig: failed (rc=%d): %s\n", rc,
|
||||
strerror(-rc));
|
||||
return rc;
|
||||
}
|
||||
|
||||
printf("Config upgrade submitted from %s (%ld bytes)\n", path, fsz);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int cmd_runtasks(int argc, char* argv[])
|
||||
{
|
||||
xce_taskhost_t hosts[MAX_TASK_HOSTS];
|
||||
|
|
|
|||
|
|
@ -1819,9 +1819,25 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
|
|||
uint64_t tagN; { uint64_t t; memcpy(&t, shaN, 8); tagN = be64toh(t); }
|
||||
TEST_ASSERT_FALSE(memcmp(shaO, shaN, 32) == 0);
|
||||
|
||||
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances). */
|
||||
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances).
|
||||
* bug-1469 fix interaction: flushing makes durable_lsn advance, and this
|
||||
* single-member fixture's commit engine (majority=1) reacts to ANY
|
||||
* durable advance by racing committed_lsn forward on its OWN background
|
||||
* fsync-worker thread (commit_on_durable_advance) -- which, since
|
||||
* apply_committed_entry's WAL_CONFIG_MARKER case now ALSO activates
|
||||
* config (bug-1469), could apply-and-activate this NEW marker for real
|
||||
* before this test's own thread gets a chance to pin committed_lsn back
|
||||
* down below it. PIN IMMEDIATELY, right here, before any other work
|
||||
* (previously this pin lived after conf_save_config below, leaving a
|
||||
* multi-syscall window — several real fsyncs — for that background race
|
||||
* to win against a single-member fixture's over-eager auto-commit,
|
||||
* something a real multi-node standby never does since its
|
||||
* committed_lsn is solely driven by the primary's commit-advance, never
|
||||
* local durability). */
|
||||
uint64_t new_mlsn = t43_append_marker_uncommitted(c, tagN, shaN);
|
||||
TEST_ASSERT_TRUE(new_mlsn > committed_at_old);
|
||||
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
|
||||
memory_order_release);
|
||||
|
||||
/* The OLD config is currently LIVE (activated in upgrade 1). Capture the live
|
||||
* decoded-config pointer as the activation witness: load_primary_config frees
|
||||
|
|
@ -1837,9 +1853,10 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
|
|||
* commit barrier, which would advance committed_lsn to the durable WAL tip, and
|
||||
* it bumps the advertised config_version_tag to tagN (this is the very reason
|
||||
* the primary's config-behind re-push net does not re-fire — Finding D); we
|
||||
* save FIRST, then PIN committed_lsn back below the NEW marker to model a
|
||||
* standby whose committed prefix (driven by the primary's commit-advance, not
|
||||
* local durability) has NOT yet advanced past the NEW marker. */
|
||||
* already pinned committed_lsn back below the NEW marker immediately above (to
|
||||
* model a standby whose committed prefix, driven by the primary's commit-
|
||||
* advance, not local durability, has NOT yet advanced past the NEW marker) —
|
||||
* re-pin again here for defense-in-depth against the same race. */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesN, blenN, shaN));
|
||||
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
|
||||
memory_order_release);
|
||||
|
|
@ -2329,6 +2346,147 @@ static void test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss(voi
|
|||
printf("test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss: PASS\n");
|
||||
}
|
||||
|
||||
/* ── bug-1469 regression: WAL_CONFIG_MARKER apply must ACTIVATE config on
|
||||
* the ORIGINATING member, not just receiving standbys ──────────────────────
|
||||
*
|
||||
* Live 3-node chaotic-kill soak (config-upgrade-snapshot task 8.6,
|
||||
* .wolf/buglog.json bug-1469): a config upgrade's WAL_CONFIG_MARKER commits
|
||||
* and even checkpoints successfully on the ORIGINATING member (the primary
|
||||
* that submitted the upgrade via do_upgrade_config_with_bytes), but
|
||||
* apply_committed_entry's WAL_CONFIG_MARKER case (persist.c) never called
|
||||
* config_activate_committed()/load_primary_config() -- so that member's
|
||||
* c->myconf stayed parsed from the OLD config bytes until some LATER,
|
||||
* independent event (a restart, or an incoming out-of-band
|
||||
* handle_repl_config_push) happened to refresh it. The live soak killed the
|
||||
* originating primary shortly after it committed the upgrade, before any
|
||||
* such later event, freezing a stale c->myconf into its last checkpoint --
|
||||
* a genuine cross-member materialized-state divergence (ctrl-1's checkpoint
|
||||
* ended up a different byte size than ctrl-2/ctrl-3's, both LSN 11).
|
||||
* config_activate_committed WAS already being called correctly on
|
||||
* RECEIVING standbys, via handle_repl_config_push -- this gap is specific
|
||||
* to the member that originates/commits the marker on itself.
|
||||
*/
|
||||
|
||||
/* Build a valid ClusterConfig blob distinguishable by config_generation +
|
||||
* clustername (mirrors t43_make_config_blob, parameterized so this test can
|
||||
* commit two genuinely different, independently-decodable configs -- an
|
||||
* "old" pre-upgrade config and a "new" post-upgrade config -- without
|
||||
* relying on a byte-mutation of an opaque capnp blob, which risks
|
||||
* corrupting the encoding; config-upgrade-snapshot's own standing gotcha
|
||||
* notes t43_make_config_blob() returns byte-IDENTICAL content on repeat
|
||||
* calls, so a genuinely different SECOND blob needs its own encode call). */
|
||||
static uint8_t *t1469_make_config_blob(uint64_t gen, const char *name,
|
||||
size_t *len_out)
|
||||
{
|
||||
cluster_config_t cfg;
|
||||
struct capn cap;
|
||||
struct capn_segment *cs;
|
||||
ClusterConfig_ptr p;
|
||||
uint8_t *buf;
|
||||
size_t sz;
|
||||
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.clustername = (char *)name;
|
||||
cfg.config_generation = gen;
|
||||
|
||||
capn_init_malloc(&cap);
|
||||
cs = capn_root(&cap).seg;
|
||||
encode_ClusterConfig_ptr(cs, &p, &cfg);
|
||||
capn_setp(capn_root(&cap), 0, p.p);
|
||||
sz = (size_t)capn_size(&cap);
|
||||
buf = (uint8_t *)malloc(sz);
|
||||
TEST_ASSERT_NOT_NULL(buf);
|
||||
capn_write_mem(&cap, buf, sz, 0);
|
||||
capn_free(&cap);
|
||||
*len_out = sz;
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* This test drives the upgrade marker through the REAL production apply
|
||||
* path (config_append_marker + commit_submit + wait-for-applied, exactly
|
||||
* like test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss
|
||||
* above) on a single-member fixture acting as its OWN primary -- i.e. the
|
||||
* ORIGINATING member applying its own committed marker, never receiving an
|
||||
* out-of-band handle_repl_config_push. It NEVER calls
|
||||
* config_activate_committed() for the upgrade marker itself, so the only
|
||||
* way mc->cluster_config can reflect the new config is via
|
||||
* apply_committed_entry's own WAL_CONFIG_MARKER dispatch.
|
||||
*
|
||||
* RED (before the fix): mc->cluster_config stays pinned to the OLD config
|
||||
* (config_generation==1, "t1469-old") after the new marker's real,
|
||||
* successful commit+apply -- proving the marker-triggered dispatch never
|
||||
* activates config on the member that originated it.
|
||||
* GREEN (after the fix): mc->cluster_config reflects the NEW config
|
||||
* (config_generation==2, "t1469-new") immediately after the marker's apply
|
||||
* returns, with no restart and no out-of-band push. */
|
||||
static void test_wal_config_marker_apply_activates_originating_member_config(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr1469_activate_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/69, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Establish the pre-upgrade steady state: an OLD config already
|
||||
* committed and ACTIVATED via the legitimate, already-tested
|
||||
* config_activate_committed path (mirrors 4.3-a above) -- this member
|
||||
* is already running under config_generation=1 before any upgrade is
|
||||
* ever submitted, exactly like a real cluster member. */
|
||||
size_t oldlen = 0;
|
||||
uint8_t *oldbytes = t1469_make_config_blob(1, "t1469-old", &oldlen);
|
||||
uint8_t oldsha[32];
|
||||
SHA256(oldbytes, oldlen, oldsha);
|
||||
uint64_t oldtag; { uint64_t t; memcpy(&t, oldsha, 8); oldtag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, oldbytes, oldlen, oldsha));
|
||||
free(oldbytes);
|
||||
t43_commit_marker(c, oldtag, oldsha);
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_UINT64(1, mc->cluster_config->config_generation);
|
||||
|
||||
/* Now the upgrade: NEW config bytes saved locally exactly as
|
||||
* do_upgrade_config_with_bytes's own step 5 (conf_save_config) does on
|
||||
* the ORIGINATING primary, then a NEW marker committed through the
|
||||
* exact same real path bug-1469's live soak exercised. Deliberately
|
||||
* NEVER calls config_activate_committed() for this marker anywhere
|
||||
* below -- activation must come from apply_committed_entry's own
|
||||
* dispatch, or this assertion fails. */
|
||||
size_t newlen = 0;
|
||||
uint8_t *newbytes = t1469_make_config_blob(2, "t1469-new", &newlen);
|
||||
uint8_t newsha[32];
|
||||
SHA256(newbytes, newlen, newsha);
|
||||
uint64_t newtag; { uint64_t t; memcpy(&t, newsha, 8); newtag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, newbytes, newlen, newsha));
|
||||
free(newbytes);
|
||||
t43_commit_marker(c, newtag, newsha);
|
||||
|
||||
/* THE ASSERTION THAT PROVES THE FIX: mc->cluster_config must now
|
||||
* reflect the NEW config -- purely from apply_committed_entry's own
|
||||
* WAL_CONFIG_MARKER dispatch, exactly the production path the
|
||||
* ORIGINATING primary runs on itself. Before the fix this stays pinned
|
||||
* at generation 1 ("t1469-old") forever -- exactly bug-1469's observed
|
||||
* cross-member divergence, reproduced at the unit level. */
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(2, mc->cluster_config->config_generation,
|
||||
"bug-1469: apply_committed_entry's WAL_CONFIG_MARKER case never "
|
||||
"activated the new config on the ORIGINATING member -- c->myconf "
|
||||
"stayed pinned to the OLD config");
|
||||
TEST_ASSERT_EQUAL_STRING("t1469-new", mc->cluster_config->clustername);
|
||||
TEST_ASSERT_EQUAL_HEX64(newtag, mc->config_version_tag);
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_wal_config_marker_apply_activates_originating_member_config: PASS\n");
|
||||
}
|
||||
|
||||
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
@ -5144,6 +5302,9 @@ void test_config_repl(void)
|
|||
* production trigger (apply_committed_entry's WAL_CONFIG_MARKER case),
|
||||
* not a direct test call -- proves the fence is reachable at all. */
|
||||
RUN_TEST(test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss);
|
||||
/* bug-1469: WAL_CONFIG_MARKER apply must activate config on the
|
||||
* ORIGINATING member too, not just receiving standbys. */
|
||||
RUN_TEST(test_wal_config_marker_apply_activates_originating_member_config);
|
||||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_load_primary_config_missing_returns_error);
|
||||
RUN_TEST(test_wait_all_with_timeout_slow_threads);
|
||||
|
|
|
|||
|
|
@ -95,6 +95,14 @@ extern int xcecli_jobctrl(xcecli_ctx_t, xce_jobctrl_t*);
|
|||
extern int xcecli_notify_jobchkpt(xcecli_ctx_t, uint64_t, char*);
|
||||
extern int xcecli_queuectrl(xcecli_ctx_t, xce_queuectrl_t*);
|
||||
extern int xcecli_hostctrl(xcecli_ctx_t, xce_hostctrl_t*);
|
||||
/* xcecli_upgradeconfig: submit a configuration upgrade (sha256 + serialised
|
||||
* ClusterConfig blob) to the primary via OP_upgradeConfig, transparently
|
||||
* surviving a primary change mid-call via XCECLI_FAILOVER_CALL (same
|
||||
* write-op failover contract as xcecli_hostctrl/xcecli_queuectrl). Returns
|
||||
* the server's reply status (0 on success). */
|
||||
extern int xcecli_upgradeconfig(xcecli_ctx_t, const uint8_t *sha256,
|
||||
size_t sha256_len, const uint8_t *bytes,
|
||||
size_t bytes_len);
|
||||
extern int xcecli_runtask(xcecli_ctx_t, xce_taskrun_t*);
|
||||
extern int xcecli_notify_taskstart(xcecli_ctx_t, xce_task_pidinfo_t*);
|
||||
extern int xcecli_notify_taskfail(xcecli_ctx_t, xce_task_failinfo_t*);
|
||||
|
|
|
|||
|
|
@ -185,6 +185,19 @@ extern int xce_upgrade_config_req_extract_raw(const uint8_t *pkt_data,
|
|||
size_t *out_len,
|
||||
uint8_t **out_sha256);
|
||||
|
||||
/*
|
||||
* xce_upgradeconfig_req_raw: send OP_upgradeConfig carrying raw byte arrays
|
||||
* (sha256 + the serialised ClusterConfig blob). Bypasses the generated
|
||||
* encode_UpgradeConfigRequest path (a no-op for the sha256/bytes Data fields
|
||||
* — same class of limitation as xce_replconfigpush_req_raw) by constructing
|
||||
* the capnp message directly via capn_new_list8 + capn_setv8 + write_Request.
|
||||
* Ordinary CLIENT request: seq/from are populated the same way xce_request()
|
||||
* populates them for every other client op.
|
||||
*/
|
||||
extern int xce_upgradeconfig_req_raw(endpoint_t *ep,
|
||||
const uint8_t *sha256, size_t sha256_len,
|
||||
const uint8_t *bytes, size_t bytes_len);
|
||||
|
||||
/* ── Client-Side Transparent Retry tunables (Req 6.1, 6.2) ──────────────────
|
||||
*
|
||||
* xce_retry_config_t: configurable retry parameters with safe production
|
||||
|
|
|
|||
|
|
@ -28,6 +28,17 @@ extern int xce_update_resvals(xce_ctx_t*, int, xce_resval_t**);
|
|||
extern int xce_jobctrl(xce_ctx_t*, xce_jobctrl_t*);
|
||||
extern int xce_queuectrl(xce_ctx_t*, xce_queuectrl_t*);
|
||||
extern int xce_hostctrl(xce_ctx_t*, xce_hostctrl_t*);
|
||||
/* xce_upgradeconfig: send OP_upgradeConfig (sha256 + serialised ClusterConfig
|
||||
* blob) via xce_upgradeconfig_req_raw and wait for the reply. Returns the
|
||||
* server's reply status (0 on success; a negative errno/E_XCE_* code on
|
||||
* failure — the server logs the human-readable reason, see
|
||||
* do_upgrade_config_with_bytes/send_upgrade_error, config_repl.c). Mirrors
|
||||
* xce_hostctrl/xce_queuectrl's shape; the richer UpgradeConfigReply fields
|
||||
* (version_tag/acked_count/error text) are not surfaced to the caller — no
|
||||
* consumer of this API currently needs them beyond success/failure. */
|
||||
extern int xce_upgradeconfig(xce_ctx_t*, const uint8_t *sha256,
|
||||
size_t sha256_len, const uint8_t *bytes,
|
||||
size_t bytes_len);
|
||||
extern int xce_runtask(xce_ctx_t*, xce_taskrun_t*);
|
||||
extern int xce_update_jusage(xce_ctx_t*, uint64_t, int, int*, int, int*, int,
|
||||
xce_rusage_t**);
|
||||
|
|
|
|||
|
|
@ -778,6 +778,25 @@ int xcecli_hostctrl(xcecli_ctx_t c, xce_hostctrl_t* ctrlinfo)
|
|||
return rc;
|
||||
}
|
||||
|
||||
int xcecli_upgradeconfig(xcecli_ctx_t c, const uint8_t *sha256,
|
||||
size_t sha256_len, const uint8_t *bytes,
|
||||
size_t bytes_len)
|
||||
{
|
||||
api_ctx_t* ctx = (api_ctx_t*)c;
|
||||
int rc;
|
||||
|
||||
if ((ctx == NULL) || (sha256 == NULL) || (sha256_len == 0)
|
||||
|| (bytes == NULL) || (bytes_len == 0)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rc = XCECLI_FAILOVER_CALL(ctx, /*is_write=*/true,
|
||||
xce_upgradeconfig(&(ctx->ctx), sha256,
|
||||
sha256_len, bytes,
|
||||
bytes_len));
|
||||
return rc;
|
||||
}
|
||||
|
||||
int xcecli_runtask(xcecli_ctx_t c, xce_taskrun_t* info)
|
||||
{
|
||||
api_ctx_t* ctx = (api_ctx_t*)c;
|
||||
|
|
|
|||
|
|
@ -499,6 +499,94 @@ int xce_replconfigpush_req_raw(endpoint_t *ep, xce_peerinfo_t *from,
|
|||
return (enq_rc < 0) ? enq_rc : 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* xce_upgradeconfig_req_raw: send OP_upgradeConfig with raw byte arrays.
|
||||
*
|
||||
* Bypasses encode_UpgradeConfigRequest (a literal no-op for the sha256/bytes
|
||||
* Data fields — see the doc comment above upgrade_config_req_s in typemap.h;
|
||||
* confirmed by inspection of the generated xce.build/lib/controller.capnp.c).
|
||||
* Instead, constructs the struct Request / struct UpgradeConfigRequest
|
||||
* directly using the capnp wire-format API (capn_new_list8 + capn_setv8),
|
||||
* mirroring xce_replconfigpush_req_raw's exact bypass pattern.
|
||||
*
|
||||
* Unlike that controller-to-controller helper, this is an ordinary CLIENT
|
||||
* request: seq and from are populated the same way the generic xce_request()
|
||||
* populates them for every other client op (endpoint_getseq(ep), ep->peer),
|
||||
* not passed in by the caller.
|
||||
*/
|
||||
int xce_upgradeconfig_req_raw(endpoint_t *ep,
|
||||
const uint8_t *sha256, size_t sha256_len,
|
||||
const uint8_t *bytes, size_t bytes_len)
|
||||
{
|
||||
struct capn c;
|
||||
struct capn_segment *cs;
|
||||
struct Request rq;
|
||||
struct UpgradeConfigRequest up_req;
|
||||
UpgradeConfigRequest_ptr up_ptr;
|
||||
Request_ptr rp;
|
||||
io_packet_t *pkt;
|
||||
int rc;
|
||||
|
||||
if (ep == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
if (sha256 == NULL || sha256_len == 0 || bytes == NULL || bytes_len == 0)
|
||||
return -EINVAL;
|
||||
|
||||
capn_init_malloc(&c);
|
||||
cs = capn_root(&c).seg;
|
||||
|
||||
memset(&rq, 0, sizeof(rq));
|
||||
memset(&up_req, 0, sizeof(up_req));
|
||||
|
||||
/* sha256: write as a List(UInt8) — capn_list8 and capn_data share {capn_ptr p} */
|
||||
{
|
||||
capn_list8 sha_list = capn_new_list8(cs, (int)sha256_len);
|
||||
capn_setv8(sha_list, 0, sha256, (int)sha256_len);
|
||||
up_req.sha256.p = sha_list.p;
|
||||
}
|
||||
|
||||
/* config bytes */
|
||||
{
|
||||
capn_list8 bytes_list = capn_new_list8(cs, (int)bytes_len);
|
||||
capn_setv8(bytes_list, 0, bytes, (int)bytes_len);
|
||||
up_req.bytes.p = bytes_list.p;
|
||||
}
|
||||
|
||||
up_ptr = new_UpgradeConfigRequest(cs);
|
||||
write_UpgradeConfigRequest(&up_req, up_ptr);
|
||||
|
||||
rq.opcode = OPCode_upgradeConfig;
|
||||
rq.seq = endpoint_getseq(ep);
|
||||
rq.body_which = Request_body_upgradeconfigrequestmsg;
|
||||
rq.body.upgradeconfigrequestmsg = up_ptr;
|
||||
encode_PeerInfo_ptr(cs, &rq.from, ep->peer);
|
||||
|
||||
rp = new_Request(cs);
|
||||
write_Request(&rq, rp);
|
||||
capn_setp(capn_root(&c), 0, rp.p);
|
||||
|
||||
pkt = (io_packet_t *)calloc(1, sizeof(*pkt));
|
||||
if (!pkt) {
|
||||
capn_free(&c);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
pkt->sz = capn_size(&c);
|
||||
rc = alloc_packet(pkt);
|
||||
if (rc < 0) {
|
||||
free(pkt);
|
||||
capn_free(&c);
|
||||
return -errno;
|
||||
}
|
||||
|
||||
capn_write_mem(&c, (uint8_t *)pkt->data, pkt->sz, 0);
|
||||
capn_free(&c);
|
||||
|
||||
int enq_rc = endpoint_enq(ep, &(pkt->link));
|
||||
return (enq_rc < 0) ? enq_rc : 0;
|
||||
}
|
||||
|
||||
/* wal_entry_to_wire: fill a wire WalCatchupEntry from a C entry's header
|
||||
* fields and raw payload bytes. Sub-entry nesting is one level deep —
|
||||
* elements written here never carry their own entries list. */
|
||||
|
|
|
|||
|
|
@ -647,6 +647,36 @@ int xce_hostctrl(xce_ctx_t* ctx, xce_hostctrl_t* ctrl)
|
|||
return status;
|
||||
}
|
||||
|
||||
int xce_upgradeconfig(xce_ctx_t* ctx, const uint8_t *sha256,
|
||||
size_t sha256_len, const uint8_t *bytes,
|
||||
size_t bytes_len)
|
||||
{
|
||||
int rc;
|
||||
int status;
|
||||
|
||||
if (ctx == NULL) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (ctx->state != XCE_CTX_RUN) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
rc = xce_upgradeconfig_req_raw(ctx->ep, sha256, sha256_len, bytes,
|
||||
bytes_len);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
struct timespec _dl; cbuf_deadline_10s(&_dl);
|
||||
rc = cbuf_timedget(&(ctx->wbuf), &status, &_dl);
|
||||
if (rc < 0) {
|
||||
return rc;
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
int xce_runtask(xce_ctx_t* ctx, xce_taskrun_t* runinfo)
|
||||
{
|
||||
int rc;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue