feat(ha-term-safety): out-of-band config bytes, hash-verify-before-activate, serve-fence (task 4.3)
Wires config_committed_marker to a production consumer (config_activate_committed): verifies pushed bytes' sha256 against the committed marker before activation, fences (loud, non-serving, stay-alive) on mismatch, and activates the latest committed marker on recovery. Enforced via ha_dispatch_gate + promotion gate. Fixes bug-981: a 2nd+ config upgrade could permanently fence a standby on a push-before-commit ordering. Adds an off-iomgr delaymgr retry scheduled from the commit-advance hook (atomic read + non-blocking enqueue only on the ACK thread) that re-activates and clears the stale fence once the new marker commits and bytes verify.
This commit is contained in:
parent
c3201a2b1e
commit
be4c1f0721
7 changed files with 1074 additions and 12 deletions
|
|
@ -952,6 +952,603 @@ static void test_committed_marker_survives_checkpoint_trim(void)
|
|||
printf("test_committed_marker_survives_checkpoint_trim: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Config byte delivery: hash-verify-before-activate + serve-fence (task 4.3) ─
|
||||
*
|
||||
* config_activate_committed is the CONSUMER of config_committed_marker (4.2's
|
||||
* derive). It keys the out-of-band config-push byte delivery to the committed
|
||||
* marker's {version_tag, sha256}: it VERIFIES the locally-held bytes' sha256
|
||||
* against the committed marker BEFORE activating (R7.8), and on a mismatch (or
|
||||
* bytes absent locally) keeps the member from serving under the new configuration
|
||||
* and KEEPS RETRYING rather than activating mismatched config (R7.9) — the loud
|
||||
* non-serving fence that NEVER self-terminates. On recovery it activates the
|
||||
* configuration referenced by the LATEST committed marker (R7.10).
|
||||
*
|
||||
* These tests reuse the 4.2 leader fixture (t42_*) but additionally wire c->myconf
|
||||
* so the locally-held config bytes path (conf_save_config / conf_read_raw /
|
||||
* load_primary_config) is exercised. */
|
||||
|
||||
/* Build a VALID Cap'n Proto ClusterConfig blob (decodable by load_primary_config)
|
||||
* so the match path can actually ACTIVATE. Mirrors cb5_make_config_blob. */
|
||||
static uint8_t *t43_make_config_blob(size_t *len_out)
|
||||
{
|
||||
cluster_config_t cfg;
|
||||
struct capn cap;
|
||||
struct capn_segment *cs;
|
||||
ClusterConfig_ptr p;
|
||||
uint8_t *buf;
|
||||
size_t sz;
|
||||
|
||||
memset(&cfg, 0, sizeof(cfg));
|
||||
cfg.clustername = (char *)"t43-upgrade";
|
||||
cfg.port = 9991;
|
||||
cfg.config_generation = 1;
|
||||
|
||||
capn_init_malloc(&cap);
|
||||
cs = capn_root(&cap).seg;
|
||||
encode_ClusterConfig_ptr(cs, &p, &cfg);
|
||||
capn_setp(capn_root(&cap), 0, p.p);
|
||||
sz = (size_t)capn_size(&cap);
|
||||
buf = (uint8_t *)malloc(sz);
|
||||
TEST_ASSERT_NOT_NULL(buf);
|
||||
capn_write_mem(&cap, buf, sz, 0);
|
||||
capn_free(&cap);
|
||||
*len_out = sz;
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Wire a myconf_t into the cluster so conf_save_config / load_primary_config
|
||||
* resolve under <tmpdir>/<name>/conf (the production layout). */
|
||||
static myconf_t *t43_attach_myconf(cluster_t *c, const char *tmpdir)
|
||||
{
|
||||
myconf_t *mc = calloc(1, sizeof(myconf_t));
|
||||
TEST_ASSERT_NOT_NULL(mc);
|
||||
mc->workdir = strdup(tmpdir);
|
||||
mc->name = strdup("ctrl-test");
|
||||
mc->conf_keep_n = 2;
|
||||
mc->conffile = NULL;
|
||||
c->myconf = mc;
|
||||
return mc;
|
||||
}
|
||||
|
||||
static void t43_free_myconf(cluster_t *c)
|
||||
{
|
||||
myconf_t *mc = c->myconf;
|
||||
if (!mc) return;
|
||||
if (mc->cluster_config) {
|
||||
free_ClusterConfig(mc->cluster_config);
|
||||
free(mc->cluster_config);
|
||||
}
|
||||
free(mc->conffile);
|
||||
free(mc->workdir);
|
||||
free(mc->name);
|
||||
free(mc);
|
||||
c->myconf = NULL;
|
||||
}
|
||||
|
||||
/* Commit a WAL_CONFIG_MARKER for `sha`/`tag` through the real commit engine so
|
||||
* config_committed_marker returns it (the committed-prefix authority 4.3 reads). */
|
||||
static void t43_commit_marker(cluster_t *c, uint64_t tag, const uint8_t sha[32])
|
||||
{
|
||||
uint64_t mlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000));
|
||||
TEST_ASSERT_TRUE(
|
||||
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire) >= mlsn);
|
||||
}
|
||||
|
||||
/* (4.3-a) MATCH activates: a committed marker whose locally-held bytes' sha256
|
||||
* MATCHES the marker activates the configuration and leaves the member UN-fenced
|
||||
* (R7.8). */
|
||||
static void test_config_activate_match_activates_and_unfences(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_match_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/3, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* The locally-held bytes (saved exactly as the config-push handler would). */
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag;
|
||||
{ uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
|
||||
/* Commit a marker that references EXACTLY those bytes. */
|
||||
t43_commit_marker(c, tag, sha);
|
||||
|
||||
/* HASH-VERIFIED → activate, fence stays clear. */
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
/* Activation loaded the config into myconf. */
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, mc->config_version_tag);
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_config_activate_match_activates_and_unfences: PASS\n");
|
||||
}
|
||||
|
||||
/* (4.3-b) MISMATCH fences + retries, NEVER activates: an injected hash mismatch
|
||||
* (locally-held bytes whose sha256 != the committed marker) keeps the member
|
||||
* FENCED (non-serving), ALIVE, and retrying, and does NOT activate the mismatched
|
||||
* config (R7.9). */
|
||||
static void test_config_activate_mismatch_fences_and_does_not_activate(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_mismatch_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/4, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Locally-held bytes are blob A (sha A). */
|
||||
size_t blenA = 0;
|
||||
uint8_t *bytesA = t43_make_config_blob(&blenA);
|
||||
uint8_t shaA[32];
|
||||
SHA256(bytesA, blenA, shaA);
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesA, blenA, shaA));
|
||||
free(bytesA);
|
||||
|
||||
/* But the committed marker references a DIFFERENT hash (injected mismatch:
|
||||
* the bytes for this marker have NOT arrived out-of-band yet). */
|
||||
uint8_t shaB[32];
|
||||
uint64_t tagB = t42_make_ref(shaB, 0x99); /* deterministic != shaA */
|
||||
TEST_ASSERT_FALSE(memcmp(shaA, shaB, 32) == 0);
|
||||
t43_commit_marker(c, tagB, shaB);
|
||||
|
||||
/* MISMATCH → fenced, NOT activated, member stays alive (returns, no abort). */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c));
|
||||
/* The mismatched marker's config was NOT activated: the active config tag is
|
||||
* still the locally-held blob A's tag, NEVER the marker's tagB. */
|
||||
TEST_ASSERT_NOT_EQUAL(tagB, mc->config_version_tag);
|
||||
|
||||
/* RETRYING: a second call while still mismatched stays fenced (idempotent,
|
||||
* never activates mismatched) — the member keeps retrying, alive. */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c));
|
||||
TEST_ASSERT_NOT_EQUAL(tagB, mc->config_version_tag);
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_config_activate_mismatch_fences_and_does_not_activate: PASS\n");
|
||||
}
|
||||
|
||||
/* (4.3-c) OBTAINED-OUT-OF-BAND bytes are verified before activate: a member is
|
||||
* fenced while the marker's bytes are absent locally; once the matching bytes
|
||||
* arrive out-of-band (saved exactly as handle_repl_config_push would), the next
|
||||
* activate hash-verifies them against the marker and activates, clearing the
|
||||
* fence (R7.5/R7.8). */
|
||||
static void test_config_activate_obtained_bytes_verified_before_activate(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_obtain_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/5, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* The marker's referenced bytes. Compute its hash; commit the marker BEFORE
|
||||
* the bytes are present locally (no conf_save_config yet). */
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag;
|
||||
{ uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
t43_commit_marker(c, tag, sha);
|
||||
|
||||
/* Bytes NOT present locally yet → fenced, not activated (R7.9). */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c));
|
||||
|
||||
/* Bytes arrive out-of-band: the config-push transport saves them locally
|
||||
* exactly as handle_repl_config_push does (conf_save_config). */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
|
||||
/* Now the obtained bytes hash-verify against the committed marker → activate,
|
||||
* fence clears. */
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, mc->config_version_tag);
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_config_activate_obtained_bytes_verified_before_activate: PASS\n");
|
||||
}
|
||||
|
||||
/* (4.3-d) RECOVERY activates the LATEST committed marker: with two committed
|
||||
* markers, config_activate_committed activates the config referenced by the
|
||||
* LATEST committed marker (the one config_committed_marker derives), obtaining
|
||||
* its bytes from local out-of-band storage (R7.10). */
|
||||
static void test_config_activate_recovery_uses_latest_committed_marker(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_recover_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/6, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Two distinct configs, each committed as a marker in turn; the LATEST wins. */
|
||||
size_t blen1 = 0, blen2 = 0;
|
||||
uint8_t *b1 = t43_make_config_blob(&blen1);
|
||||
/* Make blob 2 differ so its hash differs from blob 1. */
|
||||
uint8_t *b2 = t43_make_config_blob(&blen2);
|
||||
TEST_ASSERT_TRUE(blen2 >= 1);
|
||||
b2[blen2 - 1] ^= 0xFF; /* perturb → distinct sha */
|
||||
uint8_t sha1[32], sha2[32];
|
||||
SHA256(b1, blen1, sha1);
|
||||
SHA256(b2, blen2, sha2);
|
||||
TEST_ASSERT_FALSE(memcmp(sha1, sha2, 32) == 0);
|
||||
uint64_t tag2;
|
||||
{ uint64_t t; memcpy(&t, sha2, 8); tag2 = be64toh(t); }
|
||||
|
||||
uint64_t tag1; { uint64_t t; memcpy(&t, sha1, 8); tag1 = be64toh(t); }
|
||||
t43_commit_marker(c, tag1, sha1);
|
||||
t43_commit_marker(c, tag2, sha2);
|
||||
|
||||
/* On recovery, the LOCALLY-held bytes are the LATEST config (blob 2 — what the
|
||||
* out-of-band store holds after the latest push). */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, b2, blen2, sha2));
|
||||
free(b1);
|
||||
free(b2);
|
||||
|
||||
/* Activation targets the LATEST committed marker (tag2), hash-verifies the
|
||||
* local bytes, and activates it. */
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
TEST_ASSERT_EQUAL_HEX64(tag2, mc->config_version_tag);
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_config_activate_recovery_uses_latest_committed_marker: PASS\n");
|
||||
}
|
||||
|
||||
/* ── task 4.3 REMEDIATION r1: production integration of the consumer ───────────
|
||||
*
|
||||
* Round-1 review REJECTED 4.3 for three integration gaps, fixed below:
|
||||
* A — verify-before-activate was BYPASSED on the push path (handle_repl_config_
|
||||
* push pre-activated via load_primary_config before the gate ran).
|
||||
* B — recovery-activate had NO production call site (boot loaded blindly).
|
||||
* C — the serve-fence had NO production reader (ha_dispatch_gate ignored it).
|
||||
*/
|
||||
|
||||
/* (4.3-A) PUSH-PATH SOLE ACTIVATION: config_activate_committed is the SOLE
|
||||
* activation point. With a committed marker that references DIFFERENT bytes than
|
||||
* the ones just delivered out-of-band, the push-path sequence (conf_save_config
|
||||
* then config_activate_committed — NO pre-emptive load_primary_config) must NOT
|
||||
* make the pushed bytes live (verify-before-activate is a real GATE, not an
|
||||
* after-the-fact overlay). Regression for Finding A. */
|
||||
static void test_push_path_mismatch_does_not_preactivate(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_pushgate_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/7, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* A committed marker references blob B (sha B) — the config the cluster
|
||||
* agreed to, whose bytes have NOT yet arrived here. */
|
||||
size_t blenB = 0;
|
||||
uint8_t *bytesB = t43_make_config_blob(&blenB);
|
||||
bytesB[blenB - 1] ^= 0x5A; /* perturb → distinct sha */
|
||||
uint8_t shaB[32];
|
||||
SHA256(bytesB, blenB, shaB);
|
||||
uint64_t tagB; { uint64_t t; memcpy(&t, shaB, 8); tagB = be64toh(t); }
|
||||
t43_commit_marker(c, tagB, shaB);
|
||||
free(bytesB);
|
||||
|
||||
/* The push delivers blob A (a DIFFERENT/older config). The push handler now
|
||||
* SAVES the bytes but does NOT pre-activate; activation is gated. */
|
||||
size_t blenA = 0;
|
||||
uint8_t *bytesA = t43_make_config_blob(&blenA);
|
||||
uint8_t shaA[32];
|
||||
SHA256(bytesA, blenA, shaA);
|
||||
TEST_ASSERT_FALSE(memcmp(shaA, shaB, 32) == 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesA, blenA, shaA));
|
||||
free(bytesA);
|
||||
|
||||
/* The push-path SOLE activation: gate fences (marker references blob B, not
|
||||
* the just-saved blob A) and does NOT load blob A. */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c));
|
||||
/* Blob A was NOT made live: no decoded config, no version tag for A. */
|
||||
TEST_ASSERT_NULL(mc->cluster_config);
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_push_path_mismatch_does_not_preactivate: PASS\n");
|
||||
}
|
||||
|
||||
/* (4.3-A/B) NO-MARKER FALLBACK: when there is NO committed marker yet (legacy /
|
||||
* first boot, or the marker has not committed), config_activate_committed FALLS
|
||||
* BACK to the existing load_primary_config behavior — it activates the
|
||||
* locally-held bytes (so config_version_tag is set for the freshness gate and the
|
||||
* push ACK) and does NOT fence. Absent-marker is NOT a mismatch. */
|
||||
static void test_config_activate_no_marker_falls_back_and_loads(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_fallback_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/8, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Local bytes present, but NO committed marker exists. */
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
|
||||
/* No marker → fall back to load_primary_config (blind load), NOT a fence. */
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_EQUAL_HEX64(tag, mc->config_version_tag);
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_config_activate_no_marker_falls_back_and_loads: PASS\n");
|
||||
}
|
||||
|
||||
/* (4.3-C) FENCE IS ENFORCED by a production reader: while the config serve-fence
|
||||
* is set, ha_dispatch_gate refuses config-dependent state-changing operations on
|
||||
* the PRIMARY (loud non-serving, member stays alive) and a fenced member is
|
||||
* non-promotable (ha_config_promotion_fenced). Read-only/replication ops still
|
||||
* pass so the bytes can arrive out-of-band. Once correct bytes arrive and
|
||||
* re-verify clears the fence, the member serves again. Regression for Finding C. */
|
||||
static void test_serve_fence_enforced_by_dispatch_gate_and_promotion(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_fence_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/9, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* The committed config (blob TARGET) whose bytes have not yet arrived. */
|
||||
size_t tlen = 0;
|
||||
uint8_t *tbytes = t43_make_config_blob(&tlen);
|
||||
uint8_t tsha[32];
|
||||
SHA256(tbytes, tlen, tsha);
|
||||
uint64_t ttag; { uint64_t t; memcpy(&t, tsha, 8); ttag = be64toh(t); }
|
||||
t43_commit_marker(c, ttag, tsha);
|
||||
|
||||
/* Stale local bytes (blob A) that do NOT match the committed marker. */
|
||||
size_t alen = 0;
|
||||
uint8_t *abytes = t43_make_config_blob(&alen);
|
||||
abytes[alen - 1] ^= 0x33;
|
||||
uint8_t asha[32];
|
||||
SHA256(abytes, alen, asha);
|
||||
TEST_ASSERT_FALSE(memcmp(asha, tsha, 32) == 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, abytes, alen, asha));
|
||||
free(abytes);
|
||||
|
||||
/* Mismatch → fenced. */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c));
|
||||
|
||||
/* PRODUCTION READER: a fenced PRIMARY refuses config-dependent state-changing
|
||||
* ops (loud non-serving) but stays alive — the gate returns an error, not a
|
||||
* crash/exit. */
|
||||
TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0);
|
||||
/* Replication / read-only ops still pass so the correct bytes can arrive. */
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_replConfigPush));
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_get));
|
||||
/* Non-promotable while fenced. */
|
||||
TEST_ASSERT_EQUAL_INT(1, ha_config_promotion_fenced(&c->ha));
|
||||
|
||||
/* Correct bytes arrive out-of-band (saved as the push handler would). */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, tbytes, tlen, tsha));
|
||||
free(tbytes);
|
||||
|
||||
/* Re-verify clears the fence → member serves and is promotable again. */
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
TEST_ASSERT_EQUAL_HEX64(ttag, mc->config_version_tag);
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_jobsub));
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_config_promotion_fenced(&c->ha));
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_serve_fence_enforced_by_dispatch_gate_and_promotion: PASS\n");
|
||||
}
|
||||
|
||||
/* Append a WAL_CONFIG_MARKER for `sha`/`tag` and flush it to the WAL, but do NOT
|
||||
* advance the committed prefix past it — returns its LSN. Models a STANDBY whose
|
||||
* local WAL tip runs ahead of its committed_lsn: on a standby committed_lsn is
|
||||
* driven by the primary's commit-advance (commit_set_raft_commit_index / the
|
||||
* library), NOT by local durability, so a flushed-but-not-yet-committed marker is
|
||||
* the real push-before-commit state. The caller pins committed_lsn back to the
|
||||
* prior value (see the test) so config_committed_marker still derives the OLDER
|
||||
* marker until the NEW one is explicitly committed. */
|
||||
static uint64_t t43_append_marker_uncommitted(cluster_t *c, uint64_t tag,
|
||||
const uint8_t sha[32])
|
||||
{
|
||||
uint64_t mlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
return mlsn;
|
||||
}
|
||||
|
||||
/* (4.3-D / R7.9) COMMIT-ADVANCE RETRY clears a push-before-commit fence: the
|
||||
* config MARKER (WAL) and config BYTES (out-of-band push) arrive on INDEPENDENT
|
||||
* transports. On a 2nd upgrade the NEW bytes are saved BEFORE the NEW marker
|
||||
* enters the committed prefix, so config_committed_marker still returns the OLDER
|
||||
* marker → the freshly-saved NEW bytes hash-MISMATCH it → the member is serve-
|
||||
* FENCED. The NEW marker commits later via the WAL path, but on_commit_advanced
|
||||
* cannot re-activate (no disk I/O on the iomgr ACK thread) and the primary's
|
||||
* config-behind re-push net does not fire (the push already advanced the advertised
|
||||
* tag) — WITHOUT the commit-advance retry trigger the standby would stay
|
||||
* PERMANENTLY fenced. This test reproduces that ordering and asserts the retry
|
||||
* trigger (config_activate_retry_fn — the SAME off-thread entry the production
|
||||
* scheduler enqueues) CLEARS the fence and activates the NEW config once the NEW
|
||||
* marker is committed. Regression for Finding D.
|
||||
*
|
||||
* The single-member commit engine advances committed_lsn to the durable WAL tip,
|
||||
* so to hold the NEW marker out of the committed prefix this test PINS
|
||||
* committed_lsn at the OLD marker's LSN (modeling a standby whose committed prefix
|
||||
* lags its local WAL tip), then advances it past the NEW marker to model the NEW
|
||||
* marker committing via the WAL path. */
|
||||
static void test_commit_advance_retry_clears_push_before_commit_fence(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr43_retry_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/10, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* ── Upgrade 1 (OLD): commit OLD marker + save OLD bytes + activate. The
|
||||
* member is UN-fenced and serving the OLD config. ───────────────────────── */
|
||||
size_t blenO = 0;
|
||||
uint8_t *bytesO = t43_make_config_blob(&blenO);
|
||||
uint8_t shaO[32];
|
||||
SHA256(bytesO, blenO, shaO);
|
||||
uint64_t tagO; { uint64_t t; memcpy(&t, shaO, 8); tagO = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesO, blenO, shaO));
|
||||
free(bytesO);
|
||||
t43_commit_marker(c, tagO, shaO);
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
TEST_ASSERT_EQUAL_HEX64(tagO, mc->config_version_tag);
|
||||
|
||||
/* The OLD marker is the committed prefix boundary the standby currently sees. */
|
||||
uint64_t committed_at_old =
|
||||
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire);
|
||||
|
||||
/* ── Upgrade 2 (NEW): the BYTES arrive (and are saved) BEFORE the NEW marker
|
||||
* enters the committed prefix — the push-before-commit ordering. ────────── */
|
||||
size_t blenN = 0;
|
||||
uint8_t *bytesN = t43_make_config_blob(&blenN);
|
||||
bytesN[blenN - 1] ^= 0xA5; /* perturb → distinct sha */
|
||||
uint8_t shaN[32];
|
||||
SHA256(bytesN, blenN, shaN);
|
||||
uint64_t tagN; { uint64_t t; memcpy(&t, shaN, 8); tagN = be64toh(t); }
|
||||
TEST_ASSERT_FALSE(memcmp(shaO, shaN, 32) == 0);
|
||||
|
||||
/* The NEW marker is APPENDED + flushed to the WAL (local tip advances). */
|
||||
uint64_t new_mlsn = t43_append_marker_uncommitted(c, tagN, shaN);
|
||||
TEST_ASSERT_TRUE(new_mlsn > committed_at_old);
|
||||
|
||||
/* The OLD config is currently LIVE (activated in upgrade 1). Capture the live
|
||||
* decoded-config pointer as the activation witness: load_primary_config frees
|
||||
* and reallocates mc->cluster_config, so a CHANGED pointer proves a (re)load
|
||||
* happened. conf_save_config (below) only writes bytes + bumps the advertised
|
||||
* config_version_tag — it does NOT decode/activate — so it must NOT change this
|
||||
* pointer; only a successful config_activate_committed does. */
|
||||
cluster_config_t *live_before = mc->cluster_config;
|
||||
TEST_ASSERT_NOT_NULL(live_before);
|
||||
|
||||
/* The NEW bytes are saved out-of-band (exactly as handle_repl_config_push
|
||||
* does). Note conf_save_config in this single-member fixture re-evaluates the
|
||||
* commit barrier, which would advance committed_lsn to the durable WAL tip, and
|
||||
* it bumps the advertised config_version_tag to tagN (this is the very reason
|
||||
* the primary's config-behind re-push net does not re-fire — Finding D); we
|
||||
* save FIRST, then PIN committed_lsn back below the NEW marker to model a
|
||||
* standby whose committed prefix (driven by the primary's commit-advance, not
|
||||
* local durability) has NOT yet advanced past the NEW marker. */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesN, blenN, shaN));
|
||||
atomic_store_explicit(&c->persist.committed_lsn, committed_at_old,
|
||||
memory_order_release);
|
||||
|
||||
/* Activate now: config_committed_marker still returns the OLDER marker (tagO),
|
||||
* so the freshly-saved NEW bytes MISMATCH it → serve-FENCED, NEW config NOT
|
||||
* ACTIVATED. The live decoded config is UNCHANGED (still the OLD one) — the
|
||||
* mismatch gate never loaded the NEW bytes. This is the permanent-fence state
|
||||
* the retry trigger must rescue. */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, config_serve_fenced(c));
|
||||
TEST_ASSERT_TRUE(mc->cluster_config == live_before); /* NEW not activated */
|
||||
|
||||
/* ── The NEW marker COMMITS later via the WAL path (the standby's committed
|
||||
* prefix advances past it, as the primary's commit-advance heartbeat would
|
||||
* drive). config_committed_marker now derives the NEW marker. ───────────── */
|
||||
atomic_store_explicit(&c->persist.committed_lsn, new_mlsn,
|
||||
memory_order_release);
|
||||
|
||||
/* DRIVE THE RETRY TRIGGER: the SAME off-thread activation entry the production
|
||||
* commit-advance path schedules (ha_cb_commit_advanced_stub →
|
||||
* config_schedule_activate_retry → delaymgr → config_activate_retry_fn). In
|
||||
* production the delaymgr worker runs this OFF the iomgr ACK thread; here we
|
||||
* invoke the entry directly (the fixture's delaymgr is not started). Now
|
||||
* config_committed_marker returns the NEW marker and the saved NEW bytes
|
||||
* hash-verify → the fence CLEARS and the NEW config ACTIVATES (cluster_config
|
||||
* is reloaded — a fresh pointer). */
|
||||
TEST_ASSERT_EQUAL_INT(0, config_activate_retry_fn(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c)); /* fence CLEARED */
|
||||
TEST_ASSERT_EQUAL_HEX64(tagN, mc->config_version_tag); /* NEW now live */
|
||||
TEST_ASSERT_NOT_NULL(mc->cluster_config);
|
||||
TEST_ASSERT_TRUE(mc->cluster_config != live_before); /* (re)loaded NEW */
|
||||
|
||||
free(bytesN);
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_commit_advance_retry_clears_push_before_commit_fence: PASS\n");
|
||||
}
|
||||
|
||||
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
@ -2721,6 +3318,19 @@ void test_config_repl(void)
|
|||
/* Task 4.2 r1: live wiring + compaction-safe derive */
|
||||
RUN_TEST(test_config_commit_change_live_path_commits_and_derives);
|
||||
RUN_TEST(test_committed_marker_survives_checkpoint_trim);
|
||||
|
||||
RUN_TEST(test_config_activate_match_activates_and_unfences);
|
||||
RUN_TEST(test_config_activate_mismatch_fences_and_does_not_activate);
|
||||
RUN_TEST(test_config_activate_obtained_bytes_verified_before_activate);
|
||||
RUN_TEST(test_config_activate_recovery_uses_latest_committed_marker);
|
||||
/* Task 4.3 r1 remediation: production integration (push-path sole activation,
|
||||
* no-marker fallback, serve-fence enforced by dispatch gate + promotion). */
|
||||
RUN_TEST(test_push_path_mismatch_does_not_preactivate);
|
||||
RUN_TEST(test_config_activate_no_marker_falls_back_and_loads);
|
||||
RUN_TEST(test_serve_fence_enforced_by_dispatch_gate_and_promotion);
|
||||
/* Task 4.3 r2 remediation: commit-advance retry clears a push-before-commit
|
||||
* serve-fence (Finding D / R7.9). */
|
||||
RUN_TEST(test_commit_advance_retry_clears_push_before_commit_fence);
|
||||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_load_primary_config_missing_returns_error);
|
||||
RUN_TEST(test_wait_all_with_timeout_slow_threads);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue