wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(config-upgrade-snapshot): add materialization-and-hash-verification activation fence

Browse source 
Adds a new, independent fence (ha_state_t.upgrade_materialization_fence)
distinct from the existing config-byte serve-fence: that fence tracks
whether the locally-held config bytes hash-match the committed marker;
this one tracks whether THIS MEMBER'S OWN upgrade-snapshot
materialization (task 5's marker-triggered checkpoint) has actually
completed for the current committed marker, with the live on-disk
config bytes still hash-verifying against it right now.

upgrade_materialization_verify(c) mirrors config_activate_committed's
shape: reads the current committed marker, checks whether
persist.cfgref_present/cfgref_tag/cfgref_sha already reflect it (stamped
by every checkpoint via scan_latest_committed_config_ref -- proof this
member's own materialization ran for this exact marker), and separately
re-reads the LIVE on-disk config bytes fresh on every call (never a
cached/embedded copy) to hash-verify them too. Both must hold for the
fence to clear. Logs loud only on the 0->1 transition, never
exits/aborts, always fail-loud + fence + stay alive + retriable.
upgrade_materialization_fenced(c) is the trivial atomic-read accessor.

Not yet wired into the dispatch gate, promotion gate, or an automatic
retry trigger -- that's task 6.2. Task 6.1 of 19.

Requirements: 1.8, 3.2, 3.4
This commit is contained in:
Rongsong Shen 2026-07-25 02:51:49 +00:00
parent 1da26b043e
commit 7b507156a0
4 changed files with 509 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

250
src/controller/testsuite/test_config_repl.c
View file

@ -1382,6 +1382,251 @@ static void test_config_activate_no_marker_falls_back_and_loads(void)
printf("test_config_activate_no_marker_falls_back_and_loads: PASS\n");
}
/* ── config-upgrade-snapshot task 6.1: Activation Fence
* (upgrade_materialization_verify / upgrade_materialization_fenced, ha.c)
* ──────────────────────────────────────────────────────────────────────
* These tests reuse this file's own t42_/t43_ fixture helpers rather than
* duplicating them into a new file — the Activation Fence's verify function
* is a close sibling of config_activate_committed (tested just above), and
* every fixture piece it needs (a real cluster_t, a real commit engine, a
* real myconf/config file, real committed WAL_CONFIG_MARKERs) already lives
* here. task 8.3 owns broader dispatch-gate/promotion-gate fence coverage in
* its own new file (test_upgrade_activation_fence.c per design.md); this
* task's own boundary is ha.c/ha.h only, so its tests stay narrowly scoped
* to upgrade_materialization_verify/upgrade_materialization_fenced
* themselves. */
extern int upgrade_materialization_verify(cluster_t *c);
extern int upgrade_materialization_fenced(const cluster_t *c);
/* (6.1-a) No committed marker yet: benign, not a divergence — fence stays
* clear, verify succeeds (mirrors config_activate_committed's own -ENOENT
* handling). */
static void test_upgrade_materialization_verify_no_marker_clears_fence(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ucaf61_nomarker_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/20, /*n_peers*/1);
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, 1, 8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
myconf_t *mc = t43_attach_myconf(c, tmp);
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
t43_free_myconf(c);
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_upgrade_materialization_verify_no_marker_clears_fence: PASS\n");
}
/* (6.1-b) A committed marker exists and its bytes are locally correct and
* hash-verify, AND this member's own checkpoint has already captured a
* reference to it (task 5's marker-triggered checkpoint runs synchronously
* on apply, so t43_commit_marker's wait-for-applied already guarantees this)
* — materialization is genuinely complete, fence clears. */
static void test_upgrade_materialization_verify_matching_materialization_clears_fence(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ucaf61_match_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/21, /*n_peers*/1);
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, 1, 8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
myconf_t *mc = t43_attach_myconf(c, tmp);
size_t blen = 0;
uint8_t *bytes = t43_make_config_blob(&blen);
uint8_t sha[32];
SHA256(bytes, blen, sha);
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
free(bytes);
t43_commit_marker(c, tag, sha);
/* Task 5's synchronous marker-triggered checkpoint already ran by the
* time t43_commit_marker (which waits for apply) returns — cfgref_*
* already reflects this exact marker. */
TEST_ASSERT_TRUE(c->persist.cfgref_present);
TEST_ASSERT_EQUAL_HEX64(tag, c->persist.cfgref_tag);
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
t43_free_myconf(c);
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_upgrade_materialization_verify_matching_materialization_clears_fence: PASS\n");
}
/* (6.1-c) A NEW marker commits while an earlier checkpoint is (simulated)
* still in-flight, so THIS marker's own materialization never actually runs
* (task 5's -EBUSY path, Req 4.4) -- config_committed_marker now returns the
* NEW marker but c->persist.cfgref_* still reflects the OLDER one.
* Materialization is genuinely incomplete for the CURRENT marker -> fence
* SETS (distinct from config_serve_fence, which would stay clear here since
* the bytes for the new marker are already present and correct). */
static void test_upgrade_materialization_verify_incomplete_materialization_fences(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ucaf61_incomplete_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/22, /*n_peers*/1);
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, 1, 8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
myconf_t *mc = t43_attach_myconf(c, tmp);
/* Marker A: commits, applies, materializes normally (baseline). */
size_t blenA = 0;
uint8_t *bytesA = t43_make_config_blob(&blenA);
uint8_t shaA[32];
SHA256(bytesA, blenA, shaA);
uint64_t tagA; { uint64_t t; memcpy(&t, shaA, 8); tagA = be64toh(t); }
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesA, blenA, shaA));
free(bytesA);
t43_commit_marker(c, tagA, shaA);
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
TEST_ASSERT_EQUAL_HEX64(tagA, c->persist.cfgref_tag);
/* Marker B: its bytes are saved correctly (config_serve_fence would stay
* clear), but its own checkpoint is forced to skip (-EBUSY) by holding
* the shared checkpoint_in_progress CAS open across the commit -- the
* exact real mechanism task 5's marker-apply case hits under Req 4.4. */
size_t blenB = 0;
uint8_t *bytesB = t43_make_config_blob(&blenB);
/* t43_make_config_blob always produces byte-identical content -- mutate
* a trailing byte so blob B genuinely differs from blob A (this test
* only ever hashes/saves the raw bytes via upgrade_materialization_verify,
* which never parses them as capnp, so a one-byte mutation is safe). */
bytesB[blenB - 1] ^= 0xFF;
uint8_t shaB[32];
SHA256(bytesB, blenB, shaB);
uint64_t tagB; { uint64_t t; memcpy(&t, shaB, 8); tagB = be64toh(t); }
TEST_ASSERT_FALSE(memcmp(shaA, shaB, 32) == 0);
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesB, blenB, shaB));
free(bytesB);
TEST_ASSERT_EQUAL_INT(
0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1,
memory_order_acq_rel));
uint64_t mlsnB = 0;
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tagB, shaB, &mlsnB));
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
TEST_ASSERT_EQUAL_INT(0,
commit_submit(&c->commit, mlsnB, COMMIT_MODE_WAIT, 1000));
/* apply runs (marker B's WAL_CONFIG_MARKER case fires and hits -EBUSY on
* the still-held CAS -- treated as expected/self-resolving, not a
* failure, so apply itself succeeds); wait for it directly since
* t43_commit_marker's own wait helper is not used here. */
uint64_t deadline_iters = 200;
while (deadline_iters-- > 0 &&
atomic_load_explicit(&c->persist.last_applied_lsn,
memory_order_acquire) < mlsnB) {
struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000 };
nanosleep(&ts, NULL);
}
TEST_ASSERT_TRUE(atomic_load_explicit(&c->persist.last_applied_lsn,
memory_order_acquire) >= mlsnB);
/* Release the simulated in-flight checkpoint now that marker B's apply
* (and its -EBUSY'd checkpoint attempt) has already happened. */
atomic_store_explicit(&c->persist.checkpoint_in_progress, 0,
memory_order_release);
/* config_committed_marker now authoritatively returns marker B, but this
* member's own materialization never ran for it (cfgref_tag is still A)
* -- genuinely incomplete, not a bytes mismatch. */
TEST_ASSERT_EQUAL_HEX64(tagA, c->persist.cfgref_tag);
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
/* Idempotent retry: calling again while still genuinely unresolved stays
* fenced, does not error/crash/toggle unexpectedly. */
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
t43_free_myconf(c);
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_upgrade_materialization_verify_incomplete_materialization_fences: PASS\n");
}
/* (6.1-d) Materialization for the current marker DID complete (cfgref_*
* correctly captured it, matching), but the LOCAL config file has since been
* overwritten with different bytes (without a new marker committing) --
* simulates the exact "stale embedded copy" trap design.md warns about.
* A correct implementation re-reads the LIVE bytes fresh every call rather
* than trusting cfgref_sha (a checkpoint-time reference) -- so this MUST
* fence, proving the fresh re-read actually happens. */
static void test_upgrade_materialization_verify_stale_local_bytes_after_materialization_fences(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_ucaf61_stale_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t *c = t42_make_cluster(tmp);
TEST_ASSERT_NOT_NULL(c);
t42_make_leader(c, /*term*/23, /*n_peers*/1);
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&c->commit, &c->persist, 1, 8));
repl_set_commit_engine(&c->persist.repl, &c->commit);
myconf_t *mc = t43_attach_myconf(c, tmp);
size_t blen = 0;
uint8_t *bytes = t43_make_config_blob(&blen);
uint8_t sha[32];
SHA256(bytes, blen, sha);
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
free(bytes);
t43_commit_marker(c, tag, sha);
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
/* The local config file is overwritten with DIFFERENT bytes, without any
* new marker committing (simulates local corruption/drift, or a config
* push that lands out of order) -- config_committed_marker still
* returns the SAME marker/sha as before. */
size_t blen2 = 0;
uint8_t *bytes2 = t43_make_config_blob(&blen2);
/* t43_make_config_blob always produces byte-identical content -- mutate
* a trailing byte so the "overwritten" bytes genuinely differ. */
bytes2[blen2 - 1] ^= 0xFF;
uint8_t sha2[32];
SHA256(bytes2, blen2, sha2);
TEST_ASSERT_FALSE(memcmp(sha, sha2, 32) == 0);
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2));
free(bytes2);
/* cfgref_sha (captured at checkpoint time) still shows the OLD matching
* value -- a naive implementation trusting it would wrongly stay clear.
* The correct implementation re-reads the file fresh and fences. */
TEST_ASSERT_EQUAL_HEX64(tag, c->persist.cfgref_tag);
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
t43_free_myconf(c);
commit_engine_fini(&c->commit);
t42_teardown(c);
printf("test_upgrade_materialization_verify_stale_local_bytes_after_materialization_fences: PASS\n");
}
/* (4.3-C) FENCE IS ENFORCED by a production reader: while the config serve-fence
* is set, ha_dispatch_gate refuses config-dependent state-changing operations on
* the PRIMARY (loud non-serving, member stays alive) and a fenced member is
@ -4407,6 +4652,11 @@ void test_config_repl(void)
* no-marker fallback, serve-fence enforced by dispatch gate + promotion). */
RUN_TEST(test_push_path_mismatch_does_not_preactivate);
RUN_TEST(test_config_activate_no_marker_falls_back_and_loads);
/* config-upgrade-snapshot task 6.1: Activation Fence. */
RUN_TEST(test_upgrade_materialization_verify_no_marker_clears_fence);
RUN_TEST(test_upgrade_materialization_verify_matching_materialization_clears_fence);
RUN_TEST(test_upgrade_materialization_verify_incomplete_materialization_fences);
RUN_TEST(test_upgrade_materialization_verify_stale_local_bytes_after_materialization_fences);
RUN_TEST(test_serve_fence_enforced_by_dispatch_gate_and_promotion);
/* Task 4.3 r2 remediation: commit-advance retry clears a push-before-commit
* serve-fence (Finding D / R7.9). */