forked from lei.chen/myxce
feat(config-upgrade-snapshot): gate primary serving and promotion on the activation fence
Wires task 6.1's upgrade_materialization_verify/upgrade_materialization_fenced into ha_dispatch_gate's CTRL_PRIMARY branch (sequential block alongside, not merged into, the existing config_serve_fenced check -- same replication/read-only opcode allowlist, refuses state-changing ops with -XCE_ERR_NOT_READY while fenced) and the promotion gate (raft_cb_become_leader refuses to become CTRL_PRIMARY while fenced, introduces no new WAL/log-completeness eligibility mechanism, never self-terminates). Adds a 4-function self-triggering retry trampoline (upgrade_materialization_retry_fn/_vtask_entry/_delaymgr_trampoline/ schedule_upgrade_materialization_retry) mirroring config_activate_retry_fn's exact shape, hooked into ha_cb_commit_advanced_stub alongside the existing config-byte retry. Fixes a real bug caught on review: the fence had no production path that could ever engage it from its zero-initialized default -- every setter call site was gated on the fence already being set (closed loop), making the new gate checks permanently dead code. Fixed with two unconditional upgrade_materialization_verify() call sites: persist.c's apply_committed_entry WAL_CONFIG_MARKER case (right after persist_checkpoint_marker_triggered returns, regardless of its rc) and myapp.c's boot sequence (right after config_activate_committed, for Req 5.2 crash/restart resume). New test drives the fence through the real production trigger (a forced -EBUSY checkpoint miss on a real marker commit+apply), not a direct verify() call, proving the end-to-end path actually works. Task 6.2 of 19. Requirements: 3.1, 3.2, 3.3, 3.4, 4.3, 5.5
This commit is contained in:
parent
7b507156a0
commit
89b85e1680
6 changed files with 805 additions and 7 deletions
|
|
@ -1687,6 +1687,29 @@ static void test_serve_fence_enforced_by_dispatch_gate_and_promotion(void)
|
|||
TEST_ASSERT_EQUAL_INT(0, config_activate_committed(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, config_serve_fenced(c));
|
||||
TEST_ASSERT_EQUAL_HEX64(ttag, mc->config_version_tag);
|
||||
|
||||
/* config-upgrade-snapshot task 6.2 remediation: t43_commit_marker above
|
||||
* drives the TARGET marker through the REAL apply path, which (as of
|
||||
* the 6.2 fix) makes apply_committed_entry's WAL_CONFIG_MARKER case
|
||||
* (persist.c) unconditionally re-evaluate the separate, independent
|
||||
* upgrade_materialization_fence too -- not just config_serve_fence,
|
||||
* which is all this test is otherwise about. At that moment the TARGET
|
||||
* bytes had not arrived locally yet (this test's own deliberate setup,
|
||||
* to exercise config_serve_fence), so that checkpoint's
|
||||
* scan_latest_committed_config_ref already stamped cfgref_{tag,sha}
|
||||
* from the marker itself (materialized-reference present), but the
|
||||
* live-bytes hash-check genuinely failed (no local bytes yet) --
|
||||
* correctly setting upgrade_materialization_fence too, per Req 3.2.
|
||||
* That is orthogonal, correct behavior, not a bug in either fence, but
|
||||
* this test never previously accounted for it (the fence was
|
||||
* unreachable before the 6.2 fix). Re-verify it here now that the
|
||||
* correct bytes are locally present, mirroring config_activate_
|
||||
* committed's own re-verify just above for config_serve_fence, so the
|
||||
* remaining assertions below exercise only the fence this test is
|
||||
* actually about. */
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_jobsub));
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_config_promotion_fenced(&c->ha));
|
||||
|
||||
|
|
@ -1857,6 +1880,455 @@ static void test_commit_advance_retry_clears_push_before_commit_fence(void)
|
|||
printf("test_commit_advance_retry_clears_push_before_commit_fence: PASS\n");
|
||||
}
|
||||
|
||||
/* ── config-upgrade-snapshot task 6.2: Activation Fence gate composition +
|
||||
* self-triggering retry (upgrade_materialization_fenced / ha_dispatch_gate /
|
||||
* raft_cb_become_leader / schedule_upgrade_materialization_retry, ha.c) ────
|
||||
*
|
||||
* These tests reuse this file's own t42_/t43_ fixture helpers and mirror
|
||||
* the existing (4.3-C/D) config_serve_fence gate-composition/retry tests
|
||||
* above as the direct template, for the new independent
|
||||
* upgrade_materialization_fence instead.
|
||||
*/
|
||||
|
||||
/* raft_cb_become_leader is file-static; drive it through the LIVE-cluster
|
||||
* XCE_TESTING seam (fix-promotion-standby-registration) so this test proves
|
||||
* the REAL election-time promotion decision refuses while fenced, not just a
|
||||
* proxy accessor read. */
|
||||
extern void raft_cb_become_leader_with_cluster_for_test(cluster_t *c,
|
||||
uint64_t term);
|
||||
/* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback
|
||||
* (ha_cb_commit_advanced_stub) so the wiring test below exercises the real
|
||||
* commit-advance trigger, not a hand-rolled call. */
|
||||
extern void ha_cb_commit_advanced_for_test(ha_state_t *ha,
|
||||
uint64_t commit_index);
|
||||
|
||||
/* (6.2-a) UPGRADE MATERIALIZATION FENCE IS ENFORCED by two production
|
||||
* readers: (1) ha_dispatch_gate refuses a config-dependent state-changing
|
||||
* operation on the PRIMARY while fenced (loud non-serving, member stays
|
||||
* alive) but keeps replication/read-only ops flowing so materialization can
|
||||
* proceed and re-verify; (2) a REAL election-time promotion attempt
|
||||
* (raft_cb_become_leader_with_cluster_for_test — the actual gate, not a proxy
|
||||
* accessor) refuses to become CTRL_PRIMARY while this member's own
|
||||
* materialization for the current committed marker is incomplete. Regression
|
||||
* for the dispatch-gate/promotion-gate composition (Req 3.2, 3.3, 4.3, 5.5).
|
||||
*/
|
||||
static void test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr62_fence_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/40, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Commit a marker but never checkpoint against it -- this member's own
|
||||
* materialization is genuinely incomplete for it (identical setup to
|
||||
* test_upgrade_materialization_verify_matching_materialization_clears_fence's
|
||||
* sibling incomplete-materialization test, just without the -EBUSY
|
||||
* choreography -- a fresh marker with no checkpoint yet run is already
|
||||
* incomplete by construction). */
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
|
||||
/* Force materialization to stay incomplete for THIS marker: hold the
|
||||
* shared checkpoint_in_progress CAS open across the commit (the same
|
||||
* mechanism test_upgrade_materialization_verify_incomplete_materialization_
|
||||
* fences uses), so the marker-apply checkpoint hits -EBUSY and cfgref_*
|
||||
* never advances to it. */
|
||||
TEST_ASSERT_EQUAL_INT(
|
||||
0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1,
|
||||
memory_order_acq_rel));
|
||||
uint64_t mlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000));
|
||||
uint64_t deadline_iters = 200;
|
||||
while (deadline_iters-- > 0 &&
|
||||
atomic_load_explicit(&c->persist.last_applied_lsn,
|
||||
memory_order_acquire) < mlsn) {
|
||||
struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000 * 1000 };
|
||||
nanosleep(&ts, NULL);
|
||||
}
|
||||
atomic_store_explicit(&c->persist.checkpoint_in_progress, 0,
|
||||
memory_order_release);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
|
||||
/* PRODUCTION READER 1: ha_dispatch_gate refuses a state-changing op on
|
||||
* the fenced PRIMARY but stays alive (error, not crash/exit). */
|
||||
TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0);
|
||||
/* Replication / read-only ops still pass. */
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_heartbeat));
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_get));
|
||||
|
||||
/* PRODUCTION READER 2: a real election-time promotion attempt refuses to
|
||||
* become CTRL_PRIMARY while fenced. ha_primary_log_append above only
|
||||
* required c->ha.raft.role == RAFT_LEADER (set by t42_make_leader), not
|
||||
* ha->role == CTRL_PRIMARY, so resetting ha->role to CTRL_ELECTING here
|
||||
* to observe a genuine promotion decision does not disturb the marker
|
||||
* already committed above. */
|
||||
c->ha.phase2_done = 1; /* skip cluster_init_phase2 -- unrelated first-boot path */
|
||||
atomic_store_explicit(&c->ha.role, (int32_t)CTRL_ELECTING,
|
||||
memory_order_release);
|
||||
raft_cb_become_leader_with_cluster_for_test(c, /*term*/40);
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(
|
||||
(int32_t)CTRL_PRIMARY,
|
||||
atomic_load_explicit(&c->ha.role, memory_order_acquire),
|
||||
"config-upgrade-snapshot task 6.2 (Req 3.2/3.3/4.3/5.5): a member "
|
||||
"whose own upgrade-snapshot materialization is incomplete must NOT "
|
||||
"be promoted to CTRL_PRIMARY");
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* unaffected */
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion: PASS\n");
|
||||
}
|
||||
|
||||
/* (6.2-b) schedule_upgrade_materialization_retry is a no-op when NOT fenced
|
||||
* -- steady state schedules nothing (no busy-spin), mirroring
|
||||
* config_schedule_activate_retry's own documented no-op behavior. */
|
||||
static void test_schedule_upgrade_materialization_retry_noop_when_not_fenced(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr62_retry_noop_unfenced_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/41, /*n_peers*/1);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c)); /* fixture default: unfenced */
|
||||
TEST_ASSERT_FALSE(c->delaymgr.running); /* fixture never starts delaymgr */
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, schedule_upgrade_materialization_retry(c));
|
||||
|
||||
t42_teardown(c);
|
||||
printf("test_schedule_upgrade_materialization_retry_noop_when_not_fenced: PASS\n");
|
||||
}
|
||||
|
||||
/* (6.2-c) schedule_upgrade_materialization_retry is a no-op when the
|
||||
* delaymgr is not running, even while fenced -- the unit-test fixture path
|
||||
* (mirrors config_schedule_activate_retry's own documented delaymgr-not-
|
||||
* running no-op; test_commit_advance_retry_clears_push_before_commit_fence
|
||||
* relies on this same fixture property to drive its retry entry directly). */
|
||||
static void test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr62_retry_noop_nodmgr_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/42, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
t43_commit_marker(c, tag, sha);
|
||||
|
||||
/* Overwrite the local bytes without a new marker committing -- the same
|
||||
* "stale local bytes" fence trigger
|
||||
* test_upgrade_materialization_verify_stale_local_bytes_after_
|
||||
* materialization_fences uses -- so the fence is genuinely SET without
|
||||
* needing the -EBUSY checkpoint choreography. */
|
||||
size_t blen2 = 0;
|
||||
uint8_t *bytes2 = t43_make_config_blob(&blen2);
|
||||
bytes2[blen2 - 1] ^= 0xFF;
|
||||
uint8_t sha2[32];
|
||||
SHA256(bytes2, blen2, sha2);
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2));
|
||||
free(bytes2);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
TEST_ASSERT_FALSE(c->delaymgr.running);
|
||||
|
||||
/* delaymgr not running -- no-op, returns 0, does not crash/leak a task. */
|
||||
TEST_ASSERT_EQUAL_INT(0, schedule_upgrade_materialization_retry(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c)); /* still fenced -- untouched */
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running: PASS\n");
|
||||
}
|
||||
|
||||
/* (6.2-d) upgrade_materialization_retry_fn -- the same off-thread entry
|
||||
* point schedule_upgrade_materialization_retry's trampoline would enqueue in
|
||||
* production -- stays fenced (-EAGAIN) while the genuine mismatch persists,
|
||||
* then CLEARS the fence once the live bytes are corrected to match the
|
||||
* committed marker, without a new marker needing to commit. Mirrors
|
||||
* config_activate_retry_fn's own direct-drive regression
|
||||
* (test_commit_advance_retry_clears_push_before_commit_fence). */
|
||||
static void test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr62_retry_fn_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/43, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
t43_commit_marker(c, tag, sha);
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
|
||||
|
||||
/* Local bytes are overwritten with DIFFERENT content, without a new
|
||||
* marker committing -- cfgref_* still shows the old matching reference,
|
||||
* but a fresh live re-read must fence (same trap 6.1-d proves for
|
||||
* upgrade_materialization_verify itself). */
|
||||
size_t blenX = 0;
|
||||
uint8_t *bytesX = t43_make_config_blob(&blenX);
|
||||
bytesX[blenX - 1] ^= 0xA5;
|
||||
uint8_t shaX[32];
|
||||
SHA256(bytesX, blenX, shaX);
|
||||
TEST_ASSERT_FALSE(memcmp(sha, shaX, 32) == 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytesX, blenX, shaX));
|
||||
free(bytesX);
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
|
||||
/* Drive the retry entry directly while the mismatch is still genuine:
|
||||
* stays fenced, does not crash/toggle incorrectly. */
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_retry_fn(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
|
||||
/* Restore the CORRECT bytes (matching the still-current committed
|
||||
* marker) without any new marker commit -- models the out-of-band
|
||||
* config-push bytes finally arriving/correcting. */
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
|
||||
/* Drive the retry entry again: re-verify now succeeds -- fence CLEARS. */
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_retry_fn(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence: PASS\n");
|
||||
}
|
||||
|
||||
/* (6.2-e) WIRING: the commit-advance path (ha_cb_commit_advanced_stub, the
|
||||
* WIRED on_commit_advanced callback) consults upgrade_materialization_fenced
|
||||
* and calls schedule_upgrade_materialization_retry when fenced -- driven
|
||||
* through the REAL wired callback (ha_cb_commit_advanced_for_test), not a
|
||||
* hand-rolled call, so this proves the actual call-site wiring executes
|
||||
* without crashing/corrupting state. schedule_upgrade_materialization_retry
|
||||
* itself is a no-op in this fixture (delaymgr not running -- proven in
|
||||
* isolation by 6.2-c above), so the fence is expected to remain SET
|
||||
* afterward; this test's contribution is exercising the NEW branch at the
|
||||
* real call site itself, not re-proving the no-op contract. */
|
||||
static void test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr62_wiring_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/44, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
t43_commit_marker(c, tag, sha);
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
|
||||
|
||||
/* Overwrite local bytes (stale-bytes trigger, same as 6.2-c) to fence
|
||||
* without a new marker. */
|
||||
size_t blen2 = 0;
|
||||
uint8_t *bytes2 = t43_make_config_blob(&blen2);
|
||||
bytes2[blen2 - 1] ^= 0xFF;
|
||||
uint8_t sha2[32];
|
||||
SHA256(bytes2, blen2, sha2);
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes2, blen2, sha2));
|
||||
free(bytes2);
|
||||
TEST_ASSERT_EQUAL_INT(-EAGAIN, upgrade_materialization_verify(c));
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
|
||||
uint64_t committed_lsn =
|
||||
atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire);
|
||||
|
||||
/* Drive the REAL wired commit-advance callback -- must not crash, must
|
||||
* not corrupt the fence state (schedule_upgrade_materialization_retry's
|
||||
* own no-op-when-delaymgr-not-running contract keeps it SET). */
|
||||
ha_cb_commit_advanced_for_test(&c->ha, committed_lsn);
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry: PASS\n");
|
||||
}
|
||||
|
||||
/* ── config-upgrade-snapshot task 6.2 remediation: Activation Fence set by a
|
||||
* REAL production trigger, not a direct test call ──────────────────────────
|
||||
*
|
||||
* Every test above (6.2-a..e, and 6.1's own 4 tests earlier in this file)
|
||||
* reaches the fenced state by calling upgrade_materialization_verify()
|
||||
* directly in the test body. That proves the fence's own logic is correct,
|
||||
* but NOT that a running production controller can ever reach the fenced
|
||||
* state in the first place: every PRODUCTION call path to
|
||||
* upgrade_materialization_verify() -- upgrade_materialization_retry_fn, and
|
||||
* both of ITS callers (schedule_upgrade_materialization_retry and the
|
||||
* commit-advance hook in ha_cb_commit_advanced_stub) -- is itself gated on
|
||||
* upgrade_materialization_fenced() already being true. Starting from the
|
||||
* fence's zero-initialized (unfenced) default, that is a closed loop with
|
||||
* no entry point: nothing in production could ever set the fence for the
|
||||
* first time, which made the dispatch-gate/promotion-gate checks in 6.2-a
|
||||
* permanently dead code against a real controller.
|
||||
*
|
||||
* The actual fix (persist.c's apply_committed_entry, WAL_CONFIG_MARKER
|
||||
* case) calls upgrade_materialization_verify() UNCONDITIONALLY right after
|
||||
* persist_checkpoint_marker_triggered() returns, regardless of that call's
|
||||
* own rc -- checkpoint completion (or a self-resolving skip) is the
|
||||
* design-intended re-evaluation trigger (design.md's Activation Fence
|
||||
* component). This test reaches the fenced state purely through that real
|
||||
* seam -- config_append_marker + commit_submit + wait-for-applied, with
|
||||
* persist_checkpoint_marker_triggered() forced to miss via the exact
|
||||
* checkpoint_in_progress technique test_upgrade_materialization_verify_
|
||||
* incomplete_materialization_fences (6.1-c) and test_upgrade_materialization
|
||||
* _fence_enforced_by_dispatch_gate_and_promotion (6.2-a) use -- and NEVER
|
||||
* calls upgrade_materialization_verify() itself. */
|
||||
static void test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_cr62_real_trigger_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t *c = t42_make_cluster(tmp);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
t42_make_leader(c, /*term*/45, /*n_peers*/1);
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_engine_init(&c->commit, &c->persist, 1, 8));
|
||||
repl_set_commit_engine(&c->persist.repl, &c->commit);
|
||||
myconf_t *mc = t43_attach_myconf(c, tmp);
|
||||
|
||||
/* Fixture default: unfenced (fresh cluster_t, zero-initialized atomic)
|
||||
* -- the exact "closed loop, no entry point" starting condition the
|
||||
* rejection identified. This test body never calls
|
||||
* upgrade_materialization_verify() anywhere below -- the fence must be
|
||||
* set purely by the real apply-time trigger. */
|
||||
TEST_ASSERT_EQUAL_INT(0, upgrade_materialization_fenced(c));
|
||||
|
||||
size_t blen = 0;
|
||||
uint8_t *bytes = t43_make_config_blob(&blen);
|
||||
uint8_t sha[32];
|
||||
SHA256(bytes, blen, sha);
|
||||
uint64_t tag; { uint64_t t; memcpy(&t, sha, 8); tag = be64toh(t); }
|
||||
TEST_ASSERT_EQUAL_INT(0, conf_save_config(mc, bytes, blen, sha));
|
||||
free(bytes);
|
||||
|
||||
/* Force materialization to genuinely miss for THIS marker: hold the
|
||||
* shared checkpoint_in_progress CAS open across the commit, so
|
||||
* persist_checkpoint_marker_triggered() (called from
|
||||
* apply_committed_entry's WAL_CONFIG_MARKER case) hits the real -EBUSY
|
||||
* path (Req 4.4) -- self-resolving, not a failure, so apply itself
|
||||
* still succeeds and last_applied_lsn still advances. */
|
||||
TEST_ASSERT_EQUAL_INT(
|
||||
0, atomic_exchange_explicit(&c->persist.checkpoint_in_progress, 1,
|
||||
memory_order_acq_rel));
|
||||
uint64_t mlsn = 0;
|
||||
TEST_ASSERT_EQUAL_INT(0, config_append_marker(c, tag, sha, &mlsn));
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_flush(&c->persist));
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
commit_submit(&c->commit, mlsn, COMMIT_MODE_WAIT, 1000));
|
||||
|
||||
/* Wait for the REAL apply_committed_entry() dispatch to finish for this
|
||||
* marker. persist_apply_pump_run advances last_applied_lsn only AFTER
|
||||
* that dispatch call returns (see apply_committed_entry's own
|
||||
* WAL_CONFIG_MARKER case doc comment), so by the time
|
||||
* t42_wait_applied observes last_applied_lsn >= mlsn, the new
|
||||
* unconditional upgrade_materialization_verify(c) call at the end of
|
||||
* that case has already run synchronously on this same call stack. */
|
||||
t42_wait_applied(&c->persist, mlsn, 2000);
|
||||
TEST_ASSERT_TRUE(persist_last_applied_lsn(&c->persist) >= mlsn);
|
||||
|
||||
/* Release the simulated in-flight checkpoint now that the marker's own
|
||||
* apply (and its -EBUSY'd checkpoint attempt) has already happened. */
|
||||
atomic_store_explicit(&c->persist.checkpoint_in_progress, 0,
|
||||
memory_order_release);
|
||||
|
||||
/* THE ASSERTION THAT PROVES THE FIX: the fence reads SET, and this test
|
||||
* body never called upgrade_materialization_verify() itself -- the only
|
||||
* possible source is persist.c's new unconditional post-checkpoint
|
||||
* call. Before the fix, this read 0 forever: every production path to
|
||||
* upgrade_materialization_verify() was gated on the fence already being
|
||||
* 1, so nothing could ever set it for the first time. */
|
||||
TEST_ASSERT_EQUAL_INT(1, upgrade_materialization_fenced(c));
|
||||
/* Confirm this is a genuine incomplete-materialization fence (cfgref_*
|
||||
* never advanced to this marker, because the checkpoint that would have
|
||||
* stamped it was skipped) -- not a coincidental pass. */
|
||||
TEST_ASSERT_FALSE(c->persist.cfgref_present &&
|
||||
c->persist.cfgref_tag == tag &&
|
||||
memcmp(c->persist.cfgref_sha, sha, 32) == 0);
|
||||
|
||||
/* Close the loop end-to-end: the two production READERS of this fence
|
||||
* (ha_dispatch_gate and the promotion gate) both observe it too, using
|
||||
* the same real seams 6.2-a exercises -- reusing existing test
|
||||
* infrastructure, not re-deriving it. */
|
||||
TEST_ASSERT_TRUE(ha_dispatch_gate(&c->ha, OPCode_jobsub) < 0);
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_dispatch_gate(&c->ha, OPCode_heartbeat));
|
||||
|
||||
c->ha.phase2_done = 1; /* skip cluster_init_phase2 -- unrelated first-boot path */
|
||||
atomic_store_explicit(&c->ha.role, (int32_t)CTRL_ELECTING,
|
||||
memory_order_release);
|
||||
raft_cb_become_leader_with_cluster_for_test(c, /*term*/45);
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(
|
||||
(int32_t)CTRL_PRIMARY,
|
||||
atomic_load_explicit(&c->ha.role, memory_order_acquire),
|
||||
"config-upgrade-snapshot task 6.2 remediation: a fence set by the "
|
||||
"REAL apply-time trigger must refuse promotion, same as a "
|
||||
"directly-set fence does");
|
||||
|
||||
t43_free_myconf(c);
|
||||
commit_engine_fini(&c->commit);
|
||||
t42_teardown(c);
|
||||
printf("test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss: PASS\n");
|
||||
}
|
||||
|
||||
/* ── wait_all_with_timeout (XCE_TESTING shim) ───── */
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
|
|
@ -4661,6 +5133,17 @@ void test_config_repl(void)
|
|||
/* Task 4.3 r2 remediation: commit-advance retry clears a push-before-commit
|
||||
* serve-fence (Finding D / R7.9). */
|
||||
RUN_TEST(test_commit_advance_retry_clears_push_before_commit_fence);
|
||||
/* config-upgrade-snapshot task 6.2: Activation Fence gate composition +
|
||||
* self-triggering retry. */
|
||||
RUN_TEST(test_upgrade_materialization_fence_enforced_by_dispatch_gate_and_promotion);
|
||||
RUN_TEST(test_schedule_upgrade_materialization_retry_noop_when_not_fenced);
|
||||
RUN_TEST(test_schedule_upgrade_materialization_retry_noop_when_delaymgr_not_running);
|
||||
RUN_TEST(test_upgrade_materialization_retry_fn_recovers_from_stale_bytes_fence);
|
||||
RUN_TEST(test_commit_advance_wiring_calls_upgrade_materialization_schedule_retry);
|
||||
/* config-upgrade-snapshot task 6.2 remediation: fence set by a REAL
|
||||
* production trigger (apply_committed_entry's WAL_CONFIG_MARKER case),
|
||||
* not a direct test call -- proves the fence is reachable at all. */
|
||||
RUN_TEST(test_wal_config_marker_apply_sets_fence_via_real_checkpoint_miss);
|
||||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_load_primary_config_missing_returns_error);
|
||||
RUN_TEST(test_wait_all_with_timeout_slow_threads);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue