forked from lei.chen/myxce
feat(job-array-single-wal-record): gate promotion on the live apply fence
Threads !persist_apply_resource_fenced(&c->persist) as a required conjunct into both existing promotion-eligibility call sites in ha.c: the periodic fence-release check (promotion_fence_timeout_fn) and the election-time check (raft_cb_become_leader). A node with the fence engaged is refused promotion at both points; once it self-clears (task 4.2), the node is eligible again with no other action. Task 4.3 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
parent
b47cc660ba
commit
8cb8c25d4d
4 changed files with 288 additions and 22 deletions
|
|
@ -2394,6 +2394,104 @@ static void test_promotion_fence_apply_divergent_survives_structural_catchup(voi
|
|||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
|
||||
* Site 1 (periodic fence-release check, promotion_fence_timeout_fn ~ha.c:1069):
|
||||
* the Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
|
||||
* persist_mark_apply_resource_fenced / persist_clear_apply_resource_fence)
|
||||
* must gate this release check exactly as apply_divergent/self_restart_pending
|
||||
* already do — engaging it must survive a purely-structural catch-up
|
||||
* re-check, and clearing it (the apply pump's own self-heal) must restore
|
||||
* eligibility with no other action.
|
||||
*
|
||||
* Mirrors test_promotion_fence_apply_divergent_survives_structural_catchup
|
||||
* above exactly, swapping the trigger for the new fence primitive so the two
|
||||
* triggers are proven orthogonal by construction, same as that test proved
|
||||
* apply_divergent orthogonal to the structural fields.
|
||||
*
|
||||
* RED (pre-task-4.3 code): the timer's release condition has no
|
||||
* !persist_apply_resource_fenced conjunct, so once truncated/below_hwm are
|
||||
* healthy the release branch fires regardless of the live fence — in_catch_up
|
||||
* drops to 0 even though the node is still stuck on a resource-exhaustion
|
||||
* apply failure. GREEN (post-fix): the release branch is skipped while the
|
||||
* fence is engaged (in_catch_up stays 1), and clearing the fence + re-arming
|
||||
* the timer restores eligibility (in_catch_up returns to 0) with no other
|
||||
* action. */
|
||||
static void test_promotion_fence_apply_resource_fenced_survives_structural_catchup(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_fence_res_fenced_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
|
||||
/*durable_floor=*/4);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
|
||||
/* Engage the Live Apply Resource Fence (task 4.2 primitive), exactly as
|
||||
* persist_apply_pump_run does on a resource-exhaustion apply failure --
|
||||
* independent of the WAL-structural fields (truncated/durable_floor) set
|
||||
* above, proving the two conditions are orthogonal by construction. */
|
||||
persist_mark_apply_resource_fenced(&c->persist, /*lsn=*/7);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
persist_apply_resource_fenced(&c->persist),
|
||||
"fixture precondition: the live apply resource fence must be engaged");
|
||||
|
||||
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
|
||||
|
||||
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
|
||||
uint64_t);
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
|
||||
/*durable_floor=*/4, /*hwm=*/10);
|
||||
TEST_ASSERT_EQUAL_INT(
|
||||
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
|
||||
|
||||
/* Simulate the EXACT SAME structural catch-up as the sibling apply_divergent
|
||||
* test: log now intact and durable past the HWM. The live apply resource
|
||||
* fence is UNTOUCHED by this -- it can only be cleared from inside the
|
||||
* apply pump itself on a successful retry of the stuck LSN (task 4.2),
|
||||
* never by a live structural mutation like this one. */
|
||||
c->persist.recovery_verdict = RECOVER_CLEAN;
|
||||
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
|
||||
|
||||
/* Wait long enough for the window to fire. */
|
||||
usleep(300 * 1000);
|
||||
|
||||
/* THE RED/GREEN ASSERTION (Req 9.4/9.5, task 4.3): even though the
|
||||
* structural conditions the timer inspects are now fully healthy, a node
|
||||
* with the Live Apply Resource Fence engaged must NOT be released -- a
|
||||
* stuck resource-exhaustion apply cannot be proven fixed by more WAL bytes
|
||||
* arriving; only the apply pump's own successful retry may clear it. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
|
||||
"task 4.3: a node with the Live Apply Resource Fence engaged must "
|
||||
"remain fenced (non-promotable) across a purely-structural catch-up "
|
||||
"re-check (Req 9.4/9.5)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
persist_apply_resource_fenced(&c->persist),
|
||||
"the live apply resource fence itself must still read engaged -- only "
|
||||
"the apply pump's own successful retry clears it, not this timer");
|
||||
|
||||
/* Now simulate the apply pump's own self-heal (task 4.2): clear the fence
|
||||
* directly, mirroring persist_clear_apply_resource_fence's real call site
|
||||
* inside persist_apply_pump_run. Re-arm the timer to prove eligibility
|
||||
* returns with NO OTHER ACTION once the fence clears (task 4.3's
|
||||
* Observable clause). */
|
||||
persist_clear_apply_resource_fence(&c->persist);
|
||||
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&c->persist));
|
||||
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/false,
|
||||
/*durable_floor=*/10, /*hwm=*/10);
|
||||
usleep(300 * 1000);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
|
||||
"task 4.3: once the live apply resource fence clears, the node "
|
||||
"becomes eligible again with no other action");
|
||||
|
||||
ha_set_promotion_fence_window_ms(0);
|
||||
dur_destroy_fence_cluster(c);
|
||||
printf("test_promotion_fence_apply_resource_fenced_survives_structural_"
|
||||
"catchup: PASS\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
|
@ -2458,6 +2556,10 @@ void test_commit_durability(void)
|
|||
* an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
|
||||
* alone. */
|
||||
RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup);
|
||||
/* job-array-single-wal-record task 4.3 (Req 9.4/9.5) — site 1: the Live
|
||||
* Apply Resource Fence must gate the periodic release check the same way,
|
||||
* and self-heal (no other action) once the fence clears. */
|
||||
RUN_TEST(test_promotion_fence_apply_resource_fenced_survives_structural_catchup);
|
||||
#else
|
||||
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue