feat(job-array-single-wal-record): gate promotion on the live apply fence
Threads !persist_apply_resource_fenced(&c->persist) as a required conjunct into both existing promotion-eligibility call sites in ha.c: the periodic fence-release check (promotion_fence_timeout_fn) and the election-time check (raft_cb_become_leader). A node with the fence engaged is refused promotion at both points; once it self-clears (task 4.2), the node is eligible again with no other action. Task 4.3 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
parent
b47cc660ba
commit
8cb8c25d4d
4 changed files with 288 additions and 22 deletions
|
|
@ -2394,6 +2394,104 @@ static void test_promotion_fence_apply_divergent_survives_structural_catchup(voi
|
|||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
|
||||
* Site 1 (periodic fence-release check, promotion_fence_timeout_fn ~ha.c:1069):
|
||||
* the Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
|
||||
* persist_mark_apply_resource_fenced / persist_clear_apply_resource_fence)
|
||||
* must gate this release check exactly as apply_divergent/self_restart_pending
|
||||
* already do — engaging it must survive a purely-structural catch-up
|
||||
* re-check, and clearing it (the apply pump's own self-heal) must restore
|
||||
* eligibility with no other action.
|
||||
*
|
||||
* Mirrors test_promotion_fence_apply_divergent_survives_structural_catchup
|
||||
* above exactly, swapping the trigger for the new fence primitive so the two
|
||||
* triggers are proven orthogonal by construction, same as that test proved
|
||||
* apply_divergent orthogonal to the structural fields.
|
||||
*
|
||||
* RED (pre-task-4.3 code): the timer's release condition has no
|
||||
* !persist_apply_resource_fenced conjunct, so once truncated/below_hwm are
|
||||
* healthy the release branch fires regardless of the live fence — in_catch_up
|
||||
* drops to 0 even though the node is still stuck on a resource-exhaustion
|
||||
* apply failure. GREEN (post-fix): the release branch is skipped while the
|
||||
* fence is engaged (in_catch_up stays 1), and clearing the fence + re-arming
|
||||
* the timer restores eligibility (in_catch_up returns to 0) with no other
|
||||
* action. */
|
||||
static void test_promotion_fence_apply_resource_fenced_survives_structural_catchup(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_fence_res_fenced_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
|
||||
/*durable_floor=*/4);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
|
||||
/* Engage the Live Apply Resource Fence (task 4.2 primitive), exactly as
|
||||
* persist_apply_pump_run does on a resource-exhaustion apply failure --
|
||||
* independent of the WAL-structural fields (truncated/durable_floor) set
|
||||
* above, proving the two conditions are orthogonal by construction. */
|
||||
persist_mark_apply_resource_fenced(&c->persist, /*lsn=*/7);
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
persist_apply_resource_fenced(&c->persist),
|
||||
"fixture precondition: the live apply resource fence must be engaged");
|
||||
|
||||
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
|
||||
|
||||
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
|
||||
uint64_t);
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
|
||||
/*durable_floor=*/4, /*hwm=*/10);
|
||||
TEST_ASSERT_EQUAL_INT(
|
||||
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
|
||||
|
||||
/* Simulate the EXACT SAME structural catch-up as the sibling apply_divergent
|
||||
* test: log now intact and durable past the HWM. The live apply resource
|
||||
* fence is UNTOUCHED by this -- it can only be cleared from inside the
|
||||
* apply pump itself on a successful retry of the stuck LSN (task 4.2),
|
||||
* never by a live structural mutation like this one. */
|
||||
c->persist.recovery_verdict = RECOVER_CLEAN;
|
||||
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
|
||||
|
||||
/* Wait long enough for the window to fire. */
|
||||
usleep(300 * 1000);
|
||||
|
||||
/* THE RED/GREEN ASSERTION (Req 9.4/9.5, task 4.3): even though the
|
||||
* structural conditions the timer inspects are now fully healthy, a node
|
||||
* with the Live Apply Resource Fence engaged must NOT be released -- a
|
||||
* stuck resource-exhaustion apply cannot be proven fixed by more WAL bytes
|
||||
* arriving; only the apply pump's own successful retry may clear it. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
|
||||
"task 4.3: a node with the Live Apply Resource Fence engaged must "
|
||||
"remain fenced (non-promotable) across a purely-structural catch-up "
|
||||
"re-check (Req 9.4/9.5)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
persist_apply_resource_fenced(&c->persist),
|
||||
"the live apply resource fence itself must still read engaged -- only "
|
||||
"the apply pump's own successful retry clears it, not this timer");
|
||||
|
||||
/* Now simulate the apply pump's own self-heal (task 4.2): clear the fence
|
||||
* directly, mirroring persist_clear_apply_resource_fence's real call site
|
||||
* inside persist_apply_pump_run. Re-arm the timer to prove eligibility
|
||||
* returns with NO OTHER ACTION once the fence clears (task 4.3's
|
||||
* Observable clause). */
|
||||
persist_clear_apply_resource_fence(&c->persist);
|
||||
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&c->persist));
|
||||
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/false,
|
||||
/*durable_floor=*/10, /*hwm=*/10);
|
||||
usleep(300 * 1000);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
|
||||
"task 4.3: once the live apply resource fence clears, the node "
|
||||
"becomes eligible again with no other action");
|
||||
|
||||
ha_set_promotion_fence_window_ms(0);
|
||||
dur_destroy_fence_cluster(c);
|
||||
printf("test_promotion_fence_apply_resource_fenced_survives_structural_"
|
||||
"catchup: PASS\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
|
@ -2458,6 +2556,10 @@ void test_commit_durability(void)
|
|||
* an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
|
||||
* alone. */
|
||||
RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup);
|
||||
/* job-array-single-wal-record task 4.3 (Req 9.4/9.5) — site 1: the Live
|
||||
* Apply Resource Fence must gate the periodic release check the same way,
|
||||
* and self-heal (no other action) once the fence clears. */
|
||||
RUN_TEST(test_promotion_fence_apply_resource_fenced_survives_structural_catchup);
|
||||
#else
|
||||
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -1300,6 +1300,134 @@ static void test_promotion_refused_when_apply_divergent_even_if_eligible(void)
|
|||
"checked\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
|
||||
* Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the
|
||||
* Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
|
||||
* persist_mark_apply_resource_fenced) must gate this eligibility check the
|
||||
* same way apply_divergent/self_restart_pending already do.
|
||||
*
|
||||
* Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above
|
||||
* exactly — same structurally-healthy fixture (durable_floor(0) >=
|
||||
* promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision
|
||||
* == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the
|
||||
* sticky-flag conjunct contributes nothing) — swapping the trigger for the new
|
||||
* fence primitive so the new conjunct is isolated as the ONLY variable
|
||||
* changed, exactly as the round-3 apply_divergent test isolated its conjunct.
|
||||
*
|
||||
* RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced
|
||||
* conjunct, so a structurally-eligible node with the live fence engaged is
|
||||
* promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get
|
||||
* registered as standbys. GREEN (post-fix): promotion is refused — role stays
|
||||
* non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no
|
||||
* standbys are registered. */
|
||||
static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void)
|
||||
{
|
||||
/* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible:
|
||||
* durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 =>
|
||||
* genuinely ELIGIBLE on every OTHER conjunct. */
|
||||
cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
||||
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
persist_recovery_unrecoverable(&g_cv.persist),
|
||||
"fixture precondition: sticky recovery_unrecoverable must be clear so "
|
||||
"this test isolates the apply-resource-fence guard, not the "
|
||||
"sticky-flag guard");
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
g_cv.persist.apply_verdict == APPLY_DIVERGENT,
|
||||
"fixture precondition: apply_verdict must not be APPLY_DIVERGENT so "
|
||||
"this test isolates the apply-resource-fence guard from the sibling "
|
||||
"apply_divergent guard");
|
||||
|
||||
/* Engage the Live Apply Resource Fence directly -- exactly as
|
||||
* persist_apply_pump_run does on a resource-exhaustion apply failure
|
||||
* (task 4.2), and exactly as the sibling site-1 test
|
||||
* (test_commit_durability.c) sets it for that call site. */
|
||||
persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3);
|
||||
TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist));
|
||||
|
||||
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
||||
TEST_ASSERT_NOT_NULL(ctx);
|
||||
ctx->n_peers = 2;
|
||||
pthread_mutex_init(&ctx->lock, NULL);
|
||||
sem_init(&ctx->done, 0, 0);
|
||||
|
||||
xce_candidate_t peer0 = {0}, peer1 = {0};
|
||||
cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
|
||||
cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
|
||||
|
||||
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
||||
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
|
||||
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
|
||||
|
||||
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
|
||||
|
||||
/* Act: real promotion attempt with a live, structurally-healthy cluster. */
|
||||
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
|
||||
|
||||
/* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE +
|
||||
* unrecoverable==false + apply_verdict!=DIVERGENT -- the new
|
||||
* apply-resource-fence conjunct alone must block promotion. */
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(
|
||||
(int32_t)CTRL_PRIMARY,
|
||||
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
||||
"job-array-single-wal-record task 4.3 (Req 9.4/9.5): a "
|
||||
"structurally-eligible node with the Live Apply Resource Fence "
|
||||
"engaged must NOT be promoted to CTRL_PRIMARY");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire),
|
||||
"a refused apply-resource-fenced promotion must fence the member "
|
||||
"(in_catch_up=1) and keep it alive, retrying catch-up -- never "
|
||||
"self-exit");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
||||
repl_active_standby_count(&g_cv.persist.repl),
|
||||
"promotion refusal must happen BEFORE the standby-registration block "
|
||||
"-- no vote endpoint may be registered when the node never became "
|
||||
"primary");
|
||||
|
||||
/* Now clear the fence (the apply pump's own self-heal, task 4.2) and
|
||||
* retry promotion: eligibility must return with no other action (task
|
||||
* 4.3's Observable clause). */
|
||||
persist_clear_apply_resource_fence(&g_cv.persist);
|
||||
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist));
|
||||
atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release);
|
||||
|
||||
xce_candidate_t peer0b = {0}, peer1b = {0};
|
||||
ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0]));
|
||||
TEST_ASSERT_NOT_NULL(ctx2);
|
||||
ctx2->n_peers = 2;
|
||||
pthread_mutex_init(&ctx2->lock, NULL);
|
||||
sem_init(&ctx2->done, 0, 0);
|
||||
cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700);
|
||||
cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701);
|
||||
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release);
|
||||
|
||||
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2);
|
||||
|
||||
TEST_ASSERT_EQUAL_MESSAGE(
|
||||
(int32_t)CTRL_PRIMARY,
|
||||
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
||||
"task 4.3: once the live apply resource fence clears, the node is "
|
||||
"eligible for promotion again with no other action");
|
||||
|
||||
/* Cleanup */
|
||||
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
|
||||
if (g_cv.ha.standby_endpoints) {
|
||||
zarray_fini(g_cv.ha.standby_endpoints);
|
||||
free(g_cv.ha.standby_endpoints);
|
||||
g_cv.ha.standby_endpoints = NULL;
|
||||
}
|
||||
pthread_mutex_destroy(&ctx->lock);
|
||||
sem_destroy(&ctx->done);
|
||||
free(ctx);
|
||||
pthread_mutex_destroy(&ctx2->lock);
|
||||
sem_destroy(&ctx2->done);
|
||||
free(ctx2);
|
||||
cv_teardown();
|
||||
printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: "
|
||||
"checked\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
|
@ -1341,6 +1469,11 @@ void test_convergence_verification(void)
|
|||
"(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n");
|
||||
|
||||
RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible);
|
||||
|
||||
printf("\n>> Promotion eligibility gate site-2 coverage "
|
||||
"(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n");
|
||||
|
||||
RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible);
|
||||
#else
|
||||
printf("test_convergence_verification: SKIP (non-test build)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue