feat(job-array-single-wal-record): gate promotion on the live apply fence
Threads !persist_apply_resource_fenced(&c->persist) as a required conjunct into both existing promotion-eligibility call sites in ha.c: the periodic fence-release check (promotion_fence_timeout_fn) and the election-time check (raft_cb_become_leader). A node with the fence engaged is refused promotion at both points; once it self-clears (task 4.2), the node is eligible again with no other action. Task 4.3 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
parent
b47cc660ba
commit
8cb8c25d4d
4 changed files with 288 additions and 22 deletions
|
|
@ -1300,6 +1300,134 @@ static void test_promotion_refused_when_apply_divergent_even_if_eligible(void)
|
|||
"checked\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
|
||||
* Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the
|
||||
* Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
|
||||
* persist_mark_apply_resource_fenced) must gate this eligibility check the
|
||||
* same way apply_divergent/self_restart_pending already do.
|
||||
*
|
||||
* Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above
|
||||
* exactly — same structurally-healthy fixture (durable_floor(0) >=
|
||||
* promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision
|
||||
* == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the
|
||||
* sticky-flag conjunct contributes nothing) — swapping the trigger for the new
|
||||
* fence primitive so the new conjunct is isolated as the ONLY variable
|
||||
* changed, exactly as the round-3 apply_divergent test isolated its conjunct.
|
||||
*
|
||||
* RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced
|
||||
* conjunct, so a structurally-eligible node with the live fence engaged is
|
||||
* promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get
|
||||
* registered as standbys. GREEN (post-fix): promotion is refused — role stays
|
||||
* non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no
|
||||
* standbys are registered. */
|
||||
static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void)
|
||||
{
|
||||
/* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible:
|
||||
* durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 =>
|
||||
* genuinely ELIGIBLE on every OTHER conjunct. */
|
||||
cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0,
|
||||
/*appended_lsn=*/0, /*n_vcpus=*/8);
|
||||
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
persist_recovery_unrecoverable(&g_cv.persist),
|
||||
"fixture precondition: sticky recovery_unrecoverable must be clear so "
|
||||
"this test isolates the apply-resource-fence guard, not the "
|
||||
"sticky-flag guard");
|
||||
TEST_ASSERT_FALSE_MESSAGE(
|
||||
g_cv.persist.apply_verdict == APPLY_DIVERGENT,
|
||||
"fixture precondition: apply_verdict must not be APPLY_DIVERGENT so "
|
||||
"this test isolates the apply-resource-fence guard from the sibling "
|
||||
"apply_divergent guard");
|
||||
|
||||
/* Engage the Live Apply Resource Fence directly -- exactly as
|
||||
* persist_apply_pump_run does on a resource-exhaustion apply failure
|
||||
* (task 4.2), and exactly as the sibling site-1 test
|
||||
* (test_commit_durability.c) sets it for that call site. */
|
||||
persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3);
|
||||
TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist));
|
||||
|
||||
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
|
||||
TEST_ASSERT_NOT_NULL(ctx);
|
||||
ctx->n_peers = 2;
|
||||
pthread_mutex_init(&ctx->lock, NULL);
|
||||
sem_init(&ctx->done, 0, 0);
|
||||
|
||||
xce_candidate_t peer0 = {0}, peer1 = {0};
|
||||
cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
|
||||
cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
|
||||
|
||||
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
|
||||
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
|
||||
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
|
||||
|
||||
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
|
||||
|
||||
/* Act: real promotion attempt with a live, structurally-healthy cluster. */
|
||||
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
|
||||
|
||||
/* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE +
|
||||
* unrecoverable==false + apply_verdict!=DIVERGENT -- the new
|
||||
* apply-resource-fence conjunct alone must block promotion. */
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(
|
||||
(int32_t)CTRL_PRIMARY,
|
||||
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
||||
"job-array-single-wal-record task 4.3 (Req 9.4/9.5): a "
|
||||
"structurally-eligible node with the Live Apply Resource Fence "
|
||||
"engaged must NOT be promoted to CTRL_PRIMARY");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire),
|
||||
"a refused apply-resource-fenced promotion must fence the member "
|
||||
"(in_catch_up=1) and keep it alive, retrying catch-up -- never "
|
||||
"self-exit");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
|
||||
repl_active_standby_count(&g_cv.persist.repl),
|
||||
"promotion refusal must happen BEFORE the standby-registration block "
|
||||
"-- no vote endpoint may be registered when the node never became "
|
||||
"primary");
|
||||
|
||||
/* Now clear the fence (the apply pump's own self-heal, task 4.2) and
|
||||
* retry promotion: eligibility must return with no other action (task
|
||||
* 4.3's Observable clause). */
|
||||
persist_clear_apply_resource_fence(&g_cv.persist);
|
||||
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist));
|
||||
atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release);
|
||||
|
||||
xce_candidate_t peer0b = {0}, peer1b = {0};
|
||||
ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0]));
|
||||
TEST_ASSERT_NOT_NULL(ctx2);
|
||||
ctx2->n_peers = 2;
|
||||
pthread_mutex_init(&ctx2->lock, NULL);
|
||||
sem_init(&ctx2->done, 0, 0);
|
||||
cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700);
|
||||
cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701);
|
||||
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release);
|
||||
|
||||
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2);
|
||||
|
||||
TEST_ASSERT_EQUAL_MESSAGE(
|
||||
(int32_t)CTRL_PRIMARY,
|
||||
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
|
||||
"task 4.3: once the live apply resource fence clears, the node is "
|
||||
"eligible for promotion again with no other action");
|
||||
|
||||
/* Cleanup */
|
||||
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
|
||||
if (g_cv.ha.standby_endpoints) {
|
||||
zarray_fini(g_cv.ha.standby_endpoints);
|
||||
free(g_cv.ha.standby_endpoints);
|
||||
g_cv.ha.standby_endpoints = NULL;
|
||||
}
|
||||
pthread_mutex_destroy(&ctx->lock);
|
||||
sem_destroy(&ctx->done);
|
||||
free(ctx);
|
||||
pthread_mutex_destroy(&ctx2->lock);
|
||||
sem_destroy(&ctx2->done);
|
||||
free(ctx2);
|
||||
cv_teardown();
|
||||
printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: "
|
||||
"checked\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
|
@ -1341,6 +1469,11 @@ void test_convergence_verification(void)
|
|||
"(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n");
|
||||
|
||||
RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible);
|
||||
|
||||
printf("\n>> Promotion eligibility gate site-2 coverage "
|
||||
"(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n");
|
||||
|
||||
RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible);
|
||||
#else
|
||||
printf("test_convergence_verification: SKIP (non-test build)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue