feat(job-array-single-wal-record): gate promotion on the live apply fence

Threads !persist_apply_resource_fenced(&c->persist) as a required
conjunct into both existing promotion-eligibility call sites in ha.c:
the periodic fence-release check (promotion_fence_timeout_fn) and the
election-time check (raft_cb_become_leader). A node with the fence
engaged is refused promotion at both points; once it self-clears
(task 4.2), the node is eligible again with no other action.

Task 4.3 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
Rongsong Shen 2026-07-15 04:21:35 +00:00
parent b47cc660ba
commit 8cb8c25d4d
4 changed files with 288 additions and 22 deletions

View file

@ -95,7 +95,7 @@
- _Requirements: 9.4, 9.5_
- _Boundary: Live Apply Resource Fence_
- [ ] 4.3 Gate promotion on the live fence
- [x] 4.3 Gate promotion on the live fence
- Add the new fence state as a required condition at both existing points where a node's eligibility for promotion is decided (the periodic release check and the election-time check)
- Observable: a node with the fence engaged is refused promotion at both points; once the fence clears, the node is eligible again without any other change
- _Requirements: 9.4, 9.5_

View file

@ -1062,11 +1062,25 @@ static int promotion_fence_timeout_fn(void *arg)
bool self_restart_pending =
atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
/* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
* 9.4/9.5): same unconditional, never-live-cleared-by-structure refusal
* as apply_divergent/self_restart_pending above — a node stuck on a
* resource-exhaustion apply failure (task 4.1/4.2's Live Apply Resource
* Fence) can have a perfectly healthy WAL (not truncated, durable_floor
* >= HWM) by construction, since apply success and WAL durability are
* orthogonal. This purely structural re-check must not release the fence
* on WAL health alone — only the apply pump's own successful retry of the
* stuck LSN clears it (persist_clear_apply_resource_fence, called from
* inside persist_apply_pump_run itself). */
bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
/* Caught up: not truncated AND durable position reached the boundary AND
* the current apply verdict is not DIVERGENT AND no self-restart is
* pending → release the fence + clear the sticky flag; the member is
* eligible on its next election (Req 2.6). */
if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending) {
* pending AND no live apply resource fence is engaged → release the
* fence + clear the sticky flag; the member is eligible on its next
* election (Req 2.6, Req 9.5). */
if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending
&& !apply_resource_fenced) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
@ -1312,6 +1326,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
bool self_restart_pending =
atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
/* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
* 9.4/9.5): same unconditional-refuse treatment as apply_divergent/
* self_restart_pending above — a node stuck on a resource-exhaustion
* apply failure (task 4.1/4.2's Live Apply Resource Fence) must never
* become primary while the fence is engaged, regardless of how
* structurally ELIGIBLE its WAL looks (truncated/durable_floor are
* orthogonal to apply success). Only the apply pump's own successful
* retry of the stuck LSN clears it — no election/HWM condition can. */
bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
/* A healthy peer is reachable for catch-up iff some voter granted with a
* live endpoint survives this election (these become the standby
* channels promoted below). */
@ -1342,34 +1366,41 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
peer_reachable, /*catch_up_succeeded=*/false);
if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent
&& !self_restart_pending) {
/* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, AND no
* pending self-restart: release the fence (a caught-up member
* regains eligibility, Req 2.6), clear the sticky flag if set, and
* cancel any pending re-alert timer. Proceed to authoritative
* promotion below. */
&& !self_restart_pending && !apply_resource_fenced) {
/* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, no
* pending self-restart, AND no live apply resource fence engaged:
* release the fence (a caught-up member regains eligibility, Req
* 2.6), clear the sticky flag if set, and cancel any pending
* re-alert timer. Proceed to authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent, or
* self-restart-pending: refuse promotion — do NOT become
* CTRL_PRIMARY. Fence the member and arm the catch-up re-alert
* timer. The member STAYS ALIVE, non-promotable, retrying catch-up
* (Req 2.2/2.4). NEVER self-exit, never fence silently. An
* apply-divergent or self-restart-pending member is refused
* unconditionally (Req 8.4; ha-convergence-recovery Req 1.4) — no
* peer/HWM condition can clear either trigger; only a fresh boot
* (clean restart replay, or a successful re-exec) can. */
/* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent,
* self-restart-pending, or apply-resource-fenced: refuse
* promotion — do NOT become CTRL_PRIMARY. Fence the member and
* arm the catch-up re-alert timer. The member STAYS ALIVE,
* non-promotable, retrying catch-up (Req 2.2/2.4). NEVER
* self-exit, never fence silently. An apply-divergent,
* self-restart-pending, or apply-resource-fenced member is
* refused unconditionally (Req 8.4; ha-convergence-recovery Req
* 1.4; job-array-single-wal-record Req 9.4/9.5) — no peer/HWM
* condition can clear any of these triggers; apply-resource-
* fenced clears only from inside the apply pump itself on a
* successful retry (task 4.2), the others only on a fresh boot
* (clean restart replay, or a successful re-exec). */
WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d "
"apply_divergent=%d self_restart_pending=%d) — "
"apply_divergent=%d self_restart_pending=%d "
"apply_resource_fenced=%d) — "
"recovery_truncated=%d durable_floor=%" PRIu64
" cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + "
"arming catch-up re-alert window, staying ALIVE "
"(ha-commit-durability Req 2.2/2.4; raft-apply-decoupling "
"Req 8.4; ha-convergence-recovery Req 1.4)",
"Req 8.4; ha-convergence-recovery Req 1.4; "
"job-array-single-wal-record Req 9.4/9.5)",
(int)gate, unrecoverable, apply_divergent, self_restart_pending,
truncated, durable_floor, promotion_safe_lsn, peer_reachable);
apply_resource_fenced, truncated, durable_floor,
promotion_safe_lsn, peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;

View file

@ -2394,6 +2394,104 @@ static void test_promotion_fence_apply_divergent_survives_structural_catchup(voi
"PASS\n");
}
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
* Site 1 (periodic fence-release check, promotion_fence_timeout_fn ~ha.c:1069):
* the Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
* persist_mark_apply_resource_fenced / persist_clear_apply_resource_fence)
* must gate this release check exactly as apply_divergent/self_restart_pending
* already do — engaging it must survive a purely-structural catch-up
* re-check, and clearing it (the apply pump's own self-heal) must restore
* eligibility with no other action.
*
* Mirrors test_promotion_fence_apply_divergent_survives_structural_catchup
* above exactly, swapping the trigger for the new fence primitive so the two
* triggers are proven orthogonal by construction, same as that test proved
* apply_divergent orthogonal to the structural fields.
*
* RED (pre-task-4.3 code): the timer's release condition has no
* !persist_apply_resource_fenced conjunct, so once truncated/below_hwm are
* healthy the release branch fires regardless of the live fence — in_catch_up
* drops to 0 even though the node is still stuck on a resource-exhaustion
* apply failure. GREEN (post-fix): the release branch is skipped while the
* fence is engaged (in_catch_up stays 1), and clearing the fence + re-arming
* the timer restores eligibility (in_catch_up returns to 0) with no other
* action. */
static void test_promotion_fence_apply_resource_fenced_survives_structural_catchup(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_res_fenced_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
/* Engage the Live Apply Resource Fence (task 4.2 primitive), exactly as
* persist_apply_pump_run does on a resource-exhaustion apply failure --
* independent of the WAL-structural fields (truncated/durable_floor) set
* above, proving the two conditions are orthogonal by construction. */
persist_mark_apply_resource_fenced(&c->persist, /*lsn=*/7);
TEST_ASSERT_TRUE_MESSAGE(
persist_apply_resource_fenced(&c->persist),
"fixture precondition: the live apply resource fence must be engaged");
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
/* Simulate the EXACT SAME structural catch-up as the sibling apply_divergent
* test: log now intact and durable past the HWM. The live apply resource
* fence is UNTOUCHED by this -- it can only be cleared from inside the
* apply pump itself on a successful retry of the stuck LSN (task 4.2),
* never by a live structural mutation like this one. */
c->persist.recovery_verdict = RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
/* Wait long enough for the window to fire. */
usleep(300 * 1000);
/* THE RED/GREEN ASSERTION (Req 9.4/9.5, task 4.3): even though the
* structural conditions the timer inspects are now fully healthy, a node
* with the Live Apply Resource Fence engaged must NOT be released -- a
* stuck resource-exhaustion apply cannot be proven fixed by more WAL bytes
* arriving; only the apply pump's own successful retry may clear it. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"task 4.3: a node with the Live Apply Resource Fence engaged must "
"remain fenced (non-promotable) across a purely-structural catch-up "
"re-check (Req 9.4/9.5)");
TEST_ASSERT_TRUE_MESSAGE(
persist_apply_resource_fenced(&c->persist),
"the live apply resource fence itself must still read engaged -- only "
"the apply pump's own successful retry clears it, not this timer");
/* Now simulate the apply pump's own self-heal (task 4.2): clear the fence
* directly, mirroring persist_clear_apply_resource_fence's real call site
* inside persist_apply_pump_run. Re-arm the timer to prove eligibility
* returns with NO OTHER ACTION once the fence clears (task 4.3's
* Observable clause). */
persist_clear_apply_resource_fence(&c->persist);
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&c->persist));
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/false,
/*durable_floor=*/10, /*hwm=*/10);
usleep(300 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"task 4.3: once the live apply resource fence clears, the node "
"becomes eligible again with no other action");
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_apply_resource_fenced_survives_structural_"
"catchup: PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
@ -2458,6 +2556,10 @@ void test_commit_durability(void)
* an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
* alone. */
RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup);
/* job-array-single-wal-record task 4.3 (Req 9.4/9.5) — site 1: the Live
* Apply Resource Fence must gate the periodic release check the same way,
* and self-heal (no other action) once the fence clears. */
RUN_TEST(test_promotion_fence_apply_resource_fenced_survives_structural_catchup);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif

View file

@ -1300,6 +1300,134 @@ static void test_promotion_refused_when_apply_divergent_even_if_eligible(void)
"checked\n");
}
/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
* Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the
* Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
* persist_mark_apply_resource_fenced) must gate this eligibility check the
* same way apply_divergent/self_restart_pending already do.
*
* Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above
* exactly — same structurally-healthy fixture (durable_floor(0) >=
* promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision
* == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the
* sticky-flag conjunct contributes nothing) — swapping the trigger for the new
* fence primitive so the new conjunct is isolated as the ONLY variable
* changed, exactly as the round-3 apply_divergent test isolated its conjunct.
*
* RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced
* conjunct, so a structurally-eligible node with the live fence engaged is
* promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get
* registered as standbys. GREEN (post-fix): promotion is refused — role stays
* non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no
* standbys are registered. */
static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void)
{
/* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible:
* durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 =>
* genuinely ELIGIBLE on every OTHER conjunct. */
cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0,
/*appended_lsn=*/0, /*n_vcpus=*/8);
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(&g_cv.persist),
"fixture precondition: sticky recovery_unrecoverable must be clear so "
"this test isolates the apply-resource-fence guard, not the "
"sticky-flag guard");
TEST_ASSERT_FALSE_MESSAGE(
g_cv.persist.apply_verdict == APPLY_DIVERGENT,
"fixture precondition: apply_verdict must not be APPLY_DIVERGENT so "
"this test isolates the apply-resource-fence guard from the sibling "
"apply_divergent guard");
/* Engage the Live Apply Resource Fence directly -- exactly as
* persist_apply_pump_run does on a resource-exhaustion apply failure
* (task 4.2), and exactly as the sibling site-1 test
* (test_commit_durability.c) sets it for that call site. */
persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3);
TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist));
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
TEST_ASSERT_NOT_NULL(ctx);
ctx->n_peers = 2;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
xce_candidate_t peer0 = {0}, peer1 = {0};
cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
/* Act: real promotion attempt with a live, structurally-healthy cluster. */
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
/* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE +
* unrecoverable==false + apply_verdict!=DIVERGENT -- the new
* apply-resource-fence conjunct alone must block promotion. */
TEST_ASSERT_NOT_EQUAL_MESSAGE(
(int32_t)CTRL_PRIMARY,
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
"job-array-single-wal-record task 4.3 (Req 9.4/9.5): a "
"structurally-eligible node with the Live Apply Resource Fence "
"engaged must NOT be promoted to CTRL_PRIMARY");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire),
"a refused apply-resource-fenced promotion must fence the member "
"(in_catch_up=1) and keep it alive, retrying catch-up -- never "
"self-exit");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
repl_active_standby_count(&g_cv.persist.repl),
"promotion refusal must happen BEFORE the standby-registration block "
"-- no vote endpoint may be registered when the node never became "
"primary");
/* Now clear the fence (the apply pump's own self-heal, task 4.2) and
* retry promotion: eligibility must return with no other action (task
* 4.3's Observable clause). */
persist_clear_apply_resource_fence(&g_cv.persist);
TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist));
atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release);
xce_candidate_t peer0b = {0}, peer1b = {0};
ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0]));
TEST_ASSERT_NOT_NULL(ctx2);
ctx2->n_peers = 2;
pthread_mutex_init(&ctx2->lock, NULL);
sem_init(&ctx2->done, 0, 0);
cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700);
cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701);
atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release);
raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2);
TEST_ASSERT_EQUAL_MESSAGE(
(int32_t)CTRL_PRIMARY,
atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
"task 4.3: once the live apply resource fence clears, the node is "
"eligible for promotion again with no other action");
/* Cleanup */
atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
if (g_cv.ha.standby_endpoints) {
zarray_fini(g_cv.ha.standby_endpoints);
free(g_cv.ha.standby_endpoints);
g_cv.ha.standby_endpoints = NULL;
}
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
pthread_mutex_destroy(&ctx2->lock);
sem_destroy(&ctx2->done);
free(ctx2);
cv_teardown();
printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: "
"checked\n");
}
#endif /* XCE_TESTING */
/* ── Suite entry point ──────────────────────────────────────────────────── */
@ -1341,6 +1469,11 @@ void test_convergence_verification(void)
"(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n");
RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible);
printf("\n>> Promotion eligibility gate site-2 coverage "
"(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n");
RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible);
#else
printf("test_convergence_verification: SKIP (non-test build)\n");
#endif