diff --git a/.kiro/specs/job-array-single-wal-record/tasks.md b/.kiro/specs/job-array-single-wal-record/tasks.md index 849f55b..a446cd7 100644 --- a/.kiro/specs/job-array-single-wal-record/tasks.md +++ b/.kiro/specs/job-array-single-wal-record/tasks.md @@ -95,7 +95,7 @@ - _Requirements: 9.4, 9.5_ - _Boundary: Live Apply Resource Fence_ -- [ ] 4.3 Gate promotion on the live fence +- [x] 4.3 Gate promotion on the live fence - Add the new fence state as a required condition at both existing points where a node's eligibility for promotion is decided (the periodic release check and the election-time check) - Observable: a node with the fence engaged is refused promotion at both points; once the fence clears, the node is eligible again without any other change - _Requirements: 9.4, 9.5_ diff --git a/src/controller/ha.c b/src/controller/ha.c index ea71ab3..4157906 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -1062,11 +1062,25 @@ static int promotion_fence_timeout_fn(void *arg) bool self_restart_pending = atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire); + /* apply_resource_fenced (job-array-single-wal-record task 4.3, Req + * 9.4/9.5): same unconditional, never-live-cleared-by-structure refusal + * as apply_divergent/self_restart_pending above — a node stuck on a + * resource-exhaustion apply failure (task 4.1/4.2's Live Apply Resource + * Fence) can have a perfectly healthy WAL (not truncated, durable_floor + * >= HWM) by construction, since apply success and WAL durability are + * orthogonal. This purely structural re-check must not release the fence + * on WAL health alone — only the apply pump's own successful retry of the + * stuck LSN clears it (persist_clear_apply_resource_fence, called from + * inside persist_apply_pump_run itself). */ + bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist); + /* Caught up: not truncated AND durable position reached the boundary AND * the current apply verdict is not DIVERGENT AND no self-restart is - * pending → release the fence + clear the sticky flag; the member is - * eligible on its next election (Req 2.6). */ - if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending) { + * pending AND no live apply resource fence is engaged → release the + * fence + clear the sticky flag; the member is eligible on its next + * election (Req 2.6, Req 9.5). */ + if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending + && !apply_resource_fenced) { atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release); persist_clear_recovery_unrecoverable(&c->persist); INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%" @@ -1312,6 +1326,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term) bool self_restart_pending = atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire); + /* apply_resource_fenced (job-array-single-wal-record task 4.3, Req + * 9.4/9.5): same unconditional-refuse treatment as apply_divergent/ + * self_restart_pending above — a node stuck on a resource-exhaustion + * apply failure (task 4.1/4.2's Live Apply Resource Fence) must never + * become primary while the fence is engaged, regardless of how + * structurally ELIGIBLE its WAL looks (truncated/durable_floor are + * orthogonal to apply success). Only the apply pump's own successful + * retry of the stuck LSN clears it — no election/HWM condition can. */ + bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist); + /* A healthy peer is reachable for catch-up iff some voter granted with a * live endpoint survives this election (these become the standby * channels promoted below). */ @@ -1342,34 +1366,41 @@ static void raft_cb_become_leader(void *ctx, uint64_t term) peer_reachable, /*catch_up_succeeded=*/false); if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent - && !self_restart_pending) { - /* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, AND no - * pending self-restart: release the fence (a caught-up member - * regains eligibility, Req 2.6), clear the sticky flag if set, and - * cancel any pending re-alert timer. Proceed to authoritative - * promotion below. */ + && !self_restart_pending && !apply_resource_fenced) { + /* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, no + * pending self-restart, AND no live apply resource fence engaged: + * release the fence (a caught-up member regains eligibility, Req + * 2.6), clear the sticky flag if set, and cancel any pending + * re-alert timer. Proceed to authoritative promotion below. */ atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release); persist_clear_recovery_unrecoverable(&c->persist); ha_cancel_promotion_fence_timer(ha); } else { - /* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent, or - * self-restart-pending: refuse promotion — do NOT become - * CTRL_PRIMARY. Fence the member and arm the catch-up re-alert - * timer. The member STAYS ALIVE, non-promotable, retrying catch-up - * (Req 2.2/2.4). NEVER self-exit, never fence silently. An - * apply-divergent or self-restart-pending member is refused - * unconditionally (Req 8.4; ha-convergence-recovery Req 1.4) — no - * peer/HWM condition can clear either trigger; only a fresh boot - * (clean restart replay, or a successful re-exec) can. */ + /* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent, + * self-restart-pending, or apply-resource-fenced: refuse + * promotion — do NOT become CTRL_PRIMARY. Fence the member and + * arm the catch-up re-alert timer. The member STAYS ALIVE, + * non-promotable, retrying catch-up (Req 2.2/2.4). NEVER + * self-exit, never fence silently. An apply-divergent, + * self-restart-pending, or apply-resource-fenced member is + * refused unconditionally (Req 8.4; ha-convergence-recovery Req + * 1.4; job-array-single-wal-record Req 9.4/9.5) — no peer/HWM + * condition can clear any of these triggers; apply-resource- + * fenced clears only from inside the apply pump itself on a + * successful retry (task 4.2), the others only on a fresh boot + * (clean restart replay, or a successful re-exec). */ WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d " - "apply_divergent=%d self_restart_pending=%d) — " + "apply_divergent=%d self_restart_pending=%d " + "apply_resource_fenced=%d) — " "recovery_truncated=%d durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + " "arming catch-up re-alert window, staying ALIVE " "(ha-commit-durability Req 2.2/2.4; raft-apply-decoupling " - "Req 8.4; ha-convergence-recovery Req 1.4)", + "Req 8.4; ha-convergence-recovery Req 1.4; " + "job-array-single-wal-record Req 9.4/9.5)", (int)gate, unrecoverable, apply_divergent, self_restart_pending, - truncated, durable_floor, promotion_safe_lsn, peer_reachable); + apply_resource_fenced, truncated, durable_floor, + promotion_safe_lsn, peer_reachable); ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, promotion_safe_lsn); return; diff --git a/src/controller/testsuite/test_commit_durability.c b/src/controller/testsuite/test_commit_durability.c index b94fcde..d609380 100644 --- a/src/controller/testsuite/test_commit_durability.c +++ b/src/controller/testsuite/test_commit_durability.c @@ -2394,6 +2394,104 @@ static void test_promotion_fence_apply_divergent_survives_structural_catchup(voi "PASS\n"); } +/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ────────────────────── + * Site 1 (periodic fence-release check, promotion_fence_timeout_fn ~ha.c:1069): + * the Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced / + * persist_mark_apply_resource_fenced / persist_clear_apply_resource_fence) + * must gate this release check exactly as apply_divergent/self_restart_pending + * already do — engaging it must survive a purely-structural catch-up + * re-check, and clearing it (the apply pump's own self-heal) must restore + * eligibility with no other action. + * + * Mirrors test_promotion_fence_apply_divergent_survives_structural_catchup + * above exactly, swapping the trigger for the new fence primitive so the two + * triggers are proven orthogonal by construction, same as that test proved + * apply_divergent orthogonal to the structural fields. + * + * RED (pre-task-4.3 code): the timer's release condition has no + * !persist_apply_resource_fenced conjunct, so once truncated/below_hwm are + * healthy the release branch fires regardless of the live fence — in_catch_up + * drops to 0 even though the node is still stuck on a resource-exhaustion + * apply failure. GREEN (post-fix): the release branch is skipped while the + * fence is engaged (in_catch_up stays 1), and clearing the fence + re-arming + * the timer restores eligibility (in_catch_up returns to 0) with no other + * action. */ +static void test_promotion_fence_apply_resource_fenced_survives_structural_catchup(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_fence_res_fenced_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, + /*durable_floor=*/4); + TEST_ASSERT_NOT_NULL(c); + + /* Engage the Live Apply Resource Fence (task 4.2 primitive), exactly as + * persist_apply_pump_run does on a resource-exhaustion apply failure -- + * independent of the WAL-structural fields (truncated/durable_floor) set + * above, proving the two conditions are orthogonal by construction. */ + persist_mark_apply_resource_fenced(&c->persist, /*lsn=*/7); + TEST_ASSERT_TRUE_MESSAGE( + persist_apply_resource_fenced(&c->persist), + "fixture precondition: the live apply resource fence must be engaged"); + + ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */ + + extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, + uint64_t); + ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, + /*durable_floor=*/4, /*hwm=*/10); + TEST_ASSERT_EQUAL_INT( + 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire)); + + /* Simulate the EXACT SAME structural catch-up as the sibling apply_divergent + * test: log now intact and durable past the HWM. The live apply resource + * fence is UNTOUCHED by this -- it can only be cleared from inside the + * apply pump itself on a successful retry of the stuck LSN (task 4.2), + * never by a live structural mutation like this one. */ + c->persist.recovery_verdict = RECOVER_CLEAN; + atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release); + + /* Wait long enough for the window to fire. */ + usleep(300 * 1000); + + /* THE RED/GREEN ASSERTION (Req 9.4/9.5, task 4.3): even though the + * structural conditions the timer inspects are now fully healthy, a node + * with the Live Apply Resource Fence engaged must NOT be released -- a + * stuck resource-exhaustion apply cannot be proven fixed by more WAL bytes + * arriving; only the apply pump's own successful retry may clear it. */ + TEST_ASSERT_EQUAL_INT_MESSAGE( + 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), + "task 4.3: a node with the Live Apply Resource Fence engaged must " + "remain fenced (non-promotable) across a purely-structural catch-up " + "re-check (Req 9.4/9.5)"); + TEST_ASSERT_TRUE_MESSAGE( + persist_apply_resource_fenced(&c->persist), + "the live apply resource fence itself must still read engaged -- only " + "the apply pump's own successful retry clears it, not this timer"); + + /* Now simulate the apply pump's own self-heal (task 4.2): clear the fence + * directly, mirroring persist_clear_apply_resource_fence's real call site + * inside persist_apply_pump_run. Re-arm the timer to prove eligibility + * returns with NO OTHER ACTION once the fence clears (task 4.3's + * Observable clause). */ + persist_clear_apply_resource_fence(&c->persist); + TEST_ASSERT_FALSE(persist_apply_resource_fenced(&c->persist)); + + ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/false, + /*durable_floor=*/10, /*hwm=*/10); + usleep(300 * 1000); + TEST_ASSERT_EQUAL_INT_MESSAGE( + 0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), + "task 4.3: once the live apply resource fence clears, the node " + "becomes eligible again with no other action"); + + ha_set_promotion_fence_window_ms(0); + dur_destroy_fence_cluster(c); + printf("test_promotion_fence_apply_resource_fenced_survives_structural_" + "catchup: PASS\n"); +} + #endif /* XCE_TESTING */ /* ------------------------------------------------------------------ */ @@ -2458,6 +2556,10 @@ void test_commit_durability(void) * an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up * alone. */ RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup); + /* job-array-single-wal-record task 4.3 (Req 9.4/9.5) — site 1: the Live + * Apply Resource Fence must gate the periodic release check the same way, + * and self-heal (no other action) once the fence clears. */ + RUN_TEST(test_promotion_fence_apply_resource_fenced_survives_structural_catchup); #else printf("test_commit_durability: skipped (no XCE_TESTING)\n"); #endif diff --git a/src/controller/testsuite/test_convergence_verification.c b/src/controller/testsuite/test_convergence_verification.c index c14da19..aee39a3 100644 --- a/src/controller/testsuite/test_convergence_verification.c +++ b/src/controller/testsuite/test_convergence_verification.c @@ -1300,6 +1300,134 @@ static void test_promotion_refused_when_apply_divergent_even_if_eligible(void) "checked\n"); } +/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ────────────────────── + * Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the + * Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced / + * persist_mark_apply_resource_fenced) must gate this eligibility check the + * same way apply_divergent/self_restart_pending already do. + * + * Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above + * exactly — same structurally-healthy fixture (durable_floor(0) >= + * promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision + * == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the + * sticky-flag conjunct contributes nothing) — swapping the trigger for the new + * fence primitive so the new conjunct is isolated as the ONLY variable + * changed, exactly as the round-3 apply_divergent test isolated its conjunct. + * + * RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced + * conjunct, so a structurally-eligible node with the live fence engaged is + * promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get + * registered as standbys. GREEN (post-fix): promotion is refused — role stays + * non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no + * standbys are registered. */ +static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void) +{ + /* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible: + * durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 => + * genuinely ELIGIBLE on every OTHER conjunct. */ + cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0, + /*appended_lsn=*/0, /*n_vcpus=*/8); + + TEST_ASSERT_FALSE_MESSAGE( + persist_recovery_unrecoverable(&g_cv.persist), + "fixture precondition: sticky recovery_unrecoverable must be clear so " + "this test isolates the apply-resource-fence guard, not the " + "sticky-flag guard"); + TEST_ASSERT_FALSE_MESSAGE( + g_cv.persist.apply_verdict == APPLY_DIVERGENT, + "fixture precondition: apply_verdict must not be APPLY_DIVERGENT so " + "this test isolates the apply-resource-fence guard from the sibling " + "apply_divergent guard"); + + /* Engage the Live Apply Resource Fence directly -- exactly as + * persist_apply_pump_run does on a resource-exhaustion apply failure + * (task 4.2), and exactly as the sibling site-1 test + * (test_commit_durability.c) sets it for that call site. */ + persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3); + TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist)); + + ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0])); + TEST_ASSERT_NOT_NULL(ctx); + ctx->n_peers = 2; + pthread_mutex_init(&ctx->lock, NULL); + sem_init(&ctx->done, 0, 0); + + xce_candidate_t peer0 = {0}, peer1 = {0}; + cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700); + cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701); + + g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t)); + TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints); + zarray_init(g_cv.ha.standby_endpoints, 4, NULL); + + atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release); + + /* Act: real promotion attempt with a live, structurally-healthy cluster. */ + raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1); + + /* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE + + * unrecoverable==false + apply_verdict!=DIVERGENT -- the new + * apply-resource-fence conjunct alone must block promotion. */ + TEST_ASSERT_NOT_EQUAL_MESSAGE( + (int32_t)CTRL_PRIMARY, + atomic_load_explicit(&g_cv.ha.role, memory_order_acquire), + "job-array-single-wal-record task 4.3 (Req 9.4/9.5): a " + "structurally-eligible node with the Live Apply Resource Fence " + "engaged must NOT be promoted to CTRL_PRIMARY"); + TEST_ASSERT_EQUAL_INT_MESSAGE( + 1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire), + "a refused apply-resource-fenced promotion must fence the member " + "(in_catch_up=1) and keep it alive, retrying catch-up -- never " + "self-exit"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, + repl_active_standby_count(&g_cv.persist.repl), + "promotion refusal must happen BEFORE the standby-registration block " + "-- no vote endpoint may be registered when the node never became " + "primary"); + + /* Now clear the fence (the apply pump's own self-heal, task 4.2) and + * retry promotion: eligibility must return with no other action (task + * 4.3's Observable clause). */ + persist_clear_apply_resource_fence(&g_cv.persist); + TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist)); + atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release); + + xce_candidate_t peer0b = {0}, peer1b = {0}; + ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0])); + TEST_ASSERT_NOT_NULL(ctx2); + ctx2->n_peers = 2; + pthread_mutex_init(&ctx2->lock, NULL); + sem_init(&ctx2->done, 0, 0); + cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700); + cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701); + atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release); + + raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2); + + TEST_ASSERT_EQUAL_MESSAGE( + (int32_t)CTRL_PRIMARY, + atomic_load_explicit(&g_cv.ha.role, memory_order_acquire), + "task 4.3: once the live apply resource fence clears, the node is " + "eligible for promotion again with no other action"); + + /* Cleanup */ + atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release); + if (g_cv.ha.standby_endpoints) { + zarray_fini(g_cv.ha.standby_endpoints); + free(g_cv.ha.standby_endpoints); + g_cv.ha.standby_endpoints = NULL; + } + pthread_mutex_destroy(&ctx->lock); + sem_destroy(&ctx->done); + free(ctx); + pthread_mutex_destroy(&ctx2->lock); + sem_destroy(&ctx2->done); + free(ctx2); + cv_teardown(); + printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: " + "checked\n"); +} + #endif /* XCE_TESTING */ /* ── Suite entry point ──────────────────────────────────────────────────── */ @@ -1341,6 +1469,11 @@ void test_convergence_verification(void) "(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n"); RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible); + + printf("\n>> Promotion eligibility gate site-2 coverage " + "(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n"); + + RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible); #else printf("test_convergence_verification: SKIP (non-test build)\n"); #endif