feat(job-array-single-wal-record): gate promotion on the live apply fence

Threads !persist_apply_resource_fenced(&c->persist) as a required
conjunct into both existing promotion-eligibility call sites in ha.c:
the periodic fence-release check (promotion_fence_timeout_fn) and the
election-time check (raft_cb_become_leader). A node with the fence
engaged is refused promotion at both points; once it self-clears
(task 4.2), the node is eligible again with no other action.

Task 4.3 of job-array-single-wal-record (Requirements 9.4, 9.5).

.kiro/specs/job-array-single-wal-record/tasks.md

@@ -95,7 +95,7 @@
   - _Requirements: 9.4, 9.5_
   - _Boundary: Live Apply Resource Fence_
 
-- [ ] 4.3 Gate promotion on the live fence
+- [x] 4.3 Gate promotion on the live fence
   - Add the new fence state as a required condition at both existing points where a node's eligibility for promotion is decided (the periodic release check and the election-time check)
   - Observable: a node with the fence engaged is refused promotion at both points; once the fence clears, the node is eligible again without any other change
   - _Requirements: 9.4, 9.5_

src/controller/ha.c

@@ -1062,11 +1062,25 @@ static int promotion_fence_timeout_fn(void *arg)
     bool     self_restart_pending =
         atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
 
+    /* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
+     * 9.4/9.5): same unconditional, never-live-cleared-by-structure refusal
+     * as apply_divergent/self_restart_pending above — a node stuck on a
+     * resource-exhaustion apply failure (task 4.1/4.2's Live Apply Resource
+     * Fence) can have a perfectly healthy WAL (not truncated, durable_floor
+     * >= HWM) by construction, since apply success and WAL durability are
+     * orthogonal. This purely structural re-check must not release the fence
+     * on WAL health alone — only the apply pump's own successful retry of the
+     * stuck LSN clears it (persist_clear_apply_resource_fence, called from
+     * inside persist_apply_pump_run itself). */
+    bool     apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
+
     /* Caught up: not truncated AND durable position reached the boundary AND
      * the current apply verdict is not DIVERGENT AND no self-restart is
-     * pending → release the fence + clear the sticky flag; the member is
+     * pending AND no live apply resource fence is engaged → release the
-     * eligible on its next election (Req 2.6). */
+     * fence + clear the sticky flag; the member is eligible on its next
-    if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending) {
+     * election (Req 2.6, Req 9.5). */
+    if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending
+        && !apply_resource_fenced) {
         atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
         persist_clear_recovery_unrecoverable(&c->persist);
         INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
@@ -1312,6 +1326,16 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
         bool     self_restart_pending =
             atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
 
+        /* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
+         * 9.4/9.5): same unconditional-refuse treatment as apply_divergent/
+         * self_restart_pending above — a node stuck on a resource-exhaustion
+         * apply failure (task 4.1/4.2's Live Apply Resource Fence) must never
+         * become primary while the fence is engaged, regardless of how
+         * structurally ELIGIBLE its WAL looks (truncated/durable_floor are
+         * orthogonal to apply success). Only the apply pump's own successful
+         * retry of the stuck LSN clears it — no election/HWM condition can. */
+        bool     apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
+
         /* A healthy peer is reachable for catch-up iff some voter granted with a
          * live endpoint survives this election (these become the standby
          * channels promoted below). */
@@ -1342,34 +1366,41 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
             peer_reachable, /*catch_up_succeeded=*/false);
 
         if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent
-            && !self_restart_pending) {
+            && !self_restart_pending && !apply_resource_fenced) {
-            /* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, AND no
+            /* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, no
-             * pending self-restart: release the fence (a caught-up member
+             * pending self-restart, AND no live apply resource fence engaged:
-             * regains eligibility, Req 2.6), clear the sticky flag if set, and
+             * release the fence (a caught-up member regains eligibility, Req
-             * cancel any pending re-alert timer.  Proceed to authoritative
+             * 2.6), clear the sticky flag if set, and cancel any pending
-             * promotion below. */
+             * re-alert timer.  Proceed to authoritative promotion below. */
             atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
             persist_clear_recovery_unrecoverable(&c->persist);
             ha_cancel_promotion_fence_timer(ha);
         } else {
-            /* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent, or
+            /* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent,
-             * self-restart-pending: refuse promotion — do NOT become
+             * self-restart-pending, or apply-resource-fenced: refuse
-             * CTRL_PRIMARY.  Fence the member and arm the catch-up re-alert
+             * promotion — do NOT become CTRL_PRIMARY.  Fence the member and
-             * timer.  The member STAYS ALIVE, non-promotable, retrying catch-up
+             * arm the catch-up re-alert timer.  The member STAYS ALIVE,
-             * (Req 2.2/2.4).  NEVER self-exit, never fence silently.  An
+             * non-promotable, retrying catch-up (Req 2.2/2.4).  NEVER
-             * apply-divergent or self-restart-pending member is refused
+             * self-exit, never fence silently.  An apply-divergent,
-             * unconditionally (Req 8.4; ha-convergence-recovery Req 1.4) — no
+             * self-restart-pending, or apply-resource-fenced member is
-             * peer/HWM condition can clear either trigger; only a fresh boot
+             * refused unconditionally (Req 8.4; ha-convergence-recovery Req
-             * (clean restart replay, or a successful re-exec) can. */
+             * 1.4; job-array-single-wal-record Req 9.4/9.5) — no peer/HWM
+             * condition can clear any of these triggers; apply-resource-
+             * fenced clears only from inside the apply pump itself on a
+             * successful retry (task 4.2), the others only on a fresh boot
+             * (clean restart replay, or a successful re-exec). */
             WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d "
-                 "apply_divergent=%d self_restart_pending=%d) — "
+                 "apply_divergent=%d self_restart_pending=%d "
+                 "apply_resource_fenced=%d) — "
                  "recovery_truncated=%d durable_floor=%" PRIu64
                  " cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + "
                  "arming catch-up re-alert window, staying ALIVE "
                  "(ha-commit-durability Req 2.2/2.4; raft-apply-decoupling "
-                 "Req 8.4; ha-convergence-recovery Req 1.4)",
+                 "Req 8.4; ha-convergence-recovery Req 1.4; "
+                 "job-array-single-wal-record Req 9.4/9.5)",
                  (int)gate, unrecoverable, apply_divergent, self_restart_pending,
-                 truncated, durable_floor, promotion_safe_lsn, peer_reachable);
+                 apply_resource_fenced, truncated, durable_floor,
+                 promotion_safe_lsn, peer_reachable);
             ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
                                          promotion_safe_lsn);
             return;

src/controller/testsuite/test_commit_durability.c

@@ -2394,6 +2394,104 @@ static void test_promotion_fence_apply_divergent_survives_structural_catchup(voi
            "PASS\n");
 }
 
+/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
+ * Site 1 (periodic fence-release check, promotion_fence_timeout_fn ~ha.c:1069):
+ * the Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
+ * persist_mark_apply_resource_fenced / persist_clear_apply_resource_fence)
+ * must gate this release check exactly as apply_divergent/self_restart_pending
+ * already do — engaging it must survive a purely-structural catch-up
+ * re-check, and clearing it (the apply pump's own self-heal) must restore
+ * eligibility with no other action.
+ *
+ * Mirrors test_promotion_fence_apply_divergent_survives_structural_catchup
+ * above exactly, swapping the trigger for the new fence primitive so the two
+ * triggers are proven orthogonal by construction, same as that test proved
+ * apply_divergent orthogonal to the structural fields.
+ *
+ * RED (pre-task-4.3 code): the timer's release condition has no
+ * !persist_apply_resource_fenced conjunct, so once truncated/below_hwm are
+ * healthy the release branch fires regardless of the live fence — in_catch_up
+ * drops to 0 even though the node is still stuck on a resource-exhaustion
+ * apply failure. GREEN (post-fix): the release branch is skipped while the
+ * fence is engaged (in_catch_up stays 1), and clearing the fence + re-arming
+ * the timer restores eligibility (in_catch_up returns to 0) with no other
+ * action. */
+static void test_promotion_fence_apply_resource_fenced_survives_structural_catchup(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_fence_res_fenced_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
+                                          /*durable_floor=*/4);
+    TEST_ASSERT_NOT_NULL(c);
+
+    /* Engage the Live Apply Resource Fence (task 4.2 primitive), exactly as
+     * persist_apply_pump_run does on a resource-exhaustion apply failure --
+     * independent of the WAL-structural fields (truncated/durable_floor) set
+     * above, proving the two conditions are orthogonal by construction. */
+    persist_mark_apply_resource_fenced(&c->persist, /*lsn=*/7);
+    TEST_ASSERT_TRUE_MESSAGE(
+        persist_apply_resource_fenced(&c->persist),
+        "fixture precondition: the live apply resource fence must be engaged");
+
+    ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
+
+    extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
+                                                      uint64_t);
+    ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
+                                          /*durable_floor=*/4, /*hwm=*/10);
+    TEST_ASSERT_EQUAL_INT(
+        1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
+
+    /* Simulate the EXACT SAME structural catch-up as the sibling apply_divergent
+     * test: log now intact and durable past the HWM. The live apply resource
+     * fence is UNTOUCHED by this -- it can only be cleared from inside the
+     * apply pump itself on a successful retry of the stuck LSN (task 4.2),
+     * never by a live structural mutation like this one. */
+    c->persist.recovery_verdict = RECOVER_CLEAN;
+    atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
+
+    /* Wait long enough for the window to fire. */
+    usleep(300 * 1000);
+
+    /* THE RED/GREEN ASSERTION (Req 9.4/9.5, task 4.3): even though the
+     * structural conditions the timer inspects are now fully healthy, a node
+     * with the Live Apply Resource Fence engaged must NOT be released -- a
+     * stuck resource-exhaustion apply cannot be proven fixed by more WAL bytes
+     * arriving; only the apply pump's own successful retry may clear it. */
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
+        "task 4.3: a node with the Live Apply Resource Fence engaged must "
+        "remain fenced (non-promotable) across a purely-structural catch-up "
+        "re-check (Req 9.4/9.5)");
+    TEST_ASSERT_TRUE_MESSAGE(
+        persist_apply_resource_fenced(&c->persist),
+        "the live apply resource fence itself must still read engaged -- only "
+        "the apply pump's own successful retry clears it, not this timer");
+
+    /* Now simulate the apply pump's own self-heal (task 4.2): clear the fence
+     * directly, mirroring persist_clear_apply_resource_fence's real call site
+     * inside persist_apply_pump_run.  Re-arm the timer to prove eligibility
+     * returns with NO OTHER ACTION once the fence clears (task 4.3's
+     * Observable clause). */
+    persist_clear_apply_resource_fence(&c->persist);
+    TEST_ASSERT_FALSE(persist_apply_resource_fenced(&c->persist));
+
+    ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/false,
+                                          /*durable_floor=*/10, /*hwm=*/10);
+    usleep(300 * 1000);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
+        "task 4.3: once the live apply resource fence clears, the node "
+        "becomes eligible again with no other action");
+
+    ha_set_promotion_fence_window_ms(0);
+    dur_destroy_fence_cluster(c);
+    printf("test_promotion_fence_apply_resource_fenced_survives_structural_"
+           "catchup: PASS\n");
+}
+
 #endif /* XCE_TESTING */
 
 /* ------------------------------------------------------------------ */
@@ -2458,6 +2556,10 @@ void test_commit_durability(void)
      * an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
      * alone. */
     RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup);
+    /* job-array-single-wal-record task 4.3 (Req 9.4/9.5) — site 1: the Live
+     * Apply Resource Fence must gate the periodic release check the same way,
+     * and self-heal (no other action) once the fence clears. */
+    RUN_TEST(test_promotion_fence_apply_resource_fenced_survives_structural_catchup);
 #else
     printf("test_commit_durability: skipped (no XCE_TESTING)\n");
 #endif

src/controller/testsuite/test_convergence_verification.c

@@ -1300,6 +1300,134 @@ static void test_promotion_refused_when_apply_divergent_even_if_eligible(void)
            "checked\n");
 }
 
+/* ── job-array-single-wal-record task 4.3 (Req 9.4/9.5) ──────────────────────
+ * Site 2 (election-time check, raft_cb_become_leader ~ha.c:1344-1345): the
+ * Live Apply Resource Fence (task 4.2's persist_apply_resource_fenced /
+ * persist_mark_apply_resource_fenced) must gate this eligibility check the
+ * same way apply_divergent/self_restart_pending already do.
+ *
+ * Mirrors test_promotion_refused_when_apply_divergent_even_if_eligible above
+ * exactly — same structurally-healthy fixture (durable_floor(0) >=
+ * promotion_safe_lsn(0), not truncated => genuinely ha_promotion_gate_decision
+ * == ELIGIBLE, and persist_recovery_unrecoverable explicitly false so the
+ * sticky-flag conjunct contributes nothing) — swapping the trigger for the new
+ * fence primitive so the new conjunct is isolated as the ONLY variable
+ * changed, exactly as the round-3 apply_divergent test isolated its conjunct.
+ *
+ * RED (pre-task-4.3 code): the gate has no !persist_apply_resource_fenced
+ * conjunct, so a structurally-eligible node with the live fence engaged is
+ * promoted anyway — role flips to CTRL_PRIMARY and vote endpoints get
+ * registered as standbys. GREEN (post-fix): promotion is refused — role stays
+ * non-PRIMARY, the member is fenced (in_catch_up=1, never-self-exit), and no
+ * standbys are registered. */
+static void test_promotion_refused_when_apply_resource_fenced_even_if_eligible(void)
+{
+    /* Identical fixture to test_promotion_refused_when_apply_divergent_even_if_eligible:
+     * durable_floor=0, recovery_truncated=false, promotion_safe_lsn=0 =>
+     * genuinely ELIGIBLE on every OTHER conjunct. */
+    cv_setup("resfenced", /*n_peers=*/2, /*n_standbys=*/0,
+             /*appended_lsn=*/0, /*n_vcpus=*/8);
+
+    TEST_ASSERT_FALSE_MESSAGE(
+        persist_recovery_unrecoverable(&g_cv.persist),
+        "fixture precondition: sticky recovery_unrecoverable must be clear so "
+        "this test isolates the apply-resource-fence guard, not the "
+        "sticky-flag guard");
+    TEST_ASSERT_FALSE_MESSAGE(
+        g_cv.persist.apply_verdict == APPLY_DIVERGENT,
+        "fixture precondition: apply_verdict must not be APPLY_DIVERGENT so "
+        "this test isolates the apply-resource-fence guard from the sibling "
+        "apply_divergent guard");
+
+    /* Engage the Live Apply Resource Fence directly -- exactly as
+     * persist_apply_pump_run does on a resource-exhaustion apply failure
+     * (task 4.2), and exactly as the sibling site-1 test
+     * (test_commit_durability.c) sets it for that call site. */
+    persist_mark_apply_resource_fenced(&g_cv.persist, /*lsn=*/3);
+    TEST_ASSERT_TRUE(persist_apply_resource_fenced(&g_cv.persist));
+
+    ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + 2 * sizeof(ctx->votes[0]));
+    TEST_ASSERT_NOT_NULL(ctx);
+    ctx->n_peers = 2;
+    pthread_mutex_init(&ctx->lock, NULL);
+    sem_init(&ctx->done, 0, 0);
+
+    xce_candidate_t peer0 = {0}, peer1 = {0};
+    cv_set_vote(ctx, 0, &peer0, "sb0.example", 7700);
+    cv_set_vote(ctx, 1, &peer1, "sb1.example", 7701);
+
+    g_cv.ha.standby_endpoints = calloc(1, sizeof(zarray_t));
+    TEST_ASSERT_NOT_NULL(g_cv.ha.standby_endpoints);
+    zarray_init(g_cv.ha.standby_endpoints, 4, NULL);
+
+    atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx, memory_order_release);
+
+    /* Act: real promotion attempt with a live, structurally-healthy cluster. */
+    raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/1);
+
+    /* Assert (Req 9.4/9.5): refused despite genuine structural ELIGIBLE +
+     * unrecoverable==false + apply_verdict!=DIVERGENT -- the new
+     * apply-resource-fence conjunct alone must block promotion. */
+    TEST_ASSERT_NOT_EQUAL_MESSAGE(
+        (int32_t)CTRL_PRIMARY,
+        atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
+        "job-array-single-wal-record task 4.3 (Req 9.4/9.5): a "
+        "structurally-eligible node with the Live Apply Resource Fence "
+        "engaged must NOT be promoted to CTRL_PRIMARY");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        1, atomic_load_explicit(&g_cv.ha.in_catch_up, memory_order_acquire),
+        "a refused apply-resource-fenced promotion must fence the member "
+        "(in_catch_up=1) and keep it alive, retrying catch-up -- never "
+        "self-exit");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0,
+        repl_active_standby_count(&g_cv.persist.repl),
+        "promotion refusal must happen BEFORE the standby-registration block "
+        "-- no vote endpoint may be registered when the node never became "
+        "primary");
+
+    /* Now clear the fence (the apply pump's own self-heal, task 4.2) and
+     * retry promotion: eligibility must return with no other action (task
+     * 4.3's Observable clause). */
+    persist_clear_apply_resource_fence(&g_cv.persist);
+    TEST_ASSERT_FALSE(persist_apply_resource_fenced(&g_cv.persist));
+    atomic_store_explicit(&g_cv.ha.in_catch_up, 0, memory_order_release);
+
+    xce_candidate_t peer0b = {0}, peer1b = {0};
+    ha_elect_ctx_t *ctx2 = calloc(1, sizeof(*ctx2) + 2 * sizeof(ctx2->votes[0]));
+    TEST_ASSERT_NOT_NULL(ctx2);
+    ctx2->n_peers = 2;
+    pthread_mutex_init(&ctx2->lock, NULL);
+    sem_init(&ctx2->done, 0, 0);
+    cv_set_vote(ctx2, 0, &peer0b, "sb0.example", 7700);
+    cv_set_vote(ctx2, 1, &peer1b, "sb1.example", 7701);
+    atomic_store_explicit(&g_cv.ha.elect_ctx, (void *)ctx2, memory_order_release);
+
+    raft_cb_become_leader_with_cluster_for_test(&g_cv, /*term=*/2);
+
+    TEST_ASSERT_EQUAL_MESSAGE(
+        (int32_t)CTRL_PRIMARY,
+        atomic_load_explicit(&g_cv.ha.role, memory_order_acquire),
+        "task 4.3: once the live apply resource fence clears, the node is "
+        "eligible for promotion again with no other action");
+
+    /* Cleanup */
+    atomic_store_explicit(&g_cv.ha.elect_ctx, NULL, memory_order_release);
+    if (g_cv.ha.standby_endpoints) {
+        zarray_fini(g_cv.ha.standby_endpoints);
+        free(g_cv.ha.standby_endpoints);
+        g_cv.ha.standby_endpoints = NULL;
+    }
+    pthread_mutex_destroy(&ctx->lock);
+    sem_destroy(&ctx->done);
+    free(ctx);
+    pthread_mutex_destroy(&ctx2->lock);
+    sem_destroy(&ctx2->done);
+    free(ctx2);
+    cv_teardown();
+    printf("test_promotion_refused_when_apply_resource_fenced_even_if_eligible: "
+           "checked\n");
+}
+
 #endif /* XCE_TESTING */
 
 /* ── Suite entry point ──────────────────────────────────────────────────── */
@@ -1341,6 +1469,11 @@ void test_convergence_verification(void)
            "(raft-apply-decoupling task 5.3 round-3, Req 8.4)\n\n");
 
     RUN_TEST(test_promotion_refused_when_apply_divergent_even_if_eligible);
+
+    printf("\n>> Promotion eligibility gate site-2 coverage "
+           "(job-array-single-wal-record task 4.3, Req 9.4/9.5)\n\n");
+
+    RUN_TEST(test_promotion_refused_when_apply_resource_fenced_even_if_eligible);
 #else
     printf("test_convergence_verification: SKIP (non-test build)\n");
 #endif