lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(job-array-single-wal-record): add live, self-healing apply resource fence

Browse source 
Adds a new atomic LSN field on persist_t (distinct from the existing
boot-only unrecoverable-fence flag) plus three primitives:
persist_apply_resource_fenced(), persist_mark_apply_resource_fenced(),
persist_clear_apply_resource_fence(). Wired into
persist_apply_pump_run(): engages on task 4.1's halt condition, clears
unconditionally on the walk's next successful advance — provably
equivalent to "clears only when the same stuck LSN's retry succeeds"
since the walk is strictly sequential and never advances past a
failed position.

Known residual gap, out of this task's boundary (documented in
tasks.md Implementation Notes): replay_wal() during boot/restart
replay does not yet share this protection, so a node crashing while
fenced can silently re-pass the same failure on restart. Flagged for
a follow-up task, not fixed here.

Task 4.2 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
Rongsong Shen 2026-07-15 03:54:20 +00:00
parent cc7e99c2f7
commit b47cc660ba
4 changed files with 296 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

71
src/controller/persist.c
View file

@ -623,6 +623,15 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
p->apply_verdict = APPLY_CLEAN;
atomic_store(&p->last_applied_lsn, 0);
/* Live Apply Resource Fence starts clear (job-array-single-wal-record
* task 4.2, Req 9.4): a fresh store has nothing stuck failing to apply.
* Set by persist_mark_apply_resource_fenced on a resource-exhaustion
* apply failure; cleared by persist_clear_apply_resource_fence the
* instant a retry of the same LSN succeeds -- see persist_t's field doc
* comment for why this is deliberately separate from
* recovery_unrecoverable just above. */
atomic_store(&p->apply_resource_fenced_lsn, 0);
/* bug-1416: defaults to true (nothing to gate) so every fixture that
* calls persist_init() but never persist_recover() -- most of this
* spec's own test suite -- sees identical, ungated apply-pump/live-
@ -3374,8 +3383,16 @@ int persist_apply_pump_run(persist_t* p)
"committed entry lsn=%" PRIu64 " wal_type=%" PRIu32
" (rc=%d) -- stopping this pass at this position; the "
"entry stays committed and will be retried, from the same "
"position, on the next kick",
"position, on the next kick; Live Apply Resource Fence "
"ENGAGED (job-array-single-wal-record Req 9.4)",
next, wal_type, apply_rc);
/* job-array-single-wal-record task 4.2 (design.md "Live Apply
* Resource Fence"): engage the NEW, live, self-healing fence --
* distinct from recovery_unrecoverable -- so the promotion gate
* (task 4.3) can refuse this node while it stays alive and
* keeps retrying. Cleared below, from inside this same walk,
* the instant a retry of this exact LSN succeeds (Req 9.5). */
persist_mark_apply_resource_fenced(p, next);
free(payload);
pthread_mutex_unlock(&p->apply_pump_lock);
return apply_rc;
@ -3385,6 +3402,21 @@ int persist_apply_pump_run(persist_t* p)
atomic_store_explicit(&p->last_applied_lsn, next, memory_order_relaxed);
applied++;
/* job-array-single-wal-record task 4.2 (design.md "Live Apply
* Resource Fence", Req 9.5): self-heal. Called unconditionally on
* EVERY successful apply, not just after a known-fenced LSN --
* persist_clear_apply_resource_fence() is a cheap no-op when nothing
* is fenced (the overwhelming common case). This is correct and
* sufficient because the walk above is strictly sequential and
* returns immediately on any failure (see the halt branch just
* above): it can never advance PAST a fenced LSN without that exact
* LSN applying successfully first, so "the next entry to apply
* successfully after a fence engages" and "a retry of the SAME
* stuck LSN succeeding" are the same event by construction. No
* election, promotion check, or external/admin action is involved --
* the fence clears purely because this walk itself made progress. */
persist_clear_apply_resource_fence(p);
/* raft-apply-decoupling task 6.2 (Req 1.1, 2.1-2.3): consume this
* LSN's command hand-off, if any (task 3.2's
* persist_register_pending_apply(), e.g. handle_jobsubmit's
@ -4357,6 +4389,43 @@ void persist_clear_recovery_unrecoverable(persist_t* p)
}
}
/* job-array-single-wal-record task 4.2 (design.md "Live Apply Resource
* Fence", Req 9.4/9.5). See the apply_resource_fenced_lsn field's doc
* comment (persist.h) and this trio's own doc comment (persist.h) for the
* full rationale and the deliberate distinction from
* persist_recovery_unrecoverable/persist_clear_recovery_unrecoverable just
* above -- that boot-only mechanism is NOT reused here on purpose. */
bool persist_apply_resource_fenced(persist_t* p)
{
return p && atomic_load_explicit(&p->apply_resource_fenced_lsn,
memory_order_acquire) != 0;
}
void persist_mark_apply_resource_fenced(persist_t* p, uint64_t lsn)
{
if (!p)
return;
atomic_store_explicit(&p->apply_resource_fenced_lsn, lsn,
memory_order_release);
}
void persist_clear_apply_resource_fence(persist_t* p)
{
if (!p)
return;
/* Clear only on a genuine transition so the self-heal log fires once per
* episode, not on every ordinary successful apply (the overwhelming
* common case: nothing was fenced, so this is a cheap no-op). */
uint64_t prev = atomic_exchange_explicit(&p->apply_resource_fenced_lsn, 0,
memory_order_acq_rel);
if (prev != 0) {
INFO("persist_apply_pump_run: Live Apply Resource Fence RELEASED — "
"lsn=%" PRIu64 " applied successfully on retry; promotable "
"again (job-array-single-wal-record Req 9.5)",
prev);
}
}
recover_decision_t
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
const recover_policy_inputs_t* in)

153
src/controller/testsuite/test_apply_pump.c
View file

@ -767,6 +767,157 @@ static void test_apply_pump_run_halts_at_failed_lsn_without_advancing(void)
"PASS\n");
}
/* ── 2c. Task 4.2 (job-array-single-wal-record, design.md "Live Apply
* Resource Fence"): a resource-exhaustion apply failure that halts the walk
* (task 4.1) must additionally ENGAGE a new, live, self-healing fence --
* persist_apply_resource_fenced()/persist_mark_apply_resource_fenced() --
* deliberately DISTINCT from the boot-only sticky recovery_unrecoverable
* flag (Req 9.4). Reuses the same force-fail seam and array-record shape as
* test 2b above, but asserts the NEW fence accessor rather than the halt
* behavior itself (already covered by test 2b). */
static void test_apply_pump_run_engages_live_fence_on_failed_lsn(void)
{
tap_setup();
#ifdef XCE_TESTING
job_array_apply_test_clear_force_fail();
#endif
bool fenced_before = persist_apply_resource_fenced(&g_tap_c.persist);
const uint64_t base_jobid = 95000;
const uint64_t array_size = 3;
job_submit_data_t spec;
memset(&spec, 0, sizeof(spec));
spec.queue = (char*)"q";
spec.user = (char*)"tester";
spec.mintasks = 1;
spec.maxtasks = 1;
spec.job_name = (char*)"array-fence-test";
uint8_t* array_payload = NULL;
size_t array_len = 0;
int enc_rc = tap_encode_job_submit(base_jobid, array_size, &spec,
&array_payload, &array_len);
/* Fail at scratch-build index 1 of 3 -- identical seam to test 2b. */
#ifdef XCE_TESTING
job_array_apply_test_set_force_fail_at(1);
#endif
int rc_array = persist_wal_async(&g_tap_c.persist, WAL_JOB_ARRAY_SUBMIT,
array_payload, (uint32_t)array_len);
free(array_payload);
uint64_t array_lsn = g_tap_c.persist.current_lsn - 1;
atomic_store(&g_tap_c.persist.committed_lsn, array_lsn);
int walk_rc = persist_apply_pump_run(&g_tap_c.persist);
bool fenced_after_failure =
persist_apply_resource_fenced(&g_tap_c.persist);
#ifdef XCE_TESTING
job_array_apply_test_clear_force_fail();
#endif
tap_teardown();
TEST_ASSERT_EQUAL_INT(0, enc_rc);
TEST_ASSERT_EQUAL_INT(0, rc_array);
TEST_ASSERT_FALSE_MESSAGE(fenced_before,
"a fresh persist_t must start with the Live Apply Resource Fence "
"CLEAR -- nothing has failed yet, and this is a distinct piece of "
"state from the boot-only recovery_unrecoverable flag");
TEST_ASSERT_TRUE_MESSAGE(walk_rc < 0,
"the forced construction failure must still halt the walk (task "
"4.1 regression guard)");
TEST_ASSERT_TRUE_MESSAGE(fenced_after_failure,
"the Live Apply Resource Fence must ENGAGE the moment the apply "
"walk halts on a resource-exhaustion apply failure (Req 9.4)");
printf("test_apply_pump_run_engages_live_fence_on_failed_lsn: PASS\n");
}
/* ── 2d. Task 4.2 self-heal: the fence must CLEAR itself, from inside the
* apply walk, the instant a retry of the SAME stuck LSN succeeds -- with NO
* external/admin action (Req 9.5). Mirrors test 2b's engage-then-retry
* shape but asserts the fence accessor at each stage instead of job
* visibility. */
static void test_apply_pump_run_self_clears_fence_on_successful_retry(void)
{
tap_setup();
#ifdef XCE_TESTING
job_array_apply_test_clear_force_fail();
#endif
const uint64_t base_jobid = 96000;
const uint64_t array_size = 3;
job_submit_data_t spec;
memset(&spec, 0, sizeof(spec));
spec.queue = (char*)"q";
spec.user = (char*)"tester";
spec.mintasks = 1;
spec.maxtasks = 1;
spec.job_name = (char*)"array-fence-selfheal-test";
uint8_t* array_payload = NULL;
size_t array_len = 0;
int enc_rc = tap_encode_job_submit(base_jobid, array_size, &spec,
&array_payload, &array_len);
/* Fail at scratch-build index 1 of 3 -- identical seam to tests 2b/2c. */
#ifdef XCE_TESTING
job_array_apply_test_set_force_fail_at(1);
#endif
int rc_array = persist_wal_async(&g_tap_c.persist, WAL_JOB_ARRAY_SUBMIT,
array_payload, (uint32_t)array_len);
free(array_payload);
uint64_t array_lsn = g_tap_c.persist.current_lsn - 1;
atomic_store(&g_tap_c.persist.committed_lsn, array_lsn);
/* First attempt: forced failure engages the fence (proven by test 2c;
* re-checked here as this test's own precondition). */
int walk_rc_fail = persist_apply_pump_run(&g_tap_c.persist);
bool fenced_after_failure =
persist_apply_resource_fenced(&g_tap_c.persist);
/* Clear the injected fault and retry the SAME stuck LSN -- no external
* or admin action, purely the apply walk retrying on its own. */
#ifdef XCE_TESTING
job_array_apply_test_clear_force_fail();
#endif
int walk_rc_retry = persist_apply_pump_run(&g_tap_c.persist);
bool fenced_after_retry =
persist_apply_resource_fenced(&g_tap_c.persist);
tap_teardown();
TEST_ASSERT_EQUAL_INT(0, enc_rc);
TEST_ASSERT_EQUAL_INT(0, rc_array);
TEST_ASSERT_TRUE_MESSAGE(walk_rc_fail < 0,
"the forced failure must still halt the walk (task 4.1 regression "
"guard)");
TEST_ASSERT_TRUE_MESSAGE(fenced_after_failure,
"precondition: the fence must be engaged before this test can prove "
"self-heal");
TEST_ASSERT_TRUE_MESSAGE(walk_rc_retry >= 0,
"once the fault clears, the retry of the same stuck LSN must "
"succeed");
TEST_ASSERT_FALSE_MESSAGE(fenced_after_retry,
"the Live Apply Resource Fence must CLEAR itself, from inside the "
"apply walk, the moment the retry of the SAME stuck position "
"succeeds -- with no external/admin action (Req 9.5)");
printf("test_apply_pump_run_self_clears_fence_on_successful_retry: "
"PASS\n");
}
/* ── 3. Hazard 1 guard precision: job_ctrl's own live append must survive ── */
static void test_apply_pump_guard_does_not_break_job_ctrl_live_append(void)
@ -1492,6 +1643,8 @@ void test_apply_pump(void)
RUN_TEST(test_apply_pump_kick_applies_committed_burst_in_ascending_lsn_order);
RUN_TEST(test_apply_pump_run_applies_mixed_submit_and_fact_entries);
RUN_TEST(test_apply_pump_run_halts_at_failed_lsn_without_advancing);
RUN_TEST(test_apply_pump_run_engages_live_fence_on_failed_lsn);
RUN_TEST(test_apply_pump_run_self_clears_fence_on_successful_retry);
RUN_TEST(test_apply_pump_guard_does_not_break_job_ctrl_live_append);
RUN_TEST(test_apply_pump_fail_racing_finish_merges_against_current_state);
RUN_TEST(test_apply_pump_late_pending_apply_registration_self_heals);