feat(job-array-single-wal-record): add live, self-healing apply resource fence
Adds a new atomic LSN field on persist_t (distinct from the existing boot-only unrecoverable-fence flag) plus three primitives: persist_apply_resource_fenced(), persist_mark_apply_resource_fenced(), persist_clear_apply_resource_fence(). Wired into persist_apply_pump_run(): engages on task 4.1's halt condition, clears unconditionally on the walk's next successful advance — provably equivalent to "clears only when the same stuck LSN's retry succeeds" since the walk is strictly sequential and never advances past a failed position. Known residual gap, out of this task's boundary (documented in tasks.md Implementation Notes): replay_wal() during boot/restart replay does not yet share this protection, so a node crashing while fenced can silently re-pass the same failure on restart. Flagged for a follow-up task, not fixed here. Task 4.2 of job-array-single-wal-record (Requirements 9.4, 9.5).
This commit is contained in:
parent
cc7e99c2f7
commit
b47cc660ba
4 changed files with 296 additions and 2 deletions
|
|
@ -767,6 +767,157 @@ static void test_apply_pump_run_halts_at_failed_lsn_without_advancing(void)
|
|||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── 2c. Task 4.2 (job-array-single-wal-record, design.md "Live Apply
|
||||
* Resource Fence"): a resource-exhaustion apply failure that halts the walk
|
||||
* (task 4.1) must additionally ENGAGE a new, live, self-healing fence --
|
||||
* persist_apply_resource_fenced()/persist_mark_apply_resource_fenced() --
|
||||
* deliberately DISTINCT from the boot-only sticky recovery_unrecoverable
|
||||
* flag (Req 9.4). Reuses the same force-fail seam and array-record shape as
|
||||
* test 2b above, but asserts the NEW fence accessor rather than the halt
|
||||
* behavior itself (already covered by test 2b). */
|
||||
|
||||
static void test_apply_pump_run_engages_live_fence_on_failed_lsn(void)
|
||||
{
|
||||
tap_setup();
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_clear_force_fail();
|
||||
#endif
|
||||
|
||||
bool fenced_before = persist_apply_resource_fenced(&g_tap_c.persist);
|
||||
|
||||
const uint64_t base_jobid = 95000;
|
||||
const uint64_t array_size = 3;
|
||||
job_submit_data_t spec;
|
||||
memset(&spec, 0, sizeof(spec));
|
||||
spec.queue = (char*)"q";
|
||||
spec.user = (char*)"tester";
|
||||
spec.mintasks = 1;
|
||||
spec.maxtasks = 1;
|
||||
spec.job_name = (char*)"array-fence-test";
|
||||
|
||||
uint8_t* array_payload = NULL;
|
||||
size_t array_len = 0;
|
||||
int enc_rc = tap_encode_job_submit(base_jobid, array_size, &spec,
|
||||
&array_payload, &array_len);
|
||||
|
||||
/* Fail at scratch-build index 1 of 3 -- identical seam to test 2b. */
|
||||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_set_force_fail_at(1);
|
||||
#endif
|
||||
|
||||
int rc_array = persist_wal_async(&g_tap_c.persist, WAL_JOB_ARRAY_SUBMIT,
|
||||
array_payload, (uint32_t)array_len);
|
||||
free(array_payload);
|
||||
uint64_t array_lsn = g_tap_c.persist.current_lsn - 1;
|
||||
|
||||
atomic_store(&g_tap_c.persist.committed_lsn, array_lsn);
|
||||
|
||||
int walk_rc = persist_apply_pump_run(&g_tap_c.persist);
|
||||
|
||||
bool fenced_after_failure =
|
||||
persist_apply_resource_fenced(&g_tap_c.persist);
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_clear_force_fail();
|
||||
#endif
|
||||
|
||||
tap_teardown();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, enc_rc);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc_array);
|
||||
TEST_ASSERT_FALSE_MESSAGE(fenced_before,
|
||||
"a fresh persist_t must start with the Live Apply Resource Fence "
|
||||
"CLEAR -- nothing has failed yet, and this is a distinct piece of "
|
||||
"state from the boot-only recovery_unrecoverable flag");
|
||||
TEST_ASSERT_TRUE_MESSAGE(walk_rc < 0,
|
||||
"the forced construction failure must still halt the walk (task "
|
||||
"4.1 regression guard)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(fenced_after_failure,
|
||||
"the Live Apply Resource Fence must ENGAGE the moment the apply "
|
||||
"walk halts on a resource-exhaustion apply failure (Req 9.4)");
|
||||
|
||||
printf("test_apply_pump_run_engages_live_fence_on_failed_lsn: PASS\n");
|
||||
}
|
||||
|
||||
/* ── 2d. Task 4.2 self-heal: the fence must CLEAR itself, from inside the
|
||||
* apply walk, the instant a retry of the SAME stuck LSN succeeds -- with NO
|
||||
* external/admin action (Req 9.5). Mirrors test 2b's engage-then-retry
|
||||
* shape but asserts the fence accessor at each stage instead of job
|
||||
* visibility. */
|
||||
|
||||
static void test_apply_pump_run_self_clears_fence_on_successful_retry(void)
|
||||
{
|
||||
tap_setup();
|
||||
|
||||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_clear_force_fail();
|
||||
#endif
|
||||
|
||||
const uint64_t base_jobid = 96000;
|
||||
const uint64_t array_size = 3;
|
||||
job_submit_data_t spec;
|
||||
memset(&spec, 0, sizeof(spec));
|
||||
spec.queue = (char*)"q";
|
||||
spec.user = (char*)"tester";
|
||||
spec.mintasks = 1;
|
||||
spec.maxtasks = 1;
|
||||
spec.job_name = (char*)"array-fence-selfheal-test";
|
||||
|
||||
uint8_t* array_payload = NULL;
|
||||
size_t array_len = 0;
|
||||
int enc_rc = tap_encode_job_submit(base_jobid, array_size, &spec,
|
||||
&array_payload, &array_len);
|
||||
|
||||
/* Fail at scratch-build index 1 of 3 -- identical seam to tests 2b/2c. */
|
||||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_set_force_fail_at(1);
|
||||
#endif
|
||||
|
||||
int rc_array = persist_wal_async(&g_tap_c.persist, WAL_JOB_ARRAY_SUBMIT,
|
||||
array_payload, (uint32_t)array_len);
|
||||
free(array_payload);
|
||||
uint64_t array_lsn = g_tap_c.persist.current_lsn - 1;
|
||||
|
||||
atomic_store(&g_tap_c.persist.committed_lsn, array_lsn);
|
||||
|
||||
/* First attempt: forced failure engages the fence (proven by test 2c;
|
||||
* re-checked here as this test's own precondition). */
|
||||
int walk_rc_fail = persist_apply_pump_run(&g_tap_c.persist);
|
||||
bool fenced_after_failure =
|
||||
persist_apply_resource_fenced(&g_tap_c.persist);
|
||||
|
||||
/* Clear the injected fault and retry the SAME stuck LSN -- no external
|
||||
* or admin action, purely the apply walk retrying on its own. */
|
||||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_clear_force_fail();
|
||||
#endif
|
||||
int walk_rc_retry = persist_apply_pump_run(&g_tap_c.persist);
|
||||
bool fenced_after_retry =
|
||||
persist_apply_resource_fenced(&g_tap_c.persist);
|
||||
|
||||
tap_teardown();
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, enc_rc);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc_array);
|
||||
TEST_ASSERT_TRUE_MESSAGE(walk_rc_fail < 0,
|
||||
"the forced failure must still halt the walk (task 4.1 regression "
|
||||
"guard)");
|
||||
TEST_ASSERT_TRUE_MESSAGE(fenced_after_failure,
|
||||
"precondition: the fence must be engaged before this test can prove "
|
||||
"self-heal");
|
||||
TEST_ASSERT_TRUE_MESSAGE(walk_rc_retry >= 0,
|
||||
"once the fault clears, the retry of the same stuck LSN must "
|
||||
"succeed");
|
||||
TEST_ASSERT_FALSE_MESSAGE(fenced_after_retry,
|
||||
"the Live Apply Resource Fence must CLEAR itself, from inside the "
|
||||
"apply walk, the moment the retry of the SAME stuck position "
|
||||
"succeeds -- with no external/admin action (Req 9.5)");
|
||||
|
||||
printf("test_apply_pump_run_self_clears_fence_on_successful_retry: "
|
||||
"PASS\n");
|
||||
}
|
||||
|
||||
/* ── 3. Hazard 1 guard precision: job_ctrl's own live append must survive ── */
|
||||
|
||||
static void test_apply_pump_guard_does_not_break_job_ctrl_live_append(void)
|
||||
|
|
@ -1492,6 +1643,8 @@ void test_apply_pump(void)
|
|||
RUN_TEST(test_apply_pump_kick_applies_committed_burst_in_ascending_lsn_order);
|
||||
RUN_TEST(test_apply_pump_run_applies_mixed_submit_and_fact_entries);
|
||||
RUN_TEST(test_apply_pump_run_halts_at_failed_lsn_without_advancing);
|
||||
RUN_TEST(test_apply_pump_run_engages_live_fence_on_failed_lsn);
|
||||
RUN_TEST(test_apply_pump_run_self_clears_fence_on_successful_retry);
|
||||
RUN_TEST(test_apply_pump_guard_does_not_break_job_ctrl_live_append);
|
||||
RUN_TEST(test_apply_pump_fail_racing_finish_merges_against_current_state);
|
||||
RUN_TEST(test_apply_pump_late_pending_apply_registration_self_heals);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue