fix(job-array-single-wal-record): close bug-1443's bulk-insert partial-visibility gap
restore_job_array()'s bulk-insert pass could fail partway through (key_add() on realloc growth exhaustion), leaving an already-inserted prefix permanently visible in c->jobmgr but excluded from out_new_jobs -- so those elements existed and were individually addressable but never reached HOOK_DETAIL_JOB_NEW, never scheduled or pushed, not on this attempt or any later successful retry. A literal violation of Requirements 2.3 and 9.4, tracked as bug-1443. Adds kvmgr_reserve()/xzarray_reserve(), new purely-additive primitives in shared kvmgr.c/zarray.c (following the existing kvmgr_capture_keys() template -- bare extern function, no RTTI vtable slot, caller-holds- write-lock contract), that pre-grow c->jobmgr's backing store before the bulk-insert pass begins, closing the dominant N-scaling failure trigger structurally. Found a genuine off-by-one in the existing, unmodified xzarray_ensure_items(): its strict less-than sufficiency check means reserving exactly array_size and then performing array_size real stores still triggers one more growth cycle on the last store. Reserving array_size + 1 is the exact minimal correct compensation, independently re-derived and confirmed by review with concrete arithmetic. For the now-much-narrower residual case (kvtab_add()'s own small per-entry allocation, or a content collision -- both non-N-scaling, honestly documented as still theoretically possible): restore_job_array() now populates out_new_jobs with whatever prefix was genuinely inserted before a failure, and apply_committed_entry() fires HOOK_DETAIL_JOB_NEW for that non-empty partial prefix even when the overall return is negative, while still propagating the negative rc so the LSN correctly halts and the Live Apply Resource Fence still engages for the remainder. A genuinely-inserted element is never left un-pushed again. Task 7.1 of job-array-single-wal-record (Requirements 2.3, 9.4, 9.6).
This commit is contained in:
parent
575f928bbc
commit
dcf9f39c22
13 changed files with 910 additions and 36 deletions
|
|
@ -662,6 +662,70 @@ static void test_ace_job_array_submit_forced_failure_fires_no_hook(void)
|
|||
tace_teardown();
|
||||
printf("test_ace_job_array_submit_forced_failure_fires_no_hook: PASS\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 7.1 (bug-1443 partial-prefix fix):
|
||||
* a bulk-insert-phase failure (the narrower residual gap, distinct from the
|
||||
* scratch-build-phase failure above) must STILL fire HOOK_DETAIL_JOB_NEW --
|
||||
* for exactly the genuinely-inserted prefix -- while still returning the
|
||||
* negative rc so persist_apply_pump_run() halts/fences for the remainder.
|
||||
* Before this task, this exact scenario fired NO hook at all (see
|
||||
* test_ace_job_array_submit_forced_failure_fires_no_hook above, which forces
|
||||
* the OTHER, scratch-build-phase seam and correctly still expects no hook --
|
||||
* that one never inserts anything real, so no hook is correct there; this
|
||||
* test's bulk-insert-phase seam DOES insert a real, permanent prefix, so a
|
||||
* hook is now required). */
|
||||
static void test_ace_job_array_submit_bulk_insert_failure_fires_hook_for_prefix(void)
|
||||
{
|
||||
tace_setup();
|
||||
|
||||
const uint64_t base_jobid = 5400;
|
||||
const uint64_t array_size = 5;
|
||||
job_submit_data_t spec = tace_array_make_spec();
|
||||
uint8_t* payload = NULL;
|
||||
size_t plen = 0;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, tace_encode_job_submit_array(base_jobid,
|
||||
array_size, &spec,
|
||||
&payload, &plen));
|
||||
|
||||
xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||||
tace_array_spy_hook, NULL);
|
||||
|
||||
/* Force the bulk-insert pass (NOT the scratch-build pass) to fail at
|
||||
* index 2 -- indices 0 and 1 are genuinely, permanently inserted into
|
||||
* c->jobmgr via restore_job_array()'s real key_add() calls before this
|
||||
* fires. */
|
||||
job_array_apply_test_set_bulk_insert_force_fail_at(2);
|
||||
|
||||
int rc = apply_committed_entry(&g_tace_p, WAL_JOB_ARRAY_SUBMIT, 63,
|
||||
payload, (uint32_t)plen);
|
||||
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
||||
"a bulk-insert-phase failure must still propagate as a negative"
|
||||
" errno so the caller halts/fences the LSN");
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_tace_hook_fires,
|
||||
"the genuinely-inserted prefix must still get exactly ONE"
|
||||
" HOOK_DETAIL_JOB_NEW fire -- task 7.1's bug-1443 partial-prefix fix"
|
||||
" -- even though the overall call reports failure");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_tace_last_new_list_count,
|
||||
"the fire must carry exactly the 2 genuinely-inserted elements, not"
|
||||
" the full array_size and not zero");
|
||||
|
||||
for (uint64_t i = 0; i < 2; i++) {
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(tace_find_job(base_jobid + i),
|
||||
"the genuinely-inserted prefix must be visible in c->jobmgr");
|
||||
}
|
||||
for (uint64_t i = 2; i < array_size; i++) {
|
||||
TEST_ASSERT_NULL_MESSAGE(tace_find_job(base_jobid + i),
|
||||
"indices never reached by key_add() must not be visible");
|
||||
}
|
||||
|
||||
job_array_apply_test_clear_bulk_insert_force_fail();
|
||||
free(payload);
|
||||
tace_teardown();
|
||||
printf("test_ace_job_array_submit_bulk_insert_failure_fires_hook_for_"
|
||||
"prefix: PASS\n");
|
||||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Defensive / boundary ─────────────────────────────────────────────── */
|
||||
|
|
@ -688,6 +752,7 @@ void test_apply_committed_entry(void)
|
|||
RUN_TEST(test_ace_job_array_submit_idempotent_replay_no_extra_notify);
|
||||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_ace_job_array_submit_forced_failure_fires_no_hook);
|
||||
RUN_TEST(test_ace_job_array_submit_bulk_insert_failure_fires_hook_for_prefix);
|
||||
#endif
|
||||
RUN_TEST(test_ace_null_persist_returns_error);
|
||||
UNITY_END();
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@
|
|||
#include "xce/types.h"
|
||||
#include "xlist.h"
|
||||
#include "xstring.h"
|
||||
#include "zarray.h"
|
||||
|
||||
static int g_taj_jobobj_ready;
|
||||
static int g_taj_xstring_ready;
|
||||
|
|
@ -92,6 +93,7 @@ static void taj_setup(void)
|
|||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_clear_force_fail();
|
||||
job_array_apply_test_clear_bulk_insert_force_fail();
|
||||
xzarray_reserve_test_clear_force_fail();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -100,6 +102,7 @@ static void taj_teardown(void)
|
|||
#ifdef XCE_TESTING
|
||||
job_array_apply_test_clear_force_fail();
|
||||
job_array_apply_test_clear_bulk_insert_force_fail();
|
||||
xzarray_reserve_test_clear_force_fail();
|
||||
#endif
|
||||
/* Mirrors test_apply_committed_entry.c's tace_teardown: the jobmgr's own
|
||||
* datakiller is unset, so the job_t values it still holds are not
|
||||
|
|
@ -532,6 +535,142 @@ static void test_rja_bulk_insert_failure_reports_error_not_false_success(void)
|
|||
taj_teardown();
|
||||
printf("test_rja_bulk_insert_failure_reports_error_not_false_success: PASS\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 7.1 (bug-1443): restore_job_array()
|
||||
* now calls kvmgr_reserve() BEFORE the bulk-insert pass begins ──────────────
|
||||
* Proven by forcing the underlying xzarray_reserve() primitive itself to
|
||||
* fail (the same seam test_zarray_reserve.c/test_kvmgr_reserve.c use in
|
||||
* isolation, reused here at the job.c integration level): if
|
||||
* restore_job_array() genuinely calls kvmgr_reserve() and gates the
|
||||
* bulk-insert loop on its result, NO element of this attempt can ever reach
|
||||
* key_add() -- c->jobmgr must remain completely untouched, and the function
|
||||
* must report the reservation failure honestly (negative rc), exactly like
|
||||
* the OTHER existing pre-mutation failure paths in this function. If the
|
||||
* call were missing, or placed after the loop starts, at least the first
|
||||
* element(s) would have already been key_add()'d for real before this forced
|
||||
* failure could take effect -- this test would then observe a non-empty
|
||||
* partial prefix instead of nothing. */
|
||||
static void test_rja_kvmgr_reserve_gates_bulk_insert_pass(void)
|
||||
{
|
||||
taj_setup();
|
||||
|
||||
const uint64_t base_jobid = 77000;
|
||||
const uint64_t array_size = 5;
|
||||
job_submit_data_t spec = taj_make_spec();
|
||||
uint8_t* payload = NULL;
|
||||
size_t len = 0;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
||||
|
||||
xzarray_reserve_test_set_force_fail();
|
||||
|
||||
xlist_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
int rc = restore_job_array(&g_taj_c, payload, len, 70, &out);
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
||||
"a forced reservation failure must be reported as a negative rc,"
|
||||
" treated exactly like the other pre-mutation failure paths");
|
||||
|
||||
for (uint64_t i = 0; i < array_size; i++) {
|
||||
TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
||||
"kvmgr_reserve() must be called and checked BEFORE the first"
|
||||
" key_add() of the bulk-insert pass -- a forced reservation"
|
||||
" failure must leave EVERY index of this attempt untouched, not"
|
||||
" just some");
|
||||
}
|
||||
|
||||
/* Clear the seam and retry: the identical record must now succeed
|
||||
* fully, proving the earlier reservation failure did not leave any
|
||||
* poisoned state behind (c->jobmgr really was never touched). */
|
||||
xzarray_reserve_test_clear_force_fail();
|
||||
|
||||
xlist_t out2;
|
||||
memset(&out2, 0, sizeof(out2));
|
||||
int rc2 = restore_job_array(&g_taj_c, payload, len, 71, &out2);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc2,
|
||||
"a retry after the reservation seam clears must construct every"
|
||||
" element, proving the earlier failure left c->jobmgr genuinely"
|
||||
" untouched");
|
||||
|
||||
xlist_fini(&out2);
|
||||
free(payload);
|
||||
taj_teardown();
|
||||
printf("test_rja_kvmgr_reserve_gates_bulk_insert_pass: PASS\n");
|
||||
}
|
||||
|
||||
/* ── job-array-single-wal-record task 7.1 (bug-1443 partial-prefix fix):
|
||||
* a bulk-insert-phase failure (despite a SUCCESSFUL reservation) must still
|
||||
* populate out_new_jobs with the genuinely-inserted prefix ─────────────────
|
||||
* Reuses task 3.1's existing bulk-insert-phase fault-injection seam
|
||||
* unmodified -- job_array_apply_test_set_bulk_insert_force_fail_at()
|
||||
* simulates key_add() itself failing at a chosen index REGARDLESS of
|
||||
* capacity state, so it exercises the exact narrow residual case task 7.1
|
||||
* targets (key_add() still fails even though reservation succeeded), with
|
||||
* no seam extension needed. Before task 7.1, out_new_jobs was left
|
||||
* completely untouched (empty) on this failure path -- see
|
||||
* test_rja_bulk_insert_failure_reports_error_not_false_success above, which
|
||||
* never inspects `out` after a failed call for exactly that reason. This
|
||||
* test proves out_new_jobs now carries indices [0, k) -- the genuinely,
|
||||
* permanently inserted prefix -- so a caller (apply_committed_entry(), see
|
||||
* test_apply_committed_entry.c) can still notify for it. */
|
||||
static void test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix(void)
|
||||
{
|
||||
taj_setup();
|
||||
|
||||
const uint64_t base_jobid = 76500;
|
||||
const uint64_t array_size = 5;
|
||||
job_submit_data_t spec = taj_make_spec();
|
||||
uint8_t* payload = NULL;
|
||||
size_t len = 0;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
||||
|
||||
/* Force the bulk-insert pass to fail inserting the 3rd element (index 2
|
||||
* of built.count) -- indices 0 and 1 are genuinely, permanently
|
||||
* key_add()'d into c->jobmgr before this fires (reservation, task 7.1,
|
||||
* already succeeded and is irrelevant to this simulated failure). */
|
||||
job_array_apply_test_set_bulk_insert_force_fail_at(2);
|
||||
|
||||
xlist_t out;
|
||||
memset(&out, 0, sizeof(out));
|
||||
int rc = restore_job_array(&g_taj_c, payload, len, 55, &out);
|
||||
|
||||
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
||||
"a bulk-insert-phase failure must still be reported as a negative"
|
||||
" rc, even though a genuine prefix was inserted");
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(2, out.count,
|
||||
"out_new_jobs must now carry exactly the genuinely-inserted prefix"
|
||||
" [0, k) -- task 7.1's bug-1443 partial-prefix fix -- instead of"
|
||||
" being left empty/untouched as before this task");
|
||||
|
||||
for (int k = 0; k < out.count; k++) {
|
||||
job_t* j = (job_t*)xlist_get(&out, k);
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(base_jobid + (uint64_t)k, j->job.jobid,
|
||||
"the transferred prefix must be in index order, matching the"
|
||||
" genuinely-inserted indices");
|
||||
}
|
||||
|
||||
/* Cross-check against direct jobmgr lookups, mirroring the existing
|
||||
* bulk-insert-failure test's own residual-gap assertions. */
|
||||
for (uint64_t i = 0; i < 2; i++) {
|
||||
TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i));
|
||||
}
|
||||
for (uint64_t i = 2; i < array_size; i++) {
|
||||
TEST_ASSERT_NULL(taj_find_job(base_jobid + i));
|
||||
}
|
||||
|
||||
xlist_fini(&out);
|
||||
free(payload);
|
||||
job_array_apply_test_clear_bulk_insert_force_fail();
|
||||
taj_teardown();
|
||||
printf("test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix: "
|
||||
"PASS\n");
|
||||
}
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ── Defensive / boundary ─────────────────────────────────────────────── */
|
||||
|
|
@ -868,6 +1007,8 @@ void test_job_array_apply(void)
|
|||
#ifdef XCE_TESTING
|
||||
RUN_TEST(test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds);
|
||||
RUN_TEST(test_rja_bulk_insert_failure_reports_error_not_false_success);
|
||||
RUN_TEST(test_rja_kvmgr_reserve_gates_bulk_insert_pass);
|
||||
RUN_TEST(test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix);
|
||||
#endif
|
||||
RUN_TEST(test_rja_null_or_empty_payload_returns_error);
|
||||
RUN_TEST(test_rja_out_new_jobs_is_optional);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue