/* test_job_array_apply.c — job-array-single-wal-record tasks 3.1 / 3.3 * * Unit tests for restore_job_array() (job.c), the shared N-way array * expansion primitive (design.md "Array Apply Expansion" / Req 2.1, 2.2, * 2.3, 3.3, 4.1, 5.1, 5.2, 5.3, 9.3, 9.6), PLUS (task 3.3, added below the * task-3.1 tests) dispatch-level tests proving apply_wal_entry()'s new * WAL_JOB_ARRAY_SUBMIT case -- the standby catch-up / replicated-apply * dispatcher, a SEPARATE, still-live function from apply_committed_entry() * (design.md "Existing Architecture Analysis" table) -- expands a committed * array record identically to apply_committed_entry() while staying * hook-free, matching that dispatcher's existing single-job case (Req 3.1, * 3.2, 4.1). * * Covers (task 3.1, restore_job_array() called DIRECTLY): * - N-way expansion produces the correct contiguous jobid set, each * element tagged with entry_lsn (Req 5.1, 5.2, 5.3). * - Idempotent whole-record re-apply is a no-op (-EEXIST, nothing new). * - Idempotent PARTIAL re-apply skips only the indices that already * exist (Req 3.3). * - The forceable fault-injection seam: a forced failure partway through * leaves ZERO elements of THIS attempt visible in c->jobmgr (not just * the failed index), and a subsequent successful retry (seam cleared) * produces the complete, non-duplicated set (Req 9.3, 9.4, 9.6). * * Covers (task 3.3, driving apply_wal_entry()/apply_committed_entry()): * - apply_wal_entry() constructs every array element and fires NO * notification hook (Req 3.1, 3.2). * - apply_committed_entry() and apply_wal_entry() applied to the SAME * committed array record produce byte-for-byte identical resulting job * state (Req 3.1, 3.2, 4.1). * * Fixture (task 3.1 tests): a minimal cluster_t (jobmgr only -- the only * field restore_job_array reads) mirrors test_apply_committed_entry.c's * "bare fixture, no more than the function under test needs" philosophy. * job_obj_init()/xstring_table_init() are process-global (shared across * every test file in the controller_unit_test binary) -- guarded * idempotently, never torn down here, matching every sibling fixture in * this suite (test_apply_committed_entry.c, test_jobj_hardening.c). * * Fixture (task 3.3 tests, tajd_* below): a richer cluster_t (jobmgr + * hooksys) since these tests drive the dispatch functions -- not * restore_job_array() directly -- and must be able to prove the ABSENCE of * a hook fire. Mirrors test_apply_committed_entry.c's tace_setup shape. */ #include #include #include #include #include #include #include #include #include #include #include "cluster.h" #include "controller.capnp.h" #include "hook.h" #include "jobj.h" #include "persist.h" #include "xce/cluster_def.h" #include "xce/types.h" #include "xlist.h" #include "xstring.h" #include "zarray.h" static int g_taj_jobobj_ready; static int g_taj_xstring_ready; static cluster_t g_taj_c; static void taj_setup(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; if (!g_taj_xstring_ready) { TEST_ASSERT_TRUE(xstring_table_init(64) >= 0); g_taj_xstring_ready = 1; } if (!g_taj_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_taj_jobobj_ready = 1; } memset(&g_taj_c, 0, sizeof(g_taj_c)); g_taj_c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(g_taj_c.jobmgr); #ifdef XCE_TESTING job_array_apply_test_clear_force_fail(); job_array_apply_test_clear_bulk_insert_force_fail(); xzarray_reserve_test_clear_force_fail(); #endif } static void taj_teardown(void) { #ifdef XCE_TESTING job_array_apply_test_clear_force_fail(); job_array_apply_test_clear_bulk_insert_force_fail(); xzarray_reserve_test_clear_force_fail(); #endif /* Mirrors test_apply_committed_entry.c's tace_teardown: the jobmgr's own * datakiller is unset, so the job_t values it still holds are not * individually released here -- accepted, matching that established * fixture convention (the controller suite runs with * ASAN_OPTIONS=detect_leaks=0). */ if (g_taj_c.jobmgr) dereference(g_taj_c.jobmgr); } /* Look up a job_t wrapper by jobid (NULL if absent). */ static job_t* taj_find_job(uint64_t jobid) { job_t* j = NULL; lock(g_taj_c.jobmgr, RTTI_LOCK_READ); int found = key_find(g_taj_c.jobmgr, (void*)jobid, (void**)&j); unlock(g_taj_c.jobmgr); return (found > 0) ? j : NULL; } /* Seeds a job directly into the fixture's jobmgr, bypassing * restore_job_array entirely -- used to simulate "this index already * exists" for the partial-idempotency test. */ static job_submit_data_t g_taj_seed_submit; static void taj_seed_job(uint64_t jobid) { xce_job_t jdef; memset(&g_taj_seed_submit, 0, sizeof(g_taj_seed_submit)); g_taj_seed_submit.queue = (char*)"seed-q"; g_taj_seed_submit.user = (char*)"seed-user"; memset(&jdef, 0, sizeof(jdef)); jdef.jobid = jobid; jdef.state = JOB_STAT_PEND; jdef.submit = &g_taj_seed_submit; TEST_ASSERT_NOT_NULL(makeJob(g_taj_c.jobmgr, &jdef)); } /* Encodes a (base_jobid, array_size, spec) JobSubmit payload -- the exact * shape the committed WAL_JOB_ARRAY_SUBMIT record carries (mirrors * persist.c's persist_submit_job_array_async encode side, and * test_apply_committed_entry.c's tace_encode_job/tace_encode_worker * pattern for the encode helper shape). Caller frees *out. */ static int taj_encode_job_submit(uint64_t base_jobid, uint64_t array_size, job_submit_data_t* spec, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobSubmit cjs; JobSubmit_ptr jsp; xce_job_submit_t s; memset(&s, 0, sizeof(s)); s.jobid = base_jobid; s.array_size = array_size; s.submit = spec; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobSubmit(cs, &cjs, &s); jsp = new_JobSubmit(cs); write_JobSubmit(&cjs, jsp); capn_setp(capn_root(&c), 0, jsp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } static job_submit_data_t taj_make_spec(void) { job_submit_data_t s; memset(&s, 0, sizeof(s)); s.queue = (char*)"normal"; s.user = (char*)"tester"; s.mintasks = 1; s.maxtasks = 1; s.job_name = (char*)"array-job"; return s; } /* ── N-way expansion produces the correct contiguous set (Req 5.1-5.3) ── */ static void test_rja_expands_contiguous_set_and_tags_lsn(void) { taj_setup(); const uint64_t base_jobid = 71000; const uint64_t array_size = 5; const uint64_t entry_lsn = 777; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_taj_c, payload, len, entry_lsn, &out); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc, "a fresh array record must report array_size newly created elements"); TEST_ASSERT_EQUAL_INT((int)array_size, out.count); for (uint64_t i = 0; i < array_size; i++) { job_t* j = taj_find_job(base_jobid + i); TEST_ASSERT_NOT_NULL_MESSAGE(j, "every array index must be visible in c->jobmgr"); TEST_ASSERT_EQUAL_UINT64(base_jobid + i, j->job.jobid); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state); TEST_ASSERT_EQUAL_UINT64_MESSAGE(entry_lsn, j->submit_lsn, "every constructed element must be tagged with entry_lsn"); TEST_ASSERT_NOT_NULL(j->job.submit); TEST_ASSERT_EQUAL_STRING("normal", j->job.submit->queue); TEST_ASSERT_EQUAL_STRING("tester", j->job.submit->user); /* out_new_jobs carries the same objects, in index order. */ TEST_ASSERT_EQUAL_PTR(j, xlist_get(&out, (int)i)); } xlist_fini(&out); free(payload); taj_teardown(); printf("test_rja_expands_contiguous_set_and_tags_lsn: PASS\n"); } /* ── N-way expansion across SEVERAL array sizes (task 3.4 gap-closer) ─── * The test above only proves correctness at ONE size (5). Task 3.4's own * observable explicitly asks for "several array sizes" -- mirroring the * WAL-append/commit-continuation size sweep task 2.3 already added * (test_job_array_submit_one_entry_one_continuation_across_sizes, * test_commit_barrier_rollout.c) for the encode/commit side. This is the * expansion-side analog: same size set {2, 3, 100, 1000}, proving the * CONTIGUOUS jobid set (and its exact endpoints -- no off-by-one leakage * into an adjacent, never-submitted jobid) holds at small, medium, and * large array sizes, not just one. */ static void test_rja_expands_contiguous_set_across_sizes(void) { static const uint64_t sizes[] = { 2, 3, 100, 1000 }; size_t idx; taj_setup(); for (idx = 0; idx < sizeof(sizes) / sizeof(sizes[0]); idx++) { const uint64_t array_size = sizes[idx]; const uint64_t base_jobid = 90000 + (uint64_t)idx * 2000000; const uint64_t entry_lsn = 900 + (uint64_t)idx; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; char msg[160]; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_taj_c, payload, len, entry_lsn, &out); snprintf(msg, sizeof(msg), "array_size=%llu: must report array_size newly created elements", (unsigned long long)array_size); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc, msg); snprintf(msg, sizeof(msg), "array_size=%llu: out_new_jobs count must equal array_size", (unsigned long long)array_size); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, out.count, msg); for (uint64_t i = 0; i < array_size; i++) { job_t* j = taj_find_job(base_jobid + i); snprintf(msg, sizeof(msg), "array_size=%llu index=%llu: element must be visible", (unsigned long long)array_size, (unsigned long long)i); TEST_ASSERT_NOT_NULL_MESSAGE(j, msg); TEST_ASSERT_EQUAL_UINT64(base_jobid + i, j->job.jobid); TEST_ASSERT_EQUAL_UINT64(entry_lsn, j->submit_lsn); } /* Exact endpoints -- no off-by-one leakage into an adjacent, * never-submitted jobid on either side of the range. */ snprintf(msg, sizeof(msg), "array_size=%llu: one-below-base must not exist", (unsigned long long)array_size); TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid - 1), msg); snprintf(msg, sizeof(msg), "array_size=%llu: one-past-the-end must not exist", (unsigned long long)array_size); TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + array_size), msg); xlist_fini(&out); free(payload); } taj_teardown(); printf("test_rja_expands_contiguous_set_across_sizes: PASS\n"); } /* ── Idempotent whole-record re-apply is a no-op (Req 3.3) ─────────────── */ static void test_rja_idempotent_whole_record_replay_is_noop(void) { taj_setup(); const uint64_t base_jobid = 72000; const uint64_t array_size = 4; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); xlist_t out1; memset(&out1, 0, sizeof(out1)); int rc1 = restore_job_array(&g_taj_c, payload, len, 10, &out1); TEST_ASSERT_EQUAL_INT((int)array_size, rc1); xlist_fini(&out1); /* Re-apply the identical committed record (replay / retried commit). */ xlist_t out2; memset(&out2, 0, sizeof(out2)); int rc2 = restore_job_array(&g_taj_c, payload, len, 10, &out2); TEST_ASSERT_EQUAL_INT_MESSAGE(-EEXIST, rc2, "re-applying an already-fully-applied record must be a whole-record" " no-op (-EEXIST), constructing nothing new"); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i)); } free(payload); taj_teardown(); printf("test_rja_idempotent_whole_record_replay_is_noop: PASS\n"); } /* ── Idempotent PARTIAL re-apply skips only existing indices (Req 3.3) ── */ static void test_rja_partial_reapply_skips_existing_indices(void) { taj_setup(); const uint64_t base_jobid = 73000; const uint64_t array_size = 5; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; /* Index 2 already exists before the array apply ever runs. */ taj_seed_job(base_jobid + 2); TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_taj_c, payload, len, 20, &out); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size - 1, rc, "one pre-existing index must be skipped, not treated as an error"); TEST_ASSERT_EQUAL_INT((int)array_size - 1, out.count); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_NOT_NULL_MESSAGE(taj_find_job(base_jobid + i), "every index, pre-existing or newly created, must be present"); } /* The pre-existing index must not appear among the NEWLY created set. */ for (int k = 0; k < out.count; k++) { job_t* j = (job_t*)xlist_get(&out, k); TEST_ASSERT_NOT_EQUAL_UINT64(base_jobid + 2, j->job.jobid); } xlist_fini(&out); free(payload); taj_teardown(); printf("test_rja_partial_reapply_skips_existing_indices: PASS\n"); } #ifdef XCE_TESTING /* ── Forced mid-array failure: zero elements visible; retry succeeds ──── * (Req 2.1, 2.2, 2.3, 9.3, 9.4, 9.6 -- the load-bearing atomicity contract: * a mid-array failure must discard EVERY element built this attempt, * including the ones "before" the forced failure index, and must never * have touched c->jobmgr at all.) */ static void test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds(void) { taj_setup(); const uint64_t base_jobid = 74000; const uint64_t array_size = 5; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); /* Force a construction failure at index 2 -- indices 0 and 1 will have * "succeeded" in the scratch list before the forced failure fires. */ job_array_apply_test_set_force_fail_at(2); xlist_t out1; memset(&out1, 0, sizeof(out1)); int rc1 = restore_job_array(&g_taj_c, payload, len, 30, &out1); TEST_ASSERT_TRUE_MESSAGE(rc1 < 0, "a forced mid-array construction failure must return a negative rc"); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i), "ALL elements of a failed attempt must be invisible -- including" " the ones that succeeded construction before the forced" " failure index, not only the failed index itself"); } /* Clear the seam and retry the IDENTICAL record from scratch. */ job_array_apply_test_clear_force_fail(); xlist_t out2; memset(&out2, 0, sizeof(out2)); int rc2 = restore_job_array(&g_taj_c, payload, len, 30, &out2); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc2, "a retry after the injected failure clears must produce the" " complete, non-duplicated set"); TEST_ASSERT_EQUAL_INT((int)array_size, out2.count); for (uint64_t i = 0; i < array_size; i++) { job_t* j = taj_find_job(base_jobid + i); TEST_ASSERT_NOT_NULL(j); TEST_ASSERT_EQUAL_UINT64(base_jobid + i, j->job.jobid); } xlist_fini(&out2); free(payload); taj_teardown(); printf("test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds: PASS\n"); } /* ── Bulk-insert-phase failure: reported as an error, never a false * built.count success (task 3.1 review remediation) ─────────────────────── * Unlike the scratch-build-phase seam above, this forces the SECOND phase * -- the key_add(c->jobmgr, ...) bulk-insert pass itself -- to fail * partway through, simulating c->jobmgr's backing store hitting a genuine * realloc() OOM while growing under the write lock (Req 9.4). Because * xkvmgr_type exposes no key_del (kvmgr.c), the indices already inserted * for real before the forced failure index CANNOT be rolled back -- this * test proves the function is honest about that: it reports a negative * errno (never claims full built.count success), and the already-real * prefix genuinely IS visible while the never-inserted tail genuinely is * NOT -- matching the "Residual atomicity gap" paragraph documented on * restore_job_array() (job.c). A subsequent retry (seam cleared) must not * duplicate the already-real prefix -- it is picked up by the existing * per-index idempotency skip (Req 3.3). */ static void test_rja_bulk_insert_failure_reports_error_not_false_success(void) { taj_setup(); const uint64_t base_jobid = 76000; const uint64_t array_size = 5; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); /* Force the bulk-insert pass to fail inserting the 3rd element (index 2 * of built.count) -- indices 0 and 1 will already be genuinely, * permanently inserted into c->jobmgr by the time it fires. */ job_array_apply_test_set_bulk_insert_force_fail_at(2); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_taj_c, payload, len, 50, &out); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a bulk-insert-phase failure must be reported as a negative errno," " never as a false built.count success"); TEST_ASSERT_NOT_EQUAL_MESSAGE((int)array_size, rc, "the failure must not be silently masked as full success"); /* The residual gap, proven directly: indices [0, 2) are already real * (cannot be rolled back -- no key_del); indices [2, array_size) were * never inserted anywhere. */ for (uint64_t i = 0; i < 2; i++) { TEST_ASSERT_NOT_NULL_MESSAGE(taj_find_job(base_jobid + i), "indices already key_add()'d for real before the forced failure" " must remain visible -- they cannot be un-inserted"); } for (uint64_t i = 2; i < array_size; i++) { TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i), "indices never reached by key_add() must not be visible"); } xlist_fini(&out); /* Clear the seam and retry the identical record: the two already-real * indices are skipped by the existing per-index idempotency check * (Req 3.3), and the remaining three are newly created. */ job_array_apply_test_clear_bulk_insert_force_fail(); xlist_t out2; memset(&out2, 0, sizeof(out2)); int rc2 = restore_job_array(&g_taj_c, payload, len, 51, &out2); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size - 2, rc2, "a retry must skip the two already-real indices and create only the" " remaining three"); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_NOT_NULL_MESSAGE(taj_find_job(base_jobid + i), "every index must be present after the retry"); } xlist_fini(&out2); free(payload); taj_teardown(); printf("test_rja_bulk_insert_failure_reports_error_not_false_success: PASS\n"); } /* ── job-array-single-wal-record task 7.1 (bug-1443): restore_job_array() * now calls kvmgr_reserve() BEFORE the bulk-insert pass begins ────────────── * Proven by forcing the underlying xzarray_reserve() primitive itself to * fail (the same seam test_zarray_reserve.c/test_kvmgr_reserve.c use in * isolation, reused here at the job.c integration level): if * restore_job_array() genuinely calls kvmgr_reserve() and gates the * bulk-insert loop on its result, NO element of this attempt can ever reach * key_add() -- c->jobmgr must remain completely untouched, and the function * must report the reservation failure honestly (negative rc), exactly like * the OTHER existing pre-mutation failure paths in this function. If the * call were missing, or placed after the loop starts, at least the first * element(s) would have already been key_add()'d for real before this forced * failure could take effect -- this test would then observe a non-empty * partial prefix instead of nothing. */ static void test_rja_kvmgr_reserve_gates_bulk_insert_pass(void) { taj_setup(); const uint64_t base_jobid = 77000; const uint64_t array_size = 5; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); xzarray_reserve_test_set_force_fail(); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_taj_c, payload, len, 70, &out); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a forced reservation failure must be reported as a negative rc," " treated exactly like the other pre-mutation failure paths"); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i), "kvmgr_reserve() must be called and checked BEFORE the first" " key_add() of the bulk-insert pass -- a forced reservation" " failure must leave EVERY index of this attempt untouched, not" " just some"); } /* Clear the seam and retry: the identical record must now succeed * fully, proving the earlier reservation failure did not leave any * poisoned state behind (c->jobmgr really was never touched). */ xzarray_reserve_test_clear_force_fail(); xlist_t out2; memset(&out2, 0, sizeof(out2)); int rc2 = restore_job_array(&g_taj_c, payload, len, 71, &out2); TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc2, "a retry after the reservation seam clears must construct every" " element, proving the earlier failure left c->jobmgr genuinely" " untouched"); xlist_fini(&out2); free(payload); taj_teardown(); printf("test_rja_kvmgr_reserve_gates_bulk_insert_pass: PASS\n"); } /* ── job-array-single-wal-record task 7.1 (bug-1443 partial-prefix fix): * a bulk-insert-phase failure (despite a SUCCESSFUL reservation) must still * populate out_new_jobs with the genuinely-inserted prefix ───────────────── * Reuses task 3.1's existing bulk-insert-phase fault-injection seam * unmodified -- job_array_apply_test_set_bulk_insert_force_fail_at() * simulates key_add() itself failing at a chosen index REGARDLESS of * capacity state, so it exercises the exact narrow residual case task 7.1 * targets (key_add() still fails even though reservation succeeded), with * no seam extension needed. Before task 7.1, out_new_jobs was left * completely untouched (empty) on this failure path -- see * test_rja_bulk_insert_failure_reports_error_not_false_success above, which * never inspects `out` after a failed call for exactly that reason. This * test proves out_new_jobs now carries indices [0, k) -- the genuinely, * permanently inserted prefix -- so a caller (apply_committed_entry(), see * test_apply_committed_entry.c) can still notify for it. */ static void test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix(void) { taj_setup(); const uint64_t base_jobid = 76500; const uint64_t array_size = 5; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); /* Force the bulk-insert pass to fail inserting the 3rd element (index 2 * of built.count) -- indices 0 and 1 are genuinely, permanently * key_add()'d into c->jobmgr before this fires (reservation, task 7.1, * already succeeded and is irrelevant to this simulated failure). */ job_array_apply_test_set_bulk_insert_force_fail_at(2); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_taj_c, payload, len, 55, &out); TEST_ASSERT_TRUE_MESSAGE(rc < 0, "a bulk-insert-phase failure must still be reported as a negative" " rc, even though a genuine prefix was inserted"); TEST_ASSERT_EQUAL_INT_MESSAGE(2, out.count, "out_new_jobs must now carry exactly the genuinely-inserted prefix" " [0, k) -- task 7.1's bug-1443 partial-prefix fix -- instead of" " being left empty/untouched as before this task"); for (int k = 0; k < out.count; k++) { job_t* j = (job_t*)xlist_get(&out, k); TEST_ASSERT_NOT_NULL(j); TEST_ASSERT_EQUAL_UINT64_MESSAGE(base_jobid + (uint64_t)k, j->job.jobid, "the transferred prefix must be in index order, matching the" " genuinely-inserted indices"); } /* Cross-check against direct jobmgr lookups, mirroring the existing * bulk-insert-failure test's own residual-gap assertions. */ for (uint64_t i = 0; i < 2; i++) { TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i)); } for (uint64_t i = 2; i < array_size; i++) { TEST_ASSERT_NULL(taj_find_job(base_jobid + i)); } xlist_fini(&out); free(payload); job_array_apply_test_clear_bulk_insert_force_fail(); taj_teardown(); printf("test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix: " "PASS\n"); } #endif /* XCE_TESTING */ /* ── Defensive / boundary ─────────────────────────────────────────────── */ static void test_rja_null_or_empty_payload_returns_error(void) { taj_setup(); TEST_ASSERT_TRUE(restore_job_array(&g_taj_c, NULL, 0, 1, NULL) < 0); TEST_ASSERT_TRUE(restore_job_array(NULL, (const void*)"x", 1, 1, NULL) < 0); taj_teardown(); printf("test_rja_null_or_empty_payload_returns_error: PASS\n"); } static void test_rja_out_new_jobs_is_optional(void) { taj_setup(); const uint64_t base_jobid = 75000; const uint64_t array_size = 3; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); int rc = restore_job_array(&g_taj_c, payload, len, 40, NULL); TEST_ASSERT_EQUAL_INT((int)array_size, rc); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i)); } free(payload); taj_teardown(); printf("test_rja_out_new_jobs_is_optional: PASS\n"); } /* ── Task 3.3: apply_wal_entry() standby catch-up dispatch ─────────────── */ static cluster_t g_tajd_c; static persist_t g_tajd_p; static int g_tajd_hook_fires; static int tajd_spy_hook(int type, int situation, int detail, void* data, va_list ap) { (void)type; (void)detail; (void)data; (void)ap; if (situation == HOOK_AFTER_EVENT) g_tajd_hook_fires++; return 0; } static void tajd_setup(void) { kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 }; if (!g_taj_xstring_ready) { TEST_ASSERT_TRUE(xstring_table_init(64) >= 0); g_taj_xstring_ready = 1; } if (!g_taj_jobobj_ready) { TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32)); g_taj_jobobj_ready = 1; } memset(&g_tajd_c, 0, sizeof(g_tajd_c)); TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&g_tajd_c.hooksys, XCE_CONTROLLER_HOOK_MAX)); g_tajd_c.jobmgr = makeInstance(&xkvmgr_type, &int_params); TEST_ASSERT_NOT_NULL(g_tajd_c.jobmgr); memset(&g_tajd_p, 0, sizeof(g_tajd_p)); g_tajd_p.cluster = &g_tajd_c; g_tajd_hook_fires = 0; #ifdef XCE_TESTING job_array_apply_test_clear_force_fail(); job_array_apply_test_clear_bulk_insert_force_fail(); #endif } static void tajd_teardown(void) { #ifdef XCE_TESTING job_array_apply_test_clear_force_fail(); job_array_apply_test_clear_bulk_insert_force_fail(); #endif xce_hook_fini(&g_tajd_c.hooksys); if (g_tajd_c.jobmgr) dereference(g_tajd_c.jobmgr); } static job_t* tajd_find_job(cluster_t* c, uint64_t jobid) { job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); int found = key_find(c->jobmgr, (void*)jobid, (void**)&j); unlock(c->jobmgr); return (found > 0) ? j : NULL; } /* ── job_ctrl() addresses an array-expanded element by its per-element * identity, exactly as it does for a single-submitted job (Req 5.2, task * 3.4 gap-closer) ─────────────────────────────────────────────────────── * Every test above only proves an array element is FOUND via the raw * key_find(c->jobmgr, jobid) primitive -- the SAME primitive job_ctrl() * itself uses internally (job.c:880) to resolve a real client cancel/ * signal request. This test drives job_ctrl() itself (the real production * command entry point, not a bypass) against ONE array-expanded jobid, to * prove Requirement 5.2's "querying, cancelling ... exactly as it does for * job elements created via the current per-element path" holds for the * actual command path -- not only for the lookup primitive it happens to * share with every other test's assertions. */ static void test_tajd_job_ctrl_cancels_array_expanded_element_by_identity(void) { tajd_setup(); const uint64_t base_jobid = 95000; const uint64_t array_size = 4; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); xlist_t out; memset(&out, 0, sizeof(out)); int rc = restore_job_array(&g_tajd_c, payload, len, 500, &out); TEST_ASSERT_EQUAL_INT((int)array_size, rc); xlist_fini(&out); free(payload); /* Cancel exactly ONE element (index 2 of 4) via the real command entry * point job_ctrl() -- the same function a client cancel/signal request * dispatches through for a single-submitted job. */ xce_signal_t sig = { .signo = SIGKILL, .force = 0 }; xce_jobctrl_t ctrinfo = { .action = JobControl_u_siginfo, .jobid = base_jobid + 2, .u = { .siginfo = &sig }, }; int ctrl_rc = job_ctrl(&g_tajd_c, &ctrinfo); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ctrl_rc, "job_ctrl must resolve an array-expanded jobid exactly as it does" " for a single-submitted job"); job_t* cancelled = tajd_find_job(&g_tajd_c, base_jobid + 2); TEST_ASSERT_NOT_NULL(cancelled); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, cancelled->job.state, "the targeted array-expanded element must transition to EXIT via" " job_ctrl's real not-yet-started-job SIGKILL path"); /* Every OTHER array-expanded element must be untouched -- proving * job_ctrl addresses exactly the requested per-element identity, not * the whole array. */ for (uint64_t i = 0; i < array_size; i++) { if (i == 2) continue; job_t* j = tajd_find_job(&g_tajd_c, base_jobid + i); TEST_ASSERT_NOT_NULL(j); TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, "an untargeted sibling array element must be unaffected by a" " job_ctrl call against a different index"); } /* A jobid one past the array's committed range must not resolve. */ xce_jobctrl_t missing_ctrinfo = { .action = JobControl_u_siginfo, .jobid = base_jobid + array_size, .u = { .siginfo = &sig }, }; TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, job_ctrl(&g_tajd_c, &missing_ctrinfo), "a jobid one past the array's committed range must not resolve"); tajd_teardown(); printf("test_tajd_job_ctrl_cancels_array_expanded_element_by_identity:" " PASS\n"); } /* apply_wal_entry()'s WAL_JOB_ARRAY_SUBMIT case constructs every element * through restore_job_array() and fires NO notification -- mirroring the * existing WAL_JOB_SUBMIT case's hook-free behavior at this same dispatch * seam (standby catch-up does not notify the local scheduler/API server; * that is apply_committed_entry()'s job, Req 3.1, 3.2). */ static void test_tajd_apply_wal_entry_constructs_all_and_fires_no_hook(void) { tajd_setup(); const uint64_t base_jobid = 78000; const uint64_t array_size = 3; const uint64_t entry_lsn = 80; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&g_tajd_c.hooksys, XCE_CONTROLLER_HOOK_JOB, tajd_spy_hook, NULL)); int rc = apply_wal_entry(&g_tajd_p, WAL_JOB_ARRAY_SUBMIT, entry_lsn, payload, (uint32_t)len); TEST_ASSERT_EQUAL_INT(0, rc); for (uint64_t i = 0; i < array_size; i++) { job_t* j = tajd_find_job(&g_tajd_c, base_jobid + i); TEST_ASSERT_NOT_NULL_MESSAGE(j, "every array index must be constructed via restore_job_array" " through apply_wal_entry's standby catch-up dispatch"); TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state); TEST_ASSERT_EQUAL_UINT64_MESSAGE(entry_lsn, j->submit_lsn, "each constructed element must be tagged with the entry's lsn"); } TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tajd_hook_fires, "apply_wal_entry's standby catch-up dispatch must never fire a" " notification hook, matching its existing single-job WAL_JOB_SUBMIT" " case"); free(payload); tajd_teardown(); printf("test_tajd_apply_wal_entry_constructs_all_and_fires_no_hook: " "PASS\n"); } /* The task 3.3 Observable: applying the SAME committed array record through * apply_committed_entry() (live apply / boot replay) and through * apply_wal_entry() (standby catch-up) must produce byte-for-byte identical * resulting job state either way -- both dispatchers share the one * restore_job_array() expansion primitive, so they can never diverge (Req * 3.1, 3.2, 4.1). Uses two independent fixture instances (setup/apply/ * snapshot/teardown, run twice) rather than two live cluster_t at once, to * keep each dispatch call's state isolated and the comparison unambiguous. */ typedef struct { uint64_t jobid; uint32_t state; uint64_t submit_lsn; char queue[32]; char user[32]; } tajd_snapshot_t; static void tajd_snapshot_array(cluster_t* c, uint64_t base_jobid, uint64_t array_size, tajd_snapshot_t* out) { for (uint64_t i = 0; i < array_size; i++) { job_t* j = tajd_find_job(c, base_jobid + i); TEST_ASSERT_NOT_NULL(j); out[i].jobid = j->job.jobid; out[i].state = j->job.state; out[i].submit_lsn = j->submit_lsn; out[i].queue[0] = '\0'; out[i].user[0] = '\0'; if (j->job.submit != NULL) { if (j->job.submit->queue != NULL) strncpy(out[i].queue, j->job.submit->queue, sizeof(out[i].queue) - 1); if (j->job.submit->user != NULL) strncpy(out[i].user, j->job.submit->user, sizeof(out[i].user) - 1); } } } static void test_tajd_apply_committed_entry_vs_apply_wal_entry_byte_identical(void) { const uint64_t base_jobid = 79000; const uint64_t array_size = 4; const uint64_t entry_lsn = 90; job_submit_data_t spec = taj_make_spec(); uint8_t* payload = NULL; size_t len = 0; TEST_ASSERT_EQUAL_INT(0, taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len)); tajd_snapshot_t via_committed[4]; tajd_snapshot_t via_wal[4]; /* Path 1: live-apply / boot-replay dispatch. */ tajd_setup(); int rc1 = apply_committed_entry(&g_tajd_p, WAL_JOB_ARRAY_SUBMIT, entry_lsn, payload, (uint32_t)len); TEST_ASSERT_EQUAL_INT((int)array_size, rc1); tajd_snapshot_array(&g_tajd_c, base_jobid, array_size, via_committed); tajd_teardown(); /* Path 2: standby catch-up / replicated-apply dispatch. */ tajd_setup(); int rc2 = apply_wal_entry(&g_tajd_p, WAL_JOB_ARRAY_SUBMIT, entry_lsn, payload, (uint32_t)len); TEST_ASSERT_EQUAL_INT(0, rc2); tajd_snapshot_array(&g_tajd_c, base_jobid, array_size, via_wal); tajd_teardown(); for (uint64_t i = 0; i < array_size; i++) { TEST_ASSERT_EQUAL_UINT64_MESSAGE(via_committed[i].jobid, via_wal[i].jobid, "jobid must match across both dispatchers"); TEST_ASSERT_EQUAL_UINT32_MESSAGE(via_committed[i].state, via_wal[i].state, "state must match across both dispatchers"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(via_committed[i].submit_lsn, via_wal[i].submit_lsn, "submit_lsn must match across both dispatchers"); TEST_ASSERT_EQUAL_STRING_MESSAGE(via_committed[i].queue, via_wal[i].queue, "queue must match across both dispatchers"); TEST_ASSERT_EQUAL_STRING_MESSAGE(via_committed[i].user, via_wal[i].user, "user must match across both dispatchers"); } free(payload); printf("test_tajd_apply_committed_entry_vs_apply_wal_entry_byte_" "identical: PASS\n"); } /* ── Test group entry point ───────────────────────────────────────────── */ void test_job_array_apply(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_rja_expands_contiguous_set_and_tags_lsn); RUN_TEST(test_rja_expands_contiguous_set_across_sizes); RUN_TEST(test_rja_idempotent_whole_record_replay_is_noop); RUN_TEST(test_rja_partial_reapply_skips_existing_indices); #ifdef XCE_TESTING RUN_TEST(test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds); RUN_TEST(test_rja_bulk_insert_failure_reports_error_not_false_success); RUN_TEST(test_rja_kvmgr_reserve_gates_bulk_insert_pass); RUN_TEST(test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix); #endif RUN_TEST(test_rja_null_or_empty_payload_returns_error); RUN_TEST(test_rja_out_new_jobs_is_optional); RUN_TEST(test_tajd_job_ctrl_cancels_array_expanded_element_by_identity); RUN_TEST(test_tajd_apply_wal_entry_constructs_all_and_fires_no_hook); RUN_TEST(test_tajd_apply_committed_entry_vs_apply_wal_entry_byte_identical); }