Rongsong Shen
a81f2a249c
Only 21 of 86 test_*.c group functions called their own nested UNITY_BEGIN()/UNITY_END(). Each nested call reset Unity's global NumberOfTests/TestFailures counters, so controller_test.c's final `return UNITY_END();` -- the value meson test actually gates on -- only ever reflected whichever small window of tests ran since the LAST nested reset, silently discarding pass/fail evidence for everything before it (measured: 16 of 972 total tests in one run). The same nested-BEGIN calls also caused PASS/FAIL banners to show the wrong source file for tests in files that don't reset it themselves (Unity.TestFile just carries over from whichever group last called UNITY_BEGIN()). Fix: replace every nested UNITY_BEGIN() with UnitySetTestFile(__FILE__) (sets file attribution only, no counter reset) and drop the matching UNITY_END(); add the same UnitySetTestFile(__FILE__) as the first statement to the other group functions that had neither. The 2 genuinely standalone binaries (test_job_array_sched_push.c, test_job_array_apiserver_push.c -- each has its own int main(), not registered via controller_test.c's TEST() macro) are untouched. Verified: single consolidated summary now reads "972 Tests 0 Failures 0 Ignored OK", exactly matching the ground-truth RUN_TEST call-site count across all 83 registered group files (976 total minus 4 belonging to the 2 standalone binaries). Every PASS/FAIL banner now names its own true source file. Full xce-build.sh all regression pass clean (Ok: 9, Fail: 0). No production code touched -- test-harness plumbing only.
1018 lines
40 KiB
C
1018 lines
40 KiB
C
/* test_job_array_apply.c — job-array-single-wal-record tasks 3.1 / 3.3
|
|
*
|
|
* Unit tests for restore_job_array() (job.c), the shared N-way array
|
|
* expansion primitive (design.md "Array Apply Expansion" / Req 2.1, 2.2,
|
|
* 2.3, 3.3, 4.1, 5.1, 5.2, 5.3, 9.3, 9.6), PLUS (task 3.3, added below the
|
|
* task-3.1 tests) dispatch-level tests proving apply_wal_entry()'s new
|
|
* WAL_JOB_ARRAY_SUBMIT case -- the standby catch-up / replicated-apply
|
|
* dispatcher, a SEPARATE, still-live function from apply_committed_entry()
|
|
* (design.md "Existing Architecture Analysis" table) -- expands a committed
|
|
* array record identically to apply_committed_entry() while staying
|
|
* hook-free, matching that dispatcher's existing single-job case (Req 3.1,
|
|
* 3.2, 4.1).
|
|
*
|
|
* Covers (task 3.1, restore_job_array() called DIRECTLY):
|
|
* - N-way expansion produces the correct contiguous jobid set, each
|
|
* element tagged with entry_lsn (Req 5.1, 5.2, 5.3).
|
|
* - Idempotent whole-record re-apply is a no-op (-EEXIST, nothing new).
|
|
* - Idempotent PARTIAL re-apply skips only the indices that already
|
|
* exist (Req 3.3).
|
|
* - The forceable fault-injection seam: a forced failure partway through
|
|
* leaves ZERO elements of THIS attempt visible in c->jobmgr (not just
|
|
* the failed index), and a subsequent successful retry (seam cleared)
|
|
* produces the complete, non-duplicated set (Req 9.3, 9.4, 9.6).
|
|
*
|
|
* Covers (task 3.3, driving apply_wal_entry()/apply_committed_entry()):
|
|
* - apply_wal_entry() constructs every array element and fires NO
|
|
* notification hook (Req 3.1, 3.2).
|
|
* - apply_committed_entry() and apply_wal_entry() applied to the SAME
|
|
* committed array record produce byte-for-byte identical resulting job
|
|
* state (Req 3.1, 3.2, 4.1).
|
|
*
|
|
* Fixture (task 3.1 tests): a minimal cluster_t (jobmgr only -- the only
|
|
* field restore_job_array reads) mirrors test_apply_committed_entry.c's
|
|
* "bare fixture, no more than the function under test needs" philosophy.
|
|
* job_obj_init()/xstring_table_init() are process-global (shared across
|
|
* every test file in the controller_unit_test binary) -- guarded
|
|
* idempotently, never torn down here, matching every sibling fixture in
|
|
* this suite (test_apply_committed_entry.c, test_jobj_hardening.c).
|
|
*
|
|
* Fixture (task 3.3 tests, tajd_* below): a richer cluster_t (jobmgr +
|
|
* hooksys) since these tests drive the dispatch functions -- not
|
|
* restore_job_array() directly -- and must be able to prove the ABSENCE of
|
|
* a hook fire. Mirrors test_apply_committed_entry.c's tace_setup shape.
|
|
*/
|
|
|
|
#include <rtti.h>
|
|
|
|
#include <capnp_c.h>
|
|
#include <errno.h>
|
|
#include <signal.h>
|
|
#include <stdarg.h>
|
|
#include <stdbool.h>
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "cluster.h"
|
|
#include "controller.capnp.h"
|
|
#include "hook.h"
|
|
#include "jobj.h"
|
|
#include "persist.h"
|
|
#include "xce/cluster_def.h"
|
|
#include "xce/types.h"
|
|
#include "xlist.h"
|
|
#include "xstring.h"
|
|
#include "zarray.h"
|
|
|
|
static int g_taj_jobobj_ready;
|
|
static int g_taj_xstring_ready;
|
|
|
|
static cluster_t g_taj_c;
|
|
|
|
static void taj_setup(void)
|
|
{
|
|
kvmgr_params_t int_params
|
|
= { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 };
|
|
|
|
if (!g_taj_xstring_ready) {
|
|
TEST_ASSERT_TRUE(xstring_table_init(64) >= 0);
|
|
g_taj_xstring_ready = 1;
|
|
}
|
|
if (!g_taj_jobobj_ready) {
|
|
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
|
|
g_taj_jobobj_ready = 1;
|
|
}
|
|
|
|
memset(&g_taj_c, 0, sizeof(g_taj_c));
|
|
g_taj_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
|
|
TEST_ASSERT_NOT_NULL(g_taj_c.jobmgr);
|
|
|
|
#ifdef XCE_TESTING
|
|
job_array_apply_test_clear_force_fail();
|
|
job_array_apply_test_clear_bulk_insert_force_fail();
|
|
xzarray_reserve_test_clear_force_fail();
|
|
#endif
|
|
}
|
|
|
|
static void taj_teardown(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
job_array_apply_test_clear_force_fail();
|
|
job_array_apply_test_clear_bulk_insert_force_fail();
|
|
xzarray_reserve_test_clear_force_fail();
|
|
#endif
|
|
/* Mirrors test_apply_committed_entry.c's tace_teardown: the jobmgr's own
|
|
* datakiller is unset, so the job_t values it still holds are not
|
|
* individually released here -- accepted, matching that established
|
|
* fixture convention (the controller suite runs with
|
|
* ASAN_OPTIONS=detect_leaks=0). */
|
|
if (g_taj_c.jobmgr)
|
|
dereference(g_taj_c.jobmgr);
|
|
}
|
|
|
|
/* Look up a job_t wrapper by jobid (NULL if absent). */
|
|
static job_t* taj_find_job(uint64_t jobid)
|
|
{
|
|
job_t* j = NULL;
|
|
lock(g_taj_c.jobmgr, RTTI_LOCK_READ);
|
|
int found = key_find(g_taj_c.jobmgr, (void*)jobid, (void**)&j);
|
|
unlock(g_taj_c.jobmgr);
|
|
return (found > 0) ? j : NULL;
|
|
}
|
|
|
|
/* Seeds a job directly into the fixture's jobmgr, bypassing
|
|
* restore_job_array entirely -- used to simulate "this index already
|
|
* exists" for the partial-idempotency test. */
|
|
static job_submit_data_t g_taj_seed_submit;
|
|
|
|
static void taj_seed_job(uint64_t jobid)
|
|
{
|
|
xce_job_t jdef;
|
|
|
|
memset(&g_taj_seed_submit, 0, sizeof(g_taj_seed_submit));
|
|
g_taj_seed_submit.queue = (char*)"seed-q";
|
|
g_taj_seed_submit.user = (char*)"seed-user";
|
|
|
|
memset(&jdef, 0, sizeof(jdef));
|
|
jdef.jobid = jobid;
|
|
jdef.state = JOB_STAT_PEND;
|
|
jdef.submit = &g_taj_seed_submit;
|
|
|
|
TEST_ASSERT_NOT_NULL(makeJob(g_taj_c.jobmgr, &jdef));
|
|
}
|
|
|
|
/* Encodes a (base_jobid, array_size, spec) JobSubmit payload -- the exact
|
|
* shape the committed WAL_JOB_ARRAY_SUBMIT record carries (mirrors
|
|
* persist.c's persist_submit_job_array_async encode side, and
|
|
* test_apply_committed_entry.c's tace_encode_job/tace_encode_worker
|
|
* pattern for the encode helper shape). Caller frees *out. */
|
|
static int taj_encode_job_submit(uint64_t base_jobid, uint64_t array_size,
|
|
job_submit_data_t* spec, uint8_t** out,
|
|
size_t* out_len)
|
|
{
|
|
struct capn c;
|
|
struct capn_segment* cs;
|
|
struct JobSubmit cjs;
|
|
JobSubmit_ptr jsp;
|
|
xce_job_submit_t s;
|
|
|
|
memset(&s, 0, sizeof(s));
|
|
s.jobid = base_jobid;
|
|
s.array_size = array_size;
|
|
s.submit = spec;
|
|
|
|
capn_init_malloc(&c);
|
|
cs = capn_root(&c).seg;
|
|
encode_JobSubmit(cs, &cjs, &s);
|
|
jsp = new_JobSubmit(cs);
|
|
write_JobSubmit(&cjs, jsp);
|
|
capn_setp(capn_root(&c), 0, jsp.p);
|
|
|
|
*out_len = capn_size(&c);
|
|
*out = malloc(*out_len);
|
|
if (!*out) {
|
|
capn_free(&c);
|
|
return -ENOMEM;
|
|
}
|
|
capn_write_mem(&c, *out, *out_len, 0);
|
|
capn_free(&c);
|
|
return 0;
|
|
}
|
|
|
|
static job_submit_data_t taj_make_spec(void)
|
|
{
|
|
job_submit_data_t s;
|
|
memset(&s, 0, sizeof(s));
|
|
s.queue = (char*)"normal";
|
|
s.user = (char*)"tester";
|
|
s.mintasks = 1;
|
|
s.maxtasks = 1;
|
|
s.job_name = (char*)"array-job";
|
|
return s;
|
|
}
|
|
|
|
/* ── N-way expansion produces the correct contiguous set (Req 5.1-5.3) ── */
|
|
|
|
static void test_rja_expands_contiguous_set_and_tags_lsn(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 71000;
|
|
const uint64_t array_size = 5;
|
|
const uint64_t entry_lsn = 777;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_taj_c, payload, len, entry_lsn, &out);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc,
|
|
"a fresh array record must report array_size newly created elements");
|
|
TEST_ASSERT_EQUAL_INT((int)array_size, out.count);
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
job_t* j = taj_find_job(base_jobid + i);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(j, "every array index must be visible in c->jobmgr");
|
|
TEST_ASSERT_EQUAL_UINT64(base_jobid + i, j->job.jobid);
|
|
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(entry_lsn, j->submit_lsn,
|
|
"every constructed element must be tagged with entry_lsn");
|
|
TEST_ASSERT_NOT_NULL(j->job.submit);
|
|
TEST_ASSERT_EQUAL_STRING("normal", j->job.submit->queue);
|
|
TEST_ASSERT_EQUAL_STRING("tester", j->job.submit->user);
|
|
|
|
/* out_new_jobs carries the same objects, in index order. */
|
|
TEST_ASSERT_EQUAL_PTR(j, xlist_get(&out, (int)i));
|
|
}
|
|
|
|
xlist_fini(&out);
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_expands_contiguous_set_and_tags_lsn: PASS\n");
|
|
}
|
|
|
|
/* ── N-way expansion across SEVERAL array sizes (task 3.4 gap-closer) ───
|
|
* The test above only proves correctness at ONE size (5). Task 3.4's own
|
|
* observable explicitly asks for "several array sizes" -- mirroring the
|
|
* WAL-append/commit-continuation size sweep task 2.3 already added
|
|
* (test_job_array_submit_one_entry_one_continuation_across_sizes,
|
|
* test_commit_barrier_rollout.c) for the encode/commit side. This is the
|
|
* expansion-side analog: same size set {2, 3, 100, 1000}, proving the
|
|
* CONTIGUOUS jobid set (and its exact endpoints -- no off-by-one leakage
|
|
* into an adjacent, never-submitted jobid) holds at small, medium, and
|
|
* large array sizes, not just one. */
|
|
static void test_rja_expands_contiguous_set_across_sizes(void)
|
|
{
|
|
static const uint64_t sizes[] = { 2, 3, 100, 1000 };
|
|
size_t idx;
|
|
|
|
taj_setup();
|
|
|
|
for (idx = 0; idx < sizeof(sizes) / sizeof(sizes[0]); idx++) {
|
|
const uint64_t array_size = sizes[idx];
|
|
const uint64_t base_jobid = 90000 + (uint64_t)idx * 2000000;
|
|
const uint64_t entry_lsn = 900 + (uint64_t)idx;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
char msg[160];
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload,
|
|
&len));
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_taj_c, payload, len, entry_lsn, &out);
|
|
|
|
snprintf(msg, sizeof(msg),
|
|
"array_size=%llu: must report array_size newly created elements",
|
|
(unsigned long long)array_size);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc, msg);
|
|
|
|
snprintf(msg, sizeof(msg),
|
|
"array_size=%llu: out_new_jobs count must equal array_size",
|
|
(unsigned long long)array_size);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, out.count, msg);
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
job_t* j = taj_find_job(base_jobid + i);
|
|
snprintf(msg, sizeof(msg),
|
|
"array_size=%llu index=%llu: element must be visible",
|
|
(unsigned long long)array_size, (unsigned long long)i);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(j, msg);
|
|
TEST_ASSERT_EQUAL_UINT64(base_jobid + i, j->job.jobid);
|
|
TEST_ASSERT_EQUAL_UINT64(entry_lsn, j->submit_lsn);
|
|
}
|
|
|
|
/* Exact endpoints -- no off-by-one leakage into an adjacent,
|
|
* never-submitted jobid on either side of the range. */
|
|
snprintf(msg, sizeof(msg),
|
|
"array_size=%llu: one-below-base must not exist",
|
|
(unsigned long long)array_size);
|
|
TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid - 1), msg);
|
|
snprintf(msg, sizeof(msg),
|
|
"array_size=%llu: one-past-the-end must not exist",
|
|
(unsigned long long)array_size);
|
|
TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + array_size), msg);
|
|
|
|
xlist_fini(&out);
|
|
free(payload);
|
|
}
|
|
|
|
taj_teardown();
|
|
printf("test_rja_expands_contiguous_set_across_sizes: PASS\n");
|
|
}
|
|
|
|
/* ── Idempotent whole-record re-apply is a no-op (Req 3.3) ─────────────── */
|
|
|
|
static void test_rja_idempotent_whole_record_replay_is_noop(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 72000;
|
|
const uint64_t array_size = 4;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
xlist_t out1;
|
|
memset(&out1, 0, sizeof(out1));
|
|
int rc1 = restore_job_array(&g_taj_c, payload, len, 10, &out1);
|
|
TEST_ASSERT_EQUAL_INT((int)array_size, rc1);
|
|
xlist_fini(&out1);
|
|
|
|
/* Re-apply the identical committed record (replay / retried commit). */
|
|
xlist_t out2;
|
|
memset(&out2, 0, sizeof(out2));
|
|
int rc2 = restore_job_array(&g_taj_c, payload, len, 10, &out2);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-EEXIST, rc2,
|
|
"re-applying an already-fully-applied record must be a whole-record"
|
|
" no-op (-EEXIST), constructing nothing new");
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i));
|
|
}
|
|
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_idempotent_whole_record_replay_is_noop: PASS\n");
|
|
}
|
|
|
|
/* ── Idempotent PARTIAL re-apply skips only existing indices (Req 3.3) ── */
|
|
|
|
static void test_rja_partial_reapply_skips_existing_indices(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 73000;
|
|
const uint64_t array_size = 5;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
/* Index 2 already exists before the array apply ever runs. */
|
|
taj_seed_job(base_jobid + 2);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_taj_c, payload, len, 20, &out);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size - 1, rc,
|
|
"one pre-existing index must be skipped, not treated as an error");
|
|
TEST_ASSERT_EQUAL_INT((int)array_size - 1, out.count);
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
|
"every index, pre-existing or newly created, must be present");
|
|
}
|
|
|
|
/* The pre-existing index must not appear among the NEWLY created set. */
|
|
for (int k = 0; k < out.count; k++) {
|
|
job_t* j = (job_t*)xlist_get(&out, k);
|
|
TEST_ASSERT_NOT_EQUAL_UINT64(base_jobid + 2, j->job.jobid);
|
|
}
|
|
|
|
xlist_fini(&out);
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_partial_reapply_skips_existing_indices: PASS\n");
|
|
}
|
|
|
|
#ifdef XCE_TESTING
|
|
/* ── Forced mid-array failure: zero elements visible; retry succeeds ────
|
|
* (Req 2.1, 2.2, 2.3, 9.3, 9.4, 9.6 -- the load-bearing atomicity contract:
|
|
* a mid-array failure must discard EVERY element built this attempt,
|
|
* including the ones "before" the forced failure index, and must never
|
|
* have touched c->jobmgr at all.) */
|
|
static void test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 74000;
|
|
const uint64_t array_size = 5;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
/* Force a construction failure at index 2 -- indices 0 and 1 will have
|
|
* "succeeded" in the scratch list before the forced failure fires. */
|
|
job_array_apply_test_set_force_fail_at(2);
|
|
|
|
xlist_t out1;
|
|
memset(&out1, 0, sizeof(out1));
|
|
int rc1 = restore_job_array(&g_taj_c, payload, len, 30, &out1);
|
|
|
|
TEST_ASSERT_TRUE_MESSAGE(rc1 < 0,
|
|
"a forced mid-array construction failure must return a negative rc");
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
|
"ALL elements of a failed attempt must be invisible -- including"
|
|
" the ones that succeeded construction before the forced"
|
|
" failure index, not only the failed index itself");
|
|
}
|
|
|
|
/* Clear the seam and retry the IDENTICAL record from scratch. */
|
|
job_array_apply_test_clear_force_fail();
|
|
|
|
xlist_t out2;
|
|
memset(&out2, 0, sizeof(out2));
|
|
int rc2 = restore_job_array(&g_taj_c, payload, len, 30, &out2);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc2,
|
|
"a retry after the injected failure clears must produce the"
|
|
" complete, non-duplicated set");
|
|
TEST_ASSERT_EQUAL_INT((int)array_size, out2.count);
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
job_t* j = taj_find_job(base_jobid + i);
|
|
TEST_ASSERT_NOT_NULL(j);
|
|
TEST_ASSERT_EQUAL_UINT64(base_jobid + i, j->job.jobid);
|
|
}
|
|
|
|
xlist_fini(&out2);
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds: PASS\n");
|
|
}
|
|
|
|
/* ── Bulk-insert-phase failure: reported as an error, never a false
|
|
* built.count success (task 3.1 review remediation) ───────────────────────
|
|
* Unlike the scratch-build-phase seam above, this forces the SECOND phase
|
|
* -- the key_add(c->jobmgr, ...) bulk-insert pass itself -- to fail
|
|
* partway through, simulating c->jobmgr's backing store hitting a genuine
|
|
* realloc() OOM while growing under the write lock (Req 9.4). Because
|
|
* xkvmgr_type exposes no key_del (kvmgr.c), the indices already inserted
|
|
* for real before the forced failure index CANNOT be rolled back -- this
|
|
* test proves the function is honest about that: it reports a negative
|
|
* errno (never claims full built.count success), and the already-real
|
|
* prefix genuinely IS visible while the never-inserted tail genuinely is
|
|
* NOT -- matching the "Residual atomicity gap" paragraph documented on
|
|
* restore_job_array() (job.c). A subsequent retry (seam cleared) must not
|
|
* duplicate the already-real prefix -- it is picked up by the existing
|
|
* per-index idempotency skip (Req 3.3). */
|
|
static void test_rja_bulk_insert_failure_reports_error_not_false_success(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 76000;
|
|
const uint64_t array_size = 5;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
/* Force the bulk-insert pass to fail inserting the 3rd element (index 2
|
|
* of built.count) -- indices 0 and 1 will already be genuinely,
|
|
* permanently inserted into c->jobmgr by the time it fires. */
|
|
job_array_apply_test_set_bulk_insert_force_fail_at(2);
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_taj_c, payload, len, 50, &out);
|
|
|
|
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
|
"a bulk-insert-phase failure must be reported as a negative errno,"
|
|
" never as a false built.count success");
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE((int)array_size, rc,
|
|
"the failure must not be silently masked as full success");
|
|
|
|
/* The residual gap, proven directly: indices [0, 2) are already real
|
|
* (cannot be rolled back -- no key_del); indices [2, array_size) were
|
|
* never inserted anywhere. */
|
|
for (uint64_t i = 0; i < 2; i++) {
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
|
"indices already key_add()'d for real before the forced failure"
|
|
" must remain visible -- they cannot be un-inserted");
|
|
}
|
|
for (uint64_t i = 2; i < array_size; i++) {
|
|
TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
|
"indices never reached by key_add() must not be visible");
|
|
}
|
|
|
|
xlist_fini(&out);
|
|
|
|
/* Clear the seam and retry the identical record: the two already-real
|
|
* indices are skipped by the existing per-index idempotency check
|
|
* (Req 3.3), and the remaining three are newly created. */
|
|
job_array_apply_test_clear_bulk_insert_force_fail();
|
|
|
|
xlist_t out2;
|
|
memset(&out2, 0, sizeof(out2));
|
|
int rc2 = restore_job_array(&g_taj_c, payload, len, 51, &out2);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size - 2, rc2,
|
|
"a retry must skip the two already-real indices and create only the"
|
|
" remaining three");
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
|
"every index must be present after the retry");
|
|
}
|
|
|
|
xlist_fini(&out2);
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_bulk_insert_failure_reports_error_not_false_success: PASS\n");
|
|
}
|
|
|
|
/* ── job-array-single-wal-record task 7.1 (bug-1443): restore_job_array()
|
|
* now calls kvmgr_reserve() BEFORE the bulk-insert pass begins ──────────────
|
|
* Proven by forcing the underlying xzarray_reserve() primitive itself to
|
|
* fail (the same seam test_zarray_reserve.c/test_kvmgr_reserve.c use in
|
|
* isolation, reused here at the job.c integration level): if
|
|
* restore_job_array() genuinely calls kvmgr_reserve() and gates the
|
|
* bulk-insert loop on its result, NO element of this attempt can ever reach
|
|
* key_add() -- c->jobmgr must remain completely untouched, and the function
|
|
* must report the reservation failure honestly (negative rc), exactly like
|
|
* the OTHER existing pre-mutation failure paths in this function. If the
|
|
* call were missing, or placed after the loop starts, at least the first
|
|
* element(s) would have already been key_add()'d for real before this forced
|
|
* failure could take effect -- this test would then observe a non-empty
|
|
* partial prefix instead of nothing. */
|
|
static void test_rja_kvmgr_reserve_gates_bulk_insert_pass(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 77000;
|
|
const uint64_t array_size = 5;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
xzarray_reserve_test_set_force_fail();
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_taj_c, payload, len, 70, &out);
|
|
|
|
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
|
"a forced reservation failure must be reported as a negative rc,"
|
|
" treated exactly like the other pre-mutation failure paths");
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_NULL_MESSAGE(taj_find_job(base_jobid + i),
|
|
"kvmgr_reserve() must be called and checked BEFORE the first"
|
|
" key_add() of the bulk-insert pass -- a forced reservation"
|
|
" failure must leave EVERY index of this attempt untouched, not"
|
|
" just some");
|
|
}
|
|
|
|
/* Clear the seam and retry: the identical record must now succeed
|
|
* fully, proving the earlier reservation failure did not leave any
|
|
* poisoned state behind (c->jobmgr really was never touched). */
|
|
xzarray_reserve_test_clear_force_fail();
|
|
|
|
xlist_t out2;
|
|
memset(&out2, 0, sizeof(out2));
|
|
int rc2 = restore_job_array(&g_taj_c, payload, len, 71, &out2);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)array_size, rc2,
|
|
"a retry after the reservation seam clears must construct every"
|
|
" element, proving the earlier failure left c->jobmgr genuinely"
|
|
" untouched");
|
|
|
|
xlist_fini(&out2);
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_kvmgr_reserve_gates_bulk_insert_pass: PASS\n");
|
|
}
|
|
|
|
/* ── job-array-single-wal-record task 7.1 (bug-1443 partial-prefix fix):
|
|
* a bulk-insert-phase failure (despite a SUCCESSFUL reservation) must still
|
|
* populate out_new_jobs with the genuinely-inserted prefix ─────────────────
|
|
* Reuses task 3.1's existing bulk-insert-phase fault-injection seam
|
|
* unmodified -- job_array_apply_test_set_bulk_insert_force_fail_at()
|
|
* simulates key_add() itself failing at a chosen index REGARDLESS of
|
|
* capacity state, so it exercises the exact narrow residual case task 7.1
|
|
* targets (key_add() still fails even though reservation succeeded), with
|
|
* no seam extension needed. Before task 7.1, out_new_jobs was left
|
|
* completely untouched (empty) on this failure path -- see
|
|
* test_rja_bulk_insert_failure_reports_error_not_false_success above, which
|
|
* never inspects `out` after a failed call for exactly that reason. This
|
|
* test proves out_new_jobs now carries indices [0, k) -- the genuinely,
|
|
* permanently inserted prefix -- so a caller (apply_committed_entry(), see
|
|
* test_apply_committed_entry.c) can still notify for it. */
|
|
static void test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 76500;
|
|
const uint64_t array_size = 5;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
/* Force the bulk-insert pass to fail inserting the 3rd element (index 2
|
|
* of built.count) -- indices 0 and 1 are genuinely, permanently
|
|
* key_add()'d into c->jobmgr before this fires (reservation, task 7.1,
|
|
* already succeeded and is irrelevant to this simulated failure). */
|
|
job_array_apply_test_set_bulk_insert_force_fail_at(2);
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_taj_c, payload, len, 55, &out);
|
|
|
|
TEST_ASSERT_TRUE_MESSAGE(rc < 0,
|
|
"a bulk-insert-phase failure must still be reported as a negative"
|
|
" rc, even though a genuine prefix was inserted");
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(2, out.count,
|
|
"out_new_jobs must now carry exactly the genuinely-inserted prefix"
|
|
" [0, k) -- task 7.1's bug-1443 partial-prefix fix -- instead of"
|
|
" being left empty/untouched as before this task");
|
|
|
|
for (int k = 0; k < out.count; k++) {
|
|
job_t* j = (job_t*)xlist_get(&out, k);
|
|
TEST_ASSERT_NOT_NULL(j);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(base_jobid + (uint64_t)k, j->job.jobid,
|
|
"the transferred prefix must be in index order, matching the"
|
|
" genuinely-inserted indices");
|
|
}
|
|
|
|
/* Cross-check against direct jobmgr lookups, mirroring the existing
|
|
* bulk-insert-failure test's own residual-gap assertions. */
|
|
for (uint64_t i = 0; i < 2; i++) {
|
|
TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i));
|
|
}
|
|
for (uint64_t i = 2; i < array_size; i++) {
|
|
TEST_ASSERT_NULL(taj_find_job(base_jobid + i));
|
|
}
|
|
|
|
xlist_fini(&out);
|
|
free(payload);
|
|
job_array_apply_test_clear_bulk_insert_force_fail();
|
|
taj_teardown();
|
|
printf("test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix: "
|
|
"PASS\n");
|
|
}
|
|
#endif /* XCE_TESTING */
|
|
|
|
/* ── Defensive / boundary ─────────────────────────────────────────────── */
|
|
|
|
static void test_rja_null_or_empty_payload_returns_error(void)
|
|
{
|
|
taj_setup();
|
|
|
|
TEST_ASSERT_TRUE(restore_job_array(&g_taj_c, NULL, 0, 1, NULL) < 0);
|
|
TEST_ASSERT_TRUE(restore_job_array(NULL, (const void*)"x", 1, 1, NULL) < 0);
|
|
|
|
taj_teardown();
|
|
printf("test_rja_null_or_empty_payload_returns_error: PASS\n");
|
|
}
|
|
|
|
static void test_rja_out_new_jobs_is_optional(void)
|
|
{
|
|
taj_setup();
|
|
|
|
const uint64_t base_jobid = 75000;
|
|
const uint64_t array_size = 3;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
int rc = restore_job_array(&g_taj_c, payload, len, 40, NULL);
|
|
TEST_ASSERT_EQUAL_INT((int)array_size, rc);
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_NOT_NULL(taj_find_job(base_jobid + i));
|
|
}
|
|
|
|
free(payload);
|
|
taj_teardown();
|
|
printf("test_rja_out_new_jobs_is_optional: PASS\n");
|
|
}
|
|
|
|
/* ── Task 3.3: apply_wal_entry() standby catch-up dispatch ─────────────── */
|
|
|
|
static cluster_t g_tajd_c;
|
|
static persist_t g_tajd_p;
|
|
static int g_tajd_hook_fires;
|
|
|
|
static int tajd_spy_hook(int type, int situation, int detail, void* data,
|
|
va_list ap)
|
|
{
|
|
(void)type; (void)detail; (void)data; (void)ap;
|
|
if (situation == HOOK_AFTER_EVENT)
|
|
g_tajd_hook_fires++;
|
|
return 0;
|
|
}
|
|
|
|
static void tajd_setup(void)
|
|
{
|
|
kvmgr_params_t int_params
|
|
= { .key_type = KVMGR_KEY_INT, .nslots = 32, .delta = 16 };
|
|
|
|
if (!g_taj_xstring_ready) {
|
|
TEST_ASSERT_TRUE(xstring_table_init(64) >= 0);
|
|
g_taj_xstring_ready = 1;
|
|
}
|
|
if (!g_taj_jobobj_ready) {
|
|
TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
|
|
g_taj_jobobj_ready = 1;
|
|
}
|
|
|
|
memset(&g_tajd_c, 0, sizeof(g_tajd_c));
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
xce_hook_init(&g_tajd_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
|
|
g_tajd_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
|
|
TEST_ASSERT_NOT_NULL(g_tajd_c.jobmgr);
|
|
|
|
memset(&g_tajd_p, 0, sizeof(g_tajd_p));
|
|
g_tajd_p.cluster = &g_tajd_c;
|
|
|
|
g_tajd_hook_fires = 0;
|
|
|
|
#ifdef XCE_TESTING
|
|
job_array_apply_test_clear_force_fail();
|
|
job_array_apply_test_clear_bulk_insert_force_fail();
|
|
#endif
|
|
}
|
|
|
|
static void tajd_teardown(void)
|
|
{
|
|
#ifdef XCE_TESTING
|
|
job_array_apply_test_clear_force_fail();
|
|
job_array_apply_test_clear_bulk_insert_force_fail();
|
|
#endif
|
|
xce_hook_fini(&g_tajd_c.hooksys);
|
|
if (g_tajd_c.jobmgr)
|
|
dereference(g_tajd_c.jobmgr);
|
|
}
|
|
|
|
static job_t* tajd_find_job(cluster_t* c, uint64_t jobid)
|
|
{
|
|
job_t* j = NULL;
|
|
lock(c->jobmgr, RTTI_LOCK_READ);
|
|
int found = key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
|
unlock(c->jobmgr);
|
|
return (found > 0) ? j : NULL;
|
|
}
|
|
|
|
/* ── job_ctrl() addresses an array-expanded element by its per-element
|
|
* identity, exactly as it does for a single-submitted job (Req 5.2, task
|
|
* 3.4 gap-closer) ───────────────────────────────────────────────────────
|
|
* Every test above only proves an array element is FOUND via the raw
|
|
* key_find(c->jobmgr, jobid) primitive -- the SAME primitive job_ctrl()
|
|
* itself uses internally (job.c:880) to resolve a real client cancel/
|
|
* signal request. This test drives job_ctrl() itself (the real production
|
|
* command entry point, not a bypass) against ONE array-expanded jobid, to
|
|
* prove Requirement 5.2's "querying, cancelling ... exactly as it does for
|
|
* job elements created via the current per-element path" holds for the
|
|
* actual command path -- not only for the lookup primitive it happens to
|
|
* share with every other test's assertions. */
|
|
static void test_tajd_job_ctrl_cancels_array_expanded_element_by_identity(void)
|
|
{
|
|
tajd_setup();
|
|
|
|
const uint64_t base_jobid = 95000;
|
|
const uint64_t array_size = 4;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
xlist_t out;
|
|
memset(&out, 0, sizeof(out));
|
|
int rc = restore_job_array(&g_tajd_c, payload, len, 500, &out);
|
|
TEST_ASSERT_EQUAL_INT((int)array_size, rc);
|
|
xlist_fini(&out);
|
|
free(payload);
|
|
|
|
/* Cancel exactly ONE element (index 2 of 4) via the real command entry
|
|
* point job_ctrl() -- the same function a client cancel/signal request
|
|
* dispatches through for a single-submitted job. */
|
|
xce_signal_t sig = { .signo = SIGKILL, .force = 0 };
|
|
xce_jobctrl_t ctrinfo = {
|
|
.action = JobControl_u_siginfo,
|
|
.jobid = base_jobid + 2,
|
|
.u = { .siginfo = &sig },
|
|
};
|
|
|
|
int ctrl_rc = job_ctrl(&g_tajd_c, &ctrinfo);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ctrl_rc,
|
|
"job_ctrl must resolve an array-expanded jobid exactly as it does"
|
|
" for a single-submitted job");
|
|
|
|
job_t* cancelled = tajd_find_job(&g_tajd_c, base_jobid + 2);
|
|
TEST_ASSERT_NOT_NULL(cancelled);
|
|
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, cancelled->job.state,
|
|
"the targeted array-expanded element must transition to EXIT via"
|
|
" job_ctrl's real not-yet-started-job SIGKILL path");
|
|
|
|
/* Every OTHER array-expanded element must be untouched -- proving
|
|
* job_ctrl addresses exactly the requested per-element identity, not
|
|
* the whole array. */
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
if (i == 2)
|
|
continue;
|
|
job_t* j = tajd_find_job(&g_tajd_c, base_jobid + i);
|
|
TEST_ASSERT_NOT_NULL(j);
|
|
TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
|
|
"an untargeted sibling array element must be unaffected by a"
|
|
" job_ctrl call against a different index");
|
|
}
|
|
|
|
/* A jobid one past the array's committed range must not resolve. */
|
|
xce_jobctrl_t missing_ctrinfo = {
|
|
.action = JobControl_u_siginfo,
|
|
.jobid = base_jobid + array_size,
|
|
.u = { .siginfo = &sig },
|
|
};
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT,
|
|
job_ctrl(&g_tajd_c, &missing_ctrinfo),
|
|
"a jobid one past the array's committed range must not resolve");
|
|
|
|
tajd_teardown();
|
|
printf("test_tajd_job_ctrl_cancels_array_expanded_element_by_identity:"
|
|
" PASS\n");
|
|
}
|
|
|
|
/* apply_wal_entry()'s WAL_JOB_ARRAY_SUBMIT case constructs every element
|
|
* through restore_job_array() and fires NO notification -- mirroring the
|
|
* existing WAL_JOB_SUBMIT case's hook-free behavior at this same dispatch
|
|
* seam (standby catch-up does not notify the local scheduler/API server;
|
|
* that is apply_committed_entry()'s job, Req 3.1, 3.2). */
|
|
static void test_tajd_apply_wal_entry_constructs_all_and_fires_no_hook(void)
|
|
{
|
|
tajd_setup();
|
|
|
|
const uint64_t base_jobid = 78000;
|
|
const uint64_t array_size = 3;
|
|
const uint64_t entry_lsn = 80;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
xce_add_hook(&g_tajd_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
|
|
tajd_spy_hook, NULL));
|
|
|
|
int rc = apply_wal_entry(&g_tajd_p, WAL_JOB_ARRAY_SUBMIT, entry_lsn,
|
|
payload, (uint32_t)len);
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
job_t* j = tajd_find_job(&g_tajd_c, base_jobid + i);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(j,
|
|
"every array index must be constructed via restore_job_array"
|
|
" through apply_wal_entry's standby catch-up dispatch");
|
|
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(entry_lsn, j->submit_lsn,
|
|
"each constructed element must be tagged with the entry's lsn");
|
|
}
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tajd_hook_fires,
|
|
"apply_wal_entry's standby catch-up dispatch must never fire a"
|
|
" notification hook, matching its existing single-job WAL_JOB_SUBMIT"
|
|
" case");
|
|
|
|
free(payload);
|
|
tajd_teardown();
|
|
printf("test_tajd_apply_wal_entry_constructs_all_and_fires_no_hook: "
|
|
"PASS\n");
|
|
}
|
|
|
|
/* The task 3.3 Observable: applying the SAME committed array record through
|
|
* apply_committed_entry() (live apply / boot replay) and through
|
|
* apply_wal_entry() (standby catch-up) must produce byte-for-byte identical
|
|
* resulting job state either way -- both dispatchers share the one
|
|
* restore_job_array() expansion primitive, so they can never diverge (Req
|
|
* 3.1, 3.2, 4.1). Uses two independent fixture instances (setup/apply/
|
|
* snapshot/teardown, run twice) rather than two live cluster_t at once, to
|
|
* keep each dispatch call's state isolated and the comparison unambiguous. */
|
|
typedef struct {
|
|
uint64_t jobid;
|
|
uint32_t state;
|
|
uint64_t submit_lsn;
|
|
char queue[32];
|
|
char user[32];
|
|
} tajd_snapshot_t;
|
|
|
|
static void tajd_snapshot_array(cluster_t* c, uint64_t base_jobid,
|
|
uint64_t array_size, tajd_snapshot_t* out)
|
|
{
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
job_t* j = tajd_find_job(c, base_jobid + i);
|
|
TEST_ASSERT_NOT_NULL(j);
|
|
out[i].jobid = j->job.jobid;
|
|
out[i].state = j->job.state;
|
|
out[i].submit_lsn = j->submit_lsn;
|
|
out[i].queue[0] = '\0';
|
|
out[i].user[0] = '\0';
|
|
if (j->job.submit != NULL) {
|
|
if (j->job.submit->queue != NULL)
|
|
strncpy(out[i].queue, j->job.submit->queue,
|
|
sizeof(out[i].queue) - 1);
|
|
if (j->job.submit->user != NULL)
|
|
strncpy(out[i].user, j->job.submit->user,
|
|
sizeof(out[i].user) - 1);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
test_tajd_apply_committed_entry_vs_apply_wal_entry_byte_identical(void)
|
|
{
|
|
const uint64_t base_jobid = 79000;
|
|
const uint64_t array_size = 4;
|
|
const uint64_t entry_lsn = 90;
|
|
job_submit_data_t spec = taj_make_spec();
|
|
uint8_t* payload = NULL;
|
|
size_t len = 0;
|
|
|
|
TEST_ASSERT_EQUAL_INT(0,
|
|
taj_encode_job_submit(base_jobid, array_size, &spec, &payload, &len));
|
|
|
|
tajd_snapshot_t via_committed[4];
|
|
tajd_snapshot_t via_wal[4];
|
|
|
|
/* Path 1: live-apply / boot-replay dispatch. */
|
|
tajd_setup();
|
|
int rc1 = apply_committed_entry(&g_tajd_p, WAL_JOB_ARRAY_SUBMIT, entry_lsn,
|
|
payload, (uint32_t)len);
|
|
TEST_ASSERT_EQUAL_INT((int)array_size, rc1);
|
|
tajd_snapshot_array(&g_tajd_c, base_jobid, array_size, via_committed);
|
|
tajd_teardown();
|
|
|
|
/* Path 2: standby catch-up / replicated-apply dispatch. */
|
|
tajd_setup();
|
|
int rc2 = apply_wal_entry(&g_tajd_p, WAL_JOB_ARRAY_SUBMIT, entry_lsn,
|
|
payload, (uint32_t)len);
|
|
TEST_ASSERT_EQUAL_INT(0, rc2);
|
|
tajd_snapshot_array(&g_tajd_c, base_jobid, array_size, via_wal);
|
|
tajd_teardown();
|
|
|
|
for (uint64_t i = 0; i < array_size; i++) {
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(via_committed[i].jobid,
|
|
via_wal[i].jobid, "jobid must match across both dispatchers");
|
|
TEST_ASSERT_EQUAL_UINT32_MESSAGE(via_committed[i].state,
|
|
via_wal[i].state, "state must match across both dispatchers");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(via_committed[i].submit_lsn,
|
|
via_wal[i].submit_lsn,
|
|
"submit_lsn must match across both dispatchers");
|
|
TEST_ASSERT_EQUAL_STRING_MESSAGE(via_committed[i].queue,
|
|
via_wal[i].queue, "queue must match across both dispatchers");
|
|
TEST_ASSERT_EQUAL_STRING_MESSAGE(via_committed[i].user,
|
|
via_wal[i].user, "user must match across both dispatchers");
|
|
}
|
|
|
|
free(payload);
|
|
printf("test_tajd_apply_committed_entry_vs_apply_wal_entry_byte_"
|
|
"identical: PASS\n");
|
|
}
|
|
|
|
/* ── Test group entry point ───────────────────────────────────────────── */
|
|
|
|
void test_job_array_apply(void)
|
|
{
|
|
UnitySetTestFile(__FILE__);
|
|
RUN_TEST(test_rja_expands_contiguous_set_and_tags_lsn);
|
|
RUN_TEST(test_rja_expands_contiguous_set_across_sizes);
|
|
RUN_TEST(test_rja_idempotent_whole_record_replay_is_noop);
|
|
RUN_TEST(test_rja_partial_reapply_skips_existing_indices);
|
|
#ifdef XCE_TESTING
|
|
RUN_TEST(test_rja_forced_failure_leaves_nothing_visible_then_retry_succeeds);
|
|
RUN_TEST(test_rja_bulk_insert_failure_reports_error_not_false_success);
|
|
RUN_TEST(test_rja_kvmgr_reserve_gates_bulk_insert_pass);
|
|
RUN_TEST(test_rja_bulk_insert_failure_populates_out_new_jobs_with_prefix);
|
|
#endif
|
|
RUN_TEST(test_rja_null_or_empty_payload_returns_error);
|
|
RUN_TEST(test_rja_out_new_jobs_is_optional);
|
|
RUN_TEST(test_tajd_job_ctrl_cancels_array_expanded_element_by_identity);
|
|
RUN_TEST(test_tajd_apply_wal_entry_constructs_all_and_fires_no_hook);
|
|
RUN_TEST(test_tajd_apply_committed_entry_vs_apply_wal_entry_byte_identical);
|
|
}
|