lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(scheduler): defer job-array expansion off the vtask thread pool

Browse source 
sched_on_job_array() expanded a compact array-job push (up to
hundreds of elements: a per-element dedup lookup plus a full deep
clone) inline, synchronously, on whatever vtaskmgr thread handled the
incoming request -- the same shared pool the scheduler's own
controller connection traffic depends on, violating ha-async-io.md's
"bounded handler work only" contract for that pool (a project-wide
design invariant). A prior finding already showed a burst of
6x50-element array pushes (300 job events landing within ~2ms of each
other) causing the scheduler to lose its controller connection within
245ms and enter a reconnect/failover loop -- directly consistent with
this mechanism, and the same reconnect storm whose resync races
motivated the companion JOBS_PUSH fix in this branch.

The request handler now does O(1) work only: validate the descriptor
and enqueue it unexpanded (XCE_SCHED_OP_UPDATE_JOB_ARRAY, mirroring
the existing XCE_SCHED_OP_ADMIN_RECORD deferred-dispatch pattern).
The actual per-element expansion and dedup check moved to
translate_one_pending_update(), which runs on the round-processing
thread (main.c's xce_sched_driver_run_normal_mode_round(), before
xce_core_begin_session() each round) -- a thread with no connection-
liveness responsibility. As a side effect this also closes the
dedup check's own pre-existing cross-thread TOCTOU race (previously
documented as bug-1714 follow-up "Bug B"): only one thread ever runs
it now.

test_job_array_expand.c and test_job_array_wire_e2e.c, which asserted
the old inline-expansion behavior, are updated to assert the new
deferred-enqueue behavior instead; expansion, dedup, and full
drain-push-release ownership coverage moved to new tests in
test_driver_translate.c.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
Rongsong Shen 2026-09-15 09:01:47 +00:00
parent 6c201bb3e8
commit d5ec1eb3ae
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
10 changed files with 838 additions and 517 deletions
Download patch file Download diff file Expand all files Collapse all files

187
src/scheduler/testsuite/test_driver_translate.c
View file

@ -353,6 +353,29 @@ static xce_job_t* make_wire_job(uint64_t jobid, uint32_t state,
return j;
}
/* Compact job-array descriptor, same shape init_job_array() (test_job_
* array_expand.c) builds -- heap-owned strings, matching what a real
* XCE_SCHED_OP_UPDATE_JOB_ARRAY request now carries unexpanded (connection-
* loss-under-burst fix). */
static xce_job_submit_t* make_wire_job_array(uint64_t base_jobid,
uint64_t array_size)
{
job_submit_data_t* submit = (job_submit_data_t*)calloc(1, sizeof(*submit));
xce_job_submit_t* ja = (xce_job_submit_t*)calloc(1, sizeof(*ja));
submit->queue = strdup("normal");
submit->user = strdup("tester");
submit->mintasks = 1;
submit->maxtasks = 1;
submit->resreq = strdup("type == local");
ja->jobid = base_jobid;
ja->array_size = array_size;
ja->submit = submit;
return ja;
}
/* ---------------------------------------------------------------------- */
/* Per-op unit tests of translate_one_pending_update() (white-box static */
/* call) -- one per op type, asserting event type + every field matches. */
@ -633,6 +656,132 @@ static void test_translate_job_rusage_null_rusages_skips(void)
TEST_ASSERT_EQUAL_INT(0, rc);
}
/* Connection-loss-under-burst fix (XCE_SCHED_OP_UPDATE_JOB_ARRAY, moved
* from handler.c's former sched_on_job_array()): a compact Data.jobArray
* push expands into N ordinary schedulable tasks via base_jobid + i
* arithmetic, HERE on the round-processing thread instead of the
* request-handling thread. */
static void test_translate_job_array_expands_into_n_events(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja = make_wire_job_array(9000, 3);
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = ja;
rc = translate_one_pending_update(&core, req, 42, &ev);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_EQUAL_INT(XCE_CORE_EVENT_JOBS_PUSH, ev.type);
TEST_ASSERT_EQUAL_UINT64(42, ev.ts);
TEST_ASSERT_EQUAL_INT(3, ev.u.jobs_push.njobs);
TEST_ASSERT_EQUAL_UINT64(9000, ev.u.jobs_push.jobs[0]->jobid);
TEST_ASSERT_EQUAL_UINT64(9001, ev.u.jobs_push.jobs[1]->jobid);
TEST_ASSERT_EQUAL_UINT64(9002, ev.u.jobs_push.jobs[2]->jobid);
TEST_ASSERT_EQUAL_STRING("normal", ev.u.jobs_push.jobs[0]->submit->queue);
TEST_ASSERT_EQUAL_STRING("normal", ev.u.jobs_push.jobs[2]->submit->queue);
/* every element gets its OWN independent clone, not a shared pointer --
* required so each is independently free_Job_ptr()-able without a
* double-free. */
TEST_ASSERT_TRUE(ev.u.jobs_push.jobs[0]->submit
!= ev.u.jobs_push.jobs[1]->submit);
TEST_ASSERT_TRUE(ev.u.jobs_push.jobs[1]->submit
!= ev.u.jobs_push.jobs[2]->submit);
/* Expansion result stashed back on req (same pointers as ev.u.jobs_push
* above) so release_translated_payload() can free it later -- see that
* function's own new case. */
TEST_ASSERT_EQUAL_INT(3, req->u.jobarrayupdate.expanded_njobs);
TEST_ASSERT_EQUAL_PTR(ev.u.jobs_push.jobs,
req->u.jobarrayupdate.expanded_jobs);
}
/* Req 6.5/8.4 (moved from handler.c, same rationale): deduplicate a
* redelivered push by element identity -- only genuinely-new indices are
* expanded. Seeds 2 of 3 indices into the REAL core->jobs kvmgr this file's
* fixture_init() sets up (via the real xce_sched_core_init()), simulating
* "already materialized from an earlier round". */
static void test_translate_job_array_skips_already_existing_elements(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja = make_wire_job_array(9100, 3);
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
xce_sched_job_t existing0;
xce_sched_job_t existing1;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = ja;
memset(&existing0, 0, sizeof(existing0));
existing0.jobid = 9100;
sched_add_job(&core, &existing0);
memset(&existing1, 0, sizeof(existing1));
existing1.jobid = 9101;
sched_add_job(&core, &existing1);
rc = translate_one_pending_update(&core, req, 1, &ev);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_EQUAL_INT(XCE_CORE_EVENT_JOBS_PUSH, ev.type);
TEST_ASSERT_EQUAL_INT(1, ev.u.jobs_push.njobs);
TEST_ASSERT_EQUAL_UINT64(9102, ev.u.jobs_push.jobs[0]->jobid);
}
/* Every index already exists: a fully-duplicate redelivery -- nothing new
* to schedule, translate reports "nothing to push" (rc=0), same as any
* other skip case, not an error. */
static void test_translate_job_array_all_duplicates_returns_zero_noop(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja = make_wire_job_array(9200, 2);
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
xce_sched_job_t existing0;
xce_sched_job_t existing1;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = ja;
memset(&existing0, 0, sizeof(existing0));
existing0.jobid = 9200;
sched_add_job(&core, &existing0);
memset(&existing1, 0, sizeof(existing1));
existing1.jobid = 9201;
sched_add_job(&core, &existing1);
rc = translate_one_pending_update(&core, req, 1, &ev);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(0, req->u.jobarrayupdate.expanded_njobs);
TEST_ASSERT_NULL(req->u.jobarrayupdate.expanded_jobs);
}
/* A never-decoded/malformed descriptor (NULL job_array, the enqueue side
* should never produce this, guarded defensively here anyway, same
* discipline every other case in this function follows) skips cleanly. */
static void test_translate_job_array_null_payload_returns_zero(void)
{
xce_sched_core_t core;
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = NULL;
rc = translate_one_pending_update(&core, req, 1, &ev);
TEST_ASSERT_EQUAL_INT(0, rc);
}
static void test_translate_unknown_op_skips(void)
{
xce_sched_core_t core;
@ -854,6 +1003,39 @@ static void test_drain_translates_and_pushes_all_ten_op_types(void)
TEST_ASSERT_EQUAL_PTR(config, core.config);
}
/* Connection-loss-under-burst fix, full drain-push-release cycle: proves
* XCE_SCHED_OP_UPDATE_JOB_ARRAY's ownership handling is correct end to end
* under a REAL ASAN-instrumented run -- both the original job_array
* descriptor (release_translated_payload()'s unconditional free) and the N
* freshly-expanded clones (adopted by core_build_job(), so never
* double-freed) survive the real xce_sched_driver_translate_pending_events()
* -> xce_core_push_events() -> release_translated_payload() sequence
* cleanly -- the same proof test_drain_translates_and_pushes_all_ten_op_types
* gives every other op type, dedicated here since job-array's cleanup path
* (two distinct owned allocations per request, not one) is unique to it. */
static void test_drain_translates_and_pushes_job_array_op(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja;
xce_sched_update_request_t* req;
int rc;
fixture_init(&core);
ja = make_wire_job_array(9300, 3);
req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
req->u.jobarrayupdate.job_array = ja;
enqueue(&core, req);
rc = xce_sched_driver_translate_pending_events(&core, NULL);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(sched_get_job(&core, 9300));
TEST_ASSERT_NOT_NULL(sched_get_job(&core, 9301));
TEST_ASSERT_NOT_NULL(sched_get_job(&core, 9302));
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, sched_get_job(&core, 9300)->state);
}
/* ---------------------------------------------------------------------- */
/* Buffering discipline (Req 5 AC 8): events accumulate between drains; */
/* an empty drain is a valid no-op returning 0, not an error. */
@ -932,8 +1114,13 @@ int main(void)
RUN_TEST(test_translate_resvals_delta);
RUN_TEST(test_translate_job_rusage);
RUN_TEST(test_translate_job_rusage_null_rusages_skips);
RUN_TEST(test_translate_job_array_expands_into_n_events);
RUN_TEST(test_translate_job_array_skips_already_existing_elements);
RUN_TEST(test_translate_job_array_all_duplicates_returns_zero_noop);
RUN_TEST(test_translate_job_array_null_payload_returns_zero);
RUN_TEST(test_translate_unknown_op_skips);
RUN_TEST(test_drain_translates_and_pushes_all_ten_op_types);
RUN_TEST(test_drain_translates_and_pushes_job_array_op);
RUN_TEST(test_accumulates_between_drains_and_empty_drain_is_noop);
RUN_TEST(test_null_core_returns_einval);
RUN_TEST(test_uninitialized_pend_updates_returns_einval);

182
src/scheduler/testsuite/test_job_array_expand.c
View file

@ -3,12 +3,22 @@
* job-array push receive path (job-array-single-wal-record task 5.3, Req
* 6.3, 6.4, 6.5, 8.4).
*
* REVISED (connection-loss-under-burst fix, see XCE_SCHED_OP_UPDATE_JOB_
* ARRAY's own comment, scheduler.h): sched_on_job_array() no longer expands
* the compact descriptor into N tasks itself -- it just takes ownership and
* enqueues it as a single XCE_SCHED_OP_UPDATE_JOB_ARRAY request. The actual
* per-element expansion (and its dedup check) now runs in driver.c's
* translate_one_pending_update(), covered by test_driver_translate.c's own
* UPDATE_JOB_ARRAY tests. This file now covers only: (a) sched_on_job_array()
* /sched_handle_dataset()'s own input validation and ownership-transfer
* bookkeeping, and (b) that the wire-decode -> handler -> queue chain
* delivers the descriptor's fields correctly, unmodified.
*
* Strategy: pull the real (static) sched_handle_dataset / sched_on_job_array
* into this TU via #include "../src/handler.c" (same white-box pattern as
* test_handlers.c), driven against a REAL sched->jobs kvmgr so the
* per-element dedup check (sched_get_job) is exercised for real rather than
* faked. sched_add_job/sched_get_job below are the SAME kvmgr key_add/
* key_find path production uses (mirrors test_job_update.c's precedent).
* test_handlers.c). sched_add_job/sched_get_job stubs below are unused by
* anything this file still exercises (kept, harmless, for parity with other
* handler.c white-box test files that DO need them).
*/
#define UNIT_TEST 1
#include <errno.h>
@ -94,13 +104,6 @@ static void fixture_init(xce_scheduler_t* sched)
sched->core.jobs = makeInstance(&xkvmgr_type, &jobs_params);
}
static void seed_existing_job(xce_scheduler_t* sched, uint64_t jobid)
{
xce_sched_job_t* job = (xce_sched_job_t*)calloc(1, sizeof(*job));
job->jobid = jobid;
sched_add_job(&(sched->core), job);
}
/* Builds a heap-allocated, heap-owned-strings (jobArray, submit) pair -- the
* same shape decode_JobSubmit_ptr() produces on the real wire-decode path
* (controller.capnp.c), not a stack/string-literal stand-in. Task 7.3 made
@ -128,41 +131,42 @@ static xce_job_submit_t* init_job_array(uint64_t base_jobid,
return ja;
}
/* Pop one pending update request, asserting it is an UPDATE_JOB op; hands
* back njobs/jobs (caller now owns them -- free via free_Job_ptr + free() to
* mirror sched_pend_update_killer). Returns -1 if the queue is empty. */
static int pop_job_update(xce_scheduler_t* sched, xce_job_t*** jobs_out)
/* Pop one pending update request, asserting it is an UPDATE_JOB_ARRAY op;
* hands back the raw job_array descriptor (caller now owns it -- free via
* free_JobSubmit_ptr to mirror release_translated_payload()'s own cleanup on
* the failure/never-translated path). Returns NULL if the queue is empty. */
static xce_job_submit_t* pop_job_array_update(xce_scheduler_t* sched)
{
data_item_t* it = dataq_deq(sched->core.pend_updates[0], NULL);
xce_sched_update_request_t* req;
int njobs;
xce_job_submit_t* job_array;
if (it == NULL) {
return -1;
return NULL;
}
req = ITPTR_TO_UPDATE_REQ(it);
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_JOB, req->op);
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_JOB_ARRAY, req->op);
njobs = req->u.jobupdate.njobs;
if (jobs_out != NULL) {
*jobs_out = req->u.jobupdate.jobs;
}
job_array = req->u.jobarrayupdate.job_array;
TEST_ASSERT_EQUAL_INT(0, req->u.jobarrayupdate.expanded_njobs);
TEST_ASSERT_NULL(req->u.jobarrayupdate.expanded_jobs);
free(req);
return njobs;
return job_array;
}
/* ── Req 6.3/6.4: a compact Data.jobArray push expands locally into N
* ordinary schedulable tasks via base_jobid + i arithmetic ── */
/* ── Connection-loss-under-burst fix: a compact Data.jobArray push is now
* queued UNEXPANDED -- ownership of the raw descriptor transfers to the
* XCE_SCHED_OP_UPDATE_JOB_ARRAY request as-is, no per-element work happens
* on this (request-handling) thread. Expansion into N tasks is
* test_driver_translate.c's concern now (translate_one_pending_update()). ── */
static void test_job_array_push_expands_into_n_tasks(void)
static void test_job_array_push_enqueues_raw_descriptor_unexpanded(void)
{
xce_scheduler_t sched;
xce_job_submit_t* ja;
dataset_t dataset;
xce_job_t** jobs = NULL;
int njobs;
int i;
xce_job_submit_t* popped;
fixture_init(&sched);
ja = init_job_array(5000, 3);
@ -171,71 +175,19 @@ static void test_job_array_push_expands_into_n_tasks(void)
dataset.table = XceData_data_jobArray;
dataset.rows = (void*)ja;
/* sched_handle_dataset() owns and frees dataset.rows on return (task
* 7.3) -- do not touch ja/dataset.rows again after this call. */
TEST_ASSERT_TRUE(sched_handle_dataset(&sched, &dataset) >= 0);
njobs = pop_job_update(&sched, &jobs);
TEST_ASSERT_EQUAL_INT(3, njobs);
TEST_ASSERT_EQUAL_UINT64(5000, jobs[0]->jobid);
TEST_ASSERT_EQUAL_UINT64(5001, jobs[1]->jobid);
TEST_ASSERT_EQUAL_UINT64(5002, jobs[2]->jobid);
TEST_ASSERT_EQUAL_STRING("normal", jobs[0]->submit->queue);
TEST_ASSERT_EQUAL_STRING("normal", jobs[2]->submit->queue);
/* every element gets its OWN independent clone, not a shared pointer --
* required so each is independently free_Job_ptr()-able downstream
* (sched_pend_update_killer) without a double-free. */
TEST_ASSERT_TRUE(jobs[0]->submit != jobs[1]->submit);
TEST_ASSERT_TRUE(jobs[1]->submit != jobs[2]->submit);
/* Ownership transferred to the queued request -- sched_handle_dataset()
* must not have freed it and must have nulled dataset.rows out. */
TEST_ASSERT_NULL(dataset.rows);
for (i = 0; i < njobs; i++) {
free_Job_ptr(&jobs[i]);
}
free(jobs);
}
popped = pop_job_array_update(&sched);
TEST_ASSERT_EQUAL_PTR(ja, popped); /* same descriptor, not a clone */
TEST_ASSERT_EQUAL_UINT64(5000, popped->jobid);
TEST_ASSERT_EQUAL_UINT64(3, popped->array_size);
TEST_ASSERT_EQUAL_STRING("normal", popped->submit->queue);
/* ── Req 6.5/8.4: a redelivered push (e.g. after a scheduler reconnect)
* must not create duplicate tasks -- reuse the jobid-keyed exists-check
* (sched_get_job) the ordinary single-job path relies on ── */
static void test_job_array_redelivery_does_not_duplicate(void)
{
xce_scheduler_t sched;
dataset_t dataset;
xce_job_t** jobs = NULL;
int njobs;
int i;
fixture_init(&sched);
memset(&dataset, 0, sizeof(dataset));
dataset.table = XceData_data_jobArray;
dataset.rows = (void*)init_job_array(6000, 2);
/* first delivery: 2 new tasks enqueued. sched_handle_dataset() owns and
* frees dataset.rows on return (task 7.3). */
TEST_ASSERT_TRUE(sched_handle_dataset(&sched, &dataset) >= 0);
njobs = pop_job_update(&sched, &jobs);
TEST_ASSERT_EQUAL_INT(2, njobs);
for (i = 0; i < njobs; i++) {
/* simulate the async consumer (scheduler.c's sched_update_jobs ->
* sched_add_job, not linked into this white-box TU) materializing
* the element into sched->jobs, so the redelivery's dedup check
* sees them as already existing -- exactly what a real reconnect
* scenario looks like from the handler's point of view. */
seed_existing_job(&sched, jobs[i]->jobid);
free_Job_ptr(&jobs[i]);
}
free(jobs);
/* redelivery of the IDENTICAL push: a genuinely new decode of the
* retransmitted wire message (same jobid/array_size/submit content, but
* its own freshly-allocated descriptor -- exactly what a real reconnect
* redelivers; the first descriptor is already freed by the call above).
* Every index already exists. */
dataset.rows = (void*)init_job_array(6000, 2);
TEST_ASSERT_EQUAL_INT(0, sched_handle_dataset(&sched, &dataset));
TEST_ASSERT_EQUAL_INT(-1, pop_job_update(&sched, NULL)); /* nothing new */
free_JobSubmit_ptr(&popped);
}
/* ── Guard-clause coverage on the decode step itself ── */
@ -279,7 +231,8 @@ static void test_job_array_null_descriptor_returns_einval(void)
TEST_ASSERT_EQUAL_INT(-EINVAL, sched_on_job_array(&sched, NULL));
}
/* ── job-array-single-wal-record task 5.3b: full wire-dispatch chain ──────
/* ── job-array-single-wal-record task 5.3b: full wire-dispatch chain
* (REVISED for the connection-loss-under-burst fix) ──────────────────────
* Every test above feeds a stack-built xce_job_submit_t straight into
* sched_handle_dataset(), bypassing the transport layer entirely -- the same
* white-box convention every sibling handler.c test uses (task 5.3's own
@ -290,10 +243,11 @@ static void test_job_array_null_descriptor_returns_einval(void)
* sends -- OPCode_push / Request_body_pushmsg carrying one Data.jobArray
* descriptor -- then hands extract_data()'s reassembled dataset_t to
* sched_handle_dataset(), proving the sender-payload -> wire-dispatch ->
* receiver-expand chain is intact end to end, not just sched_on_job_array()
* in isolation. */
* receiver-enqueue chain delivers the descriptor's fields correctly, intact
* end to end -- expansion itself is no longer this chain's job (driver.c's
* concern now), so this only proves the queued descriptor. */
static void test_wire_dispatch_chain_delivers_n_tasks(void)
static void test_wire_dispatch_chain_delivers_job_array_descriptor(void)
{
xce_scheduler_t sched;
job_submit_data_t submit;
@ -303,9 +257,7 @@ static void test_wire_dispatch_chain_delivers_n_tasks(void)
xce_request_t decoded;
io_packet_t* pkt;
dataset_t dataset;
xce_job_t** jobs = NULL;
int njobs;
int i;
xce_job_submit_t* popped;
fixture_init(&sched);
@ -350,27 +302,22 @@ static void test_wire_dispatch_chain_delivers_n_tasks(void)
* handler (task 5.3, unmodified) -- proving the two layers compose. */
TEST_ASSERT_TRUE(sched_handle_dataset(&sched, &dataset) >= 0);
/* Assert: N schedulable tasks, correctly addressed via base_jobid + i. */
njobs = pop_job_update(&sched, &jobs);
TEST_ASSERT_EQUAL_INT(4, njobs);
TEST_ASSERT_EQUAL_UINT64(8000, jobs[0]->jobid);
TEST_ASSERT_EQUAL_UINT64(8001, jobs[1]->jobid);
TEST_ASSERT_EQUAL_UINT64(8002, jobs[2]->jobid);
TEST_ASSERT_EQUAL_UINT64(8003, jobs[3]->jobid);
TEST_ASSERT_EQUAL_STRING("normal", jobs[0]->submit->queue);
TEST_ASSERT_EQUAL_STRING("normal", jobs[3]->submit->queue);
for (i = 0; i < njobs; i++) {
free_Job_ptr(&jobs[i]);
}
free(jobs);
/* Assert: the decoded descriptor was queued unexpanded, fields intact,
* correctly addressed via base_jobid + array_size. */
popped = pop_job_array_update(&sched);
TEST_ASSERT_NOT_NULL(popped);
TEST_ASSERT_EQUAL_UINT64(8000, popped->jobid);
TEST_ASSERT_EQUAL_UINT64(4, popped->array_size);
TEST_ASSERT_EQUAL_STRING("normal", popped->submit->queue);
free_JobSubmit_ptr(&popped);
/* Cleanup: extract_data() transferred ownership of the decoded
* xce_job_submit_t into dataset.rows (nulling the source pointer inside
* decoded.body.pushmsg); sched_handle_dataset()'s jobArray case now
* frees dataset.rows itself and nulls it out (task 7.3), so there is
* nothing left for this test to free there. decoded.body.pushmsg is a
* hollow container, safe to free via the normal request teardown. */
* decoded.body.pushmsg); sched_handle_dataset()'s jobArray case
* transfers that ownership again, into the queued request, and nulls
* dataset.rows out -- nothing left for this test to free there.
* decoded.body.pushmsg is a hollow container, safe to free via the
* normal request teardown. */
TEST_ASSERT_NULL(dataset.rows);
xce_free_request(&decoded);
}
@ -380,11 +327,10 @@ int main(int argc, char* argv[])
(void)argc;
(void)argv;
UNITY_BEGIN();
RUN_TEST(test_job_array_push_expands_into_n_tasks, 0);
RUN_TEST(test_job_array_redelivery_does_not_duplicate, 0);
RUN_TEST(test_job_array_push_enqueues_raw_descriptor_unexpanded, 0);
RUN_TEST(test_job_array_null_submit_returns_einval, 0);
RUN_TEST(test_job_array_zero_size_returns_einval, 0);
RUN_TEST(test_job_array_null_descriptor_returns_einval, 0);
RUN_TEST(test_wire_dispatch_chain_delivers_n_tasks, 0);
RUN_TEST(test_wire_dispatch_chain_delivers_job_array_descriptor, 0);
return UNITY_END();
}

221
src/scheduler/testsuite/test_job_array_wire_e2e.c
View file

@ -16,12 +16,23 @@
* -> sched_handle_dataset()
* -> sched_on_job_array() (task 5.3)
*
* ...delivers exactly N correctly-addressed scheduler tasks and fires task
* 5.3's confirmation log line exactly once (Req 6.3, 6.4, 8.4); and that an
* identical byte-for-byte SECOND delivery (independently re-decoded from the
* same encoded bytes -- the wire-level equivalent of "the same push arrives
* twice", e.g. after a reconnect) creates zero new tasks and logs zero
* additional confirmation lines (Req 6.5, 8.4).
* ...delivers the compact descriptor into the pend-updates queue intact and
* unexpanded (Req 6.3, 6.4), correctly addressed by base jobid/array_size.
*
* REVISED (connection-loss-under-burst fix, see XCE_SCHED_OP_UPDATE_JOB_
* ARRAY's own comment, scheduler.h): sched_on_job_array() no longer expands
* the descriptor into N tasks, deduplicates a redelivery, or logs a
* confirmation line itself -- all three now happen in driver.c's
* translate_one_pending_update(), on the round-processing thread, covered by
* test_driver_translate.c's own UPDATE_JOB_ARRAY tests
* (test_translate_job_array_expands_into_n_events,
* test_translate_job_array_skips_already_existing_elements,
* test_translate_job_array_all_duplicates_returns_zero_noop). This file's
* own two tests were revised accordingly: a redelivery of byte-identical
* wire bytes through THIS layer alone is correctly and safely a second,
* independent, unexpanded descriptor -- the wire/handler layer has no
* dedup check of its own to lose; the actual dedup coverage lives in
* test_driver_translate.c now.
*
* Existing coverage this test does NOT duplicate:
* - test_job_array_sched_push.c (task 5.1) already proves the CONTROLLER
@ -29,14 +40,13 @@
* - test_request.c's test_extract_data_job_array_passes_through_intact
* (task 5.3b) already proves extract_data() itself decodes the payload
* correctly via a real encode/decode round trip.
* - test_job_array_expand.c's test_wire_dispatch_chain_delivers_n_tasks
* (task 5.3b) already proves extract_data() -> sched_handle_dataset()
* compose, but stops one layer below the real registered handler and
* never exercises redelivery through the wire path or the log line.
* - test_job_array_expand.c's test_wire_dispatch_chain_delivers_job_array_
* descriptor (task 5.3b) already proves extract_data() ->
* sched_handle_dataset() compose, but stops one layer below the real
* registered handler.
* This test's unique contribution: the REAL top-level dispatcher
* (scheduler_req_handler(), not sched_handle_dataset() directly), the
* redelivery-under-real-wire-bytes case, and the confirmation log line's
* actual emission count.
* (scheduler_req_handler(), not sched_handle_dataset() directly) and the
* redelivery-under-real-wire-bytes case.
*
* vtask_entry() (xcectx.c) is `static` and not exported for testing -- it is
* also the trigger normally driven by a live socket + iomgr epoll loop, which
@ -51,13 +61,11 @@
* Bug-catching demonstration (this task's "RED phase"): temporarily
* reverting extract_data()'s `case XceData_data_jobArray:` (task 5.3b's fix)
* back to falling through to `default: rc = -EINVAL` makes this test's first
* assertion (`callnext != 0` while dataset->rows stays NULL) fail loudly,
* and the subsequent job-count assertions fail too (zero tasks created) --
* assertion (`callnext != 0` while dataset->rows stays NULL) fail loudly --
* confirming this test has teeth against the exact class of bug 5.3b fixed.
*/
#define UNIT_TEST 1
#include <errno.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -139,96 +147,28 @@ static void fixture_init(xce_scheduler_t* sched)
sched->core.jobs = makeInstance(&xkvmgr_type, &jobs_params);
}
/* Pop one pending update request, asserting it is an UPDATE_JOB op; hands
* back njobs/jobs (caller now owns them -- free via free_Job_ptr + free() to
* mirror sched_pend_update_killer). Returns -1 if the queue is empty. Mirrors
/* Pop one pending update request, asserting it is an UPDATE_JOB_ARRAY op;
* hands back the raw job_array descriptor (caller now owns it -- free via
* free_JobSubmit_ptr). Returns NULL if the queue is empty. Mirrors
* test_job_array_expand.c's identical helper. */
static int pop_job_update(xce_scheduler_t* sched, xce_job_t*** jobs_out)
static xce_job_submit_t* pop_job_array_update(xce_scheduler_t* sched)
{
data_item_t* it = dataq_deq(sched->core.pend_updates[0], NULL);
xce_sched_update_request_t* req;
int njobs;
xce_sched_update_request_t* req;
xce_job_submit_t* job_array;
if (it == NULL) {
return -1;
return NULL;
}
req = ITPTR_TO_UPDATE_REQ(it);
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_JOB, req->op);
TEST_ASSERT_EQUAL_INT(XCE_SCHED_OP_UPDATE_JOB_ARRAY, req->op);
njobs = req->u.jobupdate.njobs;
if (jobs_out != NULL) {
*jobs_out = req->u.jobupdate.jobs;
}
job_array = req->u.jobarrayupdate.job_array;
free(req);
return njobs;
return job_array;
}
/* Materializes a popped job array into sched->jobs (mirroring what the
* async consumer, scheduler.c's sched_update_jobs -> sched_add_job, not
* linked into this white-box TU, would do) -- so a SECOND, redelivered
* push's dedup check sees the first delivery's elements as already
* existing. Frees each popped xce_job_t and the array itself, same
* simulated-reconnect technique test_job_array_expand.c's
* test_job_array_redelivery_does_not_duplicate uses (there via
* seed_existing_job, here folded into one step since njobs is already
* known). */
static void materialize_popped_jobs(xce_scheduler_t* sched, xce_job_t** jobs,
int njobs)
{
int i;
for (i = 0; i < njobs; i++) {
xce_sched_job_t* job = (xce_sched_job_t*)calloc(1, sizeof(*job));
job->jobid = jobs[i]->jobid;
sched_add_job(&(sched->core), job);
free_Job_ptr(&jobs[i]);
}
free(jobs);
}
/* ---------------------------------------------------------------------- */
/* Early-print log capture: sched_on_job_array()'s confirmation line uses */
/* INFO() (tlog_info -> TLOG_INFO). No test binary in this codebase calls */
/* tlog_init(), so every INFO() call routes through tlog's pre-init */
/* "early print" fallback (tlog.c's _tlog_early_print(), confirmed via the */
/* vendored tlog source). tlog_reg_early_printf_callback() lets a test */
/* intercept that fallback synchronously, on the calling thread -- no */
/* tlog_init()/tlog_exit() lifecycle, so it cannot disturb any other test */
/* sharing this process. va_copy() is used so the caller's own va_list is */
/* never consumed, in case tlog's own early-print path also reads it. */
/* ---------------------------------------------------------------------- */
static int g_confirm_log_count = 0;
static void wire_e2e_early_print_cb(struct tlog_loginfo* loginfo,
const char* format, va_list ap)
{
char buf[512];
va_list ap_copy;
if (loginfo == NULL || loginfo->level != TLOG_INFO) {
return;
}
va_copy(ap_copy, ap);
vsnprintf(buf, sizeof(buf), format, ap_copy);
va_end(ap_copy);
if (strstr(buf, "job array push received and expanded") != NULL) {
g_confirm_log_count++;
}
}
static void start_log_capture(void)
{
g_confirm_log_count = 0;
tlog_set_early_printf(1, 1, 0);
tlog_reg_early_printf_callback(wire_e2e_early_print_cb);
}
static void stop_log_capture(void) { tlog_reg_early_printf_callback(NULL); }
/* ---------------------------------------------------------------------- */
/* Real wire construction: the exact message xcesched_push_jobarray() */
/* (sched.c) sends -- OPCode_push / Request_body_pushmsg carrying one */
@ -328,52 +268,48 @@ static int deliver_via_real_dispatch_chain(io_packet_t* pkt,
return rc;
}
/* ── Test 1: one real wire push -> N correctly-addressed tasks, log fires */
/* exactly once (Req 6.3, 6.4, 8.4) */
/* ── Test 1: one real wire push -> the compact descriptor is queued intact */
/* and unexpanded (Req 6.3, 6.4) -- expansion itself is driver.c's concern */
/* now, covered by test_driver_translate.c. */
static void test_one_push_produces_n_tasks_and_logs_once(void)
static void test_one_push_enqueues_job_array_descriptor(void)
{
xce_scheduler_t sched;
job_submit_data_t submit_storage;
io_packet_t* pkt = NULL;
dataset_t* ds = NULL;
xce_job_t** jobs = NULL;
int njobs;
int i;
xce_job_submit_t* popped;
fixture_init(&sched);
start_log_capture();
build_encoded_push(&pkt, 42000, 6, &submit_storage);
TEST_ASSERT_TRUE(
deliver_via_real_dispatch_chain(pkt, &sched, &ds) >= 0);
njobs = pop_job_update(&sched, &jobs);
TEST_ASSERT_EQUAL_INT(6, njobs);
for (i = 0; i < njobs; i++) {
TEST_ASSERT_EQUAL_UINT64(42000 + (uint64_t)i, jobs[i]->jobid);
TEST_ASSERT_EQUAL_STRING("normal", jobs[i]->submit->queue);
free_Job_ptr(&jobs[i]);
}
free(jobs);
popped = pop_job_array_update(&sched);
TEST_ASSERT_NOT_NULL(popped);
TEST_ASSERT_EQUAL_UINT64(42000, popped->jobid);
TEST_ASSERT_EQUAL_UINT64(6, popped->array_size);
TEST_ASSERT_EQUAL_STRING("normal", popped->submit->queue);
free_JobSubmit_ptr(&popped);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_confirm_log_count,
"task 5.3's confirmation log line must fire exactly once for one "
"successful array expansion");
stop_log_capture();
free_JobSubmit_ptr((xce_job_submit_t**)&ds->rows);
free(ds);
printf("test_one_push_produces_n_tasks_and_logs_once: PASS\n");
printf("test_one_push_enqueues_job_array_descriptor: PASS\n");
}
/* ── Test 2: redelivery of the IDENTICAL encoded bytes (independently */
/* re-decoded, simulating a reconnect replay) creates zero new tasks and */
/* zero additional log lines (Req 6.5, 8.4) */
/* re-decoded, simulating a reconnect replay) -- REVISED (connection-loss- */
/* under-burst fix): the wire/handler layer no longer deduplicates at all */
/* (see XCE_SCHED_OP_UPDATE_JOB_ARRAY's own comment, scheduler.h), so a */
/* redelivery through THIS layer alone correctly and safely produces a */
/* second, independent, unexpanded descriptor -- dedup coverage now lives */
/* in test_driver_translate.c's own */
/* test_translate_job_array_skips_already_existing_elements / */
/* test_translate_job_array_all_duplicates_returns_zero_noop (Req 6.5). */
static void test_redelivery_creates_no_new_tasks_and_logs_nothing(void)
static void test_redelivery_enqueues_independent_second_descriptor(void)
{
xce_scheduler_t sched;
job_submit_data_t submit_storage;
@ -381,8 +317,8 @@ static void test_redelivery_creates_no_new_tasks_and_logs_nothing(void)
io_packet_t* pkt2 = NULL;
dataset_t* ds1 = NULL;
dataset_t* ds2 = NULL;
xce_job_t** jobs = NULL;
int njobs;
xce_job_submit_t* popped1;
xce_job_submit_t* popped2;
fixture_init(&sched);
@ -392,34 +328,31 @@ static void test_redelivery_creates_no_new_tasks_and_logs_nothing(void)
build_encoded_push(&pkt1, 43000, 4, &submit_storage);
build_encoded_push(&pkt2, 43000, 4, &submit_storage);
start_log_capture();
/* First delivery: 4 new tasks. */
/* First delivery. */
TEST_ASSERT_TRUE(deliver_via_real_dispatch_chain(pkt1, &sched, &ds1) >= 0);
njobs = pop_job_update(&sched, &jobs);
TEST_ASSERT_EQUAL_INT(4, njobs);
materialize_popped_jobs(&sched, jobs, njobs);
TEST_ASSERT_EQUAL_INT(1, g_confirm_log_count);
popped1 = pop_job_array_update(&sched);
TEST_ASSERT_NOT_NULL(popped1);
TEST_ASSERT_EQUAL_UINT64(43000, popped1->jobid);
TEST_ASSERT_EQUAL_UINT64(4, popped1->array_size);
free_JobSubmit_ptr(&popped1);
free_JobSubmit_ptr((xce_job_submit_t**)&ds1->rows);
free(ds1);
/* Second, byte-identical delivery (simulated redelivery/reconnect):
* every index (43000..43003) already exists in sched->jobs -- must be a
* pure no-op, and must NOT log a second confirmation line. */
TEST_ASSERT_EQUAL_INT(
0, deliver_via_real_dispatch_chain(pkt2, &sched, &ds2));
TEST_ASSERT_EQUAL_INT_MESSAGE(
-1, pop_job_update(&sched, &jobs),
"redelivery must enqueue nothing new");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_confirm_log_count,
"redelivery must not add a second confirmation log line -- still "
"exactly 1 total, from the first delivery only");
/* Second, byte-identical delivery (simulated redelivery/reconnect): a
* second, independent descriptor -- correct and safe by design (the
* downstream dedup check, now in driver.c, runs against core->jobs,
* which this white-box TU never populates from a queued-but-not-yet-
* translated request). */
TEST_ASSERT_TRUE(deliver_via_real_dispatch_chain(pkt2, &sched, &ds2) >= 0);
popped2 = pop_job_array_update(&sched);
TEST_ASSERT_NOT_NULL(popped2);
TEST_ASSERT_EQUAL_UINT64(43000, popped2->jobid);
TEST_ASSERT_EQUAL_UINT64(4, popped2->array_size);
free_JobSubmit_ptr(&popped2);
stop_log_capture();
free_JobSubmit_ptr((xce_job_submit_t**)&ds2->rows);
free(ds2);
printf("test_redelivery_creates_no_new_tasks_and_logs_nothing: PASS\n");
printf("test_redelivery_enqueues_independent_second_descriptor: PASS\n");
}
int main(int argc, char* argv[])
@ -427,7 +360,7 @@ int main(int argc, char* argv[])
(void)argc;
(void)argv;
UNITY_BEGIN();
RUN_TEST(test_one_push_produces_n_tasks_and_logs_once, 0);
RUN_TEST(test_redelivery_creates_no_new_tasks_and_logs_nothing, 0);
RUN_TEST(test_one_push_enqueues_job_array_descriptor, 0);
RUN_TEST(test_redelivery_enqueues_independent_second_descriptor, 0);
return UNITY_END();
}