lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(scheduler): defer job-array expansion off the vtask thread pool

Browse source 
sched_on_job_array() expanded a compact array-job push (up to
hundreds of elements: a per-element dedup lookup plus a full deep
clone) inline, synchronously, on whatever vtaskmgr thread handled the
incoming request -- the same shared pool the scheduler's own
controller connection traffic depends on, violating ha-async-io.md's
"bounded handler work only" contract for that pool (a project-wide
design invariant). A prior finding already showed a burst of
6x50-element array pushes (300 job events landing within ~2ms of each
other) causing the scheduler to lose its controller connection within
245ms and enter a reconnect/failover loop -- directly consistent with
this mechanism, and the same reconnect storm whose resync races
motivated the companion JOBS_PUSH fix in this branch.

The request handler now does O(1) work only: validate the descriptor
and enqueue it unexpanded (XCE_SCHED_OP_UPDATE_JOB_ARRAY, mirroring
the existing XCE_SCHED_OP_ADMIN_RECORD deferred-dispatch pattern).
The actual per-element expansion and dedup check moved to
translate_one_pending_update(), which runs on the round-processing
thread (main.c's xce_sched_driver_run_normal_mode_round(), before
xce_core_begin_session() each round) -- a thread with no connection-
liveness responsibility. As a side effect this also closes the
dedup check's own pre-existing cross-thread TOCTOU race (previously
documented as bug-1714 follow-up "Bug B"): only one thread ever runs
it now.

test_job_array_expand.c and test_job_array_wire_e2e.c, which asserted
the old inline-expansion behavior, are updated to assert the new
deferred-enqueue behavior instead; expansion, dedup, and full
drain-push-release ownership coverage moved to new tests in
test_driver_translate.c.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
Rongsong Shen 2026-09-15 09:01:47 +00:00
parent 6c201bb3e8
commit d5ec1eb3ae
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
10 changed files with 838 additions and 517 deletions
Download patch file Download diff file Expand all files Collapse all files

187
src/scheduler/testsuite/test_driver_translate.c
View file

@ -353,6 +353,29 @@ static xce_job_t* make_wire_job(uint64_t jobid, uint32_t state,
return j;
}
/* Compact job-array descriptor, same shape init_job_array() (test_job_
* array_expand.c) builds -- heap-owned strings, matching what a real
* XCE_SCHED_OP_UPDATE_JOB_ARRAY request now carries unexpanded (connection-
* loss-under-burst fix). */
static xce_job_submit_t* make_wire_job_array(uint64_t base_jobid,
uint64_t array_size)
{
job_submit_data_t* submit = (job_submit_data_t*)calloc(1, sizeof(*submit));
xce_job_submit_t* ja = (xce_job_submit_t*)calloc(1, sizeof(*ja));
submit->queue = strdup("normal");
submit->user = strdup("tester");
submit->mintasks = 1;
submit->maxtasks = 1;
submit->resreq = strdup("type == local");
ja->jobid = base_jobid;
ja->array_size = array_size;
ja->submit = submit;
return ja;
}
/* ---------------------------------------------------------------------- */
/* Per-op unit tests of translate_one_pending_update() (white-box static */
/* call) -- one per op type, asserting event type + every field matches. */
@ -633,6 +656,132 @@ static void test_translate_job_rusage_null_rusages_skips(void)
TEST_ASSERT_EQUAL_INT(0, rc);
}
/* Connection-loss-under-burst fix (XCE_SCHED_OP_UPDATE_JOB_ARRAY, moved
* from handler.c's former sched_on_job_array()): a compact Data.jobArray
* push expands into N ordinary schedulable tasks via base_jobid + i
* arithmetic, HERE on the round-processing thread instead of the
* request-handling thread. */
static void test_translate_job_array_expands_into_n_events(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja = make_wire_job_array(9000, 3);
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = ja;
rc = translate_one_pending_update(&core, req, 42, &ev);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_EQUAL_INT(XCE_CORE_EVENT_JOBS_PUSH, ev.type);
TEST_ASSERT_EQUAL_UINT64(42, ev.ts);
TEST_ASSERT_EQUAL_INT(3, ev.u.jobs_push.njobs);
TEST_ASSERT_EQUAL_UINT64(9000, ev.u.jobs_push.jobs[0]->jobid);
TEST_ASSERT_EQUAL_UINT64(9001, ev.u.jobs_push.jobs[1]->jobid);
TEST_ASSERT_EQUAL_UINT64(9002, ev.u.jobs_push.jobs[2]->jobid);
TEST_ASSERT_EQUAL_STRING("normal", ev.u.jobs_push.jobs[0]->submit->queue);
TEST_ASSERT_EQUAL_STRING("normal", ev.u.jobs_push.jobs[2]->submit->queue);
/* every element gets its OWN independent clone, not a shared pointer --
* required so each is independently free_Job_ptr()-able without a
* double-free. */
TEST_ASSERT_TRUE(ev.u.jobs_push.jobs[0]->submit
!= ev.u.jobs_push.jobs[1]->submit);
TEST_ASSERT_TRUE(ev.u.jobs_push.jobs[1]->submit
!= ev.u.jobs_push.jobs[2]->submit);
/* Expansion result stashed back on req (same pointers as ev.u.jobs_push
* above) so release_translated_payload() can free it later -- see that
* function's own new case. */
TEST_ASSERT_EQUAL_INT(3, req->u.jobarrayupdate.expanded_njobs);
TEST_ASSERT_EQUAL_PTR(ev.u.jobs_push.jobs,
req->u.jobarrayupdate.expanded_jobs);
}
/* Req 6.5/8.4 (moved from handler.c, same rationale): deduplicate a
* redelivered push by element identity -- only genuinely-new indices are
* expanded. Seeds 2 of 3 indices into the REAL core->jobs kvmgr this file's
* fixture_init() sets up (via the real xce_sched_core_init()), simulating
* "already materialized from an earlier round". */
static void test_translate_job_array_skips_already_existing_elements(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja = make_wire_job_array(9100, 3);
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
xce_sched_job_t existing0;
xce_sched_job_t existing1;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = ja;
memset(&existing0, 0, sizeof(existing0));
existing0.jobid = 9100;
sched_add_job(&core, &existing0);
memset(&existing1, 0, sizeof(existing1));
existing1.jobid = 9101;
sched_add_job(&core, &existing1);
rc = translate_one_pending_update(&core, req, 1, &ev);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_EQUAL_INT(XCE_CORE_EVENT_JOBS_PUSH, ev.type);
TEST_ASSERT_EQUAL_INT(1, ev.u.jobs_push.njobs);
TEST_ASSERT_EQUAL_UINT64(9102, ev.u.jobs_push.jobs[0]->jobid);
}
/* Every index already exists: a fully-duplicate redelivery -- nothing new
* to schedule, translate reports "nothing to push" (rc=0), same as any
* other skip case, not an error. */
static void test_translate_job_array_all_duplicates_returns_zero_noop(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja = make_wire_job_array(9200, 2);
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
xce_sched_job_t existing0;
xce_sched_job_t existing1;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = ja;
memset(&existing0, 0, sizeof(existing0));
existing0.jobid = 9200;
sched_add_job(&core, &existing0);
memset(&existing1, 0, sizeof(existing1));
existing1.jobid = 9201;
sched_add_job(&core, &existing1);
rc = translate_one_pending_update(&core, req, 1, &ev);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(0, req->u.jobarrayupdate.expanded_njobs);
TEST_ASSERT_NULL(req->u.jobarrayupdate.expanded_jobs);
}
/* A never-decoded/malformed descriptor (NULL job_array, the enqueue side
* should never produce this, guarded defensively here anyway, same
* discipline every other case in this function follows) skips cleanly. */
static void test_translate_job_array_null_payload_returns_zero(void)
{
xce_sched_core_t core;
xce_sched_update_request_t* req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
xce_core_event_t ev;
int rc;
fixture_init(&core);
req->u.jobarrayupdate.job_array = NULL;
rc = translate_one_pending_update(&core, req, 1, &ev);
TEST_ASSERT_EQUAL_INT(0, rc);
}
static void test_translate_unknown_op_skips(void)
{
xce_sched_core_t core;
@ -854,6 +1003,39 @@ static void test_drain_translates_and_pushes_all_ten_op_types(void)
TEST_ASSERT_EQUAL_PTR(config, core.config);
}
/* Connection-loss-under-burst fix, full drain-push-release cycle: proves
* XCE_SCHED_OP_UPDATE_JOB_ARRAY's ownership handling is correct end to end
* under a REAL ASAN-instrumented run -- both the original job_array
* descriptor (release_translated_payload()'s unconditional free) and the N
* freshly-expanded clones (adopted by core_build_job(), so never
* double-freed) survive the real xce_sched_driver_translate_pending_events()
* -> xce_core_push_events() -> release_translated_payload() sequence
* cleanly -- the same proof test_drain_translates_and_pushes_all_ten_op_types
* gives every other op type, dedicated here since job-array's cleanup path
* (two distinct owned allocations per request, not one) is unique to it. */
static void test_drain_translates_and_pushes_job_array_op(void)
{
xce_sched_core_t core;
xce_job_submit_t* ja;
xce_sched_update_request_t* req;
int rc;
fixture_init(&core);
ja = make_wire_job_array(9300, 3);
req = make_req(XCE_SCHED_OP_UPDATE_JOB_ARRAY);
req->u.jobarrayupdate.job_array = ja;
enqueue(&core, req);
rc = xce_sched_driver_translate_pending_events(&core, NULL);
TEST_ASSERT_EQUAL_INT(1, rc);
TEST_ASSERT_NOT_NULL(sched_get_job(&core, 9300));
TEST_ASSERT_NOT_NULL(sched_get_job(&core, 9301));
TEST_ASSERT_NOT_NULL(sched_get_job(&core, 9302));
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, sched_get_job(&core, 9300)->state);
}
/* ---------------------------------------------------------------------- */
/* Buffering discipline (Req 5 AC 8): events accumulate between drains; */
/* an empty drain is a valid no-op returning 0, not an error. */
@ -932,8 +1114,13 @@ int main(void)
RUN_TEST(test_translate_resvals_delta);
RUN_TEST(test_translate_job_rusage);
RUN_TEST(test_translate_job_rusage_null_rusages_skips);
RUN_TEST(test_translate_job_array_expands_into_n_events);
RUN_TEST(test_translate_job_array_skips_already_existing_elements);
RUN_TEST(test_translate_job_array_all_duplicates_returns_zero_noop);
RUN_TEST(test_translate_job_array_null_payload_returns_zero);
RUN_TEST(test_translate_unknown_op_skips);
RUN_TEST(test_drain_translates_and_pushes_all_ten_op_types);
RUN_TEST(test_drain_translates_and_pushes_job_array_op);
RUN_TEST(test_accumulates_between_drains_and_empty_drain_is_noop);
RUN_TEST(test_null_core_returns_einval);
RUN_TEST(test_uninitialized_pend_updates_returns_einval);