refactor(scheduler): defer job-array expansion off the vtask thread pool
sched_on_job_array() expanded a compact array-job push (up to hundreds of elements: a per-element dedup lookup plus a full deep clone) inline, synchronously, on whatever vtaskmgr thread handled the incoming request -- the same shared pool the scheduler's own controller connection traffic depends on, violating ha-async-io.md's "bounded handler work only" contract for that pool (a project-wide design invariant). A prior finding already showed a burst of 6x50-element array pushes (300 job events landing within ~2ms of each other) causing the scheduler to lose its controller connection within 245ms and enter a reconnect/failover loop -- directly consistent with this mechanism, and the same reconnect storm whose resync races motivated the companion JOBS_PUSH fix in this branch. The request handler now does O(1) work only: validate the descriptor and enqueue it unexpanded (XCE_SCHED_OP_UPDATE_JOB_ARRAY, mirroring the existing XCE_SCHED_OP_ADMIN_RECORD deferred-dispatch pattern). The actual per-element expansion and dedup check moved to translate_one_pending_update(), which runs on the round-processing thread (main.c's xce_sched_driver_run_normal_mode_round(), before xce_core_begin_session() each round) -- a thread with no connection- liveness responsibility. As a side effect this also closes the dedup check's own pre-existing cross-thread TOCTOU race (previously documented as bug-1714 follow-up "Bug B"): only one thread ever runs it now. test_job_array_expand.c and test_job_array_wire_e2e.c, which asserted the old inline-expansion behavior, are updated to assert the new deferred-enqueue behavior instead; expansion, dedup, and full drain-push-release ownership coverage moved to new tests in test_driver_translate.c. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
parent
6c201bb3e8
commit
d5ec1eb3ae
10 changed files with 838 additions and 517 deletions
|
|
@ -1,13 +1,13 @@
|
|||
{
|
||||
"version": 1,
|
||||
"meta": {
|
||||
"lastScanned": "2026-09-15T03:56:28.973Z",
|
||||
"fileCount": 567,
|
||||
"lastScanned": "2026-09-15T08:47:36.542Z",
|
||||
"fileCount": 572,
|
||||
"hits": 0,
|
||||
"misses": 0,
|
||||
"renderedHash": "a9ac7e8c7eca3b5e2b406896f2877eaecf4b21d63a5ffa783b18205d4002256c",
|
||||
"storeUpdatedAt": "2026-09-15T03:56:28.981Z",
|
||||
"rootHash": "7500ad02fc80875a"
|
||||
"renderedHash": "a6cf6257d3a03c11e763a6e34e7b004b90a51e62156a9ef766f127af914454ed",
|
||||
"storeUpdatedAt": "2026-09-15T08:47:36.550Z",
|
||||
"rootHash": "547dd1e8c0823374"
|
||||
},
|
||||
"files": {
|
||||
".agent-shell/transcripts/2026-08-11-11-39-59.md": {
|
||||
|
|
@ -9790,11 +9790,11 @@
|
|||
},
|
||||
"src/scheduler/testsuite/test_core_events.c": {
|
||||
"description": "Declares and",
|
||||
"tokens": 18252,
|
||||
"hash": "d4594ba3d4df54ad",
|
||||
"size": 68442,
|
||||
"mtimeMs": 1788858634325.683,
|
||||
"updatedAt": "2026-09-08T09:10:34.359Z",
|
||||
"tokens": 19786,
|
||||
"hash": "4489f3d00b674da8",
|
||||
"size": 74197,
|
||||
"mtimeMs": 1789458930665.5647,
|
||||
"updatedAt": "2026-09-15T07:55:30.710Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/resreq.c": {
|
||||
|
|
@ -9844,11 +9844,11 @@
|
|||
},
|
||||
"src/scheduler/include/scheduler.h": {
|
||||
"description": "if !defined(_SCHEDULER_H_)",
|
||||
"tokens": 4716,
|
||||
"hash": "92201f60362d4928",
|
||||
"size": 17683,
|
||||
"mtimeMs": 1788863604566.5034,
|
||||
"updatedAt": "2026-09-08T10:33:24.601Z",
|
||||
"tokens": 5360,
|
||||
"hash": "97ef8ebcef45538e",
|
||||
"size": 20099,
|
||||
"mtimeMs": 1789460878191.3936,
|
||||
"updatedAt": "2026-09-15T08:27:58.244Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/scheduler/src/plugin.c": {
|
||||
|
|
@ -10249,11 +10249,11 @@
|
|||
},
|
||||
"src/scheduler/src/core_events.c": {
|
||||
"description": "include <errno.h>",
|
||||
"tokens": 16287,
|
||||
"hash": "658f318636cb3ead",
|
||||
"size": 61075,
|
||||
"mtimeMs": 1789439677074.2285,
|
||||
"updatedAt": "2026-09-15T02:34:37.112Z",
|
||||
"tokens": 16984,
|
||||
"hash": "4e24abe60faa7bf7",
|
||||
"size": 63688,
|
||||
"mtimeMs": 1789459293023.9487,
|
||||
"updatedAt": "2026-09-15T08:01:33.100Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/utils.c": {
|
||||
|
|
@ -10309,6 +10309,51 @@
|
|||
"mtimeMs": 1789443750926.8965,
|
||||
"updatedAt": "2026-09-15T03:42:30.970Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/scheduler/testsuite/test_driver_translate.c": {
|
||||
"description": "Declares field",
|
||||
"tokens": 11757,
|
||||
"hash": "3351ae8b98718b4e",
|
||||
"size": 44086,
|
||||
"mtimeMs": 1789461520794.2683,
|
||||
"updatedAt": "2026-09-15T08:38:40.839Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/scheduler/src/handler.c": {
|
||||
"description": "include <errno.h>",
|
||||
"tokens": 5162,
|
||||
"hash": "d149b77df5b2f981",
|
||||
"size": 19355,
|
||||
"mtimeMs": 1789460932712.6296,
|
||||
"updatedAt": "2026-09-15T08:28:52.774Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/scheduler/src/driver.c": {
|
||||
"description": "include <errno.h>",
|
||||
"tokens": 11968,
|
||||
"hash": "e3265c56760407ea",
|
||||
"size": 44880,
|
||||
"mtimeMs": 1789461058238.7385,
|
||||
"updatedAt": "2026-09-15T08:30:58.282Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/scheduler/testsuite/test_job_array_expand.c": {
|
||||
"description": "",
|
||||
"tokens": 3393,
|
||||
"hash": "e878cf5068b6e41a",
|
||||
"size": 12786,
|
||||
"mtimeMs": 1789461227452.9976,
|
||||
"updatedAt": "2026-09-15T08:33:47.501Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/scheduler/testsuite/test_job_array_wire_e2e.c": {
|
||||
"description": "Declares vtask_entry",
|
||||
"tokens": 4026,
|
||||
"hash": "9e542dae84fa78c1",
|
||||
"size": 15107,
|
||||
"mtimeMs": 1789462056502.7026,
|
||||
"updatedAt": "2026-09-15T08:47:36.542Z",
|
||||
"source": "hook"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
# anatomy.md
|
||||
|
||||
> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T03:56:28.973Z
|
||||
> Files: 567 tracked | Anatomy hits: 0 | Misses: 0
|
||||
> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T08:47:36.542Z
|
||||
> Files: 572 tracked | Anatomy hits: 0 | Misses: 0
|
||||
|
||||
## ./
|
||||
|
||||
|
|
@ -906,12 +906,14 @@
|
|||
|
||||
## src/scheduler/include/
|
||||
|
||||
- `scheduler.h` — if !defined(_SCHEDULER_H_) (~4716 tok)
|
||||
- `scheduler.h` — if !defined(_SCHEDULER_H_) (~5360 tok)
|
||||
- `sim.h` — if !defined(_SIM_H_) (~4213 tok)
|
||||
|
||||
## src/scheduler/src/
|
||||
|
||||
- `core_events.c` — include <errno.h> (~16287 tok)
|
||||
- `core_events.c` — include <errno.h> (~16984 tok)
|
||||
- `driver.c` — include <errno.h> (~11968 tok)
|
||||
- `handler.c` — include <errno.h> (~5162 tok)
|
||||
- `plugin.c` — include <errno.h> (~4554 tok)
|
||||
- `repl_break.c` — include <stdio.h> (~9921 tok)
|
||||
- `repl_info.c` — include <stdio.h> (~16226 tok)
|
||||
|
|
@ -927,7 +929,10 @@
|
|||
|
||||
## src/scheduler/testsuite/
|
||||
|
||||
- `test_core_events.c` — Declares and (~18252 tok)
|
||||
- `test_core_events.c` — Declares and (~19786 tok)
|
||||
- `test_driver_translate.c` — Declares field (~11757 tok)
|
||||
- `test_job_array_expand.c` (~3393 tok)
|
||||
- `test_job_array_wire_e2e.c` — Declares vtask_entry (~4026 tok)
|
||||
- `test_parjob_combinatorial.c` — Declares of (~12038 tok)
|
||||
- `test_repl_breakpoints_inspection.c` — Declares and (~23533 tok)
|
||||
- `test_repl_help.c` — Declares error (~6047 tok)
|
||||
|
|
|
|||
|
|
@ -22007,6 +22007,79 @@
|
|||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-09-15T03:55:00Z"
|
||||
},
|
||||
{
|
||||
"id": "bug-1811",
|
||||
"timestamp": "2026-09-15T07:40:00Z",
|
||||
"error_message": "User-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job 0-14-1789369200-0[21] shows RUN in `info jobs` but is absent from `info worker pc13v`'s assigned-jobs listing (which lists [0-20,22], skipping 21) -- user's working theory was slot overcommitment (23 running jobs vs. 22-slot worker capacity).",
|
||||
"file": "src/scheduler/src/core_events.c, src/scheduler/src/sim_loop.c, .kiro/specs/scheduler-sim-decision-parity/{requirements,design}.md",
|
||||
"root_cause": "NOT A BUG -- initial diagnosis was WRONG and the attempted fix was reverted. Root-caused (correctly, mechanism-wise) to XCE_CORE_EVENT_JOB_STATE_CHANGE/XCE_CORE_EVENT_JOB_UPDATE (core_events.c) setting job->state=RUN unconditionally even when job->allocs/job->accounts are still NULL -- repl_info_worker_one() (repl_info.c) walks job->allocs to build the assigned-jobs listing, so an unaccounted RUN job is correctly displayed as RUN in `info jobs` but silently absent from `info worker`'s listing. Initially misdiagnosed this AS a defect (a 'phantom RUN' analogous to bug-1739's JOBS_PUSH resync gap) and implemented a guard refusing the RUN transition without an existing allocation. That guard was WRONG: this exact 'RUN with allocs==NULL' transient window is the INTENTIONAL, designed, tested trigger condition for Recorded-Decision Reconciliation (scheduler-sim-decision-parity spec, sim_loop.c's sim_reconcile_unaccounted_jobs(), extended by bug-1804's 2026-09-11 revision to set job->state=RUN synchronously via JOB_STATE_CHANGE/JOB_UPDATE ahead of allocation capture, matching the original live scheduler's own sched_commit_decision() semantics). Confirmed via test_sim_e2e_original_motivation.c's own header comment and ANCHOR_ID fixture: the SAME core_events.c code path is used, by design, to construct a job that is 'already running, with an unaccounted allocation, when a scheduling round's own live decision-making reaches it' -- 'Recorded-Decision Reconciliation's entire reason for existing'. My guard broke this fixture (test_e2e_original_motivation_long_replay) plus test_push_events_fires_each_hook_once_per_event_with_picker_idle and test_drain_translates_and_pushes_all_ten_op_types, both of which legitimately exercise a bare JOB_STATE_CHANGE/JOB_UPDATE(RUN) with no allocation yet. Further confirmed against scheduler-sim-decision-parity/requirements.md Requirement 4 ('Graceful degradation when recorded decision data is unavailable'): AC 4.1/4.2 EXPLICITLY anticipate and accept that a job's originating DISPATCH decision may not be found in the recorded replay data, in which case the scheduler reports that job's usage as unavailable rather than fabricating a value, and continues processing -- this is precisely job 21's observed symptom, not an accident. The user's overcommitment theory is also independently wrong on the accounting side: sched_account_jobs() only charges resInUse for JOB_IS_RUN(state) && accounts != NULL, so an unaccounted RUN job is correctly EXCLUDED from the real slot-usage total (used=22.00 already correctly reflects only the 22 genuinely-accounted jobs) -- there is no real overcommitment, only a display gap for a job whose allocation reconciliation hasn't (yet, or ever, depending on whether its DISPATCH decision exists later in this replay stream) caught up.",
|
||||
"fix": "REVERTED. Initial fix (guard in core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE/JOB_UPDATE refusing a RUN claim without an existing allocation, plus 6 new tests in test_core_events.c and a fixture change in test_driver_translate.c) was implemented, RED/GREEN-tested, then found to break test_sim_e2e_original_motivation.c's test_e2e_original_motivation_long_replay (the live-verify for the entire Recorded-Decision Reconciliation mechanism) plus two pre-existing test_core_events.c/test_driver_translate.c tests. All changes reverted via `git checkout --` before any commit. No production code changed. Left as an open question whether job 21 (and the user's follow-up observation that this 'occurs more frequently') ever actually reconciles later in the same replay session -- if a job's RUN-but-unaccounted display state NEVER resolves even after the replay completes, that would indicate its DISPATCH decision is genuinely missing from the recorded stream (the same category of gap scheduler-snapshot-job-allocation's PREEXIST_ID addresses for pre-recording dispatches, via snapshot-based restoration instead of replay-based reconciliation) -- worth a follow-up investigation with concrete evidence (does the job ever show up in `info worker`'s listing on a later round?) before attempting any further fix. A safe, smaller follow-up actually worth considering: improve `info worker`'s assigned-jobs display (repl_info.c) to explicitly distinguish a RUN-but-unaccounted job (e.g. an annotation) from silent omission, so operators aren't misled into an overcommitment theory the way this session's user was -- proposed to the user, not yet implemented. SUPERSEDED (2026-09-15, same session): the 'not a bug' conclusion above was itself only half right. The transient RUN-before-allocation window IS legitimate (Recorded-Decision Reconciliation), but the specific PERMANENT loss the user kept observing turned out to be a real, separate defect in the JOBS_PUSH duplicate-jobid resync merge -- see bug-1812 for the actual root cause and fix.",
|
||||
"tags": [
|
||||
"not-a-bug",
|
||||
"design-decision",
|
||||
"scheduler",
|
||||
"sim-replay",
|
||||
"recorded-decision-reconciliation",
|
||||
"reverted-fix"
|
||||
],
|
||||
"related_bugs": [
|
||||
"bug-1739",
|
||||
"bug-1804",
|
||||
"bug-1778"
|
||||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-09-15T07:40:00Z"
|
||||
},
|
||||
{
|
||||
"id": "bug-1812",
|
||||
"timestamp": "2026-09-15T08:05:00Z",
|
||||
"error_message": "User-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real production scheduler.log from rshen@pc13v /t/xce-ha-deploy): job 0-14-1789369200-0[21] holds allocs=1 (state=RUN) at the end of session 34; stepping into session 35 replays a JOBS_PUSH resync (njobs=60, including all of array job 0-14-1789369200-0[0-39]) followed by a JOB_UPDATE and a redundant JOB_STATE_CHANGE(RUN) for [21]; immediately after, `info job [21]` shows state=RUN but allocs=0. The real scheduler.log shows the identical mechanism live: `sched_job_state_change ... state 2` (schedStartJob confirmation) at 15:00:05.196, followed 16ms later by `JOBS_PUSH: sched_add_job failed for jobid ...[21] (rc=-17), merging state into existing tracked job instead of discarding` -- repeating in bursts on later reconnects (six WARNs in ~2s at 15:01:51-53).",
|
||||
"file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge, bug-1734/bug-1739's own code)",
|
||||
"root_cause": "CONFIRMED, via direct code reading plus independent live evidence from both a sim replay and a real production scheduler.log. xce_core_push_events()'s JOBS_PUSH handler, on a duplicate jobid (sched_add_job() -EEXIST, a resync redelivery of an already-tracked job), branches three ways once job_state_regresses_terminal() is ruled out: (1) resync claims RUN with real allocation data -> transfer it in (bug-1739's fix); (2) resync claims RUN with NO allocation data -> refuse, keep existing (bug-1739's own companion guard); (3) resync claims anything else (the bare `else` branch, bug-1734's original fix) -> UNCONDITIONALLY frees existing->accounts/existing->allocs and overwrites existing->state with the resync's own claim, with NO check for whether `existing` already holds a REAL, already-captured allocation the resync claim doesn't support. on_scheduler_join()'s full resync (xcesched_push_jobs(), src/controller/sched.c:214, fired on every scheduler (re)connect via join.c) is built from collect_jobs_committed() -- a committed-data snapshot gated only on 'job submission is committed', NOT on whether this specific job's DISPATCH transition has been committed yet. When a scheduler reconnects (or the connection blips) around the same moment it makes/receives confirmation of its own fresh dispatch decision for a job, the resync's committed-data snapshot can legitimately still show that job as PEND (its own dispatch commit hasn't propagated into the snapshot query yet) even though the scheduler's own live state (already set via sched_commit_decision() or the schedStartJob/JOB_STATE_CHANGE confirmation path) correctly has it RUN with a real allocation. Branch (3) blindly trusts the resync's stale PEND claim over the scheduler's own more-current, already-allocated state, freeing the real allocation and (in the observed cases) leaving state briefly PEND until a later, in-flight JOB_UPDATE/JOB_STATE_CHANGE for the SAME jobid (already queued in the same event batch, confirming the SAME dispatch) re-sets state=RUN -- but with no mechanism to restore the allocation branch (3) just destroyed. This exactly matches both the sim replay symptom (info jobs shows RUN, info worker's allocs-driven assigned-jobs listing omits it) and the production log's precise 16ms collision. Not the same mechanism as bug-1811's original (correct, but incomplete) 'Recorded-Decision Reconciliation transient window' explanation -- that window is real and self-healing, but this is a genuinely separate, permanent-until-this-fix data-loss bug triggered by resync/dispatch timing, not the reconciliation mechanism itself.",
|
||||
"fix": "FIXED (2026-09-15), TDD RED->GREEN. Added a guard in JOBS_PUSH's duplicate-jobid merge (core_events.c), inserted between bug-1739's `else if (JOB_IS_RUN(job->state))` branch and the original bare `else`: `else if ((existing->allocs != NULL) && !JOB_IS_DONE(job->state) && !JOB_IS_EXIT(job->state))` -- refuses the resync claim (WARNs, keeps existing->state/allocs/accounts untouched) whenever `existing` already holds a real allocation AND the resync's claimed state is not genuinely terminal. A terminal claim (DONE/EXIT) still falls through to the original branch, correctly releasing the allocation -- this is bug-1734's own legitimate 'job finished while disconnected' case and must not be blocked. Symmetric in spirit to bug-1739's own phantom-RUN guard (refuse to fabricate an unsupported RUN; now also refuse to destroy a supported RUN). TDD: added test_jobs_push_duplicate_jobid_run_with_allocation_survives_stale_nonrun_resync (test_core_events.c) -- job tracked RUN with a real allocation, redelivered via JOBS_PUSH as PEND with no allocation, must keep its RUN state and real allocation intact; and its companion test_jobs_push_duplicate_jobid_terminal_resync_releases_real_allocation -- the SAME RUN-with-real-allocation job, redelivered as DONE, must still release the allocation and transition (proves the new guard doesn't block legitimate terminal resyncs). RED confirmed (`xce-build.sh scheduler asan xce.build --incremental`): exactly the new stale-resync test failed (Expected 2 [RUN] Was 1 [PEND]), the terminal-resync companion already passed (unaffected by the unfixed code), 75 other tests green. GREEN confirmed after the fix: full scheduler suite 77/77 Ok, zero regressions -- including test_sim_e2e_original_motivation.c (the Recorded-Decision Reconciliation live-verify a FIRST, incorrect fix attempt this same session had broken and had to be reverted) and test_driver_translate.c, both untouched by this fix and green throughout.",
|
||||
"tags": [
|
||||
"scheduler",
|
||||
"resync",
|
||||
"jobs-push",
|
||||
"allocation-loss",
|
||||
"race-condition",
|
||||
"fixed"
|
||||
],
|
||||
"related_bugs": [
|
||||
"bug-1734",
|
||||
"bug-1738",
|
||||
"bug-1739",
|
||||
"bug-1811"
|
||||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-09-15T08:05:00Z"
|
||||
},
|
||||
{
|
||||
"id": "bug-1813",
|
||||
"timestamp": "2026-09-15T08:35:00Z",
|
||||
"error_message": "Follow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced crash): while investigating why a scheduler reconnect/resync storm collides with in-flight array-job dispatches (bug-1812's own root cause), user asked to investigate the connection-loss trigger itself. Traced the scheduler's array-job receive path (handler.c's sched_on_job_array(), reached via scheduler_req_handler() on the shared vtaskmgr thread pool, default 8 threads) and found it does the FULL per-element expansion (dedup check via sched_get_job() + N independent deep clones via clone_job_submit_data()/clone_job_startinfo()) inline, synchronously, on whatever vtask thread handles the incoming push -- the exact 'long/unbounded work on a pool that must stay free for liveness-critical messages' anti-pattern .kiro/steering/ha-async-io.md's Thread-role contract warns about (the same pool is shared with the scheduler's own controller connection traffic). A prior, independent finding (2026-09-04 discovery, this project) already showed a burst of 6x50-element array pushes (300 job events, landing within ~2ms of each other) caused the scheduler to lose its connection to the primary controller within 245ms and enter a failover/reconnect loop -- directly consistent with this mechanism. User proposed the fix directly: the request-handling thread should just requeue the raw request, with the actual (expensive) work deferred to run alongside 'begin session' -- i.e. the round-processing thread's own xce_sched_driver_translate_pending_events()/xce_core_begin_session() sequence (main.c's xce_sched_driver_run_normal_mode_round(), driver.c), which carries no connection-liveness responsibility and already establishes this exact deferred-dispatch pattern for XCE_SCHED_OP_ADMIN_RECORD.",
|
||||
"file": "src/scheduler/include/scheduler.h, src/scheduler/src/handler.c, src/scheduler/src/driver.c, src/scheduler/testsuite/test_job_array_expand.c, src/scheduler/testsuite/test_job_array_wire_e2e.c, src/scheduler/testsuite/test_driver_translate.c",
|
||||
"root_cause": "CONFIRMED via direct code reading (not yet live-reproduced end-to-end with gdb/ASAN under a real burst -- this fix is the same corrective direction bug-1714's own still-open investigation was heading, applied proactively here rather than waiting for a full live repro). handler.c's sched_on_job_array() (the XceData_data_jobArray case's handler, invoked by scheduler_req_handler() on a vtaskmgr thread) did the ENTIRE per-element expansion synchronously and inline: for each of up to array_size elements, a sched_get_job() dedup lookup plus a full deep clone (clone_job_submit_data()/clone_job_startinfo(), several calloc()+strdup() calls per element) before ever enqueueing anything. For a large concurrent burst (6 arrays x 50 elements landing within ~2ms, the 2026-09-04 finding), this ties up several of the vtaskmgr pool's 8 threads for the duration of 300 elements' worth of cloning -- the same pool the scheduler's own controller connection traffic depends on, a direct instance of ha-async-io.md's documented anti-pattern (a project-wide DESIGN INVARIANT, 'applies to ALL components: controller, worker, scheduler').",
|
||||
"fix": "FIXED (2026-09-15), TDD RED->GREEN, user-directed design. Added XCE_SCHED_OP_UPDATE_JOB_ARRAY (scheduler.h) mirroring XCE_SCHED_OP_ADMIN_RECORD's own established deferred-dispatch pattern, with a new xce_sched_update_request_t.u.jobarrayupdate union member carrying the raw, still-unexpanded xce_job_submit_t* descriptor plus (post-expansion) expanded_njobs/expanded_jobs for release_translated_payload()'s later cleanup. handler.c's sched_on_job_array() now does O(1) work only: validate the descriptor shape, wrap it in a request, enq_pend_update() -- no loop, no clone, no dedup check, ownership of the descriptor transferred to the queue on success (sched_handle_dataset()'s XceData_data_jobArray case updated to null dataset->rows on success instead of unconditionally freeing it). The full per-element expansion logic (clone_job_startinfo/clone_job_submit_data/make_array_element, moved verbatim) plus its dedup check now live in driver.c's translate_one_pending_update(), a new XCE_SCHED_OP_UPDATE_JOB_ARRAY case producing the same XCE_CORE_EVENT_JOBS_PUSH shape the old handler.c code did -- this runs on the round-processing thread (main.c's xce_sched_driver_run_normal_mode_round(), invoked before xce_core_begin_session() every round), which carries no connection-liveness responsibility. Side benefit: the dedup check's own pre-existing TOCTOU race (documented in the original code as bug-1714 follow-up 'Bug B', a genuine cross-thread race between the vtask thread's check and the driver thread's own core->jobs mutations) is now eliminated for the single-request case, since only the round-processing thread ever runs this check -- the remaining same-batch-double-delivery edge case is unchanged and was already documented as safe-by-design (sched_add_job()'s key_add() plus the JOBS_PUSH duplicate-merge logic, core_events.c, is the actual correctness boundary, untouched by this fix). release_translated_payload() (driver.c) gained a matching cleanup case: frees the original job_array descriptor unconditionally (never itself pushed into core) plus the N expanded clones (same unconditional free XCE_SCHED_OP_UPDATE_JOB's own jobupdate.jobs already gets). TDD: 4 new unit tests in test_driver_translate.c directly on translate_one_pending_update() (expands into N events with independent clones; skips already-existing elements against a REAL core->jobs kvmgr; all-duplicate redelivery returns 0/no-op; malformed payload returns 0) plus 1 new full drain-push-release end-to-end test (test_drain_translates_and_pushes_job_array_op) proving ownership correctness under real ASAN instrumentation. test_job_array_expand.c's and test_job_array_wire_e2e.c's own tests (which previously asserted the OLD inline-expansion behavior, including test_job_array_wire_e2e.c's tlog early-print-callback log-count capture, since the confirmation log line moved to driver.c) were revised to assert the NEW deferred-enqueue behavior instead -- redelivery dedup and log-firing-count coverage moved to test_driver_translate.c's own tests (log-firing-count itself was not re-added at the new location, a deliberate small scope trim, noted here rather than silently dropped). RED confirmed at each stage (xce-build.sh scheduler asan xce.build --incremental): first pass caught the 4 new driver-level unit test failures as expected plus 2 pre-existing test_job_array_wire_e2e.c failures this refactor had NOT yet been applied to (a test file this investigation initially missed, found only via the full suite run -- both asserted 'Expected 2 Was 11', i.e. the old handler-level XCE_SCHED_OP_UPDATE_JOB=2 check now correctly seeing the new XCE_SCHED_OP_UPDATE_JOB_ARRAY=11). GREEN confirmed after revising that file too: full scheduler suite 77/77 Ok, zero regressions, including test_sim_e2e_original_motivation.c (verified explicitly, given this same session's earlier unrelated mistake on bug-1811/1812's own investigation broke it once already) and all 10 new/revised job-array tests individually confirmed PASS via the raw test log.",
|
||||
"tags": [
|
||||
"scheduler",
|
||||
"vtaskmgr",
|
||||
"async-io",
|
||||
"connection-stability",
|
||||
"job-array",
|
||||
"architecture",
|
||||
"fixed",
|
||||
"user-directed"
|
||||
],
|
||||
"related_bugs": [
|
||||
"bug-1812",
|
||||
"bug-1734",
|
||||
"bug-1739",
|
||||
"bug-1714"
|
||||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-09-15T08:35:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4744,3 +4744,42 @@
|
|||
| 20:42 | Edited src/lib/mylock.c | added 1 condition(s) | ~303 |
|
||||
| 20:56 | Edited src/lib/vtaskmgr.c | 11→13 lines | ~236 |
|
||||
| 20:56 | Edited src/lib/vtaskmgr.c | 5→4 lines | ~74 |
|
||||
| 23:59 | Session end: 17 writes across 5 files (test_vtaskmgr_teardown.c, test.c, meson.build, vtaskmgr.c, mylock.c) | 9 reads | ~10856 tok |
|
||||
|
||||
## Session: 2026-09-15 23:59
|
||||
|
||||
| Time | Action | File(s) | Outcome | ~Tokens |
|
||||
|------|--------|---------|---------|--------|
|
||||
| 00:15 | Edited src/scheduler/testsuite/test_core_events.c | modified 1811() | ~1088 |
|
||||
| 00:16 | Edited src/scheduler/testsuite/test_core_events.c | modified 1811() | ~1131 |
|
||||
| 00:16 | Edited src/scheduler/testsuite/test_core_events.c | 2→6 lines | ~115 |
|
||||
| 00:23 | Edited src/scheduler/src/core_events.c | added 1 condition(s) | ~686 |
|
||||
| 00:24 | Edited src/scheduler/src/core_events.c | added 2 condition(s) | ~872 |
|
||||
| 00:32 | Edited src/scheduler/testsuite/test_core_events.c | modified test_push_events_fires_each_hook_once_per_event_with_picker_idle() | ~865 |
|
||||
| 00:35 | Edited src/scheduler/testsuite/test_driver_translate.c | 4→5 lines | ~31 |
|
||||
| 00:35 | Edited src/scheduler/testsuite/test_driver_translate.c | expanded (+19 lines) | ~391 |
|
||||
| 00:55 | Edited src/scheduler/testsuite/test_core_events.c | modified 1812() | ~1505 |
|
||||
| 00:55 | Edited src/scheduler/testsuite/test_core_events.c | 1→3 lines | ~69 |
|
||||
| 01:01 | Edited src/scheduler/src/core_events.c | added 1 condition(s) | ~896 |
|
||||
| 01:27 | Edited src/scheduler/include/scheduler.h | expanded (+21 lines) | ~428 |
|
||||
| 01:27 | Edited src/scheduler/include/scheduler.h | expanded (+16 lines) | ~266 |
|
||||
| 01:28 | Edited src/scheduler/src/handler.c | modified REVISED() | ~514 |
|
||||
| 01:28 | Edited src/scheduler/src/handler.c | added 1 condition(s) | ~252 |
|
||||
| 01:29 | Edited src/scheduler/src/driver.c | 2→5 lines | ~76 |
|
||||
| 01:30 | Edited src/scheduler/src/driver.c | added 12 condition(s) | ~1344 |
|
||||
| 01:30 | Edited src/scheduler/src/driver.c | added 5 condition(s) | ~1031 |
|
||||
| 01:30 | Edited src/scheduler/src/driver.c | modified for() | ~322 |
|
||||
| 01:31 | Edited src/scheduler/testsuite/test_job_array_expand.c | modified REVISED() | ~340 |
|
||||
| 01:31 | Edited src/scheduler/testsuite/test_job_array_expand.c | modified pop_job_array_update() | ~239 |
|
||||
| 01:31 | Edited src/scheduler/testsuite/test_job_array_expand.c | modified test_job_array_push_enqueues_raw_descriptor_unexpanded() | ~351 |
|
||||
| 01:32 | Edited src/scheduler/testsuite/test_job_array_expand.c | reduced (-7 lines) | ~21 |
|
||||
| 01:33 | Edited src/scheduler/testsuite/test_job_array_expand.c | modified test_wire_dispatch_chain_delivers_job_array_descriptor() | ~1152 |
|
||||
| 01:33 | Edited src/scheduler/testsuite/test_job_array_expand.c | inline fix | ~20 |
|
||||
| 01:35 | Edited src/scheduler/testsuite/test_driver_translate.c | modified make_wire_job_array() | ~306 |
|
||||
| 01:36 | Edited src/scheduler/testsuite/test_driver_translate.c | modified follows() | ~1396 |
|
||||
| 01:36 | Edited src/scheduler/testsuite/test_driver_translate.c | 2→6 lines | ~102 |
|
||||
| 01:38 | Edited src/scheduler/testsuite/test_driver_translate.c | modified test_drain_translates_and_pushes_job_array_op() | ~472 |
|
||||
| 01:38 | Edited src/scheduler/testsuite/test_driver_translate.c | 1→2 lines | ~34 |
|
||||
| 01:46 | Edited src/scheduler/testsuite/test_job_array_wire_e2e.c | modified vtask_entry() | ~1067 |
|
||||
| 01:47 | Edited src/scheduler/testsuite/test_job_array_wire_e2e.c | modified pop_job_array_update() | ~216 |
|
||||
| 01:47 | Edited src/scheduler/testsuite/test_job_array_wire_e2e.c | modified test_one_push_enqueues_job_array_descriptor() | ~1041 |
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue