lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(bug-1738,bug-1739): terminal-state regression + phantom RUN in JOBS_PUSH merge

Browse source 
bug-1738: a scheduler-side notify could clobber an already-terminal
job state. The controller's own terminal-transition guard
(job_state_is_terminal(), job.c) treats DONE/EXIT as permanent once
set -- apply_job_fail_transition()'s POST_EXEC/Q_POST_EXEC branch only
ever MERGES a JOB_STAT_PERR flag onto whatever DONE/EXIT already
holds, never re-deciding DONE vs EXIT after the fact. The scheduler
had no equivalent guard: HOOK_DETAIL_JOB_FAIL's schedFailJob notify
(sched_job_fail_fun, sched.c) unconditionally requests JOB_STAT_EXIT
with no regard for the job's current state, so a job that finished
successfully (DONE) and then hit an unrelated post-exec script failure
would have the scheduler's view forcibly clobbered back to EXIT.
Fixed with one shared helper, job_state_regresses_terminal(), applied
at both XCE_CORE_EVENT_JOB_STATE_CHANGE and the JOBS_PUSH duplicate-
jobid resync merge (a full resync can just as easily redeliver a stale
snapshot as a genuinely newer one) -- once a job is DONE/EXIT, neither
path can move it to a different terminal state.

bug-1739 (Symptom 1, "phantom RUN"): the bug-1734 fix's own JOBS_PUSH
merge adopted a redelivery's RUN state without transferring the real
allocation data core_build_job() had already attached to the
redelivered clone -- leaving existing->state==RUN with
existing->allocs==NULL, a job the scheduler's own resource/slot
bookkeeping has no record of. Live-reproduced: `info summary` showed
30 elements of an array marked RUN, but `info worker <name>` only
listed 22 real slot assignments for a 22-core worker. Fixed by
transferring job->accounts/job->allocs onto the existing job on a RUN
merge (freeing whatever it previously held), and refusing the merge
entirely (WARN-logged, existing state kept) if the redelivery carries
no allocation data -- treating an unbacked RUN claim as untrustworthy
rather than manufacturing a phantom.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019QxaSzHRB7pmYFp4cVnwkA
This commit is contained in:
Rongsong Shen 2026-08-31 04:20:46 +00:00
parent 026fe97b5a
commit 0e78f86377
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
2 changed files with 365 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

235
src/scheduler/testsuite/test_core_events.c
View file

@ -598,6 +598,237 @@ test_jobs_push_duplicate_jobid_with_newer_state_updates_existing_job(void)
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, tracked->state);
}
/* bug-1739 (Symptom 1, "phantom RUN"): bug-1734's own JOBS_PUSH duplicate-
* jobid merge (test above) adopted a redelivery's RUN state without
* transferring any real allocation, leaving a tracked job with state==RUN
* but allocs==NULL -- a job the scheduler's own resource/slot bookkeeping
* has no record of at all (live-reported: `info worker` listed fewer
* "assigned job:" entries than `info jobs` showed RUN, exceeding the
* worker's real core count). core_build_job() already adopts a wire Job's
* `allocs` field into the freshly-built xce_sched_job_t (job->accounts =
* make_job_account(job->allocs)) exactly like a brand-new RUN job would --
* this asserts a RUN-state resync redelivery that DOES carry real
* allocation data transfers it into the EXISTING tracked job instead of
* discarding it. */
static void
test_jobs_push_duplicate_jobid_run_state_transfers_allocation(void)
{
xce_sched_core_t core;
xce_core_event_t event;
xce_job_t* job1;
xce_job_t* resynced_job;
xce_sched_job_t* tracked;
xce_alloc_t* a_slots;
xce_host_alloc_t* h0;
xce_host_alloc_t* hallocs[1];
xce_job_alloc_t* jalloc;
int rc;
fixture_init(&core, &g_full_plugin);
job1 = make_job(57, JOB_STAT_PEND, "normal", "alice", "proj1");
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &job1;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
tracked = sched_get_job(&core, 57);
TEST_ASSERT_NOT_NULL(tracked);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, tracked->state);
TEST_ASSERT_NULL(tracked->allocs);
/* A resync redelivery carrying RUN AND the controller's real dispatch
* allocation for this job -- exactly what on_scheduler_join()'s
* xcesched_push_jobs() sends for a genuinely running job. */
a_slots = make_alloc((char*)"slots", NULL, 0, 1.0f, 0);
h0 = make_host_alloc((char*)"hostA", 1, make_allocs(1, a_slots));
hallocs[0] = h0;
jalloc = make_job_alloc(57, 1, make_host_allocs(1, hallocs[0]));
resynced_job = make_job(57, JOB_STAT_RUN, "normal", "alice",
"proj1");
resynced_job->allocs = jalloc;
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &resynced_job;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_PTR(tracked, sched_get_job(&core, 57));
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, tracked->state);
/* The real allocation must have been transferred, not discarded --
* this is the actual fix: a RUN job must never end up with allocs ==
* NULL via this path. */
TEST_ASSERT_NOT_NULL(tracked->allocs);
TEST_ASSERT_NOT_NULL(tracked->accounts);
}
/* bug-1739 companion: if a RUN-state resync redelivery does NOT carry any
* allocation data (no `allocs` on the wire), merging it in anyway would
* manufacture exactly the same phantom-RUN phenotype from a different
* cause. Asserts the merge is refused (existing job keeps its prior state)
* rather than trusting an unallocated RUN claim. */
static void
test_jobs_push_duplicate_jobid_run_state_without_allocation_is_refused(void)
{
xce_sched_core_t core;
xce_core_event_t event;
xce_job_t* job1;
xce_job_t* resynced_job;
xce_sched_job_t* tracked;
int rc;
fixture_init(&core, &g_full_plugin);
job1 = make_job(58, JOB_STAT_PEND, "normal", "alice", "proj1");
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &job1;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
tracked = sched_get_job(&core, 58);
TEST_ASSERT_NOT_NULL(tracked);
/* RUN claim with NO allocation data attached. */
resynced_job = make_job(58, JOB_STAT_RUN, "normal", "alice", "proj1");
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &resynced_job;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_PTR(tracked, sched_get_job(&core, 58));
/* Refused -- stays at its prior (safe) state, never becomes a phantom
* RUN with no allocation. */
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, tracked->state);
TEST_ASSERT_NULL(tracked->allocs);
}
/* bug-1738: the JOBS_PUSH duplicate-jobid merge must never regress an
* already-terminal job to a DIFFERENT terminal state either -- a resync can
* just as easily carry a stale snapshot as a genuinely newer one. Pushes
* jobid to DONE (as the bug-1734 test above already proves works), then a
* SECOND redelivery claiming EXIT must be ignored. */
static void
test_jobs_push_duplicate_jobid_does_not_regress_terminal_state(void)
{
xce_sched_core_t core;
xce_core_event_t event;
xce_job_t* job1;
xce_job_t* done_job;
xce_job_t* stale_exit_job;
xce_sched_job_t* tracked;
int rc;
fixture_init(&core, &g_full_plugin);
job1 = make_job(59, JOB_STAT_PEND, "normal", "alice", "proj1");
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &job1;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
done_job = make_job(59, JOB_STAT_DONE, "normal", "alice", "proj1");
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &done_job;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
tracked = sched_get_job(&core, 59);
TEST_ASSERT_NOT_NULL(tracked);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, tracked->state);
/* A stale/out-of-order redelivery claiming EXIT for a job already
* correctly recorded as DONE. */
stale_exit_job = make_job(59, JOB_STAT_EXIT, "normal", "alice", "proj1");
memset(&event, 0, sizeof(event));
event.type = XCE_CORE_EVENT_JOBS_PUSH;
event.u.jobs_push.njobs = 1;
event.u.jobs_push.jobs = &stale_exit_job;
rc = xce_core_push_events(&core, &event, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_PTR(tracked, sched_get_job(&core, 59));
/* Must still be DONE -- the stale EXIT must not have clobbered it. */
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, tracked->state);
}
/* bug-1738 (single-job notify path): XCE_CORE_EVENT_JOB_STATE_CHANGE must
* apply the SAME "terminal is permanent" guard as the JOBS_PUSH merge above
* -- this is the path a live schedFailJob notify actually reaches
* (sched_job_fail_fun, sched.c, unconditionally requests JOB_STAT_EXIT with
* no regard for the job's current state; the controller's own
* apply_job_fail_transition() only merges JOB_STAT_PERR onto an
* already-DONE/EXIT job in the equivalent real scenario, never re-decides
* DONE vs EXIT). Pushes a job to RUN then DONE via JOB_STATE_CHANGE, then a
* second JOB_STATE_CHANGE claiming EXIT must be ignored. */
static void test_job_state_change_does_not_regress_terminal_state(void)
{
xce_sched_core_t core;
xce_core_event_t events[3];
xce_job_t* job1;
xce_sched_job_t* job;
int rc;
fixture_init(&core, &g_full_plugin);
job1 = make_job(60, JOB_STAT_PEND, "normal", "alice", "proj1");
memset(events, 0, sizeof(events));
events[0].type = XCE_CORE_EVENT_JOBS_PUSH;
events[0].u.jobs_push.njobs = 1;
events[0].u.jobs_push.jobs = &job1;
events[1].type = XCE_CORE_EVENT_JOB_STATE_CHANGE;
events[1].u.job_state_change.jobid = 60;
events[1].u.job_state_change.new_state = JOB_STAT_DONE;
rc = xce_core_push_events(&core, events, 2);
TEST_ASSERT_EQUAL_INT(0, rc);
job = sched_get_job(&core, 60);
TEST_ASSERT_NOT_NULL(job);
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, job->state);
/* A schedFailJob-shaped notify claiming EXIT for the SAME job, arriving
* after it already correctly reached DONE. */
events[2].type = XCE_CORE_EVENT_JOB_STATE_CHANGE;
events[2].u.job_state_change.jobid = 60;
events[2].u.job_state_change.new_state = JOB_STAT_EXIT;
rc = xce_core_push_events(&core, &events[2], 1);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Must still be DONE -- the stale/narrower EXIT notify must not have
* clobbered it. */
TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, job->state);
}
/* JOB_UPDATE (core_events.c's live wire-shape mirror of JOB_STATE_CHANGE --
* the notify-driven single-job path a real controller's HOOK_DETAIL_JOB_UPDATE
* fire reaches, per handler.c's own XCE_SCHED_OP_UPDATE_JOB_FIELDS comment)
@ -1173,6 +1404,10 @@ int main(void)
RUN_TEST(test_jobs_push_fires_on_job_push_once_per_job);
RUN_TEST(test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak);
RUN_TEST(test_jobs_push_duplicate_jobid_with_newer_state_updates_existing_job);
RUN_TEST(test_jobs_push_duplicate_jobid_run_state_transfers_allocation);
RUN_TEST(test_jobs_push_duplicate_jobid_run_state_without_allocation_is_refused);
RUN_TEST(test_jobs_push_duplicate_jobid_does_not_regress_terminal_state);
RUN_TEST(test_job_state_change_does_not_regress_terminal_state);
RUN_TEST(test_job_update_to_terminal_state_frees_accounts_and_allocs);
RUN_TEST(test_non_picker_events_mutate_state_but_never_call_hooks);
RUN_TEST(test_workers_push_twice_updates_existing_worker_in_place);