lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(scheduler): exclude terminal jobs from assigned-jobs listing (bug-1778)

Browse source 
info worker's assigned-jobs listing gated only on job->allocs != NULL,
so a job already at a terminal state (DONE/EXIT) could still appear as
"assigned" if its allocs pointer was ever left stale -- live-observed via
a real captured replay, three DONE jobs kept showing as assigned to a
worker despite the worker's own slot-usage accounting correctly excluding
them. Root cause of the stale allocs itself is not yet pinned (suspected
snapshot save/load inconsistency); this adds the same defensive "terminal
is permanent" exclusion job_state_regresses_terminal()/sim_reconcile_
unaccounted_jobs() already apply elsewhere, at the display layer.
This commit is contained in:
Rongsong Shen 2026-09-07 07:32:08 +00:00
parent e395eaa3ad
commit ef3c4670a6
No known key found for this signature in database
GPG key ID: 5BCDAAE0123E9FE2
2 changed files with 126 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
src/scheduler/src/repl_info.c
View file

@ -729,7 +729,14 @@ static void repl_info_worker_one(xce_sched_sim_driver_t* sim,
if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) {
break;
}
if (job->allocs == NULL) {
/* bug-1778: a terminal job's allocs can still be non-NULL (root
* cause not yet pinned -- suspected snapshot save/load
* inconsistency, see buglog.json) -- exclude it defensively
* here too, mirroring the same "terminal is permanent" guard
* job_state_regresses_terminal()/sim_reconcile_unaccounted_
* jobs() already apply elsewhere. */
if ((job->allocs == NULL) || JOB_IS_DONE(job->state)
|| JOB_IS_EXIT(job->state)) {
continue;
}
for (i = 0; i < job->allocs->nhosts; i++) {

118
src/scheduler/testsuite/test_repl_resource_table.c
View file

@ -1159,6 +1159,123 @@ test_info_worker_assigned_job_listing_appends_slot_count_when_uniform(void)
"uniform: PASS\n");
}
/*
* bug-1778 (interim display-layer mitigation): a job already at a terminal
* state (DONE/EXIT) must never appear in the assigned-jobs listing, even if
* its allocs pointer is still (incorrectly) non-NULL -- live-observed via a
* real captured replay where three DONE jobs kept showing as assigned
* despite the worker's own slot-usage accounting (used=) correctly
* excluding them. Root cause of WHY allocs stays non-NULL for a terminal
* job is not yet pinned (suspected snapshot save/load inconsistency, see
* buglog.json) -- this test only pins the defensive display-layer
* exclusion, mirroring the same "terminal is permanent" guard
* job_state_regresses_terminal()/sim_reconcile_unaccounted_jobs() already
* apply elsewhere.
*/
static void
test_info_worker_assigned_job_listing_excludes_terminal_job_despite_stale_allocs(
void)
{
xce_sched_sim_driver_t sim;
xce_core_event_t events[2];
xce_queue_t wire_queue;
xce_worker_t wire_worker;
xce_worker_t* worker_ptrs[1];
xce_job_t* job_done;
xce_job_t* job_run;
xce_job_t* all_jobs[2];
uint64_t jobid_done;
uint64_t jobid_run;
xce_sched_job_t* sjob;
xce_host_alloc_t ha_host;
xce_host_alloc_t* ha_host_ptrs[1];
xce_job_alloc_t alloc;
char* captured;
char jobidStr_done[64];
char jobidStr_run[64];
int rc;
memset(&sim, 0, sizeof(sim));
rc = xce_sched_core_init(&(sim.core));
TEST_ASSERT_EQUAL_INT(0, rc);
memset(&wire_queue, 0, sizeof(wire_queue));
wire_queue.queue = "normal";
wire_queue.priority = 5;
wire_queue.status = QUEUE_STAT_OPEN;
jobid_done = mk_plain_jobid_bpi(6, 2, 1700000030, 1);
jobid_run = mk_plain_jobid_bpi(6, 2, 1700000031, 2);
job_done = make_job(jobid_done, JOB_STAT_RUN, "normal", "alice", "proj1");
job_run = make_job(jobid_run, JOB_STAT_RUN, "normal", "alice", "proj1");
all_jobs[0] = job_done;
all_jobs[1] = job_run;
memset(events, 0, sizeof(events));
events[0].type = XCE_CORE_EVENT_QUEUE_UPDATE;
events[0].u.queue_update.queue = &wire_queue;
events[1].type = XCE_CORE_EVENT_JOBS_PUSH;
events[1].u.jobs_push.njobs = 2;
events[1].u.jobs_push.jobs = all_jobs;
rc = xce_core_push_events(&(sim.core), events, 2);
TEST_ASSERT_EQUAL_INT(0, rc);
memset(&wire_worker, 0, sizeof(wire_worker));
wire_worker.name = "hostA";
wire_worker.status = 0;
worker_ptrs[0] = &wire_worker;
memset(events, 0, sizeof(events));
events[0].type = XCE_CORE_EVENT_WORKERS_PUSH;
events[0].u.workers_push.nworkers = 1;
events[0].u.workers_push.workers = worker_ptrs;
rc = xce_core_push_events(&(sim.core), events, 1);
TEST_ASSERT_EQUAL_INT(0, rc);
memset(&ha_host, 0, sizeof(ha_host));
ha_host.host = "hostA";
ha_host_ptrs[0] = &ha_host;
memset(&alloc, 0, sizeof(alloc));
alloc.nhosts = 1;
alloc.allocs = ha_host_ptrs;
/* Both jobs get the SAME allocs struct assigned directly (bypassing the
* normal event-driven cleanup) -- job_done's state is then forced to
* JOB_STAT_DONE directly, reproducing bug-1778's exact observed
* inconsistency (state=DONE, allocs still non-NULL) regardless of which
* code path actually produces it live. */
sjob = sched_get_job(&(sim.core), jobid_done);
TEST_ASSERT_NOT_NULL(sjob);
sjob->allocs = &alloc;
sjob->state = JOB_STAT_DONE;
sjob = sched_get_job(&(sim.core), jobid_run);
TEST_ASSERT_NOT_NULL(sjob);
sjob->allocs = &alloc;
captured = capture_dispatch(&sim, "info worker hostA");
fmt_jobid(jobidStr_done, sizeof(jobidStr_done), jobid_done);
fmt_jobid(jobidStr_run, sizeof(jobidStr_run), jobid_run);
TEST_ASSERT_NULL_MESSAGE(
strstr(captured, jobidStr_done),
"bug-1778: a job already at a terminal state (DONE) must never "
"appear in the assigned-jobs listing, even if its allocs pointer "
"is still (incorrectly) non-NULL");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, jobidStr_run),
"the genuinely-running job on the same worker must still be "
"listed as assigned");
free(captured);
printf("test_info_worker_assigned_job_listing_excludes_terminal_job_"
"despite_stale_allocs: PASS\n");
}
/*
* Two jobs on the SAME worker consuming DIFFERENT slot counts cannot be
* annotated with a single trailing "*N" on a compressed range (which count
@ -1606,6 +1723,7 @@ int main(void)
RUN_TEST(test_info_worker_units_mode_converts_swp_megabytes_before_human_scaling);
RUN_TEST(test_info_worker_assigned_job_listing_compresses_contiguous_array_indices);
RUN_TEST(test_info_worker_assigned_job_listing_appends_slot_count_when_uniform);
RUN_TEST(test_info_worker_assigned_job_listing_excludes_terminal_job_despite_stale_allocs);
RUN_TEST(test_info_worker_assigned_job_listing_falls_back_per_job_when_slots_differ);
RUN_TEST(test_info_resource_groups_resources_into_labeled_table_sections_in_order);
RUN_TEST(test_info_resource_load_metric_whole_number_omits_trailing_zeros);