diff --git a/src/scheduler/src/repl_info.c b/src/scheduler/src/repl_info.c index 856331b5..9772572f 100644 --- a/src/scheduler/src/repl_info.c +++ b/src/scheduler/src/repl_info.c @@ -729,7 +729,14 @@ static void repl_info_worker_one(xce_sched_sim_driver_t* sim, if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) { break; } - if (job->allocs == NULL) { + /* bug-1778: a terminal job's allocs can still be non-NULL (root + * cause not yet pinned -- suspected snapshot save/load + * inconsistency, see buglog.json) -- exclude it defensively + * here too, mirroring the same "terminal is permanent" guard + * job_state_regresses_terminal()/sim_reconcile_unaccounted_ + * jobs() already apply elsewhere. */ + if ((job->allocs == NULL) || JOB_IS_DONE(job->state) + || JOB_IS_EXIT(job->state)) { continue; } for (i = 0; i < job->allocs->nhosts; i++) { diff --git a/src/scheduler/testsuite/test_repl_resource_table.c b/src/scheduler/testsuite/test_repl_resource_table.c index 13fb70e7..150312c4 100644 --- a/src/scheduler/testsuite/test_repl_resource_table.c +++ b/src/scheduler/testsuite/test_repl_resource_table.c @@ -1159,6 +1159,123 @@ test_info_worker_assigned_job_listing_appends_slot_count_when_uniform(void) "uniform: PASS\n"); } +/* + * bug-1778 (interim display-layer mitigation): a job already at a terminal + * state (DONE/EXIT) must never appear in the assigned-jobs listing, even if + * its allocs pointer is still (incorrectly) non-NULL -- live-observed via a + * real captured replay where three DONE jobs kept showing as assigned + * despite the worker's own slot-usage accounting (used=) correctly + * excluding them. Root cause of WHY allocs stays non-NULL for a terminal + * job is not yet pinned (suspected snapshot save/load inconsistency, see + * buglog.json) -- this test only pins the defensive display-layer + * exclusion, mirroring the same "terminal is permanent" guard + * job_state_regresses_terminal()/sim_reconcile_unaccounted_jobs() already + * apply elsewhere. + */ +static void +test_info_worker_assigned_job_listing_excludes_terminal_job_despite_stale_allocs( + void) +{ + xce_sched_sim_driver_t sim; + xce_core_event_t events[2]; + xce_queue_t wire_queue; + xce_worker_t wire_worker; + xce_worker_t* worker_ptrs[1]; + xce_job_t* job_done; + xce_job_t* job_run; + xce_job_t* all_jobs[2]; + uint64_t jobid_done; + uint64_t jobid_run; + xce_sched_job_t* sjob; + xce_host_alloc_t ha_host; + xce_host_alloc_t* ha_host_ptrs[1]; + xce_job_alloc_t alloc; + char* captured; + char jobidStr_done[64]; + char jobidStr_run[64]; + int rc; + + memset(&sim, 0, sizeof(sim)); + rc = xce_sched_core_init(&(sim.core)); + TEST_ASSERT_EQUAL_INT(0, rc); + + memset(&wire_queue, 0, sizeof(wire_queue)); + wire_queue.queue = "normal"; + wire_queue.priority = 5; + wire_queue.status = QUEUE_STAT_OPEN; + + jobid_done = mk_plain_jobid_bpi(6, 2, 1700000030, 1); + jobid_run = mk_plain_jobid_bpi(6, 2, 1700000031, 2); + job_done = make_job(jobid_done, JOB_STAT_RUN, "normal", "alice", "proj1"); + job_run = make_job(jobid_run, JOB_STAT_RUN, "normal", "alice", "proj1"); + all_jobs[0] = job_done; + all_jobs[1] = job_run; + + memset(events, 0, sizeof(events)); + events[0].type = XCE_CORE_EVENT_QUEUE_UPDATE; + events[0].u.queue_update.queue = &wire_queue; + events[1].type = XCE_CORE_EVENT_JOBS_PUSH; + events[1].u.jobs_push.njobs = 2; + events[1].u.jobs_push.jobs = all_jobs; + + rc = xce_core_push_events(&(sim.core), events, 2); + TEST_ASSERT_EQUAL_INT(0, rc); + + memset(&wire_worker, 0, sizeof(wire_worker)); + wire_worker.name = "hostA"; + wire_worker.status = 0; + worker_ptrs[0] = &wire_worker; + + memset(events, 0, sizeof(events)); + events[0].type = XCE_CORE_EVENT_WORKERS_PUSH; + events[0].u.workers_push.nworkers = 1; + events[0].u.workers_push.workers = worker_ptrs; + + rc = xce_core_push_events(&(sim.core), events, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + memset(&ha_host, 0, sizeof(ha_host)); + ha_host.host = "hostA"; + ha_host_ptrs[0] = &ha_host; + memset(&alloc, 0, sizeof(alloc)); + alloc.nhosts = 1; + alloc.allocs = ha_host_ptrs; + + /* Both jobs get the SAME allocs struct assigned directly (bypassing the + * normal event-driven cleanup) -- job_done's state is then forced to + * JOB_STAT_DONE directly, reproducing bug-1778's exact observed + * inconsistency (state=DONE, allocs still non-NULL) regardless of which + * code path actually produces it live. */ + sjob = sched_get_job(&(sim.core), jobid_done); + TEST_ASSERT_NOT_NULL(sjob); + sjob->allocs = &alloc; + sjob->state = JOB_STAT_DONE; + + sjob = sched_get_job(&(sim.core), jobid_run); + TEST_ASSERT_NOT_NULL(sjob); + sjob->allocs = &alloc; + + captured = capture_dispatch(&sim, "info worker hostA"); + + fmt_jobid(jobidStr_done, sizeof(jobidStr_done), jobid_done); + fmt_jobid(jobidStr_run, sizeof(jobidStr_run), jobid_run); + + TEST_ASSERT_NULL_MESSAGE( + strstr(captured, jobidStr_done), + "bug-1778: a job already at a terminal state (DONE) must never " + "appear in the assigned-jobs listing, even if its allocs pointer " + "is still (incorrectly) non-NULL"); + TEST_ASSERT_NOT_NULL_MESSAGE( + strstr(captured, jobidStr_run), + "the genuinely-running job on the same worker must still be " + "listed as assigned"); + + free(captured); + + printf("test_info_worker_assigned_job_listing_excludes_terminal_job_" + "despite_stale_allocs: PASS\n"); +} + /* * Two jobs on the SAME worker consuming DIFFERENT slot counts cannot be * annotated with a single trailing "*N" on a compressed range (which count @@ -1606,6 +1723,7 @@ int main(void) RUN_TEST(test_info_worker_units_mode_converts_swp_megabytes_before_human_scaling); RUN_TEST(test_info_worker_assigned_job_listing_compresses_contiguous_array_indices); RUN_TEST(test_info_worker_assigned_job_listing_appends_slot_count_when_uniform); + RUN_TEST(test_info_worker_assigned_job_listing_excludes_terminal_job_despite_stale_allocs); RUN_TEST(test_info_worker_assigned_job_listing_falls_back_per_job_when_slots_differ); RUN_TEST(test_info_resource_groups_resources_into_labeled_table_sections_in_order); RUN_TEST(test_info_resource_load_metric_whole_number_omits_trailing_zeros);