From c69705ae5437d078a16cb970b666c24c628f4ea0 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sun, 27 Sep 2026 04:32:37 +0000 Subject: [PATCH] fix: the controller is the ground truth for job state on scheduler rejoin (bug-1860) A job the controller requeued to PEND while the scheduler was disconnected stayed RUN (with its old allocation) in the scheduler forever: the bug-1812 guard in core_events.c's JOBS_PUSH merge refused any non-RUN, non-terminal resync claim over a job holding a real allocation, because it could not tell a genuine requeue from a push built before the scheduler's own fresh dispatch. Design (per review): the scheduler persists nothing, so on rejoin the controller's state wins, and the race the guard protected against is closed where the truth lives: - scheduler JOBS_PUSH: the bug-1812 refusal is removed; the controller's state replaces the scheduler's and its allocation is released. The terminal-is-permanent and phantom-RUN (RUN without allocation data) guards stay. - controller dispatch_a_job_(): refuses a job that is not PEND (RUN or finished) before any worker is touched -- previously nothing checked, so a stale second dispatch would start a running job again (or, through a retryable failure, requeue it). The job is left as is, no WAL fact, and its true state is re-sent (HOOK_DETAIL_JOB_UPDATE). A PEND job with a leftover allocation stays dispatchable. - scheduler JOB_UPDATE: a RUN report carrying the controller's committed allocation is adopted, so a corrected job is never RUN with nothing charged. Tests (all RED before): test_jobs_push_duplicate_jobid_controller_pend_ wins_over_local_run (replaces the bug-1812 test that pinned the refusal), test_job_update_run_adopts_controller_allocation, test_dispatch_of_non_pending_job_is_refused. Two tests that seeded a RUN job to reach retry branches (a precondition their own headers called unreachable) now use the real paths: the counter-reset test requeues via record_job_reconcile_pending_fact() first; the disconnected-worker RUN test asserts the refusal contract. Controller suite 1238/0, scheduler 87/87. HA soaks: chaotic 6 rounds PASS (0 divergence/loss, 6 worker-resync jobs), sequential 6 rounds PASS (SIMVERIFY 7 files, 0 mismatches). The race did not occur in either run (0 refusals logged), so the new paths are proven by the unit tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/controller/job.c | 30 ++++ .../testsuite/test_dispatch_fail_retry.c | 145 ++++++++++++------ src/scheduler/src/core_events.c | 61 +++----- src/scheduler/testsuite/test_core_events.c | 119 +++++++++----- 4 files changed, 236 insertions(+), 119 deletions(-) diff --git a/src/controller/job.c b/src/controller/job.c index 2d0d10fc..716c47d7 100644 --- a/src/controller/job.c +++ b/src/controller/job.c @@ -1313,6 +1313,36 @@ static int dispatch_a_job_(cluster_t* c, endpoint_t* ep, xce_job_alloc_t* alloc) return -errno; } + /* bug-1860: only a PEND job may be dispatched. Nothing checked this, so + * a second dispatch for a job already RUN -- e.g. from a scheduler + * acting on a rejoin push older than the job's own dispatch -- would + * start it on a worker again (or, via a retryable failure, requeue a + * running job). Refuse it before any worker is touched, leave the job + * exactly as it is, and re-send its true state to the scheduler, which + * adopts it (the controller is the ground truth). */ + { + int state; + bool dispatchable; + + lock(j, RTTI_LOCK_READ); + state = j->job.state; + /* State only: a PEND job still carrying a leftover allocation + * stays dispatchable (dispatch replaces it), never refused + * forever. */ + dispatchable = JOB_IS_PEND(state) && !JOB_IS_RUN(state) + && !JOB_IS_FINISH(state); + unlock(j); + + if (!dispatchable) { + WARN("dispatch_a_job_: job %s is not pending (state 0x%x) -- " + "refusing a stale dispatch and re-sending its state", + jobidStr, state); + persist_notify_job_updated_already_persisted(c, j); + dereference(j); + return -EALREADY; + } + } + nc = alloc->nhosts; if (nc <= 0) { /* scheduler-dispatch-commit task 3.2 (Req 4.2, 4.5): resolves this diff --git a/src/controller/testsuite/test_dispatch_fail_retry.c b/src/controller/testsuite/test_dispatch_fail_retry.c index b718395f..a35de541 100644 --- a/src/controller/testsuite/test_dispatch_fail_retry.c +++ b/src/controller/testsuite/test_dispatch_fail_retry.c @@ -621,29 +621,16 @@ static void test_disconnected_worker_fails_immediately_no_retry(void) printf("test_disconnected_worker_fails_immediately_no_retry: PASS\n"); } -/* ── worker-rejoin-resync task 2.3: JOB_TRANSITION_APPLIED wiring proof - * ──────────────────────────────────────────────────────────────────── - * The NOOP_DUPLICATE test above exercises the only precondition dispatch_ - * a_job_()'s real caller can currently construct (a PEND job -- the - * scheduler only ever dispatches from its PEND pool, so dispatch_a_job_() - * is never actually invoked on a RUN job today; the same "not reachable - * via any real caller in the currently-implemented system" caveat - * test_rpc_failure_bound_resets_after_intervening_success's header - * documents for the four RETRYABLE sites applies here too -- see bug-1631). - * This test instead directly seeds a RUN job with a real, committed - * job->job.allocs (mirroring test_worker_resync.c's own technique) to - * exercise record_job_reconcile_pending_fact()'s OTHER outcome - * (JOB_TRANSITION_APPLIED, not NOOP_DUPLICATE) THROUGH this exact new - * dispatch_a_job_() call site -- proving the branch's result-handling - * (unlock -> notify-only-on-APPLIED -> dereference) is wired correctly, - * complementing task 2.1's test_worker_resync.c (which already proves - * record_job_reconcile_pending_fact() itself is correct in isolation, not - * through this caller). Not a claim that THIS exact precondition (RUN job - * hitting the disconnected branch) is reachable in production today -- - * design.md's own bug-1631 narrative is that record_job_reconcile_pending_ - * fact() releases allocs at the point of RECONCILIATION (a separate, - * later-task call site, resync.c), so by the time any job reaches a fresh - * dispatch_a_job_() call it is already PEND with allocs already NULL. */ +/* ── bug-1860: a RUN job is never re-dispatched ────────────────────────── + * Originally (worker-rejoin-resync task 2.3) this seeded a RUN job with a + * committed allocation to drive record_job_reconcile_pending_fact()'s + * APPLIED outcome through the disconnected-worker branch -- a precondition + * its own header already called unreachable from any real caller. Since + * bug-1860, dispatch_a_job_() refuses any job that is not PEND before a + * worker is touched, so it now pins that contract for the disconnected- + * worker case: refused, left RUN with its allocation, no WAL fact, state + * re-sent. (record_job_reconcile_pending_fact() itself stays covered by + * test_worker_resync.c.) */ #define DFR_JOBID_DISCONNECTED_RUN 91207ULL static uint64_t dfr_current_lsn(void) @@ -656,7 +643,7 @@ static uint64_t dfr_current_lsn(void) return lsn; } -static void test_disconnected_worker_reconciles_run_job_releases_allocs(void) +static void test_disconnected_worker_run_job_redispatch_is_refused(void) { job_t* j; worker_t* w = NULL; @@ -708,29 +695,24 @@ static void test_disconnected_worker_reconciles_run_job_releases_allocs(void) dfr_arm_worker_canary(w); rc = dispatch_a_job_test(&g_dfr, NULL, redispatch_alloc); + + /* bug-1860: a job that is already RUN is refused before any worker is + * touched -- dispatching it again would start it twice. It stays + * exactly as it is (no requeue, no WAL fact), and its true state is + * re-sent to the scheduler. */ TEST_ASSERT_TRUE_MESSAGE(rc < 0, - "a dispatch to a disconnected worker must still fail"); - - TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state, - "Req 4.6/5.4: a RUN job redispatched to a disconnected worker must " - "be returned to PEND"); - TEST_ASSERT_NULL_MESSAGE(j->job.allocs, - "bug-1631: the job's prior resource allocation must be released, " - "so a later redispatch never inherits a stale allocation"); - TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, j->dispatch_fail_count, - "Req 5.4: dispatch_fail_count must stay untouched on this path"); - + "a re-dispatch of a RUN job must be refused"); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state, + "bug-1860: a RUN job must not be requeued by a stale dispatch"); + TEST_ASSERT_EQUAL_PTR(prior_alloc, j->job.allocs); + TEST_ASSERT_EQUAL_UINT32(0, j->dispatch_fail_count); TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, cbrp_notify_capture_count_ext(), "notify_dispatch() (the retry-signal path) must never fire here"); - lsn_after = dfr_current_lsn(); - TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before, - "an APPLIED reconciliation must durably append a WAL_JOB_UPDATE " - "fact"); - + TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after, + "a refused dispatch appends no WAL fact"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dfr_update_hook_fires, - "bug-1679: an APPLIED reconciliation must notify the scheduler " - "exactly once, via this exact dispatch_a_job_() call site"); + "bug-1860: the job's true state must be re-sent to the scheduler"); dfr_assert_ref_not_over_released(j, DFR_JOBID_DISCONNECTED_RUN); dfr_assert_worker_ref_not_over_released(w, "dfr-wkr-disc2"); @@ -739,7 +721,7 @@ static void test_disconnected_worker_reconciles_run_job_releases_allocs(void) dfr_job_update_spy_hook); free_JobAlloc_ptr(&redispatch_alloc); dfr_teardown(); - printf("test_disconnected_worker_reconciles_run_job_releases_allocs: " + printf("test_disconnected_worker_run_job_redispatch_is_refused: " "PASS\n"); } @@ -1024,6 +1006,18 @@ static void test_rpc_failure_bound_resets_after_intervening_success(void) dfr_drain_ep(&working_ep); + /* bug-1860: a RUN job can no longer be re-dispatched (dispatch_a_job_() + * refuses anything not PEND), so return the job to PEND the way a real + * re-admission does -- record_job_reconcile_pending_fact(), e.g. the + * worker reported it unrecognized on rejoin -- before the failures below + * exercise the post-success counter. */ + lock(j, RTTI_LOCK_WRITE); + TEST_ASSERT_EQUAL_INT((int)JOB_TRANSITION_APPLIED, + record_job_reconcile_pending_fact(&g_dfr, j)); + unlock(j); + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state); + TEST_ASSERT_NULL(j->job.allocs); + /* Step 3: swap back to the broken endpoint and fail 3 more times -- the * genuinely load-bearing proof (see this test's header). Each of these * three must still resolve retryable/PEND; if the reset above had not @@ -1128,6 +1122,68 @@ static void test_rpc_failure_bound_resets_after_intervening_success(void) "PASS\n"); } +/* bug-1860: dispatch_a_job_() never checked the job's state, so a second + * dispatch for a job already RUN (e.g. a scheduler acting on a stale rejoin + * push) would start it on a worker again. A job that is not PEND must be + * refused before any worker is touched, left exactly as it is, and its + * true state re-sent to the scheduler (HOOK_DETAIL_JOB_UPDATE). */ +#define DFR_JOBID_NOT_PEND 91208ULL + +static void test_dispatch_of_non_pending_job_is_refused(void) +{ + job_t* j; + xce_job_alloc_t* alloc; + xce_job_alloc_t* committed; + int rc; + int before_count; + + dfr_setup(); + g_dfr_update_hook_fires = 0; + TEST_ASSERT_EQUAL_INT(0, + xce_add_hook(&g_dfr.hooksys, XCE_CONTROLLER_HOOK_JOB, + dfr_job_update_spy_hook, NULL)); + + j = dfr_seed_job(DFR_JOBID_NOT_PEND, "dfr-q-notpend"); + committed = make_job_alloc(DFR_JOBID_NOT_PEND, 1, + make_host_allocs(1, + make_host_alloc((char*)"dfr-wkr-first", 1, + make_allocs(1, + make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); + lock(j, RTTI_LOCK_WRITE); + j->job.state = JOB_STAT_RUN; + j->job.allocs = committed; + unlock(j); + + alloc = make_job_alloc(DFR_JOBID_NOT_PEND, 1, + make_host_allocs(1, + make_host_alloc((char*)"dfr-wkr-second", 1, + make_allocs(1, + make_alloc((char*)"slots", NULL, 0, 1.0f, 0))))); + TEST_ASSERT_NOT_NULL(alloc); + + cbrp_notify_capture_reset_ext(); + before_count = cbrp_notify_capture_count_ext(); + dfr_arm_ref_canary(j); + + rc = dispatch_a_job_test(&g_dfr, NULL, alloc); + + TEST_ASSERT_TRUE_MESSAGE(rc < 0, + "bug-1860: dispatching a job that is already RUN must be refused"); + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state); + TEST_ASSERT_EQUAL_PTR_MESSAGE(committed, j->job.allocs, + "the job's committed allocation must be left untouched"); + TEST_ASSERT_EQUAL_UINT32(0, j->dispatch_fail_count); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_dfr_update_hook_fires, + "bug-1860: the job's true state must be re-sent to the scheduler"); + TEST_ASSERT_EQUAL_INT_MESSAGE(before_count, + cbrp_notify_capture_count_ext(), + "a refused stale dispatch is not a retryable failure"); + dfr_assert_ref_not_over_released(j, DFR_JOBID_NOT_PEND); + + free_JobAlloc_ptr(&alloc); + dfr_teardown(); +} + void test_dispatch_fail_retry(void) { UnitySetTestFile(__FILE__); @@ -1139,10 +1195,11 @@ void test_dispatch_fail_retry(void) RUN_TEST(test_rpc_failure_resolves_retryable); RUN_TEST(test_rpc_failure_bounded_escalation); RUN_TEST(test_rpc_failure_bound_resets_after_intervening_success); + RUN_TEST(test_dispatch_of_non_pending_job_is_refused); printf("\n>> dispatch_a_job_() disconnected-worker fail-fast tests " "(worker-rejoin-resync task 2.3, Req 5.1, 5.4)\n\n"); RUN_TEST(test_disconnected_worker_fails_immediately_no_retry); - RUN_TEST(test_disconnected_worker_reconciles_run_job_releases_allocs); + RUN_TEST(test_disconnected_worker_run_job_redispatch_is_refused); } diff --git a/src/scheduler/src/core_events.c b/src/scheduler/src/core_events.c index f21c9dea..e2032634 100644 --- a/src/scheduler/src/core_events.c +++ b/src/scheduler/src/core_events.c @@ -1004,42 +1004,17 @@ int xce_core_push_events(xce_sched_core_t* core, xce_core_event_t* events, picker, existing, job->state); } } - } else if ((existing->allocs != NULL) - && !JOB_IS_DONE(job->state) - && !JOB_IS_EXIT(job->state)) { - /* bug-1812 ("stale resync regression"): symmetric - * to bug-1739's own phantom-RUN guard above, but - * for the descent direction. `existing` is - * currently RUN with a REAL, already-captured - * allocation -- this resync redelivery claims - * some OTHER, non-terminal state (e.g. PEND) with - * no allocation of its own to justify it. - * on_scheduler_join()'s full resync - * (xcesched_push_jobs(), controller/sched.c) - * fires on every scheduler reconnect, built from - * collect_jobs_committed() -- a committed-data - * snapshot that can race the scheduler's own very - * recent, self-initiated dispatch (live-confirmed - * via a production scheduler.log: a JOBS_PUSH - * resync WARN landed just 16ms after this exact - * job's own schedStartJob confirmation). Trusting - * this claim would destroy a real allocation the - * scheduler's own resource/slot bookkeeping - * already correctly has -- the same phantom-RUN- - * shaped data loss bug-1739 already guards - * against, just reached from the opposite state - * transition. A genuinely TERMINAL claim - * (DONE/EXIT) is NOT covered by this guard: that - * is legitimate "job finished while this - * scheduler was disconnected" information - * (bug-1734's own motivating case) and must - * still release the allocation and apply. */ - WARN("%s JOBS_PUSH: refusing resync state 0x%x " - "for jobid %s -- existing job is RUN with a " - "real allocation on record (existing state " - "0x%x kept)", - __func__, job->state, jobidStr, - existing->state); + /* bug-1860: no refusal for a non-RUN, non-terminal + * claim any more (bug-1812's guard kept a job RUN + * forever when the controller had requeued it while + * this scheduler was disconnected). The controller is + * the ground truth -- this scheduler persists nothing + * -- so its state replaces ours and our allocation is + * released below. The race bug-1812 guarded against, + * a push older than our own fresh dispatch, is closed + * on the controller side: dispatch_a_job_() refuses a + * job that is not PEND and re-sends the job's true + * state, adopted by JOB_UPDATE below. */ } else { if (existing->accounts != NULL) { free_job_account(existing->accounts); @@ -1342,6 +1317,20 @@ int xce_core_push_events(xce_sched_core_t* core, xce_core_event_t* events, if (job->allocs != NULL) { free_JobAlloc_ptr(&(job->allocs)); } + } else if (updated->allocs != NULL) { + /* bug-1860: the controller's committed allocation for a RUN + * job is the ground truth -- adopt it (e.g. when it refused + * a stale re-dispatch and re-sent the job's true state), so + * a RUN job is never left with no resources charged. */ + if (job->accounts != NULL) { + free_job_account(job->accounts); + job->accounts = NULL; + } + if (job->allocs != NULL) { + free_JobAlloc_ptr(&(job->allocs)); + } + MOVE_PTR(job->allocs, updated->allocs); + job->accounts = make_job_account(job->allocs); } if (updated->submit != NULL) { diff --git a/src/scheduler/testsuite/test_core_events.c b/src/scheduler/testsuite/test_core_events.c index 36a2c593..d0ceffa1 100644 --- a/src/scheduler/testsuite/test_core_events.c +++ b/src/scheduler/testsuite/test_core_events.c @@ -774,38 +774,27 @@ test_jobs_push_duplicate_jobid_run_state_without_allocation_is_refused(void) TEST_ASSERT_NULL(tracked->allocs); } -/* bug-1812 ("stale resync regression"): symmetric to bug-1739's own - * "phantom RUN" guard just above (refusing to FABRICATE an allocation-less - * RUN claim), but for the descent direction -- a JOBS_PUSH resync - * redelivery must not DESTROY a job's REAL, already-captured allocation by - * trusting an unsupported non-RUN, non-terminal claim over it. - * Live-confirmed via a real production scheduler.log: on_scheduler_join()'s - * full resync (xcesched_push_jobs(), controller/sched.c) fires on every - * scheduler reconnect, built from collect_jobs_committed() -- a - * committed-data snapshot that can race the scheduler's own very recent, - * self-initiated dispatch. The log showed a JOBS_PUSH resync WARN landing - * just 16ms after this exact job's own schedStartJob confirmation - * (sched_job_state_change -> state 2), repeating in bursts on later - * reconnects -- each collision wiping the job's real allocation while its - * state stayed RUN, exactly matching the simulator's live-reported symptom - * ("info jobs" shows RUN, "info worker"'s assigned-jobs listing -- which - * walks job->allocs -- omits it). Asserts a job tracked RUN with a real - * allocation, redelivered via resync as PEND with no allocation, KEEPS its - * RUN state and real allocation -- the resync's unsupported claim must be - * refused, not trusted. */ +/* bug-1860 (supersedes bug-1812's refusal): on a scheduler rejoin the + * controller's full JOBS_PUSH is the ground truth -- the scheduler keeps no + * persistent state of its own. A job the scheduler holds as RUN with an + * allocation, redelivered as PEND (e.g. requeued by the controller while + * the scheduler was disconnected), must become PEND and release that + * allocation; the bug-1812 guard used to refuse this, leaving the job RUN + * forever. The race bug-1812 guarded against (a push older than the + * scheduler's own fresh dispatch) is now closed on the controller side: it + * refuses to dispatch a job that is not PEND and re-sends the job's true + * state, which the JOB_UPDATE handler adopts (allocation included). */ static void -test_jobs_push_duplicate_jobid_run_with_allocation_survives_stale_nonrun_resync( - void) +test_jobs_push_duplicate_jobid_controller_pend_wins_over_local_run(void) { xce_sched_core_t core; xce_core_event_t event; xce_job_t* job1; - xce_job_t* stale_resync_job; + xce_job_t* requeued_job; xce_sched_job_t* tracked; xce_alloc_t* a_slots; xce_host_alloc_t* h0; xce_host_alloc_t* hallocs[1]; - xce_job_alloc_t* jalloc; int rc; fixture_init(&core, &g_full_plugin); @@ -823,35 +812,86 @@ test_jobs_push_duplicate_jobid_run_with_allocation_survives_stale_nonrun_resync( tracked = sched_get_job(&core, 65); TEST_ASSERT_NOT_NULL(tracked); - /* Give it a real, captured allocation -- mirrors what a genuine - * dispatch (scheduler.c's sched_commit_decision()) would already have - * set by the time a stale resync arrives. */ a_slots = make_alloc((char*)"slots", NULL, 0, 1.0f, 0); h0 = make_host_alloc((char*)"hostA", 1, make_allocs(1, a_slots)); hallocs[0] = h0; - jalloc = make_job_alloc(65, 1, make_host_allocs(1, hallocs[0])); - tracked->allocs = jalloc; + tracked->allocs = make_job_alloc(65, 1, make_host_allocs(1, hallocs[0])); tracked->accounts = make_job_account(tracked->allocs); - /* A stale on_scheduler_join() resync redelivery -- the controller's - * committed-data snapshot hasn't yet caught up to this job's own very - * recent dispatch, so it still reports PEND with no allocation. */ - stale_resync_job = make_job(65, JOB_STAT_PEND, "normal", "alice", "proj1"); + /* The controller requeued it while this scheduler was disconnected. */ + requeued_job = make_job(65, JOB_STAT_PEND, "normal", "alice", "proj1"); memset(&event, 0, sizeof(event)); event.type = XCE_CORE_EVENT_JOBS_PUSH; event.u.jobs_push.njobs = 1; - event.u.jobs_push.jobs = &stale_resync_job; + event.u.jobs_push.jobs = &requeued_job; rc = xce_core_push_events(&core, &event, 1); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_PTR(tracked, sched_get_job(&core, 65)); - /* Must still be RUN with its real allocation intact -- the stale resync - * claim must not have clobbered it. */ - TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, tracked->state); - TEST_ASSERT_EQUAL_PTR(jalloc, tracked->allocs); - TEST_ASSERT_NOT_NULL(tracked->accounts); + TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, tracked->state, + "bug-1860: the controller's rejoin push is the ground truth"); + TEST_ASSERT_NULL(tracked->allocs); + TEST_ASSERT_NULL(tracked->accounts); +} + +/* bug-1860: the controller's own report that a job is RUN, with the + * allocation it committed, replaces whatever the scheduler holds -- e.g. + * after the controller refused a stale re-dispatch and re-sent the job's + * true state. Without adopting the allocation the job would be a RUN job + * with no resources charged. */ +static void test_job_update_run_adopts_controller_allocation(void) +{ + xce_sched_core_t core; + xce_core_event_t event; + xce_job_t* job1; + xce_job_t updated; + xce_sched_job_t* job; + xce_alloc_t* a_slots; + xce_host_alloc_t* h0; + xce_host_alloc_t* hallocs[1]; + xce_job_alloc_t* jalloc; + int rc; + + fixture_init(&core, &g_full_plugin); + + job1 = make_job(67, JOB_STAT_PEND, "normal", "alice", "proj1"); + + memset(&event, 0, sizeof(event)); + event.type = XCE_CORE_EVENT_JOBS_PUSH; + event.u.jobs_push.njobs = 1; + event.u.jobs_push.jobs = &job1; + rc = xce_core_push_events(&core, &event, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + job = sched_get_job(&core, 67); + TEST_ASSERT_NOT_NULL(job); + TEST_ASSERT_NULL(job->allocs); + + a_slots = make_alloc((char*)"slots", NULL, 0, 2.0f, 0); + h0 = make_host_alloc((char*)"hostB", 1, make_allocs(1, a_slots)); + hallocs[0] = h0; + jalloc = make_job_alloc(67, 1, make_host_allocs(1, hallocs[0])); + + memset(&updated, 0, sizeof(updated)); + updated.jobid = 67; + updated.state = JOB_STAT_RUN; + updated.allocs = jalloc; + + memset(&event, 0, sizeof(event)); + event.type = XCE_CORE_EVENT_JOB_UPDATE; + event.u.job_update.jobid = 67; + event.u.job_update.job = &updated; + rc = xce_core_push_events(&core, &event, 1); + TEST_ASSERT_EQUAL_INT(0, rc); + + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, job->state); + TEST_ASSERT_EQUAL_PTR_MESSAGE(jalloc, job->allocs, + "bug-1860: a controller RUN report must bring its allocation along"); + TEST_ASSERT_NOT_NULL(job->accounts); + TEST_ASSERT_NULL_MESSAGE(updated.allocs, + "ownership of the allocation moves to the scheduler's job"); } /* Companion: a resync claiming a genuinely TERMINAL state (DONE/EXIT) for a @@ -1924,7 +1964,8 @@ int main(void) RUN_TEST(test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak); RUN_TEST(test_jobs_push_duplicate_jobid_with_newer_state_updates_existing_job); RUN_TEST(test_jobs_push_duplicate_jobid_run_state_transfers_allocation); - RUN_TEST(test_jobs_push_duplicate_jobid_run_with_allocation_survives_stale_nonrun_resync); + RUN_TEST(test_jobs_push_duplicate_jobid_controller_pend_wins_over_local_run); + RUN_TEST(test_job_update_run_adopts_controller_allocation); RUN_TEST(test_jobs_push_duplicate_jobid_terminal_resync_releases_real_allocation); RUN_TEST(test_jobs_push_duplicate_jobid_run_state_without_allocation_is_refused); RUN_TEST(test_jobs_push_duplicate_jobid_does_not_regress_terminal_state);