From 7481cedb3a135711cc2544b4259f00c950f24fde Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 27 Aug 2026 15:22:04 +0000 Subject: [PATCH] test(scheduler-snapshot-job-allocation): full cross-instance round-trip proof Task 3.1: comprehensive integration test proving tasks 1.1/1.2/2.1/2.2 work together end to end -- a RUNNING job (multi-host, multi-resource, plus a genuine shared-resource-pool component) and a PENDING job saved together and loaded into a second, independent core. Asserts: - Both jobs get a valid, non-NULL schedobj in the restored core. - The running job's allocation/accounting content-match the source, including the shared-pool component. - Worker resInUse and core sharedInUse, computed via the real sched_charge_account() chain (white-box included from resource.c, following the established test_resource_accounting.c/test.c link-only-stub convention), match exactly between source and restored cores -- the actual proof of Req 1.2/1.3, not just that job->accounts holds the right values. - Destroying the source core's job list afterward does not corrupt or crash the restored core (Req 1.4's independence guarantee exercised via real teardown, not just pointer-distinctness checks). Test-only task; no production code changed. Independently verified via a live RED/GREEN toggle against task 2.2's load-side restoration block. Requirements: 1.1, 1.2, 1.3, 1.4, 2.1, 2.3, 2.4 --- .../tasks.md | 2 +- .../testsuite/test_snapshot_save_load.c | 567 +++++++++++++++++- 2 files changed, 563 insertions(+), 6 deletions(-) diff --git a/.kiro/specs/scheduler-snapshot-job-allocation/tasks.md b/.kiro/specs/scheduler-snapshot-job-allocation/tasks.md index c4da1877..03a20cec 100644 --- a/.kiro/specs/scheduler-snapshot-job-allocation/tasks.md +++ b/.kiro/specs/scheduler-snapshot-job-allocation/tasks.md @@ -38,7 +38,7 @@ - _Boundary: snapshot load (allocation and accounting restoration)_ - [ ] 3. Integration: prove the save/load round trip end to end -- [ ] 3.1 Verify a full cross-instance save/load round trip preserves allocation, accounting, and scheduling representation together +- [x] 3.1 Verify a full cross-instance save/load round trip preserves allocation, accounting, and scheduling representation together - Save a scheduler instance containing a mix of running jobs (spanning multiple hosts, multiple resources, and a shared-resource-pool allocation) and pending jobs, then load that saved state into a second, independent scheduler instance. - Assert every running job's allocation and accounting match the source instance, every job (running or pending) has a valid scheduling representation, and that ending either instance's job list afterward does not corrupt or crash the other. - Observable completion: the round-trip test passes, demonstrating the restored instance's resource-in-use totals exactly match what the source instance reported before saving. diff --git a/src/scheduler/testsuite/test_snapshot_save_load.c b/src/scheduler/testsuite/test_snapshot_save_load.c index 75a695d7..3d3ee7dd 100644 --- a/src/scheduler/testsuite/test_snapshot_save_load.c +++ b/src/scheduler/testsuite/test_snapshot_save_load.c @@ -49,6 +49,29 @@ * for both: sched_resolve_job_picker(core, ...) naturally resolves to * whichever plugin instance is registered in THAT core's own * core->plugins kvmgr. + * + * scheduler-snapshot-job-allocation task 3.1's own test (below, the file's + * newest addition) ALSO white-box #includes ../src/resource.c -- the real + * sched_charge_account()/sched_update_account()/sched_update_res_accounts() + * chain sched_account_jobs() (scheduler.c, NOT included here -- too heavy a + * dependency surface for this otherwise-lean core-only TU) calls downstream. + * Task 3.1's test provides its own real, verbatim-copied reimplementation of + * sched_account_jobs() itself, the same "real reimplementation" technique + * this file already uses for sched_resolve_job_picker() above and + * test_sim_contention_parity.c already establishes for this exact function + * (see that file's own header comment: "a no-op stub would defeat the + * test's own premise"). Pulling in resource.c supplies sched_charge_account() + * for real but ALSO supplies a REAL sched_worker_release_host_ctx() + * (superseding this file's own former no-op stub -- removed below, since a + * duplicate definition would be a link error); it also needs two new + * link-only stubs (plugin_prepare_allocation, sched_ctx_get_uncomit_host) + * for resource.c's own never-invoked sched_try_alloc_on_host_() dependency + * chain -- the same two-stub pair test_resource_accounting.c (and test.c) + * already establish needing for the identical reason. (Note: + * test_sim_contention_parity.c only needs one of the two, + * sched_ctx_get_uncomit_host -- it gets plugin_prepare_allocation for real + * via its own #include "../src/plugin.c", so it is not the matching + * precedent for this pair together.) */ #include #include @@ -75,6 +98,7 @@ #include "../src/core_events.c" #include "../src/core_session.c" #include "../src/decision.c" +#include "../src/resource.c" #include "../src/snapshot.c" #include "../src/snapshot_load.c" #include "../src/snapshot_wire.c" @@ -87,11 +111,41 @@ void kill_sched_project(void* arg) { (void)arg; } void kill_sched_resval(xce_val_t* val) { (void)val; } void plugin_unregister(void* arg) { (void)arg; } -/* core_reset_workers()'s (core_session.c) linker dependency. UNLIKE - * test_core_full_cycle.c's own identical stub, this one IS actually - * invoked here (both core_a and core_b get real workers below) -- still a - * safe no-op, see file header comment. */ -void sched_worker_release_host_ctx(xce_sched_worker_t* w) { (void)w; } +/* core_reset_workers()'s (core_session.c) linker dependency -- REAL now + * (resource.c, #include'd above for task 3.1's own test), superseding what + * used to be this file's own no-op stub of the identical name (a duplicate + * definition would be a link error now that resource.c supplies the real + * one). Still behaviorally transparent to every pre-existing test in this + * file: every worker's host_eval_ctx starts NULL (calloc()'d), and + * free_host_eval_ctx(NULL) is a documented no-op (resource.c), so the real + * body performs exactly the same "no-op" every earlier test already relied + * on -- it is only genuinely exercised by task 3.1's own test below. + * + * resource.c's sched_try_alloc_on_host_() (never invoked by this file -- + * no test here drives live host allocation) still needs two of its own + * callees resolved at link time: plugin_prepare_allocation (plugin.c, not + * included here) and sched_ctx_get_uncomit_host (schedobj.c, not included + * here). Same two link-only, never-invoked stubs test_resource_accounting.c + * (and test.c) already establish needing for the identical reason -- + * test_sim_contention_parity.c is not this pair's precedent, since it only + * needs sched_ctx_get_uncomit_host itself (it gets plugin_prepare_allocation + * for real via its own #include "../src/plugin.c"). */ +int plugin_prepare_allocation(void* arg, xce_sched_ctx_t* ctx) +{ + (void)arg; + (void)ctx; + + return 0; +} + +int sched_ctx_get_uncomit_host(xvec_t** v, xce_sched_ctx_t* c, int i) +{ + (void)v; + (void)c; + (void)i; + + return 0; +} /* Real reimplementation of schedobj.c's sched_resolve_job_picker -- see * file header comment (same rationale/algorithm as @@ -1569,6 +1623,507 @@ test_snapshot_load_leaves_allocs_and_accounts_null_for_job_without_allocation( TEST_ASSERT_NULL(restored->accounts); } +/* ---------------------------------------------------------------------- */ +/* scheduler-snapshot-job-allocation task 3.1: the full cross-instance */ +/* round-trip proof -- everything task 2.2's own single-running-job test */ +/* (above) and task 1.2's own no-allocation schedobj tests (above) stop */ +/* short of, exercised together in ONE save/load round trip: */ +/* (1) a RUNNING job (multi-host, multi-resource, PLUS a genuine */ +/* shared-resource-pool component) saved TOGETHER with a PENDING job */ +/* (no allocation at all) in the SAME source core, loaded together */ +/* into a second, independent core; */ +/* (2) job->schedobj non-NULL for BOTH the running and the pending job; */ +/* (3) the restored allocation proven, via the REAL */ +/* sched_account_jobs()/sched_charge_account()/sched_update_account()*/ +/* chain (resource.c, white-box #include'd above), to feed the */ +/* restored core's worker resInUse AND shared-pool sharedInUse */ +/* totals to EXACTLY what the identical real chain produces on the */ +/* SOURCE core before saving -- not merely that job->accounts' own */ +/* fields look right (Req 1.2, 1.3's actual claim: "included in */ +/* in-use totals immediately" / "reported identically to normal */ +/* dispatch"); */ +/* (4) a REAL teardown of one core's job list (destroy(), the same RTTI */ +/* primitive xce_sched_core_fini() itself uses for its own kvmgrs --*/ +/* xce_sched_core_fini() itself is NOT callable against a bare- */ +/* xce_sched_core_init()'d fixture: core->pend_updates[0]/[1] are */ +/* only dataq_init()'d by scheduler.c's sched_init(), never by */ +/* xce_sched_core_init() itself, so dataq_fini() on them crashes -- */ +/* see test_scheduler_decision_redelivery.c's own header comment for */ +/* this exact, already-documented hazard) leaves the OTHER, */ +/* independently-alive core fully intact and usable afterward. */ +/* */ +/* sched_account_jobs() itself (scheduler.c) is not white-box #include'd */ +/* into this lean core-only TU (too heavy a dependency surface); this test */ +/* instead provides its own real, verbatim reimplementation of its exact */ +/* body -- the same "real reimplementation" technique this file already */ +/* uses for sched_resolve_job_picker() above, and the precedent */ +/* test_sim_contention_parity.c already establishes for this identical */ +/* function ("a no-op stub would defeat the test's own premise"). */ +/* ---------------------------------------------------------------------- */ + +/* Real reimplementation of scheduler.c's sched_account_jobs() -- verbatim + * body copy (see block comment above for why this file does not white-box + * #include scheduler.c itself). Calls the REAL sched_charge_account() + * (resource.c, #include'd above), so this genuinely exercises the same + * accounting chain live dispatch and sim_reconcile_unaccounted_jobs() both + * rely on downstream. */ +static int test_sched_account_jobs(xce_sched_core_t* core) +{ + iterator_t it; + int rc = 0; + + rc = iter_make(&it, core->jobs); + if (rc < 0) { + return rc; + } + + for (;;) { + xce_sched_job_t* job = NULL; + + rc = iter_next(&it, (void**)&job); + if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) { + break; + } + + if (!JOB_IS_RUN(job->state)) { + continue; + } + + if (job->accounts != NULL) { + sched_charge_account(core, job->accounts); + } + } + + if (rc == RTTI_ITER_NO_DATA) { + rc = 0; + } + + iter_destroy(&it); + + return rc; +} + +static void +test_snapshot_round_trip_mixed_jobs_shared_pool_resinuse_and_teardown_independence( + void) +{ + xce_sched_core_t core_a; + xce_sched_core_t core_b; + sched_snapshot_data_t snap; + xce_core_event_t events[3]; + xce_conf4sched_t config; + xce_resource_t res_slots; + xce_resource_t res_mem; + xce_resource_t res_license; + xce_resource_t* resources[3]; + xce_queue_t wire_queue; + xce_queue_t* queues[1]; + xce_worker_t wire_worker_a; + xce_worker_t wire_worker_b; + xce_worker_t* workers[2]; + xce_val_t worker_a_slots_val; + xce_val_t worker_a_mem_val; + xce_val_t worker_b_slots_val; + xce_name_val_t worker_a_slots_nv; + xce_name_val_t worker_a_mem_nv; + xce_name_val_t worker_b_slots_nv; + xce_name_val_t* worker_a_resvals[2]; + xce_name_val_t* worker_b_resvals[1]; + xce_job_t* job_running; + xce_job_t* job_pending; + xce_job_t* jobs[2]; + xce_sched_job_t* job_a_running; + xce_alloc_t* a_slots_hostA; + xce_alloc_t* a_mem_hostA; + xce_alloc_t* a_slots_hostB; + xce_alloc_t* a_license_shared; + xce_host_alloc_t* h0; + xce_host_alloc_t* h1; + xce_host_alloc_t** hallocs; + xce_job_alloc_t* jalloc; + xce_resource_t* slots_res_a; + xce_resource_t* slots_res_b; + xce_resource_t* mem_res_a; + xce_resource_t* mem_res_b; + xce_resource_t* license_res_a; + xce_resource_t* license_res_b; + xce_sched_worker_t* worker_a_a; + xce_sched_worker_t* worker_b_a; + xce_sched_worker_t* worker_a_b; + xce_sched_worker_t* worker_b_b; + xce_sched_job_t* restored_running; + xce_sched_job_t* restored_pending; + float source_hostA_slots = -1.0f; + float source_hostA_mem = -1.0f; + float source_hostB_slots = -1.0f; + float source_shared_lic = -1.0f; + float restored_hostA_slots = -1.0f; + float restored_hostA_mem = -1.0f; + float restored_hostB_slots = -1.0f; + float restored_shared_lic = -1.0f; + int rc; + + memset(&g_stub_state_a, 0, sizeof(g_stub_state_a)); + memset(&g_stub_state_b, 0, sizeof(g_stub_state_b)); + + /* =================================================================== + * Phase 1: build core_a -- register resources/queue/workers, then push + * ONE running job (destined for a multi-host, multi-resource, + * shared-pool allocation) TOGETHER with ONE pending job (no allocation + * at all), in the SAME JOBS_PUSH batch, into the SAME core -- the + * "mix in the same round trip" task 2.2's own single-job test never + * exercised. + * =================================================================== */ + fixture_init(&core_a, &g_stub_plugin_a); + + memset(&res_slots, 0, sizeof(res_slots)); + res_slots.name = "slots"; + memset(&res_mem, 0, sizeof(res_mem)); + res_mem.name = "mem"; + memset(&res_license, 0, sizeof(res_license)); + res_license.name = "license"; + resources[0] = &res_slots; + resources[1] = &res_mem; + resources[2] = &res_license; + + memset(&wire_queue, 0, sizeof(wire_queue)); + wire_queue.queue = "normal"; + wire_queue.priority = 1; + wire_queue.status = QUEUE_STAT_OPEN; + queues[0] = &wire_queue; + + memset(&config, 0, sizeof(config)); + config.n_resources = 3; + config.resources = resources; + config.n_queues = 1; + config.queues = queues; + + /* .which MUST be XCE_VAL_NUMERIC (bug-1497) -- dup_val()/get_resvals() + * both branch on it, and XCE_VAL_STRING is literally 0 (xce/types.h). */ + memset(&worker_a_slots_val, 0, sizeof(worker_a_slots_val)); + worker_a_slots_val.which = XCE_VAL_NUMERIC; + worker_a_slots_val.u.ivalue32 = 16; + memset(&worker_a_slots_nv, 0, sizeof(worker_a_slots_nv)); + worker_a_slots_nv.name = "slots"; + worker_a_slots_nv.val = &worker_a_slots_val; + + memset(&worker_a_mem_val, 0, sizeof(worker_a_mem_val)); + worker_a_mem_val.which = XCE_VAL_NUMERIC; + worker_a_mem_val.u.ivalue32 = 1000; + memset(&worker_a_mem_nv, 0, sizeof(worker_a_mem_nv)); + worker_a_mem_nv.name = "mem"; + worker_a_mem_nv.val = &worker_a_mem_val; + + worker_a_resvals[0] = &worker_a_slots_nv; + worker_a_resvals[1] = &worker_a_mem_nv; + + memset(&worker_b_slots_val, 0, sizeof(worker_b_slots_val)); + worker_b_slots_val.which = XCE_VAL_NUMERIC; + worker_b_slots_val.u.ivalue32 = 16; + memset(&worker_b_slots_nv, 0, sizeof(worker_b_slots_nv)); + worker_b_slots_nv.name = "slots"; + worker_b_slots_nv.val = &worker_b_slots_val; + + worker_b_resvals[0] = &worker_b_slots_nv; + + memset(&wire_worker_a, 0, sizeof(wire_worker_a)); + wire_worker_a.name = "hostA"; + wire_worker_a.status = 0; /* available */ + wire_worker_a.nresvals_to_sched = 2; + wire_worker_a.resvals_to_sched = worker_a_resvals; + + memset(&wire_worker_b, 0, sizeof(wire_worker_b)); + wire_worker_b.name = "hostB"; + wire_worker_b.status = 0; + wire_worker_b.nresvals_to_sched = 1; + wire_worker_b.resvals_to_sched = worker_b_resvals; + + workers[0] = &wire_worker_a; + workers[1] = &wire_worker_b; + + job_running = make_job(701, JOB_STAT_PEND, "normal", "alice", "proj1"); + job_pending = make_job(702, JOB_STAT_PEND, "normal", "bob", "proj1"); + jobs[0] = job_running; + jobs[1] = job_pending; + + memset(events, 0, sizeof(events)); + /* CONFIG_PUSH first (registers slots/mem/license into core_a->resmgr, + * which WORKERS_PUSH needs already present to resolve resvals_to_sched + * by name), then WORKERS_PUSH, then JOBS_PUSH -- same ordering the + * cross-core equivalence test above already establishes. */ + events[0].type = XCE_CORE_EVENT_CONFIG_PUSH; + events[0].u.config_push.config = &config; + events[1].type = XCE_CORE_EVENT_WORKERS_PUSH; + events[1].u.workers_push.nworkers = 2; + events[1].u.workers_push.workers = workers; + events[2].type = XCE_CORE_EVENT_JOBS_PUSH; + events[2].u.jobs_push.njobs = 2; + events[2].u.jobs_push.jobs = jobs; + + rc = xce_core_push_events(&core_a, events, 3); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(2, g_stub_state_a.ncandidates); + + /* =================================================================== + * Phase 2: promote job 701 to RUNNING with a real multi-host (hostA: + * slots+mem, hostB: slots), multi-resource allocation PLUS a genuine + * shared-resource-pool component (nres_shared/allocs_shared, "license") + * -- the direct-field-poke convention task 2.2's own test above already + * uses (no wire event exists to dispatch a job outside a real + * scheduling session in this test harness). Job 702 is left PENDING + * with no allocation at all. + * =================================================================== */ + job_a_running = sched_get_job(&core_a, 701); + TEST_ASSERT_NOT_NULL(job_a_running); + + a_slots_hostA = make_alloc((char*)"slots", NULL, 0, 2.0f, 0); + a_mem_hostA = make_alloc((char*)"mem", NULL, 0, 100.0f, 0); + a_slots_hostB = make_alloc((char*)"slots", NULL, 0, 3.0f, 0); + TEST_ASSERT_NOT_NULL(a_slots_hostA); + TEST_ASSERT_NOT_NULL(a_mem_hostA); + TEST_ASSERT_NOT_NULL(a_slots_hostB); + + h0 = make_host_alloc((char*)"hostA", 2, + make_allocs(2, a_slots_hostA, a_mem_hostA)); + h1 = make_host_alloc((char*)"hostB", 1, make_allocs(1, a_slots_hostB)); + TEST_ASSERT_NOT_NULL(h0); + TEST_ASSERT_NOT_NULL(h1); + + hallocs = make_host_allocs(2, h0, h1); + TEST_ASSERT_NOT_NULL(hallocs); + + jalloc = make_job_alloc(701, 2, hallocs); + TEST_ASSERT_NOT_NULL(jalloc); + + a_license_shared = make_alloc((char*)"license", NULL, 0, 1.0f, 0); + TEST_ASSERT_NOT_NULL(a_license_shared); + jalloc->nres_shared = 1; + jalloc->allocs_shared = make_allocs(1, a_license_shared); + TEST_ASSERT_NOT_NULL(jalloc->allocs_shared); + + job_a_running->state = JOB_STAT_RUN; + job_a_running->allocs = jalloc; + /* make_job_account() is the SAME constructor the real dispatch path + * uses -- calling it directly here, on the SOURCE core, puts core_a in + * exactly the accounted state a normal live dispatch would have left + * it in at the moment of saving, which is what makes the resInUse + * comparison in Phase 3/8 below a meaningful "reported identically to + * normal dispatch" proof (Req 1.3), not a trivial self-comparison. */ + job_a_running->accounts = make_job_account(job_a_running->allocs); + TEST_ASSERT_NOT_NULL(job_a_running->accounts); + + /* =================================================================== + * Phase 3: the actual proof point (Req 1.2/1.3) -- call the REAL + * sched_account_jobs() chain (test_sched_account_jobs() above, which + * calls the REAL sched_charge_account(), resource.c) on core_a BEFORE + * saving, capturing what "resInUse/sharedInUse, reported the normal + * way" looks like for the source instance. + * =================================================================== */ + rc = test_sched_account_jobs(&core_a); + TEST_ASSERT_EQUAL_INT(0, rc); + + worker_a_a = sched_get_worker(&core_a, "hostA"); + worker_b_a = sched_get_worker(&core_a, "hostB"); + TEST_ASSERT_NOT_NULL(worker_a_a); + TEST_ASSERT_NOT_NULL(worker_b_a); + + rc = key_find(core_a.resmgr, (void*)"slots", (void**)&slots_res_a); + TEST_ASSERT_GREATER_THAN_INT(0, rc); + rc = key_find(core_a.resmgr, (void*)"mem", (void**)&mem_res_a); + TEST_ASSERT_GREATER_THAN_INT(0, rc); + rc = key_find(core_a.resmgr, (void*)"license", (void**)&license_res_a); + TEST_ASSERT_GREATER_THAN_INT(0, rc); + TEST_ASSERT_NOT_NULL(slots_res_a); + TEST_ASSERT_NOT_NULL(mem_res_a); + TEST_ASSERT_NOT_NULL(license_res_a); + + get_resvals(&(worker_a_a->resInUse), 1, &(slots_res_a->idx), + &source_hostA_slots); + get_resvals(&(worker_a_a->resInUse), 1, &(mem_res_a->idx), + &source_hostA_mem); + get_resvals(&(worker_b_a->resInUse), 1, &(slots_res_a->idx), + &source_hostB_slots); + get_resvals(&(core_a.sharedInUse), 1, &(license_res_a->idx), + &source_shared_lic); + + TEST_ASSERT_EQUAL_FLOAT(2.0f, source_hostA_slots); + TEST_ASSERT_EQUAL_FLOAT(100.0f, source_hostA_mem); + TEST_ASSERT_EQUAL_FLOAT(3.0f, source_hostB_slots); + TEST_ASSERT_EQUAL_FLOAT(1.0f, source_shared_lic); + + /* =================================================================== + * Phase 4: save core_a's snapshot -- both jobs, mixed running/pending, + * saved together in the same batch. + * =================================================================== */ + memset(&snap, 0, sizeof(snap)); + rc = xce_sched_core_save_snapshot(&core_a, &snap); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_NOT_NULL(snap.jobs); + TEST_ASSERT_EQUAL_INT(2, snap.jobs->count); + + /* =================================================================== + * Phase 5: load into core_b -- a completely separate, independently- + * alive instance. + * =================================================================== */ + fixture_init(&core_b, &g_stub_plugin_b); + + rc = xce_sched_core_load_snapshot(&core_b, &snap); + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_EQUAL_INT(2, g_stub_state_b.ncandidates); + + restored_running = sched_get_job(&core_b, 701); + restored_pending = sched_get_job(&core_b, 702); + TEST_ASSERT_NOT_NULL(restored_running); + TEST_ASSERT_NOT_NULL(restored_pending); + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, restored_running->state); + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, restored_pending->state); + + /* =================================================================== + * Phase 6: EVERY restored job -- running AND pending -- gets a valid, + * non-NULL schedobj (Req 2.1-2.4) -- the "every job" claim task 1.2's + * own tests never exercised alongside a real allocation in the same + * core. + * =================================================================== */ + TEST_ASSERT_NOT_NULL(restored_running->schedobj); + TEST_ASSERT_EQUAL_INT(XCE_RESREQ_PRIMARY, restored_running->schedobj->tag); + TEST_ASSERT_NOT_NULL(restored_running->schedobj->u.primary); + TEST_ASSERT_NOT_NULL(restored_pending->schedobj); + TEST_ASSERT_EQUAL_INT(XCE_RESREQ_PRIMARY, restored_pending->schedobj->tag); + TEST_ASSERT_NOT_NULL(restored_pending->schedobj->u.primary); + + /* =================================================================== + * Phase 7: allocation/accounting content matches, host-for-host, + * resource-for-resource, INCLUDING the shared-pool component. Job 702 + * (pending) restored with allocs/accounts left NULL -- the normal, + * non-error case (Req 3.5). + * =================================================================== */ + TEST_ASSERT_NOT_NULL(restored_running->allocs); + TEST_ASSERT_EQUAL_UINT64(701, restored_running->allocs->jobid); + TEST_ASSERT_EQUAL_INT(2, restored_running->allocs->nhosts); + TEST_ASSERT_EQUAL_STRING("hostA", restored_running->allocs->allocs[0]->host); + TEST_ASSERT_EQUAL_STRING("hostB", restored_running->allocs->allocs[1]->host); + TEST_ASSERT_EQUAL_INT(1, restored_running->allocs->nres_shared); + TEST_ASSERT_NOT_NULL(restored_running->allocs->allocs_shared); + TEST_ASSERT_EQUAL_STRING("license", + restored_running->allocs->allocs_shared[0]->resname); + TEST_ASSERT_EQUAL_FLOAT( + 1.0f, (float)restored_running->allocs->allocs_shared[0]->value); + TEST_ASSERT_TRUE(restored_running->allocs->allocs_shared[0] + != a_license_shared); + + TEST_ASSERT_NOT_NULL(restored_running->accounts); + TEST_ASSERT_EQUAL_INT(2, restored_running->accounts->nhosts); + TEST_ASSERT_EQUAL_INT(1, restored_running->accounts->nres_shared); + TEST_ASSERT_NOT_NULL(restored_running->accounts->accounts_shared); + TEST_ASSERT_EQUAL_FLOAT( + 1.0f, (float)restored_running->accounts->accounts_shared[0]->allocated); + + TEST_ASSERT_NULL(restored_pending->allocs); + TEST_ASSERT_NULL(restored_pending->accounts); + + /* Req 1.4 independence -- three genuinely distinct pointers throughout + * the chain (source live job, saved wire job, restored job), same + * technique task 2.2's own test above establishes. */ + TEST_ASSERT_TRUE(restored_running->allocs != job_a_running->allocs); + TEST_ASSERT_TRUE(restored_running->accounts != job_a_running->accounts); + TEST_ASSERT_TRUE(snap.jobs->jobs[0]->allocs != job_a_running->allocs); + TEST_ASSERT_TRUE(restored_running->allocs != snap.jobs->jobs[0]->allocs); + + /* =================================================================== + * Phase 8: the actual proof (Req 1.2/1.3) -- call the SAME real + * accounting chain on core_b (the RESTORED instance) and confirm its + * resInUse/sharedInUse totals EXACTLY match what the identical chain + * produced on core_a (Phase 3) before saving -- proving the restored + * allocation is "included in in-use totals immediately" and "reported + * identically to ... normal dispatch" (Req 1.2, 1.3), not merely that + * job->accounts' own fields happen to look right. + * =================================================================== */ + rc = test_sched_account_jobs(&core_b); + TEST_ASSERT_EQUAL_INT(0, rc); + + worker_a_b = sched_get_worker(&core_b, "hostA"); + worker_b_b = sched_get_worker(&core_b, "hostB"); + TEST_ASSERT_NOT_NULL(worker_a_b); + TEST_ASSERT_NOT_NULL(worker_b_b); + + rc = key_find(core_b.resmgr, (void*)"slots", (void**)&slots_res_b); + TEST_ASSERT_GREATER_THAN_INT(0, rc); + rc = key_find(core_b.resmgr, (void*)"mem", (void**)&mem_res_b); + TEST_ASSERT_GREATER_THAN_INT(0, rc); + rc = key_find(core_b.resmgr, (void*)"license", (void**)&license_res_b); + TEST_ASSERT_GREATER_THAN_INT(0, rc); + TEST_ASSERT_NOT_NULL(slots_res_b); + TEST_ASSERT_NOT_NULL(mem_res_b); + TEST_ASSERT_NOT_NULL(license_res_b); + + get_resvals(&(worker_a_b->resInUse), 1, &(slots_res_b->idx), + &restored_hostA_slots); + get_resvals(&(worker_a_b->resInUse), 1, &(mem_res_b->idx), + &restored_hostA_mem); + get_resvals(&(worker_b_b->resInUse), 1, &(slots_res_b->idx), + &restored_hostB_slots); + get_resvals(&(core_b.sharedInUse), 1, &(license_res_b->idx), + &restored_shared_lic); + + TEST_ASSERT_EQUAL_FLOAT(source_hostA_slots, restored_hostA_slots); + TEST_ASSERT_EQUAL_FLOAT(source_hostA_mem, restored_hostA_mem); + TEST_ASSERT_EQUAL_FLOAT(source_hostB_slots, restored_hostB_slots); + TEST_ASSERT_EQUAL_FLOAT(source_shared_lic, restored_shared_lic); + + /* spot-pin the concrete expected values too, not just A==B equality. */ + TEST_ASSERT_EQUAL_FLOAT(2.0f, restored_hostA_slots); + TEST_ASSERT_EQUAL_FLOAT(100.0f, restored_hostA_mem); + TEST_ASSERT_EQUAL_FLOAT(3.0f, restored_hostB_slots); + TEST_ASSERT_EQUAL_FLOAT(1.0f, restored_shared_lic); + + /* =================================================================== + * Phase 9: teardown safety (Req 1.4's "later activity or shutdown in + * either instance does not corrupt or crash the other") -- a REAL + * cleanup path on core_a's job list: destroy(), the same generic RTTI + * teardown primitive xce_sched_core_fini() itself uses for + * resmgr/ugmgr/qgmgr/pgmgr/hgmgr (core.c), applied here directly to + * core_a.jobs. (xce_sched_core_fini() itself is deliberately NOT called + * here -- core_a was built via bare xce_sched_core_init(), and + * core->pend_updates[0]/[1] are only dataq_init()'d by scheduler.c's + * sched_init(), never by xce_sched_core_init() itself, so + * xce_sched_core_fini()'s own dataq_fini() calls would dereference + * still-NULL pointers and crash -- test_scheduler_decision_redelivery.c + * already documents this exact hazard for the identical minimal-fixture + * technique this file also uses.) core_b must remain fully intact and + * usable afterward -- not merely non-crashing: every value re-read + * below must still match Phase 8's, proving core_b's lifetime is + * genuinely independent of core_a's, not merely pointer-distinct. + * =================================================================== */ + destroy(core_a.jobs); + + TEST_ASSERT_NOT_NULL(sched_get_job(&core_b, 701)); + TEST_ASSERT_NOT_NULL(sched_get_job(&core_b, 702)); + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, sched_get_job(&core_b, 701)->state); + TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, + sched_get_job(&core_b, 702)->state); + TEST_ASSERT_NOT_NULL(sched_get_job(&core_b, 701)->schedobj); + TEST_ASSERT_NOT_NULL(sched_get_job(&core_b, 702)->schedobj); + + worker_a_b = sched_get_worker(&core_b, "hostA"); + worker_b_b = sched_get_worker(&core_b, "hostB"); + TEST_ASSERT_NOT_NULL(worker_a_b); + TEST_ASSERT_NOT_NULL(worker_b_b); + + get_resvals(&(worker_a_b->resInUse), 1, &(slots_res_b->idx), + &restored_hostA_slots); + get_resvals(&(worker_a_b->resInUse), 1, &(mem_res_b->idx), + &restored_hostA_mem); + get_resvals(&(worker_b_b->resInUse), 1, &(slots_res_b->idx), + &restored_hostB_slots); + get_resvals(&(core_b.sharedInUse), 1, &(license_res_b->idx), + &restored_shared_lic); + + TEST_ASSERT_EQUAL_FLOAT(2.0f, restored_hostA_slots); + TEST_ASSERT_EQUAL_FLOAT(100.0f, restored_hostA_mem); + TEST_ASSERT_EQUAL_FLOAT(3.0f, restored_hostB_slots); + TEST_ASSERT_EQUAL_FLOAT(1.0f, restored_shared_lic); +} + int main(void) { UNITY_BEGIN(); @@ -1589,5 +2144,7 @@ int main(void) test_snapshot_load_restores_allocs_and_accounts_for_running_job_across_cores); RUN_TEST( test_snapshot_load_leaves_allocs_and_accounts_null_for_job_without_allocation); + RUN_TEST( + test_snapshot_round_trip_mixed_jobs_shared_pool_resinuse_and_teardown_independence); return UNITY_END(); }